Skip to content

brainfood / k3s-apiserver

Go to a project
Toggle navigation
Toggle navigation pinning
f3a5070a by Adam Heath
Options

Add ability to set the number of agent nodes.

1 parent 5eeee2dd
Inline Side-by-side
Showing 3 changed files with 88 additions and 58 deletions
1 version: '3.4'
2
3 x-extra-hosts: &_x_extra_hosts
4 extra_hosts:
5 - "${APP_REGISTRY_NAME}:${APP_REGISTRY_ADDRESS}"
6
7 x-root-cert-volume: &_x-root-cert-volume
8 type: bind
9 source: ${CONTEXT_DIR}/certs/root.crt
10 target: /etc/ssl/certs/root.crt
11 read_only: true
12 bind:
13 create_host_path: false
14
15 x-kubelet-volume: &_x-kubelet-volume
16 type: volume
17 target: /var/lib/kubelet
18 # volume:
19 # propagation: shared
20
21 x-k3s-agent-env: &_x-k3s-agent-env
22 K3S_URL: https://k3s-master:6443
23 K3S_TOKEN_FILE: /var/lib/rancher/k3s/server/node-token
24 K3S_NODE_NAME: k3s-agent
25 VIRTUAL_HOST: ${VHOST_STUB},*${VHOST_SUFFIX}${APP_EXTRA_VHOSTS}
26 VIRTUAL_PROTO: https
27 VIRTUAL_PORT: "443"
28 SELF_SIGNED_HOST: ${VHOST_STUB},*${VHOST_SUFFIX}${APP_EXTRA_VHOSTS}
29 HTTPS_METHOD: noredirect
30
31 x-k3s-agent-base: &_x-k3s-agent-base
32 <<: *_x_extra_hosts
33 image: "docker.io/rancher/k3s:${K3S_VERSION:-latest}"
34 tmpfs:
35 - /run
36 - /var/run
37 ulimits:
38 nproc: 65535
39 nofile:
40 soft: 65535
41 hard: 65535
42 volumes:
43 - ${APP_ROOT_DIR?Please set APP_ROOT_DIR}:${APP_ROOT_MOUNT?Please specify where to mount $PWD}
44 - *_x-root-cert-volume
45 - *_x-kubelet-volume
46 - server:/var/lib/rancher/k3s/server:ro
47 - local-path-provisioner:/opt/local-path-provisioner
48 privileged: true
49 restart: always
50 networks:
51 default:
52 nginx:
53 ports:
54 - 443
55 environment:
56 <<: *_x-k3s-agent-env
57
58 services:
59 k3s-agent-1:
60 <<: *_x-k3s-agent-base
61 command: [
62 "agent",
63 ]
64 environment:
65 <<: *_x-k3s-agent-env
66 K3S_NODE_NAME: agent-1
67
...@@ -77,44 +77,6 @@ x-k3s-master-base: &_x-k3s-master-base ...@@ -77,44 +77,6 @@ x-k3s-master-base: &_x-k3s-master-base
77 - *_x-root-cert-volume 77 - *_x-root-cert-volume
78 - *_x-kubelet-volume 78 - *_x-kubelet-volume
79 79
80 x-k3s-agent-env: &_x-k3s-agent-env
81 K3S_URL: https://k3s-master:6443
82 K3S_TOKEN_FILE: /var/lib/rancher/k3s/server/node-token
83 K3S_NODE_NAME: k3s-agent
84 VIRTUAL_HOST: ${VHOST_STUB},*${VHOST_SUFFIX}${APP_EXTRA_VHOSTS}
85 VIRTUAL_PROTO: https
86 VIRTUAL_PORT: "443"
87 SELF_SIGNED_HOST: ${VHOST_STUB},*${VHOST_SUFFIX}${APP_EXTRA_VHOSTS}
88 HTTPS_METHOD: noredirect
89
90 x-k3s-agent-base: &_x-k3s-agent-base
91 <<: *_x_extra_hosts
92 image: "docker.io/rancher/k3s:${K3S_VERSION:-latest}"
93 tmpfs:
94 - /run
95 - /var/run
96 ulimits:
97 nproc: 65535
98 nofile:
99 soft: 65535
100 hard: 65535
101 volumes:
102 - ${APP_ROOT_DIR?Please set APP_ROOT_DIR}:${APP_ROOT_MOUNT?Please specify where to mount $PWD}
103 # - *_x-registries-yaml-volume
104 - *_x-root-cert-volume
105 - *_x-kubelet-volume
106 - server:/var/lib/rancher/k3s/server:ro
107 - local-path-provisioner:/opt/local-path-provisioner
108 privileged: true
109 restart: always
110 networks:
111 default:
112 nginx:
113 ports:
114 - 443
115 environment:
116 <<: *_x-k3s-agent-env
117
118 x-coredns-base: &_x-coredns-base 80 x-coredns-base: &_x-coredns-base
119 image: docker.io/coredns/coredns 81 image: docker.io/coredns/coredns
120 command: ['-conf', '/etc/coredns/Corefile'] 82 command: ['-conf', '/etc/coredns/Corefile']
...@@ -212,24 +174,6 @@ services: ...@@ -212,24 +174,6 @@ services:
212 k3s-coredns-3: 174 k3s-coredns-3:
213 <<: *_x-coredns-base 175 <<: *_x-coredns-base
214 176
215 k3s-agent-1:
216 <<: *_x-k3s-agent-base
217 command: [
218 "agent",
219 ]
220 environment:
221 <<: *_x-k3s-agent-env
222 K3S_NODE_NAME: agent-1
223
224 k3s-agent-2:
225 <<: *_x-k3s-agent-base
226 command: [
227 "agent",
228 ]
229 environment:
230 <<: *_x-k3s-agent-env
231 K3S_NODE_NAME: agent-2
232
233 volumes: 177 volumes:
234 etcd1-data: 178 etcd1-data:
235 etcd2-data: 179 etcd2-data:
......
...@@ -13,6 +13,7 @@ declare -A features=( ...@@ -13,6 +13,7 @@ declare -A features=(
13 ) 13 )
14 declare -a compose_files=(-f "$APISERVER_DIR/docker-compose.yaml") 14 declare -a compose_files=(-f "$APISERVER_DIR/docker-compose.yaml")
15 declare -a k8s_nodes=() 15 declare -a k8s_nodes=()
16 declare -i agent_count=2
16 17
17 while [[ $# -gt 0 ]]; do 18 while [[ $# -gt 0 ]]; do
18 arg="$1" 19 arg="$1"
...@@ -34,6 +35,10 @@ while [[ $# -gt 0 ]]; do ...@@ -34,6 +35,10 @@ while [[ $# -gt 0 ]]; do
34 compose_files+=(-f "$1") 35 compose_files+=(-f "$1")
35 shift 36 shift
36 ;; 37 ;;
38 (--agent-count)
39 agent_count="$1"
40 shift
41 ;;
37 (-n) 42 (-n)
38 k8s_nodes+=("$1") 43 k8s_nodes+=("$1")
39 shift 44 shift
...@@ -43,6 +48,11 @@ while [[ $# -gt 0 ]]; do ...@@ -43,6 +48,11 @@ while [[ $# -gt 0 ]]; do
43 ;; 48 ;;
44 esac 49 esac
45 done 50 done
51
52 _create_agent_compose() {
53 sed -e "s/agent-1/agent-$1/g" < "$APISERVER_DIR/agent-compose.yaml"
54 }
55
46 set -- "${args[@]}" 56 set -- "${args[@]}"
47 57
48 for feature in "${!features[@]}"; do 58 for feature in "${!features[@]}"; do
...@@ -62,8 +72,17 @@ declare -a animations=('-' '\' '|' '/') ...@@ -62,8 +72,17 @@ declare -a animations=('-' '\' '|' '/')
62 declare -i animation_index=0 72 declare -i animation_index=0
63 73
64 _compose() { 74 _compose() {
65 docker-compose --project-directory "$CONTEXT_DIR" "${compose_files[@]}" "$@" 75 declare -a agent_files
76 declare -i agent_index=0
77 for ((agent_index=1; agent_index < $(($agent_count + 1)); agent_index++)); do
78 agent_files+=("-f <(_create_agent_compose $agent_index)")
79 done
80 eval docker-compose --project-directory "$CONTEXT_DIR" "${compose_files[@]}" "${agent_files[@]}" "$@"
66 } 81 }
82 declare -i agent_index=0
83 for ((agent_index=1; agent_index < $(($agent_count + 1)); agent_index++)); do
84 k8s_nodes+=(k3s-agent-$agent_index)
85 done
67 86
68 etcdctl() { 87 etcdctl() {
69 _compose exec etcd1 etcdctl "$@" 88 _compose exec etcd1 etcdctl "$@"
...@@ -192,7 +211,7 @@ case "$cmd" in ...@@ -192,7 +211,7 @@ case "$cmd" in
192 211
193 _compose up -d k3s-coredns-1 k3s-coredns-2 k3s-coredns-3 212 _compose up -d k3s-coredns-1 k3s-coredns-2 k3s-coredns-3
194 "$APISERVER_DIR/scripts/install-cluster-dns.sh" "$CONTEXT_DIR" 213 "$APISERVER_DIR/scripts/install-cluster-dns.sh" "$CONTEXT_DIR"
195 _compose up -d k3s-agent-1 k3s-agent-2 "${k8s_nodes[@]}" 214 _compose up -d "${k8s_nodes[@]}"
196 _compose up -d k3s-master-2 k3s-master-3 215 _compose up -d k3s-master-2 k3s-master-3
197 #"$APISERVER_DIR/scripts/wait-for-system-pods.sh" 1 216 #"$APISERVER_DIR/scripts/wait-for-system-pods.sh" 1
198 _wait_for_system 217 _wait_for_system
......