Skip to content

brainfood / k3s-apiserver

Go to a project
Toggle navigation
Toggle navigation pinning
739a52bd by Adam Heath
Options

Importing initial set of files.

1 parent 35a09488
Inline Side-by-side
Showing 21 changed files with 572 additions and 0 deletions
1 .*.sw?
2 /certs/root.crt
3 /certs/root.key
4 /certs/root.srl
5 /certs/registry.crt
6 /certs/registry.csr
7 /certs/registry.key
1 ---
2 releases:
3 - name: cert-manager
4 chart: .
5 wait: true
6 ---
1 apiVersion: kustomize.config.k8s.io/v1beta1
2 kind: Kustomization
3
4 resources:
5 - https://github.com/jetstack/cert-manager/releases/download/v1.0.4/cert-manager.yaml
6
7 generatorOptions:
8 disableNameSuffixHash: true
9 secretGenerator:
10 - name: root-ca
11 namespace: cert-manager
12 files:
13 - tls.crt=../certs/root.crt
14 - tls.key=../certs/root.key
15 - ca.crt=../certs/root.crt
16 - ca.key=../certs/root.key
17
1 ---
2 apiVersion: cert-manager.io/v1
3 kind: ClusterIssuer
4 metadata:
5 namespace: cert-manager
6 name: ca-issuer
7 spec:
8 ca:
9 secretName: root-ca
10 ---
11
1 ---
2 releases:
3 - name: cluster-issuer
4 namespace: cert-manager
5 chart: .
6 wait: true
7 ---
1 apiVersion: kustomize.config.k8s.io/v1beta1
2 kind: Kustomization
3
4 resources:
5 - ./cluster-issuer.yaml
6
1 version: '2.4'
2
3 x-etcd-environment: &_x-etcd-environment
4 ALLOW_NONE_AUTHENTICATION: "yes"
5 ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380
6 ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
7 ETCD_INITIAL_CLUSTER_TOKEN: etcd-cluster
8 ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380
9 ETCD_INITIAL_CLUSTER_STATE: new
10 ETCD_DATA_DIR: /bitnami/etcd/data/db
11 ETCD_WAL_DIR: /bitnami/etcd/data/wal
12
13 x-etcd-base: &_x-etcd-base
14 image: docker.io/bitnami/etcd:3
15 entrypoint: ["/etcd-entrypoint.sh", "/opt/bitnami/scripts/etcd/entrypoint.sh"]
16 command: ["/opt/bitnami/scripts/etcd/run.sh"]
17 user: root
18
19 x-k3s-master-base: &_x-k3s-master-base
20 image: "docker.io/rancher/k3s:${K3S_VERSION:-latest}"
21 networks:
22 default:
23 aliases:
24 - k3s-master
25 tmpfs:
26 - /run
27 - /var/run
28 ulimits:
29 nproc: 65535
30 nofile:
31 soft: 65535
32 hard: 65535
33 privileged: true
34 restart: always
35 ports:
36 - 6443
37 environment:
38 - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml
39 - K3S_KUBECONFIG_MODE=666
40 - K3S_NODE_NAME=master
41 volumes:
42 - server:/var/lib/rancher/k3s/server
43 - output:/output
44 - ./etc/registries.yaml:/etc/rancher/k3s/registries.yaml:ro
45 - ./certs/root.crt:/etc/ssl/certs/root.crt:ro
46 - ./certs/registry.crt:/etc/ssl/certs/registry.crt:ro
47 - .:${APP_ROOT_MOUNT?Please set APP_ROOT_MOUNT(where to mount $PWD)}
48
49 x-k3s-agent-base: &_x-k3s-agent-base
50 image: "docker.io/rancher/k3s:${K3S_VERSION:-latest}"
51 tmpfs:
52 - /run
53 - /var/run
54 ulimits:
55 nproc: 65535
56 nofile:
57 soft: 65535
58 hard: 65535
59 volumes:
60 - .:${APP_ROOT_MOUNT?Please specify where to mount $PWD}
61 - ./etc/registries.yaml:/etc/rancher/k3s/registries.yaml:ro
62 - ./certs/root.crt:/etc/ssl/certs/root.crt:ro
63 - ./certs/registry.crt:/etc/ssl/certs/registry.crt:ro
64 - server:/var/lib/rancher/k3s/server:ro
65 privileged: true
66 restart: always
67 networks:
68 default:
69 nginx:
70 environment:
71 - K3S_URL=https://k3s-master:6443
72 - K3S_TOKEN_FILE=/var/lib/rancher/k3s/server/node-token
73 - K3S_NODE_NAME=k3s-agent
74 - VIRTUAL_HOST=${VHOST_STUB},*${VHOST_SUFFIX}
75 - VIRTUAL_PROTO=https
76 - VIRTUAL_PORT=443
77 - SELF_SIGNED_HOST=${VHOST_STUB},*${VHOST_SUFFIX}
78 - HTTPS_METHOD=noredirect
79
80 x-coredns-base: &_x-coredns-base
81 image: docker.io/coredns/coredns
82 command: ['-conf', '/etc/coredns/Corefile']
83 restart: always
84 volumes:
85 - server:/var/lib/rancher/k3s/server
86 - output:/output
87 - ./etc/coredns:/etc/coredns:ro
88
89 networks:
90 default:
91 nginx:
92 external:
93 name: nginx
94
95 services:
96 etcd1:
97 <<: *_x-etcd-base
98 environment:
99 <<: *_x-etcd-environment
100 ETCD_NAME: etcd1
101 ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd1:2380
102 ETCD_ADVERTISE_CLIENT_URLS: http://etcd1:2379
103 volumes:
104 - ./scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
105 - etcd1-data:/bitnami/etcd/data
106
107 etcd2:
108 <<: *_x-etcd-base
109 environment:
110 <<: *_x-etcd-environment
111 ETCD_NAME: etcd2
112 ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd2:2380
113 ETCD_ADVERTISE_CLIENT_URLS: http://etcd2:2379
114 volumes:
115 - ./scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
116 - etcd2-data:/bitnami/etcd/data
117
118 etcd3:
119 <<: *_x-etcd-base
120 environment:
121 <<: *_x-etcd-environment
122 ETCD_NAME: etcd3
123 ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd3:2380
124 ETCD_ADVERTISE_CLIENT_URLS: http://etcd3:2379
125 volumes:
126 - ./scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
127 - etcd3-data:/bitnami/etcd/data
128
129 k3s-master-1:
130 <<: *_x-k3s-master-base
131 command: [
132 "server",
133 "--with-node-id",
134 "--disable=traefik,coredns",
135 "--node-taint", "master=true:NoSchedule",
136 "--datastore-endpoint=http://etcd1:2379",
137 "--cluster-init",
138 ]
139
140 k3s-master-2:
141 <<: *_x-k3s-master-base
142 command: [
143 "server",
144 "--with-node-id",
145 "--disable=traefik,coredns",
146 "--node-taint", "master=true:NoSchedule",
147 "--datastore-endpoint=http://etcd2:2379",
148 "--server=http://k3s-master-1:6443",
149 ]
150
151 k3s-master-3:
152 <<: *_x-k3s-master-base
153 command: [
154 "server",
155 "--with-node-id",
156 "--disable=traefik,coredns",
157 "--node-taint", "master=true:NoSchedule",
158 "--datastore-endpoint=http://etcd3:2379",
159 "--server=http://k3s-master-1:6443",
160 ]
161
162 k3s-coredns-1:
163 <<: *_x-coredns-base
164
165 k3s-coredns-2:
166 <<: *_x-coredns-base
167
168 k3s-coredns-3:
169 <<: *_x-coredns-base
170
171 k3s-agent-1:
172 <<: *_x-k3s-agent-base
173 command: [
174 "agent",
175 "--with-node-id",
176 ]
177
178 k3s-agent-2:
179 <<: *_x-k3s-agent-base
180 command: [
181 "agent",
182 "--with-node-id",
183 ]
184
185 registry:
186 image: registry:2
187 networks:
188 default:
189 aliases:
190 - ${REGISTRY_ID?Please set REGISTRY_ID}.registry
191 nginx:
192 ports:
193 - 443
194 volumes:
195 - registry:/var/lib/registry
196 - ./certs/registry.crt:/certs/registry.crt:ro
197 - ./certs/registry.key:/certs/registry.key:ro
198
199 environment:
200 - VIRTUAL_HOST=${REGISTRY_ID?Please set REGISTRY_ID}.registry
201 - VIRTUAL_PORT=443
202 - VIRTUAL_PROTO=https
203 - HTTPS_METHOD=noredirect
204 - CERT_NAME=default
205 - REGISTRY_HTTP_ADDR=0.0.0.0:443
206 - REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt
207 - REGISTRY_HTTP_TLS_KEY=/certs/registry.key
208
209 volumes:
210 etcd1-data:
211 etcd2-data:
212 etcd3-data:
213 server: {}
214 output: {}
215 registry:
216 external: true
217 name: ${REGISTRY_VOLUME_NAME?Please set REGISTRY_VOLUME_NAME}
218
1 .:53 {
2 errors
3 health {
4 lameduck 5s
5 }
6 ready
7 kubernetes cluster.local in-addr.arpa ip6.arpa {
8 tls /var/lib/rancher/k3s/server/tls/client-admin.crt /var/lib/rancher/k3s/server/tls/client-admin.key /var/lib/rancher/k3s/server/tls/server-ca.crt
9 #kubeconfig /output/kubeconfig.yaml
10 endpoint https://k3s-master:6443
11 pods insecure
12 fallthrough in-addr.arpa ip6.arpa
13 ttl 30
14 }
15 # hosts /etc/coredns/NodeHosts {
16 # ttl 60
17 # reload 15s
18 # fallthrough
19 # }
20 prometheus :9153
21 forward . /etc/resolv.conf
22 cache 30
23 loop
24 reload
25 loadbalance
26 }
27
1 mirrors:
2 "registry.uniquely-me.local":
3 endpoint:
4 - https://registry.uniquely-me.local
5
6 configs:
7 registry.uniquely-me.local:
8 tls:
9 ca_file: "/etc/ssl/certs/registry.crt"
1 subjectAltName = @alt_names
2
3 [alt_names]
4 DNS.1 = registry.uniquely-me.local
5 DNS.2 = registry.uniquely.me
1 [req]
2 default_bits = 2048
3 default_keyfile = registry.key
4 distinguished_name = req_distinguished_name
5 req_extensions = req_ext
6 prompt = no
7 encrypt_key = no
8
9 [req_distinguished_name]
10 countryName = US
11 stateOrProvinceName = Texas
12 localityName = Dallas
13 organizationName = UNIQUELY ME
14 organizationalUnitName = IT
15 commonName = registry.uniquely-me.local
16
17 [req_ext]
18 subjectAltName = @alt_names
19
20 [alt_names]
21 DNS.1 = registry.uniquely-me.local
22 DNS.2 = registry.uniquely.me
1 helmfiles:
2 - cert-manager/helmfile.yaml
3 - cluster-issuer/helmfile.yaml
4
1 #!/bin/sh
2
3 set -e
4
5 TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
6 export TOP_DIR
7
8 mkdir -p "$TOP_DIR"/certs
9 if ! [ -e "$TOP_DIR/certs/root.key" ]; then
10 openssl genrsa -out "$TOP_DIR/certs/root.key.tmp" 2048
11 mv "$TOP_DIR/certs/root.key.tmp" "$TOP_DIR/certs/root.key"
12 fi
13 if ! [ -e "$TOP_DIR/certs/root.crt" ]; then
14 openssl req -x509 -new -nodes -key "$TOP_DIR/certs/root.key" -subj "/CN=app.local" -days 1024 -reqexts v3_req -extensions v3_ca -out "$TOP_DIR/certs/root.crt.tmp"
15 mv "$TOP_DIR/certs/root.crt.tmp" "$TOP_DIR/certs/root.crt"
16 fi
17
18 if ! [ -e "$TOP_DIR/certs/registry.key" ]; then
19 openssl genrsa -out "$TOP_DIR/certs/registry.key.tmp" 4096
20 mv "$TOP_DIR/certs/registry.key.tmp" "$TOP_DIR/certs/registry.key"
21 fi
22 if ! [ -e "$TOP_DIR/certs/registry.crt" ]; then
23 openssl req -new -key "$TOP_DIR/certs/registry.key" -config "$TOP_DIR/etc/ssl/registry.conf" -out "$TOP_DIR/certs/registry.csr"
24 openssl x509 -req -days 365 -in "$TOP_DIR/certs/registry.csr" -CA "$TOP_DIR/certs/root.crt" -CAkey "$TOP_DIR/certs/root.key" -CAcreateserial -out "$TOP_DIR/certs/registry.crt.tmp" -extfile "$TOP_DIR/etc/ssl/registry-sign.conf"
25 mv "$TOP_DIR/certs/registry.crt.tmp" "$TOP_DIR/certs/registry.crt"
26 fi
1 #!/bin/sh
2 set -ex
3 chown -R 1000:1000 /bitnami/etcd/data
4 exec "$@"
1 #!/bin/sh
2
3 set -e
4 TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
5 export TOP_DIR
6
7 COREDNS_IP_1=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-coredns-1 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
8 COREDNS_IP_2=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-coredns-2 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
9 COREDNS_IP_3=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-coredns-3 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
10
11 kubectl apply -f /dev/stdin << _EOF_
12 apiVersion: v1
13 kind: Service
14 metadata:
15 name: compose-dns-external-service
16 spec:
17 clusterIP: 10.43.0.10
18 ports:
19 - protocol: TCP
20 name: dns-tcp
21 port: 53
22 targetPort: 53
23 - protocol: UDP
24 name: dns-udp
25 port: 53
26 targetPort: 53
27 ---
28 apiVersion: v1
29 kind: Endpoints
30 metadata:
31 name: compose-dns-external-service
32 subsets:
33 - addresses:
34 - ip: $COREDNS_IP_1
35 - ip: $COREDNS_IP_2
36 - ip: $COREDNS_IP_3
37 ports:
38 - protocol: TCP
39 name: dns-tcp
40 port: 53
41 - protocol: UDP
42 name: dns-udp
43 port: 53
44 _EOF_
45
1 #!/bin/sh
2
3 set -e
4
5 TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
6 export TOP_DIR
7
8 "$TOP_DIR/scripts/ensure-certs.sh"
9 docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d registry
10 "$TOP_DIR/scripts/wait-for-etcd.sh"
11
12 docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-master-1
13 "$TOP_DIR/scripts/update-docker-kubeconfig.sh"
14 "$TOP_DIR/scripts/wait-for-master-1.sh"
15
16 docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-coredns-1 k3s-coredns-2 k3s-coredns-3
17 "$TOP_DIR/scripts/install-cluster-dns.sh"
18 docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-agent-1 k3s-agent-2
19 docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-master-2 k3s-master-3
20 "$TOP_DIR/scripts/wait-for-system-pods.sh" 2
21
22 #docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-proxy
23
24 cd "$TOP_DIR"
25 helmfile apply
1 #!/bin/sh
2
3 set -e
4
5 TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
6 export TOP_DIR
7
8 docker-compose -f "$TOP_DIR/docker-compose.yaml" down "$@"
9
1 #!/bin/bash
2
3 set -e
4
5 TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
6 export TOP_DIR
7
8 tmpd="$(mktemp -d)"
9 onexit() {
10 [[ $tmpd ]] && rm -rf "$tmpd"
11 }
12
13 trap onexit EXIT
14
15 # TODO: Check $TOP_DIR
16
17 declare -i count=10
18 while [[ $count > 0 ]]; do
19 if docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 cat /output/kubeconfig.yaml > "$tmpd/config.docker" 2>/dev/null; then
20 break
21 fi
22 sleep 1
23 count=$(($count - 1))
24 done
25 chmod 600 "$tmpd/config.docker"
26
27 MASTER_IP=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-master-1 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
28
29 kubectl config --kubeconfig="$tmpd/config.docker" view --raw=true -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d > "$tmpd/cluster-certificate-authority"
30 kubectl config --kubeconfig="$tmpd/config.docker" view --raw=true -o jsonpath='{.users[].user.client-certificate-data}' | base64 -d > "$tmpd/client-certificate"
31 kubectl config --kubeconfig="$tmpd/config.docker" view --raw=true -o jsonpath='{.users[].user.client-key-data}' | base64 -d > "$tmpd/client-key"
32
33 kubectl config set-cluster "$TOP_DIR" --embed-certs=true --server="https://$MASTER_IP:6443" --certificate-authority="$tmpd/cluster-certificate-authority" > /dev/null
34 kubectl config set-credentials "$TOP_DIR" --embed-certs=true --client-certificate="$tmpd/client-certificate" --client-key="$tmpd/client-key" > /dev/null
35 kubectl config set-context "$TOP_DIR" --cluster="$TOP_DIR" --user="$TOP_DIR" > /dev/null
36 kubectl config use-context "$TOP_DIR"
37
1 #!/bin/sh
2
3 set -e
4
5 TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
6 export TOP_DIR
7
8 ETCD_ENDPOINTS="http://etcd1:2380,http://etcd2:2380,http://etcd3:2380"
9
10 docker_compose() {
11 docker-compose -f "$TOP_DIR/docker-compose.yaml" "$@"
12 }
13
14 etcdctl() {
15 docker_compose exec etcd1 etcdctl "$@"
16 }
17
18 cnt=5
19 printf 'Waiting for etcd cluster: '
20
21 while [ $cnt -ne 0 ]; do
22 docker_compose up -d etcd1 etcd2 etcd3 1>/dev/null 2>/dev/null
23 if etcdctl --endpoints "$ETCD_ENDPOINTS" endpoint health 1>/dev/null 2>/dev/null; then
24 if [ $cnt -ne 5 ]; then
25 printf ' '
26 fi
27 printf 'done\n'
28 exit
29 fi
30 printf '.'
31 sleep 1
32 cnt=$(($cnt - 1))
33 done
34 printf ' error\n'
35
36 echo "etcd failed to initialize!" 1>&2
37 exit 1
1 #!/bin/sh
2
3 cnt=10
4 printf 'Waiting for k3s-master-1: '
5 while [ $cnt -ne 0 ]; do
6 if kubectl get --raw '/readyz' > /dev/null 2>/dev/null; then
7 if [ $cnt -ne 10 ]; then
8 printf ' '
9 fi
10 printf 'done\n'
11 exit
12 fi
13 printf '.'
14 sleep 1
15 cnt=$(($cnt - 1))
16 done
17 printf ' error\n'
18
19 echo 'k3s-master-1 failed to initialize!' 1>&2
20 exit 1
1 #!/bin/bash
2
3 set -e
4
5 needed_pods="$1"
6
7 echo "Waiting for cluster to be ready"
8 declare -i column_count=0 system_pod_count=0
9 while :; do
10 system_pods="$(kubectl get pods --namespace kube-system --no-headers 2>/dev/null || true)"
11 column_count="$(($column_count + 1))"
12 if [[ -z $system_pods ]]; then
13 echo -n "."
14 else
15 system_pod_count="$(egrep -ci '1/1[[:space:]]+Running' <<< "$system_pods" || true)"
16 echo -n "$system_pod_count"
17 if [[ $system_pod_count -eq ${needed_pods} ]]; then
18 break
19 fi
20 fi
21 if [[ $column_count -eq 40 ]]; then
22 echo
23 column_count=0
24 fi
25 sleep 1
26 done
27 if [[ $column_count -ne 0 ]]; then
28 echo
29 column_count=0
30 fi