Skip to content

brainfood / k3s-apiserver

Go to a project
Toggle navigation
Toggle navigation pinning
739a52bd by Adam Heath
Options

Importing initial set of files.

1 parent 35a09488
Inline Side-by-side
Showing 21 changed files with 572 additions and 0 deletions
.*.sw?
/certs/root.crt
/certs/root.key
/certs/root.srl
/certs/registry.crt
/certs/registry.csr
/certs/registry.key
---
releases:
- name: cert-manager
chart: .
wait: true
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- https://github.com/jetstack/cert-manager/releases/download/v1.0.4/cert-manager.yaml
generatorOptions:
disableNameSuffixHash: true
secretGenerator:
- name: root-ca
namespace: cert-manager
files:
- tls.crt=../certs/root.crt
- tls.key=../certs/root.key
- ca.crt=../certs/root.crt
- ca.key=../certs/root.key
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
namespace: cert-manager
name: ca-issuer
spec:
ca:
secretName: root-ca
---
---
releases:
- name: cluster-issuer
namespace: cert-manager
chart: .
wait: true
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./cluster-issuer.yaml
version: '2.4'
x-etcd-environment: &_x-etcd-environment
ALLOW_NONE_AUTHENTICATION: "yes"
ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380
ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
ETCD_INITIAL_CLUSTER_TOKEN: etcd-cluster
ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380
ETCD_INITIAL_CLUSTER_STATE: new
ETCD_DATA_DIR: /bitnami/etcd/data/db
ETCD_WAL_DIR: /bitnami/etcd/data/wal
x-etcd-base: &_x-etcd-base
image: docker.io/bitnami/etcd:3
entrypoint: ["/etcd-entrypoint.sh", "/opt/bitnami/scripts/etcd/entrypoint.sh"]
command: ["/opt/bitnami/scripts/etcd/run.sh"]
user: root
x-k3s-master-base: &_x-k3s-master-base
image: "docker.io/rancher/k3s:${K3S_VERSION:-latest}"
networks:
default:
aliases:
- k3s-master
tmpfs:
- /run
- /var/run
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
privileged: true
restart: always
ports:
- 6443
environment:
- K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml
- K3S_KUBECONFIG_MODE=666
- K3S_NODE_NAME=master
volumes:
- server:/var/lib/rancher/k3s/server
- output:/output
- ./etc/registries.yaml:/etc/rancher/k3s/registries.yaml:ro
- ./certs/root.crt:/etc/ssl/certs/root.crt:ro
- ./certs/registry.crt:/etc/ssl/certs/registry.crt:ro
- .:${APP_ROOT_MOUNT?Please set APP_ROOT_MOUNT(where to mount $PWD)}
x-k3s-agent-base: &_x-k3s-agent-base
image: "docker.io/rancher/k3s:${K3S_VERSION:-latest}"
tmpfs:
- /run
- /var/run
ulimits:
nproc: 65535
nofile:
soft: 65535
hard: 65535
volumes:
- .:${APP_ROOT_MOUNT?Please specify where to mount $PWD}
- ./etc/registries.yaml:/etc/rancher/k3s/registries.yaml:ro
- ./certs/root.crt:/etc/ssl/certs/root.crt:ro
- ./certs/registry.crt:/etc/ssl/certs/registry.crt:ro
- server:/var/lib/rancher/k3s/server:ro
privileged: true
restart: always
networks:
default:
nginx:
environment:
- K3S_URL=https://k3s-master:6443
- K3S_TOKEN_FILE=/var/lib/rancher/k3s/server/node-token
- K3S_NODE_NAME=k3s-agent
- VIRTUAL_HOST=${VHOST_STUB},*${VHOST_SUFFIX}
- VIRTUAL_PROTO=https
- VIRTUAL_PORT=443
- SELF_SIGNED_HOST=${VHOST_STUB},*${VHOST_SUFFIX}
- HTTPS_METHOD=noredirect
x-coredns-base: &_x-coredns-base
image: docker.io/coredns/coredns
command: ['-conf', '/etc/coredns/Corefile']
restart: always
volumes:
- server:/var/lib/rancher/k3s/server
- output:/output
- ./etc/coredns:/etc/coredns:ro
networks:
default:
nginx:
external:
name: nginx
services:
etcd1:
<<: *_x-etcd-base
environment:
<<: *_x-etcd-environment
ETCD_NAME: etcd1
ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd1:2380
ETCD_ADVERTISE_CLIENT_URLS: http://etcd1:2379
volumes:
- ./scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
- etcd1-data:/bitnami/etcd/data
etcd2:
<<: *_x-etcd-base
environment:
<<: *_x-etcd-environment
ETCD_NAME: etcd2
ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd2:2380
ETCD_ADVERTISE_CLIENT_URLS: http://etcd2:2379
volumes:
- ./scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
- etcd2-data:/bitnami/etcd/data
etcd3:
<<: *_x-etcd-base
environment:
<<: *_x-etcd-environment
ETCD_NAME: etcd3
ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd3:2380
ETCD_ADVERTISE_CLIENT_URLS: http://etcd3:2379
volumes:
- ./scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
- etcd3-data:/bitnami/etcd/data
k3s-master-1:
<<: *_x-k3s-master-base
command: [
"server",
"--with-node-id",
"--disable=traefik,coredns",
"--node-taint", "master=true:NoSchedule",
"--datastore-endpoint=http://etcd1:2379",
"--cluster-init",
]
k3s-master-2:
<<: *_x-k3s-master-base
command: [
"server",
"--with-node-id",
"--disable=traefik,coredns",
"--node-taint", "master=true:NoSchedule",
"--datastore-endpoint=http://etcd2:2379",
"--server=http://k3s-master-1:6443",
]
k3s-master-3:
<<: *_x-k3s-master-base
command: [
"server",
"--with-node-id",
"--disable=traefik,coredns",
"--node-taint", "master=true:NoSchedule",
"--datastore-endpoint=http://etcd3:2379",
"--server=http://k3s-master-1:6443",
]
k3s-coredns-1:
<<: *_x-coredns-base
k3s-coredns-2:
<<: *_x-coredns-base
k3s-coredns-3:
<<: *_x-coredns-base
k3s-agent-1:
<<: *_x-k3s-agent-base
command: [
"agent",
"--with-node-id",
]
k3s-agent-2:
<<: *_x-k3s-agent-base
command: [
"agent",
"--with-node-id",
]
registry:
image: registry:2
networks:
default:
aliases:
- ${REGISTRY_ID?Please set REGISTRY_ID}.registry
nginx:
ports:
- 443
volumes:
- registry:/var/lib/registry
- ./certs/registry.crt:/certs/registry.crt:ro
- ./certs/registry.key:/certs/registry.key:ro
environment:
- VIRTUAL_HOST=${REGISTRY_ID?Please set REGISTRY_ID}.registry
- VIRTUAL_PORT=443
- VIRTUAL_PROTO=https
- HTTPS_METHOD=noredirect
- CERT_NAME=default
- REGISTRY_HTTP_ADDR=0.0.0.0:443
- REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt
- REGISTRY_HTTP_TLS_KEY=/certs/registry.key
volumes:
etcd1-data:
etcd2-data:
etcd3-data:
server: {}
output: {}
registry:
external: true
name: ${REGISTRY_VOLUME_NAME?Please set REGISTRY_VOLUME_NAME}
.:53 {
errors
health {
lameduck 5s
}
ready
kubernetes cluster.local in-addr.arpa ip6.arpa {
tls /var/lib/rancher/k3s/server/tls/client-admin.crt /var/lib/rancher/k3s/server/tls/client-admin.key /var/lib/rancher/k3s/server/tls/server-ca.crt
#kubeconfig /output/kubeconfig.yaml
endpoint https://k3s-master:6443
pods insecure
fallthrough in-addr.arpa ip6.arpa
ttl 30
}
# hosts /etc/coredns/NodeHosts {
# ttl 60
# reload 15s
# fallthrough
# }
prometheus :9153
forward . /etc/resolv.conf
cache 30
loop
reload
loadbalance
}
mirrors:
"registry.uniquely-me.local":
endpoint:
- https://registry.uniquely-me.local
configs:
registry.uniquely-me.local:
tls:
ca_file: "/etc/ssl/certs/registry.crt"
subjectAltName = @alt_names
[alt_names]
DNS.1 = registry.uniquely-me.local
DNS.2 = registry.uniquely.me
[req]
default_bits = 2048
default_keyfile = registry.key
distinguished_name = req_distinguished_name
req_extensions = req_ext
prompt = no
encrypt_key = no
[req_distinguished_name]
countryName = US
stateOrProvinceName = Texas
localityName = Dallas
organizationName = UNIQUELY ME
organizationalUnitName = IT
commonName = registry.uniquely-me.local
[req_ext]
subjectAltName = @alt_names
[alt_names]
DNS.1 = registry.uniquely-me.local
DNS.2 = registry.uniquely.me
helmfiles:
- cert-manager/helmfile.yaml
- cluster-issuer/helmfile.yaml
#!/bin/sh
set -e
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
mkdir -p "$TOP_DIR"/certs
if ! [ -e "$TOP_DIR/certs/root.key" ]; then
openssl genrsa -out "$TOP_DIR/certs/root.key.tmp" 2048
mv "$TOP_DIR/certs/root.key.tmp" "$TOP_DIR/certs/root.key"
fi
if ! [ -e "$TOP_DIR/certs/root.crt" ]; then
openssl req -x509 -new -nodes -key "$TOP_DIR/certs/root.key" -subj "/CN=app.local" -days 1024 -reqexts v3_req -extensions v3_ca -out "$TOP_DIR/certs/root.crt.tmp"
mv "$TOP_DIR/certs/root.crt.tmp" "$TOP_DIR/certs/root.crt"
fi
if ! [ -e "$TOP_DIR/certs/registry.key" ]; then
openssl genrsa -out "$TOP_DIR/certs/registry.key.tmp" 4096
mv "$TOP_DIR/certs/registry.key.tmp" "$TOP_DIR/certs/registry.key"
fi
if ! [ -e "$TOP_DIR/certs/registry.crt" ]; then
openssl req -new -key "$TOP_DIR/certs/registry.key" -config "$TOP_DIR/etc/ssl/registry.conf" -out "$TOP_DIR/certs/registry.csr"
openssl x509 -req -days 365 -in "$TOP_DIR/certs/registry.csr" -CA "$TOP_DIR/certs/root.crt" -CAkey "$TOP_DIR/certs/root.key" -CAcreateserial -out "$TOP_DIR/certs/registry.crt.tmp" -extfile "$TOP_DIR/etc/ssl/registry-sign.conf"
mv "$TOP_DIR/certs/registry.crt.tmp" "$TOP_DIR/certs/registry.crt"
fi
#!/bin/sh
set -ex
chown -R 1000:1000 /bitnami/etcd/data
exec "$@"
#!/bin/sh
set -e
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
COREDNS_IP_1=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-coredns-1 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
COREDNS_IP_2=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-coredns-2 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
COREDNS_IP_3=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-coredns-3 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
kubectl apply -f /dev/stdin << _EOF_
apiVersion: v1
kind: Service
metadata:
name: compose-dns-external-service
spec:
clusterIP: 10.43.0.10
ports:
- protocol: TCP
name: dns-tcp
port: 53
targetPort: 53
- protocol: UDP
name: dns-udp
port: 53
targetPort: 53
---
apiVersion: v1
kind: Endpoints
metadata:
name: compose-dns-external-service
subsets:
- addresses:
- ip: $COREDNS_IP_1
- ip: $COREDNS_IP_2
- ip: $COREDNS_IP_3
ports:
- protocol: TCP
name: dns-tcp
port: 53
- protocol: UDP
name: dns-udp
port: 53
_EOF_
#!/bin/sh
set -e
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
"$TOP_DIR/scripts/ensure-certs.sh"
docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d registry
"$TOP_DIR/scripts/wait-for-etcd.sh"
docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-master-1
"$TOP_DIR/scripts/update-docker-kubeconfig.sh"
"$TOP_DIR/scripts/wait-for-master-1.sh"
docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-coredns-1 k3s-coredns-2 k3s-coredns-3
"$TOP_DIR/scripts/install-cluster-dns.sh"
docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-agent-1 k3s-agent-2
docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-master-2 k3s-master-3
"$TOP_DIR/scripts/wait-for-system-pods.sh" 2
#docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-proxy
cd "$TOP_DIR"
helmfile apply
#!/bin/sh
set -e
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
docker-compose -f "$TOP_DIR/docker-compose.yaml" down "$@"
#!/bin/bash
set -e
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
tmpd="$(mktemp -d)"
onexit() {
[[ $tmpd ]] && rm -rf "$tmpd"
}
trap onexit EXIT
# TODO: Check $TOP_DIR
declare -i count=10
while [[ $count > 0 ]]; do
if docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 cat /output/kubeconfig.yaml > "$tmpd/config.docker" 2>/dev/null; then
break
fi
sleep 1
count=$(($count - 1))
done
chmod 600 "$tmpd/config.docker"
MASTER_IP=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-master-1 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
kubectl config --kubeconfig="$tmpd/config.docker" view --raw=true -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d > "$tmpd/cluster-certificate-authority"
kubectl config --kubeconfig="$tmpd/config.docker" view --raw=true -o jsonpath='{.users[].user.client-certificate-data}' | base64 -d > "$tmpd/client-certificate"
kubectl config --kubeconfig="$tmpd/config.docker" view --raw=true -o jsonpath='{.users[].user.client-key-data}' | base64 -d > "$tmpd/client-key"
kubectl config set-cluster "$TOP_DIR" --embed-certs=true --server="https://$MASTER_IP:6443" --certificate-authority="$tmpd/cluster-certificate-authority" > /dev/null
kubectl config set-credentials "$TOP_DIR" --embed-certs=true --client-certificate="$tmpd/client-certificate" --client-key="$tmpd/client-key" > /dev/null
kubectl config set-context "$TOP_DIR" --cluster="$TOP_DIR" --user="$TOP_DIR" > /dev/null
kubectl config use-context "$TOP_DIR"
#!/bin/sh
set -e
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
ETCD_ENDPOINTS="http://etcd1:2380,http://etcd2:2380,http://etcd3:2380"
docker_compose() {
docker-compose -f "$TOP_DIR/docker-compose.yaml" "$@"
}
etcdctl() {
docker_compose exec etcd1 etcdctl "$@"
}
cnt=5
printf 'Waiting for etcd cluster: '
while [ $cnt -ne 0 ]; do
docker_compose up -d etcd1 etcd2 etcd3 1>/dev/null 2>/dev/null
if etcdctl --endpoints "$ETCD_ENDPOINTS" endpoint health 1>/dev/null 2>/dev/null; then
if [ $cnt -ne 5 ]; then
printf ' '
fi
printf 'done\n'
exit
fi
printf '.'
sleep 1
cnt=$(($cnt - 1))
done
printf ' error\n'
echo "etcd failed to initialize!" 1>&2
exit 1
#!/bin/sh
cnt=10
printf 'Waiting for k3s-master-1: '
while [ $cnt -ne 0 ]; do
if kubectl get --raw '/readyz' > /dev/null 2>/dev/null; then
if [ $cnt -ne 10 ]; then
printf ' '
fi
printf 'done\n'
exit
fi
printf '.'
sleep 1
cnt=$(($cnt - 1))
done
printf ' error\n'
echo 'k3s-master-1 failed to initialize!' 1>&2
exit 1
#!/bin/bash
set -e
needed_pods="$1"
echo "Waiting for cluster to be ready"
declare -i column_count=0 system_pod_count=0
while :; do
system_pods="$(kubectl get pods --namespace kube-system --no-headers 2>/dev/null || true)"
column_count="$(($column_count + 1))"
if [[ -z $system_pods ]]; then
echo -n "."
else
system_pod_count="$(egrep -ci '1/1[[:space:]]+Running' <<< "$system_pods" || true)"
echo -n "$system_pod_count"
if [[ $system_pod_count -eq ${needed_pods} ]]; then
break
fi
fi
if [[ $column_count -eq 40 ]]; then
echo
column_count=0
fi
sleep 1
done
if [[ $column_count -ne 0 ]]; then
echo
column_count=0
fi