Skip to content

brainfood / k3s-apiserver

Go to a project
Toggle navigation
Toggle navigation pinning
be318804 by Adam Heath
Options

Major update.

1 parent 75c860fd
Inline Side-by-side
Showing 8 changed files with 138 additions and 69 deletions
version: '2.4'
version: '3.4'
x-extra-hosts: &_x_extra_hosts
extra_hosts:
- "${APP_REGISTRY_NAME}:${APP_REGISTRY_ADDRESS}"
x-etcd-environment: &_x-etcd-environment
ALLOW_NONE_AUTHENTICATION: "yes"
......@@ -16,7 +20,35 @@ x-etcd-base: &_x-etcd-base
command: ["/opt/bitnami/scripts/etcd/run.sh"]
user: root
x-root-cert-volume: &_x-root-cert-volume
type: bind
source: ${CONTEXT_DIR}/certs/root.crt
target: /etc/ssl/certs/root.crt
read_only: true
bind:
create_host_path: false
x-kubelet-volume: &_x-kubelet-volume
type: volume
target: /var/lib/kubelet
# volume:
# propagation: shared
x-registries-yaml-volume: &_x-registries-yaml-volume
type: bind
source: ${APISERVER_DIR}/etc/registries.yaml
target: /etc/rancher/k3s/registries.yaml
read_only: true
bind:
create_host_path: false
x-k3s-master-env: &_x-k3s-master-env
K3S_KUBECONFIG_OUTPUT: /output/kubeconfig.yaml
K3S_KUBECONFIG_MODE: "666"
K3S_NODE_NAME: master
x-k3s-master-base: &_x-k3s-master-base
<<: *_x_extra_hosts
image: "docker.io/rancher/k3s:${K3S_VERSION:-latest}"
networks:
default:
......@@ -35,18 +67,27 @@ x-k3s-master-base: &_x-k3s-master-base
ports:
- 6443
environment:
- K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml
- K3S_KUBECONFIG_MODE=666
- K3S_NODE_NAME=master
<<: *_x-k3s-master-env
volumes:
- server:/var/lib/rancher/k3s/server
- output:/output
- ./etc/registries.yaml:/etc/rancher/k3s/registries.yaml:ro
- ./certs/root.crt:/etc/ssl/certs/root.crt:ro
- ./certs/registry.crt:/etc/ssl/certs/registry.crt:ro
- .:${APP_ROOT_MOUNT?Please set APP_ROOT_MOUNT(where to mount $PWD)}
- ${APP_ROOT_DIR?Please set APP_ROOT_DIR}:${APP_ROOT_MOUNT?Please set APP_ROOT_MOUNT(where to mount $PWD)}
# - *_x-registries-yaml-volume
- *_x-root-cert-volume
- *_x-kubelet-volume
x-k3s-agent-env: &_x-k3s-agent-env
K3S_URL: https://k3s-master:6443
K3S_TOKEN_FILE: /var/lib/rancher/k3s/server/node-token
K3S_NODE_NAME: k3s-agent
VIRTUAL_HOST: ${VHOST_STUB},*${VHOST_SUFFIX},${APP_REGISTRY_NAME}
VIRTUAL_PROTO: https
VIRTUAL_PORT: "443"
SELF_SIGNED_HOST: ${VHOST_STUB},*${VHOST_SUFFIX}
HTTPS_METHOD: noredirect
x-k3s-agent-base: &_x-k3s-agent-base
<<: *_x_extra_hosts
image: "docker.io/rancher/k3s:${K3S_VERSION:-latest}"
tmpfs:
- /run
......@@ -57,10 +98,10 @@ x-k3s-agent-base: &_x-k3s-agent-base
soft: 65535
hard: 65535
volumes:
- .:${APP_ROOT_MOUNT?Please specify where to mount $PWD}
- ./etc/registries.yaml:/etc/rancher/k3s/registries.yaml:ro
- ./certs/root.crt:/etc/ssl/certs/root.crt:ro
- ./certs/registry.crt:/etc/ssl/certs/registry.crt:ro
- ${APP_ROOT_DIR?Please set APP_ROOT_DIR}:${APP_ROOT_MOUNT?Please specify where to mount $PWD}
# - *_x-registries-yaml-volume
- *_x-root-cert-volume
- *_x-kubelet-volume
- server:/var/lib/rancher/k3s/server:ro
privileged: true
restart: always
......@@ -70,14 +111,7 @@ x-k3s-agent-base: &_x-k3s-agent-base
ports:
- 443
environment:
- K3S_URL=https://k3s-master:6443
- K3S_TOKEN_FILE=/var/lib/rancher/k3s/server/node-token
- K3S_NODE_NAME=k3s-agent
- VIRTUAL_HOST=${VHOST_STUB},*${VHOST_SUFFIX}
- VIRTUAL_PROTO=https
- VIRTUAL_PORT=443
- SELF_SIGNED_HOST=${VHOST_STUB},*${VHOST_SUFFIX}
- HTTPS_METHOD=noredirect
<<: *_x-k3s-agent-env
x-coredns-base: &_x-coredns-base
image: docker.io/coredns/coredns
......@@ -86,13 +120,13 @@ x-coredns-base: &_x-coredns-base
volumes:
- server:/var/lib/rancher/k3s/server
- output:/output
- ./etc/coredns:/etc/coredns:ro
- ${APISERVER_DIR}/etc/coredns:/etc/coredns:ro
networks:
default:
nginx:
external:
name: nginx
name: nginx
external: true
services:
etcd1:
......@@ -103,7 +137,7 @@ services:
ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd1:2380
ETCD_ADVERTISE_CLIENT_URLS: http://etcd1:2379
volumes:
- ./scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
- ${APISERVER_DIR}/scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
- etcd1-data:/bitnami/etcd/data
etcd2:
......@@ -114,7 +148,7 @@ services:
ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd2:2380
ETCD_ADVERTISE_CLIENT_URLS: http://etcd2:2379
volumes:
- ./scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
- ${APISERVER_DIR}/scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
- etcd2-data:/bitnami/etcd/data
etcd3:
......@@ -125,41 +159,47 @@ services:
ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd3:2380
ETCD_ADVERTISE_CLIENT_URLS: http://etcd3:2379
volumes:
- ./scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
- ${APISERVER_DIR}/scripts/etcd-entrypoint.sh:/etcd-entrypoint.sh:ro
- etcd3-data:/bitnami/etcd/data
k3s-master-1:
<<: *_x-k3s-master-base
command: [
"server",
"--with-node-id",
"--disable=traefik,coredns",
"--disable=traefik,coredns,local-storage",
"--node-taint", "node-role.kubernetes.io/master=true:NoSchedule",
"--datastore-endpoint=http://etcd1:2379",
"--cluster-init",
]
environment:
<<: *_x-k3s-agent-env
K3S_NODE_NAME: master-1
k3s-master-2:
<<: *_x-k3s-master-base
command: [
"server",
"--with-node-id",
"--disable=traefik,coredns",
"--disable=traefik,coredns,local-storage",
"--node-taint", "node-role.kubernetes.io/master=true:NoSchedule",
"--datastore-endpoint=http://etcd2:2379",
"--server=http://k3s-master-1:6443",
]
environment:
<<: *_x-k3s-agent-env
K3S_NODE_NAME: master-2
k3s-master-3:
<<: *_x-k3s-master-base
command: [
"server",
"--with-node-id",
"--disable=traefik,coredns",
"--disable=traefik,coredns,local-storage",
"--node-taint", "node-role.kubernetes.io/master=true:NoSchedule",
"--datastore-endpoint=http://etcd3:2379",
"--server=http://k3s-master-1:6443",
]
environment:
<<: *_x-k3s-agent-env
K3S_NODE_NAME: master-3
k3s-coredns-1:
<<: *_x-coredns-base
......@@ -174,15 +214,19 @@ services:
<<: *_x-k3s-agent-base
command: [
"agent",
"--with-node-id",
]
environment:
<<: *_x-k3s-agent-env
K3S_NODE_NAME: agent-1
k3s-agent-2:
<<: *_x-k3s-agent-base
command: [
"agent",
"--with-node-id",
]
environment:
<<: *_x-k3s-agent-env
K3S_NODE_NAME: agent-2
volumes:
etcd1-data:
......
mirrors:
"registry.uniquely-me.local":
"registry.local":
endpoint:
- https://registry.uniquely-me.local
- https://registry.local
configs:
registry.uniquely-me.local:
registry.local:
tls:
ca_file: "/etc/ssl/certs/registry.crt"
......
#!/bin/bash
CONTEXT_DIR="$TOP_DIR"
CONTEXT_DIR="$APISERVER_DIR"
declare -a args
declare -A features=(
[cert-manager]=1
[istio]=1
)
declare -a compose_files=(-f "$APISERVER_DIR/docker-compose.yaml")
declare -a k8s_nodes=()
while [[ $# -gt 0 ]]; do
arg="$1"
......@@ -24,6 +26,14 @@ while [[ $# -gt 0 ]]; do
features[$1]=
shift
;;
(-f)
compose_files+=(-f "$1")
shift
;;
(-n)
k8s_nodes+=("$1")
shift
;;
(*)
args+=("$arg")
;;
......@@ -41,3 +51,7 @@ for feature in "${!features[@]}"; do
done
export CONTEXT_DIR
_compose() {
docker-compose --project-directory "$CONTEXT_DIR" "${compose_files[@]}" "$@"
}
......
......@@ -4,9 +4,14 @@ set -e
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
COREDNS_IP_1=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-coredns-1 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
COREDNS_IP_2=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-coredns-2 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
COREDNS_IP_3=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-coredns-3 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
CONTEXT_DIR="$1"
_compose() {
docker-compose --project-directory "$CONTEXT_DIR" -f "$TOP_DIR/docker-compose.yaml" "$@"
}
COREDNS_IP_1=$(_compose exec -T k3s-master-1 ping -c 1 -q k3s-coredns-1 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
COREDNS_IP_2=$(_compose exec -T k3s-master-1 ping -c 1 -q k3s-coredns-2 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
COREDNS_IP_3=$(_compose exec -T k3s-master-1 ping -c 1 -q k3s-coredns-3 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
kubectl apply -f /dev/stdin << _EOF_
apiVersion: v1
......
......@@ -2,14 +2,14 @@
set -e
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
APISERVER_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export APISERVER_DIR
. "$TOP_DIR/scripts/_parse_args.bash"
. "$APISERVER_DIR/scripts/_parse_args.bash"
case "$1" in
(switch-to)
"$TOP_DIR/scripts/update-docker-kubeconfig.sh" "$CONTEXT_DIR"
"$APISERVER_DIR/scripts/update-docker-kubeconfig.sh" "$CONTEXT_DIR"
exit
;;
("")
......@@ -20,23 +20,22 @@ case "$1" in
;;
esac
"$TOP_DIR/scripts/ensure-certs.sh"
"$TOP_DIR/scripts/wait-for-etcd.sh"
"$APISERVER_DIR/scripts/ensure-certs.sh"
"$APISERVER_DIR/scripts/wait-for-etcd.sh" "$CONTEXT_DIR"
docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-master-1
"$TOP_DIR/scripts/update-docker-kubeconfig.sh" "$CONTEXT_DIR"
"$TOP_DIR/scripts/wait-for-master-1.sh"
_compose up -d k3s-master-1
"$APISERVER_DIR/scripts/update-docker-kubeconfig.sh" "$CONTEXT_DIR"
"$APISERVER_DIR/scripts/wait-for-master-1.sh"
docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-coredns-1 k3s-coredns-2 k3s-coredns-3
"$TOP_DIR/scripts/install-cluster-dns.sh"
docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-agent-1 k3s-agent-2
docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-master-2 k3s-master-3
"$TOP_DIR/scripts/wait-for-system-pods.sh" 2
_compose up -d k3s-coredns-1 k3s-coredns-2 k3s-coredns-3
"$APISERVER_DIR/scripts/install-cluster-dns.sh" "$CONTEXT_DIR"
_compose up -d k3s-agent-1 k3s-agent-2 k3s-storage-1 "${k8s_nodes[@]}"
_compose up -d k3s-master-2 k3s-master-3
"$APISERVER_DIR/scripts/wait-for-system-pods.sh" 1
#_compose up -d k3s-proxy
#docker-compose -f "$TOP_DIR/docker-compose.yaml" up -d k3s-proxy
#[[ ${features[istio]} ]] && istioctl install -yf "$APISERVER_DIR/istio-minimal-operator.yaml"
#[[ ${features[istio]} ]] && istioctl install -yf "$TOP_DIR/istio-minimal-operator.yaml"
cd "$TOP_DIR"
cd "$APISERVER_DIR"
#helmfile apply
......
......@@ -2,10 +2,10 @@
set -e
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
APISERVER_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export APISERVER_DIR
. "$TOP_DIR/scripts/_parse_args.bash"
. "$APISERVER_DIR/scripts/_parse_args.bash"
docker-compose -f "$TOP_DIR/docker-compose.yaml" down "$@"
_compose down "$@"
......
......@@ -7,6 +7,12 @@ CONTEXT_DIR="$1"
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
CONTEXT_DIR="$1"
_compose() {
docker-compose --project-directory "$CONTEXT_DIR" -f "$TOP_DIR/docker-compose.yaml" "$@"
}
tmpd="$(mktemp -d)"
onexit() {
[[ $tmpd ]] && rm -rf "$tmpd"
......@@ -18,7 +24,7 @@ trap onexit EXIT
declare -i count=10
while [[ $count > 0 ]]; do
if docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 cat /output/kubeconfig.yaml > "$tmpd/config.docker" 2>/dev/null; then
if _compose exec -T k3s-master-1 cat /output/kubeconfig.yaml > "$tmpd/config.docker" 2>/dev/null; then
break
fi
sleep 1
......@@ -26,7 +32,7 @@ while [[ $count > 0 ]]; do
done
chmod 600 "$tmpd/config.docker"
MASTER_IP=$(docker-compose -f "$TOP_DIR/docker-compose.yaml" exec -T k3s-master-1 ping -c 1 -q k3s-master-1 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
MASTER_IP=$(_compose exec -T k3s-master-1 ping -c 1 -q k3s-master-1 | sed -n 's/^PING.*(\(.*\)).*/\1/p')
kubectl config --kubeconfig="$tmpd/config.docker" view --raw=true -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d > "$tmpd/cluster-certificate-authority"
kubectl config --kubeconfig="$tmpd/config.docker" view --raw=true -o jsonpath='{.users[].user.client-certificate-data}' | base64 -d > "$tmpd/client-certificate"
......
......@@ -5,21 +5,22 @@ set -e
TOP_DIR="$(cd "$(dirname "$0")/.."; echo "$PWD")"
export TOP_DIR
CONTEXT_DIR="$1"
ETCD_ENDPOINTS="http://etcd1:2380,http://etcd2:2380,http://etcd3:2380"
docker_compose() {
docker-compose -f "$TOP_DIR/docker-compose.yaml" "$@"
_compose() {
docker-compose --project-directory "$CONTEXT_DIR" -f "$TOP_DIR/docker-compose.yaml" "$@"
}
etcdctl() {
docker_compose exec etcd1 etcdctl "$@"
_compose exec etcd1 etcdctl "$@"
}
cnt=5
printf 'Waiting for etcd cluster: '
while [ $cnt -ne 0 ]; do
docker_compose up -d etcd1 etcd2 etcd3 1>/dev/null 2>/dev/null
_compose up -d etcd1 etcd2 etcd3 1>/dev/null 2>/dev/null
if etcdctl --endpoints "$ETCD_ENDPOINTS" endpoint health 1>/dev/null 2>/dev/null; then
if [ $cnt -ne 5 ]; then
printf ' '
......