Skip to content

brainfood / k8s-helmfiles

Go to a project
Toggle navigation
Toggle navigation pinning
988a25ca by Adam Heath
Options

Letsencrypt is working, and backup/restore of the generated secrets as 

well.
1 parent f3c25da7
Inline Side-by-side
Showing 22 changed files with 1271 additions and 10 deletions
1 apiVersion: batch/v1
2 kind: CronJob
3 metadata:
4 name: backup-secrets-do-backup
5 spec:
6 schedule: "13 * * * *"
7 startingDeadlineSeconds: 3600
8 concurrencyPolicy: Forbid
9 successfulJobsHistoryLimit: 10
10 failedJobsHistoryLimit: 10
11 jobTemplate:
12 spec:
13 template:
14 spec:
15 serviceAccountName: backup-secrets
16 restartPolicy: OnFailure
17 volumes:
18 - name: work-space
19 emptyDir: {}
20 - name: config
21 configMap:
22 name: backup-secrets-config
23 - name: auth
24 secret:
25 secretName: backup-secrets-auth
26 - name: scripts
27 configMap:
28 name: backup-secrets-scripts
29 defaultMode: 0755
30 - name: aws-config
31 configMap:
32 name: backup-secrets-config
33 items:
34 - key: config
35 path: config
36 - name: aws-credentials
37 secret:
38 secretName: backup-secrets-auth
39 items:
40 - key: credentials
41 path: credentials
42 initContainers:
43 - name: list-s3
44 image: peakcom/s5cmd
45 command: ["/scripts/backup"]
46 args: ["list-s3"]
47 volumeMounts:
48 - name: work-space
49 mountPath: /work-space
50 - name: config
51 mountPath: /config
52 - name: auth
53 mountPath: /auth
54 - name: scripts
55 mountPath: /scripts
56
57 - mountPath: /root/.aws/config
58 name: aws-config
59 subPath: config
60 - mountPath: /root/.aws/credentials
61 name: aws-credentials
62 subPath: credentials
63
64 - name: calculate-delete-s3
65 image: debian
66 command: ["/scripts/backup"]
67 args: ["calculate-delete-s3"]
68 volumeMounts:
69 - name: work-space
70 mountPath: /work-space
71 - name: config
72 mountPath: /config
73 - name: scripts
74 mountPath: /scripts
75
76 - name: create-secrets-tarball
77 image: bitnami/kubectl
78 command: ["/scripts/backup"]
79 args: ["create-secrets-tarball"]
80 volumeMounts:
81 - name: work-space
82 mountPath: /work-space
83 - name: config
84 mountPath: /config
85 - name: scripts
86 mountPath: /scripts
87
88 - name: copy-to-s3
89 image: peakcom/s5cmd
90 command: ["/scripts/backup"]
91 args: ["copy-to-s3"]
92 volumeMounts:
93 - name: work-space
94 mountPath: /work-space
95 - name: config
96 mountPath: /config
97 - name: auth
98 mountPath: /auth
99 - name: scripts
100 mountPath: /scripts
101
102 - mountPath: /root/.aws/config
103 name: aws-config
104 subPath: config
105 - mountPath: /root/.aws/credentials
106 name: aws-credentials
107 subPath: credentials
108
109 containers:
110 - name: show-backup
111 image: debian
112 command: ["/scripts/backup"]
113 args: ["show-backup"]
114 volumeMounts:
115 - name: work-space
116 mountPath: /work-space
117 - name: scripts
118 mountPath: /scripts
119
1 apiVersion: kustomize.config.k8s.io/v1beta1
2 kind: Kustomization
3
4 resources:
5 - ./backup.yaml
6
1 apiVersion: batch/v1
2 kind: CronJob
3 metadata:
4 name: backup-secrets-do-backup
5 spec:
6 schedule: "13 * * * *"
7 startingDeadlineSeconds: 3600
8 concurrencyPolicy: Forbid
9 successfulJobsHistoryLimit: 10
10 failedJobsHistoryLimit: 10
11 jobTemplate:
12 spec:
13 template:
14 spec:
15 serviceAccountName: backup-secrets
16 restartPolicy: OnFailure
17 volumes:
18 - name: work-space
19 emptyDir: {}
20 - name: config
21 configMap:
22 name: backup-secrets-config
23 - name: auth
24 secret:
25 secretName: backup-secrets-auth
26 - name: scripts
27 configMap:
28 name: backup-secrets-scripts
29 defaultMode: 0755
30 - name: aws-config
31 configMap:
32 name: backup-secrets-config
33 items:
34 - key: config
35 path: config
36 - name: aws-credentials
37 secret:
38 secretName: backup-secrets-auth
39 items:
40 - key: credentials
41 path: credentials
42 initContainers:
43 - name: list-s3
44 image: peakcom/s5cmd
45 command: ["/scripts/backup"]
46 args: ["list-s3"]
47 volumeMounts:
48 - name: work-space
49 mountPath: /work-space
50 - name: config
51 mountPath: /config
52 - name: auth
53 mountPath: /auth
54 - name: scripts
55 mountPath: /scripts
56
57 - mountPath: /root/.aws/config
58 name: aws-config
59 subPath: config
60 - mountPath: /root/.aws/credentials
61 name: aws-credentials
62 subPath: credentials
63
64 - name: calculate-delete-s3
65 image: debian
66 command: ["/scripts/backup"]
67 args: ["calculate-delete-s3"]
68 volumeMounts:
69 - name: work-space
70 mountPath: /work-space
71 - name: config
72 mountPath: /config
73 - name: scripts
74 mountPath: /scripts
75
76 - name: create-secrets-tarball
77 image: bitnami/kubectl
78 command: ["/scripts/backup"]
79 args: ["create-secrets-tarball"]
80 volumeMounts:
81 - name: work-space
82 mountPath: /work-space
83 - name: config
84 mountPath: /config
85 - name: scripts
86 mountPath: /scripts
87
88 - name: copy-to-s3
89 image: peakcom/s5cmd
90 command: ["/scripts/backup"]
91 args: ["copy-to-s3"]
92 volumeMounts:
93 - name: work-space
94 mountPath: /work-space
95 - name: config
96 mountPath: /config
97 - name: auth
98 mountPath: /auth
99 - name: scripts
100 mountPath: /scripts
101
102 - mountPath: /root/.aws/config
103 name: aws-config
104 subPath: config
105 - mountPath: /root/.aws/credentials
106 name: aws-credentials
107 subPath: credentials
108
109 containers:
110 - name: show-backup
111 image: debian
112 command: ["/scripts/backup"]
113 args: ["show-backup"]
114 volumeMounts:
115 - name: work-space
116 mountPath: /work-space
117 - name: scripts
118 mountPath: /scripts
119
1 ---
2 apiVersion: v1
3 kind: ServiceAccount
4 metadata:
5 name: backup-secrets
6 ---
7 apiVersion: rbac.authorization.k8s.io/v1
8 kind: Role
9 metadata:
10 name: backup-secrets
11 rules:
12 - apiGroups: [""] # "" indicates the core API group
13 resources: ["secrets"]
14 verbs: ["get", "create", "list", "patch"]
15 ---
16 apiVersion: rbac.authorization.k8s.io/v1
17 kind: RoleBinding
18 metadata:
19 name: backup-secrets
20 subjects:
21 # You can specify more than one "subject"
22 - kind: ServiceAccount
23 name: backup-secrets
24 apiGroup: ""
25 roleRef:
26 kind: Role
27 name: backup-secrets
28 apiGroup: rbac.authorization.k8s.io
29 ---
30 apiVersion: v1
31 kind: ConfigMap
32 metadata:
33 name: backup-secrets-config
34 data:
35 APP_NAME: test
36 AWS_PROFILE: "default"
37 S3_ENABLED: "true"
38 S3_BUCKET: "brainfood-bikeshed"
39 S3_PREFIX: "um-adam/secret-backup"
40 KEEP_COUNT: "30"
41 SECRETS_TARBALL: ""
42 config: ""
43 ---
44 apiVersion: v1
45 kind: ConfigMap
46 metadata:
47 name: backup-secrets-scripts
48 ---
49 apiVersion: v1
50 kind: Secret
51 metadata:
52 name: backup-secrets-auth
53 stringData:
54 credentials: ""
55
1 apiVersion: kustomize.config.k8s.io/v1beta1
2 kind: Kustomization
3
4 resources:
5 - ./config.yaml
6 - ./backup.yaml
7
8 configMapGenerator:
9 - name: backup-secrets-scripts
10 behavior: merge
11 options:
12 disableNameSuffixHash: true
13 files:
14 - ../../scripts/backup
15
1 apiVersion: kustomize.config.k8s.io/v1beta1
2 kind: Kustomization
3
4 resources:
5 - ./restore.yaml
6
1 apiVersion: batch/v1
2 kind: Job
3 metadata:
4 annotations:
5 helm.sh/hook: post-install
6 helm.sh/hook-weight: "-5"
7 helm.sh/hook-delete-policy: before-hook-creation
8 # annotations:
9 # helm.sh/hook: pre-install
10 # helm.sh/hook-delete-policy: before-hook-creation
11 # helm.sh/hook-weight: "-5"
12 name: backup-secrets-do-restore
13 spec:
14 template:
15 spec:
16 restartPolicy: Never
17 serviceAccountName: backup-secrets
18 containers:
19 - name: show-backup
20 command: ["/scripts/backup"]
21 args: ["show-backup"]
22 image: debian
23 volumeMounts:
24 - mountPath: /work-space
25 name: work-space
26 - mountPath: /scripts
27 name: scripts
28
29 initContainers:
30 - name: list-s3
31 command: ["/scripts/backup"]
32 args: ["list-s3"]
33 image: peakcom/s5cmd
34 volumeMounts:
35 - mountPath: /work-space
36 name: work-space
37 - mountPath: /config
38 name: config
39 - mountPath: /auth
40 name: auth
41 - mountPath: /scripts
42 name: scripts
43
44 - mountPath: /root/.aws/config
45 name: aws-config
46 subPath: config
47 - mountPath: /root/.aws/credentials
48 name: aws-credentials
49 subPath: credentials
50
51 - name: calculate-restore-s3
52 command: ["/scripts/backup"]
53 args: ["calculate-restore-s3"]
54 image: debian
55 volumeMounts:
56 - mountPath: /work-space
57 name: work-space
58 - mountPath: /config
59 name: config
60 - mountPath: /auth
61 name: auth
62 - mountPath: /scripts
63 name: scripts
64 - name: copy-from-s3
65 command: ["/scripts/backup"]
66 args: ["copy-from-s3"]
67 image: peakcom/s5cmd
68 volumeMounts:
69 - mountPath: /work-space
70 name: work-space
71 - mountPath: /config
72 name: config
73 - mountPath: /auth
74 name: auth
75 - mountPath: /scripts
76 name: scripts
77
78 - mountPath: /root/.aws/config
79 name: aws-config
80 subPath: config
81 - mountPath: /root/.aws/credentials
82 name: aws-credentials
83 subPath: credentials
84 - name: restore-secrets
85 command: ["/scripts/backup"]
86 args: ["restore-secrets"]
87 image: bitnami/kubectl
88 volumeMounts:
89 - mountPath: /work-space
90 name: work-space
91 - mountPath: /scripts
92 name: scripts
93 volumes:
94 - name: work-space
95 emptyDir: {}
96 - name: aws-config
97 configMap:
98 name: backup-secrets-config
99 items:
100 - key: config
101 path: config
102 - name: aws-credentials
103 secret:
104 secretName: backup-secrets-auth
105 items:
106 - key: credentials
107 path: credentials
108 - name: config
109 configMap:
110 name: backup-secrets-config
111 - name: auth
112 secret:
113 secretName: backup-secrets-auth
114 - name: scripts
115 configMap:
116 defaultMode: 0755
117 name: backup-secrets-scripts
118
1 name: test
2 namespace: default
3
4 aws:
5 config: |-
6 [default]
7 region=us-west-2
8
9 credentials: |-
10 [default]
11 aws_access_key_id="xxx"
12 aws_secret_access_key="xxx"
13
1 bases:
2 - ../common/environments.yaml
3
4 ---
5
6 releases:
7 - name: {{ .Values.name }}-backup-secrets-base
8 chart: charts/base
9 namespace: {{ .Values.namespace }}
10 values:
11 - namePrefix: "{{ .Values.name }}-"
12 strategicMergePatches:
13 - apiVersion: v1
14 kind: ConfigMap
15 metadata:
16 namespace: {{ .Values.namespace }}
17 name: {{ .Values.name }}-backup-secrets-config
18 data:
19 APP_NAME: {{ .Values.name }}-gateway
20 config: {{ .Values.aws.config | quote }}
21 - apiVersion: v1
22 kind: Secret
23 metadata:
24 namespace: {{ .Values.namespace }}
25 name: {{ .Values.name }}-backup-secrets-auth
26 stringData:
27 credentials: {{ .Values.aws.credentials | quote }}
28
29 - name: {{ .Values.name }}-backup-secrets-restore
30 chart: charts/restore
31 namespace: {{ .Values.namespace }}
32 needs:
33 - {{ .Values.name }}-backup-secrets-base
34 values:
35 - namePrefix: "{{ .Values.name }}-"
36
37 strategicMergePatches:
38 # Have to adjust these namePrefix manually because helmfile/chartify doesn't work correctly with kustomize/helm-hooks
39 - apiVersion: batch/v1
40 kind: Job
41 metadata:
42 namespace: {{ .Values.namespace }}
43 name: {{ .Values.name }}-backup-secrets-do-restore
44 spec:
45 template:
46 spec:
47 serviceAccountName: {{ .Values.name }}-backup-secrets
48 volumes:
49 - name: aws-config
50 configMap:
51 name: {{ .Values.name }}-backup-secrets-config
52 - name: aws-credentials
53 secret:
54 secretName: {{ .Values.name }}-backup-secrets-auth
55 - name: config
56 configMap:
57 name: {{ .Values.name }}-backup-secrets-config
58 - name: auth
59 secret:
60 secretName: {{ .Values.name }}-backup-secrets-auth
61 - name: scripts
62 configMap:
63 name: {{ .Values.name }}-backup-secrets-scripts
1 #!/bin/sh
2
3 set -ex
4
5 setup_s3() {
6 AWS_PROFILE="$(cat /config/AWS_PROFILE)"
7 S3_BUCKET="$(cat /config/S3_BUCKET)"
8 S3_PREFIX="$(cat /config/S3_PREFIX)"
9 export AWS_PROFILE
10 }
11
12 case "$1" in
13 # restore/backup
14 (list-s3)
15 # s5cmd image doesn't have many tools installed
16 S3_ENABLED="$(cat /config/S3_ENABLED)"
17 if [ true = "$S3_ENABLED" ]; then
18 setup_s3
19 if /s5cmd ls "s3://$S3_BUCKET/$S3_PREFIX/*" > /work-space/s3-file-listing.tmp; then
20 mv /work-space/s3-file-listing.tmp /work-space/s3-file-listing
21 echo "$S3_BUCKET" > /work-space/S3_BUCKET
22 echo "$S3_PREFIX" > /work-space/S3_PREFIX
23 cat /work-space/s3-file-listing
24 fi
25 fi
26 ;;
27 # backup
28 (calculate-delete-s3)
29 # this is broken
30 KEEP_COUNT="$(cat /config/KEEP_COUNT)"
31 if [ -e /work-space/s3-file-listing ] && [ "z$KEEP_COUNT" != "z" ]; then
32 S3_BUCKET="$(cat /work-space/S3_BUCKET)"
33 S3_PREFIX="$(cat /work-space/S3_PREFIX)"
34 sort /work-space/s3-file-listing
35 if [ "z$SECRETS_TARBALL" = z ]; then
36 SECRETS_TARBALL="$(gawk '/wp-content.tgz$/{print $4}' < /work-space/s3-file-listing | sort | tail -1)"
37 if [ "z$SECRETS_TARBALL" != z ]; then
38 SECRETS_TARBALL="s3://$S3_BUCKET/$S3_PREFIX$SECRETS_TARBALL"
39 fi
40 fi
41 fi
42 if [ "z$SECRETS_TARBALL" != z ]; then
43 echo "$SECRETS_TARBALL" > /work-space/secrets-tarball.file
44 fi
45 ;;
46 # backup
47 (create-secrets-tarball)
48 APP_NAME="$(cat /config/APP_NAME)"
49 secret_template="{{ \$item.metadata.namespace }}:{{ \$item.metadata.name }}{{ \"\n\" }}"
50 secret_template_list="{{ range \$index, \$item := .items }}$secret_template{{ end }}"
51
52 kubectl auth can-i --list
53 _get_backup_secrets() {
54 kubectl get secret -l 'backup.cert-manager.brainfood.com=true' -l "app.backup.cert-manager.brainfood.com=$APP_NAME" -o go-template --template "$secret_template_list"
55 }
56
57 date="$(date)"
58 timestamp_year_month="$(date -d "$date" '+%Y/%m')"
59 timestamp_name="$(date -d "$date" '+%Y%m%d-%H%M%S')"
60 mkdir "/work-space/$date"
61 (_get_backup_secrets) | while IFS=":" read namespace name; do
62 mkdir -p "/work-space/$timestamp_name/$namespace"
63 kubectl get -n "$namespace" secret "$name" -o yaml > "/work-space/$timestamp_name/$namespace/$name.yaml"
64 done
65
66 tar -C "/work-space/$timestamp_name" -f - -c . | gzip -9 > "/work-space/$timestamp_name.tar.gz"
67 echo "$timestamp_name.tar.gz" > /work-space/secrets-tarball.file
68 echo "$timestamp_year_month" > /work-space/timestamp_year_month
69 ;;
70 # backup
71 (copy-to-s3)
72 # s5cmd image doesn't have many tools installed
73 S3_ENABLED="$(cat /config/S3_ENABLED)"
74 if [ true = "$S3_ENABLED" ]; then
75 SECRETS_TARBALL="$(cat /work-space/secrets-tarball.file)"
76 timestamp_year_month="$(cat /work-space/timestamp_year_month)"
77 if [ "z$SECRETS_TARBALL" != z ]; then
78 setup_s3
79 /s5cmd cp "/work-space/$SECRETS_TARBALL" "s3://$S3_BUCKET/$S3_PREFIX/$timestamp_year_month/$SECRETS_TARBALL"
80 fi
81 :
82 fi
83 ;;
84
85 # restore
86 (calculate-restore-s3)
87 if [ -e /work-space/s3-file-listing ]; then
88 S3_BUCKET="$(cat /work-space/S3_BUCKET)"
89 S3_PREFIX="$(cat /work-space/S3_PREFIX)"
90 sort /work-space/s3-file-listing
91 ls -alR /work-space
92 if [ "z$SECRETS_TARBALL" = z ]; then
93 SECRETS_TARBALL="$(grep -E '[0-9]{4}\/[0-9]{2}\/[0-9]{8}-[0-9]{6}.tar.gz' /work-space/s3-file-listing | awk '{print $4}' | sort | tail -n 1)"
94 if [ "z$SECRETS_TARBALL" != z ]; then
95 SECRETS_TARBALL="s3://$S3_BUCKET/$S3_PREFIX/$SECRETS_TARBALL"
96 fi
97 fi
98 fi
99 if [ "z$SECRETS_TARBALL" != z ]; then
100 echo "$SECRETS_TARBALL" > /work-space/secrets-tarball.file
101 fi
102 ;;
103 # restore
104 (copy-from-s3)
105 # s5cmd image doesn't have many tools installed
106 S3_ENABLED="$(cat /config/S3_ENABLED)"
107 if [ true = "$S3_ENABLED" ]; then
108 setup_s3
109 if [ -s /work-space/secrets-tarball.file ]; then
110 /s5cmd cp "$(cat /work-space/secrets-tarball.file)" /work-space/secrets.tar.gz
111 fi
112 fi
113 ;;
114 # restore
115 (restore-secrets)
116 if [ -e /work-space/secrets.tar.gz ]; then
117 rm -rf /work-space/secret-restore
118 mkdir /work-space/secret-restore
119 tar -zxC /work-space/secret-restore -f /work-space/secrets.tar.gz
120 kubectl create --dry-run=client -o yaml -f /work-space/secret-restore/* | kubectl apply -f -
121 fi
122 ;;
123 esac
124
125
1 apiVersion: batch/v1
2 kind: CronJob
3 metadata:
4 name: backup-secrets-do-backup
5 spec:
6 schedule: "13 * * * *"
7 startingDeadlineSeconds: 3600
8 concurrencyPolicy: Forbid
9 successfulJobsHistoryLimit: 10
10 failedJobsHistoryLimit: 10
11 jobTemplate:
12 spec:
13 template:
14 spec:
15 serviceAccountName: backup-secrets
16 restartPolicy: OnFailure
17 volumes:
18 - name: work-space
19 emptyDir: {}
20 - name: config
21 configMap:
22 name: backup-secrets-config
23 - name: auth
24 secret:
25 secretName: backup-secrets-auth
26 - name: scripts
27 configMap:
28 name: backup-secrets-scripts
29 defaultMode: 0755
30 - name: aws-config
31 configMap:
32 name: backup-secrets-config
33 items:
34 - key: config
35 path: config
36 - name: aws-credentials
37 secret:
38 secretName: backup-secrets-auth
39 items:
40 - key: credentials
41 path: credentials
42 initContainers:
43 - name: list-s3
44 image: peakcom/s5cmd
45 command: ["/scripts/backup"]
46 args: ["list-s3"]
47 volumeMounts:
48 - name: work-space
49 mountPath: /work-space
50 - name: config
51 mountPath: /config
52 - name: auth
53 mountPath: /auth
54 - name: scripts
55 mountPath: /scripts
56
57 - mountPath: /root/.aws/config
58 name: aws-config
59 subPath: config
60 - mountPath: /root/.aws/credentials
61 name: aws-credentials
62 subPath: credentials
63
64 - name: calculate-delete-s3
65 image: debian
66 command: ["/scripts/backup"]
67 args: ["calculate-delete-s3"]
68 volumeMounts:
69 - name: work-space
70 mountPath: /work-space
71 - name: config
72 mountPath: /config
73 - name: scripts
74 mountPath: /scripts
75
76 - name: create-secrets-tarball
77 image: bitnami/kubectl
78 command: ["/scripts/backup"]
79 args: ["create-secrets-tarball"]
80 volumeMounts:
81 - name: work-space
82 mountPath: /work-space
83 - name: config
84 mountPath: /config
85 - name: scripts
86 mountPath: /scripts
87
88 - name: copy-to-s3
89 image: peakcom/s5cmd
90 command: ["/scripts/backup"]
91 args: ["copy-to-s3"]
92 volumeMounts:
93 - name: work-space
94 mountPath: /work-space
95 - name: config
96 mountPath: /config
97 - name: auth
98 mountPath: /auth
99 - name: scripts
100 mountPath: /scripts
101
102 - mountPath: /root/.aws/config
103 name: aws-config
104 subPath: config
105 - mountPath: /root/.aws/credentials
106 name: aws-credentials
107 subPath: credentials
108
109 containers:
110 - name: show-backup
111 image: debian
112 command: ["/scripts/backup"]
113 args: ["show-backup"]
114 volumeMounts:
115 - name: work-space
116 mountPath: /work-space
117 - name: scripts
118 mountPath: /scripts
119
1 ---
2 apiVersion: v1
3 kind: ServiceAccount
4 metadata:
5 name: backup-secrets
6 ---
7 apiVersion: rbac.authorization.k8s.io/v1
8 kind: Role
9 metadata:
10 name: backup-secrets
11 rules:
12 - apiGroups: [""] # "" indicates the core API group
13 resources: ["secrets"]
14 verbs: ["get", "create", "list", "patch"]
15 ---
16 apiVersion: rbac.authorization.k8s.io/v1
17 kind: RoleBinding
18 metadata:
19 name: backup-secrets
20 subjects:
21 # You can specify more than one "subject"
22 - kind: ServiceAccount
23 name: backup-secrets
24 apiGroup: ""
25 roleRef:
26 kind: Role
27 name: backup-secrets
28 apiGroup: rbac.authorization.k8s.io
29 ---
30 apiVersion: v1
31 kind: ConfigMap
32 metadata:
33 name: backup-secrets-config
34 data:
35 APP_NAME: test
36 AWS_PROFILE: "default"
37 S3_ENABLED: "true"
38 S3_BUCKET: "brainfood-bikeshed"
39 S3_PREFIX: "um-adam/secret-backup"
40 KEEP_COUNT: "30"
41 SECRETS_TARBALL: ""
42 config: ""
43 ---
44 apiVersion: v1
45 kind: ConfigMap
46 metadata:
47 name: backup-secrets-scripts
48 ---
49 apiVersion: v1
50 kind: Secret
51 metadata:
52 name: backup-secrets-auth
53 stringData:
54 credentials: ""
55
1 apiVersion: kustomize.config.k8s.io/v1beta1
2 kind: Kustomization
3
4 resources:
5 - ./config.yaml
6 - ./backup.yaml
7
8 configMapGenerator:
9 - name: backup-secrets-scripts
10 behavior: merge
11 options:
12 disableNameSuffixHash: true
13 files:
14 - ../../scripts/backup
15
1 apiVersion: batch/v1
2 kind: CronJob
3 metadata:
4 name: backup-secrets-do-backup
5 annotations:
6 helm.sh/hook: post-install,post-upgrade
7 spec:
8 schedule: "13 * * * *"
9 startingDeadlineSeconds: 3600
10 concurrencyPolicy: Forbid
11 successfulJobsHistoryLimit: 10
12 failedJobsHistoryLimit: 10
13 jobTemplate:
14 spec:
15 template:
16 spec:
17 serviceAccountName: backup-secrets
18 restartPolicy: OnFailure
19 volumes:
20 - name: work-space
21 emptyDir: {}
22 - name: config
23 configMap:
24 name: backup-secrets-config
25 - name: auth
26 secret:
27 secretName: backup-secrets-auth
28 - name: scripts
29 configMap:
30 name: backup-secrets-scripts
31 defaultMode: 0755
32 initContainers:
33 - name: list-s3
34 image: peakcom/s5cmd
35 command: ["/scripts/backup"]
36 args: ["list-s3"]
37 volumeMounts:
38 - name: work-space
39 mountPath: /work-space
40 - name: config
41 mountPath: /config
42 - name: auth
43 mountPath: /auth
44 - name: scripts
45 mountPath: /scripts
46
47 - name: calculate-delete-s3
48 image: debian
49 command: ["/scripts/backup"]
50 args: ["calculate-delete-s3"]
51 volumeMounts:
52 - name: work-space
53 mountPath: /work-space
54 - name: config
55 mountPath: /config
56 - name: scripts
57 mountPath: /scripts
58
59 - name: create-secrets-tarball
60 image: bitnami/kubectl
61 command: ["/scripts/backup"]
62 args: ["create-secrets-tarball"]
63 volumeMounts:
64 - name: work-space
65 mountPath: /work-space
66 - name: scripts
67 mountPath: /scripts
68
69
70 - name: copy-to-s3
71 image: peakcom/s5cmd
72 command: ["/scripts/backup"]
73 args: ["copy-to-s3"]
74 volumeMounts:
75 - name: work-space
76 mountPath: /work-space
77 - name: config
78 mountPath: /config
79 - name: auth
80 mountPath: /auth
81 - name: scripts
82 mountPath: /scripts
83
84 containers:
85 - name: show-backup
86 image: debian
87 command: ["/scripts/backup"]
88 args: ["show-backup"]
89 volumeMounts:
90 - name: work-space
91 mountPath: /work-space
92 - name: scripts
93 mountPath: /scripts
94
1 ---
2 apiVersion: v1
3 kind: ServiceAccount
4 metadata:
5 name: backup-secrets
6 annotations:
7 helm.sh/hook: post-install,post-upgrade
8 helm.sh/hook-weight: "-6"
9 ---
10 apiVersion: rbac.authorization.k8s.io/v1
11 kind: Role
12 metadata:
13 name: backup-secrets
14 annotations:
15 helm.sh/hook: post-install,post-upgrade
16 helm.sh/hook-weight: "-6"
17 rules:
18 - apiGroups: [""] # "" indicates the core API group
19 resources: ["secrets"]
20 verbs: ["get", "create", "list", "patch"]
21 ---
22 apiVersion: rbac.authorization.k8s.io/v1
23 kind: RoleBinding
24 metadata:
25 name: backup-secrets
26 annotations:
27 helm.sh/hook: post-install,post-upgrade
28 helm.sh/hook-weight: "-6"
29 subjects:
30 # You can specify more than one "subject"
31 - kind: ServiceAccount
32 name: backup-secrets
33 apiGroup: ""
34 roleRef:
35 kind: Role
36 name: backup-secrets
37 apiGroup: rbac.authorization.k8s.io
38 ---
39 apiVersion: v1
40 kind: ConfigMap
41 metadata:
42 name: backup-secrets-config
43 annotations:
44 helm.sh/hook: post-install,post-upgrade
45 helm.sh/hook-weight: "-6"
46 data:
47 S3_ENABLED: "true"
48 S3_REGION: "us-west-2"
49 S3_BUCKET: "brainfood-bikeshed"
50 S3_PREFIX: "um-adam/secret-backup"
51 KEEP_COUNT: "30"
52 SECRETS_TARBALL: ""
53 ---
54 apiVersion: v1
55 kind: ConfigMap
56 metadata:
57 name: backup-secrets-scripts
58 annotations:
59 helm.sh/hook: post-install,post-upgrade
60 helm.sh/hook-weight: "-6"
61 ---
62 apiVersion: v1
63 kind: Secret
64 metadata:
65 name: backup-secrets-auth
66 annotations:
67 helm.sh/hook: post-install,post-upgrade
68 helm.sh/hook-weight: "-6"
69 stringData:
70 AWS_ACCESS_KEY: "AKIAYF7A6NI4N2CAG6V4"
71 AWS_SECRET_ACCESS_KEY: "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"
72
1 apiVersion: kustomize.config.k8s.io/v1beta1
2 kind: Kustomization
3
4 resources:
5 - ./restore.yaml
6
1 apiVersion: batch/v1
2 kind: Job
3 metadata:
4 annotations:
5 helm.sh/hook: post-install
6 helm.sh/hook-weight: "-5"
7 helm.sh/hook-delete-policy: before-hook-creation
8 # annotations:
9 # helm.sh/hook: pre-install
10 # helm.sh/hook-delete-policy: before-hook-creation
11 # helm.sh/hook-weight: "-5"
12 name: backup-secrets-do-restore
13 spec:
14 template:
15 spec:
16 restartPolicy: Never
17 serviceAccountName: backup-secrets
18 containers:
19 - name: show-backup
20 command: ["/scripts/backup"]
21 args: ["show-backup"]
22 image: debian
23 volumeMounts:
24 - mountPath: /work-space
25 name: work-space
26 - mountPath: /scripts
27 name: scripts
28
29 initContainers:
30 - name: list-s3
31 command: ["/scripts/backup"]
32 args: ["list-s3"]
33 image: peakcom/s5cmd
34 volumeMounts:
35 - mountPath: /work-space
36 name: work-space
37 - mountPath: /config
38 name: config
39 - mountPath: /auth
40 name: auth
41 - mountPath: /scripts
42 name: scripts
43
44 - mountPath: /root/.aws/config
45 name: aws-config
46 subPath: config
47 - mountPath: /root/.aws/credentials
48 name: aws-credentials
49 subPath: credentials
50
51 - name: calculate-restore-s3
52 command: ["/scripts/backup"]
53 args: ["calculate-restore-s3"]
54 image: debian
55 volumeMounts:
56 - mountPath: /work-space
57 name: work-space
58 - mountPath: /config
59 name: config
60 - mountPath: /auth
61 name: auth
62 - mountPath: /scripts
63 name: scripts
64 - name: copy-from-s3
65 command: ["/scripts/backup"]
66 args: ["copy-from-s3"]
67 image: peakcom/s5cmd
68 volumeMounts:
69 - mountPath: /work-space
70 name: work-space
71 - mountPath: /config
72 name: config
73 - mountPath: /auth
74 name: auth
75 - mountPath: /scripts
76 name: scripts
77
78 - mountPath: /root/.aws/config
79 name: aws-config
80 subPath: config
81 - mountPath: /root/.aws/credentials
82 name: aws-credentials
83 subPath: credentials
84 - name: restore-secrets
85 command: ["/scripts/backup"]
86 args: ["restore-secrets"]
87 image: bitnami/kubectl
88 volumeMounts:
89 - mountPath: /work-space
90 name: work-space
91 - mountPath: /scripts
92 name: scripts
93 volumes:
94 - name: work-space
95 emptyDir: {}
96 - name: aws-config
97 configMap:
98 name: backup-secrets-config
99 items:
100 - key: config
101 path: config
102 - name: aws-credentials
103 secret:
104 secretName: backup-secrets-auth
105 items:
106 - key: credentials
107 path: credentials
108 - name: config
109 configMap:
110 name: backup-secrets-config
111 - name: auth
112 secret:
113 secretName: backup-secrets-auth
114 - name: scripts
115 configMap:
116 defaultMode: 0755
117 name: backup-secrets-scripts
118
1 ---
1 apiVersion: cert-manager.io/v1 2 apiVersion: cert-manager.io/v1
2 kind: Issuer 3 kind: Issuer
3 metadata: 4 metadata:
......
...@@ -5,3 +5,4 @@ resources: ...@@ -5,3 +5,4 @@ resources:
5 - ./gateway.yaml 5 - ./gateway.yaml
6 - ./issuers.yaml 6 - ./issuers.yaml
7 - ./certificates.yaml 7 - ./certificates.yaml
8
......
1 name: test 1 name: test
2 namespace: default 2 namespace: default
3 gatewayClassName: istio 3 gatewayClassName: istio
4 baseDomain: example.com 4 base:
5 subDomains: 5 secretTemplate:
6 - auth.example.com 6 labels:
7 - www.example.com 7 "app.backup.cert-manager.brainfood.com": "foo-app"
8 "backup.cert-manager.brainfood.com": "true"
9 domain: app.local
10 sub:
11 secretTemplate:
12 labels:
13 "app.backup.cert-manager.brainfood.com": "foo-app"
14 "backup.cert-manager.brainfood.com": "true"
15 domains:
16 - auth.app.local
17 - www.app.local
8 18
9 letsEncrypt: 19 letsEncrypt:
10 email: name@example.com 20 email: user@example.com
21 acmeAccountSecretTemplate:
22 labels:
23 "app.backup.cert-manager.brainfood.com": "foo-app"
24 "backup.cert-manager.brainfood.com": "true"
25
......
...@@ -22,7 +22,7 @@ releases: ...@@ -22,7 +22,7 @@ releases:
22 - op: replace 22 - op: replace
23 path: /spec/dnsNames 23 path: /spec/dnsNames
24 value: 24 value:
25 - {{ .Values.baseDomain }} 25 - {{ .Values.base.domain }}
26 - target: 26 - target:
27 version: v1 27 version: v1
28 group: cert-manager.io 28 group: cert-manager.io
...@@ -32,7 +32,7 @@ releases: ...@@ -32,7 +32,7 @@ releases:
32 patch: 32 patch:
33 - op: replace 33 - op: replace
34 path: /spec/dnsNames 34 path: /spec/dnsNames
35 value: {{ .Values.subDomains | toYaml | nindent 14 }} 35 value: {{ .Values.sub.domains | toYaml | nindent 14 }}
36 - target: 36 - target:
37 version: v1beta1 37 version: v1beta1
38 group: gateway.networking.k8s.io 38 group: gateway.networking.k8s.io
...@@ -45,13 +45,13 @@ releases: ...@@ -45,13 +45,13 @@ releases:
45 value: {{ .Values.gatewayClassName }} 45 value: {{ .Values.gatewayClassName }}
46 - op: replace 46 - op: replace
47 path: /spec/listeners/1/hostname 47 path: /spec/listeners/1/hostname
48 value: {{ .Values.baseDomain }} 48 value: {{ .Values.base.domain }}
49 - op: replace 49 - op: replace
50 path: /spec/listeners/1/tls/certificateRefs/0/name 50 path: /spec/listeners/1/tls/certificateRefs/0/name
51 value: {{ .Values.name }}-exact-cert 51 value: {{ .Values.name }}-exact-cert
52 - op: replace 52 - op: replace
53 path: /spec/listeners/2/hostname 53 path: /spec/listeners/2/hostname
54 value: "*.{{ .Values.baseDomain }}" 54 value: "*.{{ .Values.base.domain }}"
55 - op: replace 55 - op: replace
56 path: /spec/listeners/2/tls/certificateRefs/0/name 56 path: /spec/listeners/2/tls/certificateRefs/0/name
57 value: {{ .Values.name }}-subs-cert 57 value: {{ .Values.name }}-subs-cert
...@@ -85,6 +85,7 @@ releases: ...@@ -85,6 +85,7 @@ releases:
85 issuerRef: 85 issuerRef:
86 name: {{ .Values.name }}-letsencrypt-staging 86 name: {{ .Values.name }}-letsencrypt-staging
87 secretName: {{ .Values.name }}-exact-cert 87 secretName: {{ .Values.name }}-exact-cert
88 secretTemplate: {{ .Values.base.secretTemplate | toYaml | nindent 12 }}
88 - apiVersion: cert-manager.io/v1 89 - apiVersion: cert-manager.io/v1
89 kind: Certificate 90 kind: Certificate
90 metadata: 91 metadata:
...@@ -94,6 +95,7 @@ releases: ...@@ -94,6 +95,7 @@ releases:
94 issuerRef: 95 issuerRef:
95 name: {{ .Values.name }}-letsencrypt-staging 96 name: {{ .Values.name }}-letsencrypt-staging
96 secretName: {{ .Values.name }}-subs-cert 97 secretName: {{ .Values.name }}-subs-cert
98 secretTemplate: {{ .Values.sub.secretTemplate | toYaml | nindent 12 }}
97 - apiVersion: cert-manager.io/v1 99 - apiVersion: cert-manager.io/v1
98 kind: Issuer 100 kind: Issuer
99 metadata: 101 metadata:
...@@ -114,4 +116,3 @@ releases: ...@@ -114,4 +116,3 @@ releases:
114 email: {{ .Values.letsEncrypt.email }} 116 email: {{ .Values.letsEncrypt.email }}
115 privateKeySecretRef: 117 privateKeySecretRef:
116 name: {{ .Values.name }}-letsencrypt-account 118 name: {{ .Values.name }}-letsencrypt-account
117
......
1 #!/bin/sh
2
3 set -ex
4
5 setup_s3() {
6 AWS_PROFILE="$(cat /config/AWS_PROFILE)"
7 S3_BUCKET="$(cat /config/S3_BUCKET)"
8 S3_PREFIX="$(cat /config/S3_PREFIX)"
9 export AWS_PROFILE
10 }
11
12 case "$1" in
13 # restore/backup
14 (list-s3)
15 # s5cmd image doesn't have many tools installed
16 S3_ENABLED="$(cat /config/S3_ENABLED)"
17 if [ true = "$S3_ENABLED" ]; then
18 setup_s3
19 if /s5cmd ls "s3://$S3_BUCKET/$S3_PREFIX/*" > /work-space/s3-file-listing.tmp; then
20 mv /work-space/s3-file-listing.tmp /work-space/s3-file-listing
21 echo "$S3_BUCKET" > /work-space/S3_BUCKET
22 echo "$S3_PREFIX" > /work-space/S3_PREFIX
23 cat /work-space/s3-file-listing
24 fi
25 fi
26 ;;
27 # backup
28 (calculate-delete-s3)
29 # this is broken
30 KEEP_COUNT="$(cat /config/KEEP_COUNT)"
31 if [ -e /work-space/s3-file-listing ] && [ "z$KEEP_COUNT" != "z" ]; then
32 S3_BUCKET="$(cat /work-space/S3_BUCKET)"
33 S3_PREFIX="$(cat /work-space/S3_PREFIX)"
34 sort /work-space/s3-file-listing
35 if [ "z$SECRETS_TARBALL" = z ]; then
36 SECRETS_TARBALL="$(gawk '/wp-content.tgz$/{print $4}' < /work-space/s3-file-listing | sort | tail -1)"
37 if [ "z$SECRETS_TARBALL" != z ]; then
38 SECRETS_TARBALL="s3://$S3_BUCKET/$S3_PREFIX$SECRETS_TARBALL"
39 fi
40 fi
41 fi
42 if [ "z$SECRETS_TARBALL" != z ]; then
43 echo "$SECRETS_TARBALL" > /work-space/secrets-tarball.file
44 fi
45 ;;
46 # backup
47 (create-secrets-tarball)
48 APP_NAME="$(cat /config/APP_NAME)"
49 secret_template="{{ \$item.metadata.namespace }}:{{ \$item.metadata.name }}{{ \"\n\" }}"
50 secret_template_list="{{ range \$index, \$item := .items }}$secret_template{{ end }}"
51
52 kubectl auth can-i --list
53 _get_backup_secrets() {
54 kubectl get secret -l 'backup.cert-manager.brainfood.com=true' -l "app.backup.cert-manager.brainfood.com=$APP_NAME" -o go-template --template "$secret_template_list"
55 }
56
57 date="$(date)"
58 timestamp_year_month="$(date -d "$date" '+%Y/%m')"
59 timestamp_name="$(date -d "$date" '+%Y%m%d-%H%M%S')"
60 mkdir "/work-space/$date"
61 (_get_backup_secrets) | while IFS=":" read namespace name; do
62 mkdir -p "/work-space/$timestamp_name/$namespace"
63 kubectl get -n "$namespace" secret "$name" -o yaml > "/work-space/$timestamp_name/$namespace/$name.yaml"
64 done
65
66 tar -C "/work-space/$timestamp_name" -f - -c . | gzip -9 > "/work-space/$timestamp_name.tar.gz"
67 echo "$timestamp_name.tar.gz" > /work-space/secrets-tarball.file
68 echo "$timestamp_year_month" > /work-space/timestamp_year_month
69 ;;
70 # backup
71 (copy-to-s3)
72 # s5cmd image doesn't have many tools installed
73 S3_ENABLED="$(cat /config/S3_ENABLED)"
74 if [ true = "$S3_ENABLED" ]; then
75 SECRETS_TARBALL="$(cat /work-space/secrets-tarball.file)"
76 timestamp_year_month="$(cat /work-space/timestamp_year_month)"
77 if [ "z$SECRETS_TARBALL" != z ]; then
78 setup_s3
79 /s5cmd cp "/work-space/$SECRETS_TARBALL" "s3://$S3_BUCKET/$S3_PREFIX/$timestamp_year_month/$SECRETS_TARBALL"
80 fi
81 :
82 fi
83 ;;
84
85 # restore
86 (calculate-restore-s3)
87 if [ -e /work-space/s3-file-listing ]; then
88 S3_BUCKET="$(cat /work-space/S3_BUCKET)"
89 S3_PREFIX="$(cat /work-space/S3_PREFIX)"
90 sort /work-space/s3-file-listing
91 ls -alR /work-space
92 if [ "z$SECRETS_TARBALL" = z ]; then
93 SECRETS_TARBALL="$(grep -E '[0-9]{4}\/[0-9]{2}\/[0-9]{8}-[0-9]{6}.tar.gz' /work-space/s3-file-listing | awk '{print $4}' | sort | tail -n 1)"
94 if [ "z$SECRETS_TARBALL" != z ]; then
95 SECRETS_TARBALL="s3://$S3_BUCKET/$S3_PREFIX/$SECRETS_TARBALL"
96 fi
97 fi
98 fi
99 if [ "z$SECRETS_TARBALL" != z ]; then
100 echo "$SECRETS_TARBALL" > /work-space/secrets-tarball.file
101 fi
102 ;;
103 # restore
104 (copy-from-s3)
105 # s5cmd image doesn't have many tools installed
106 S3_ENABLED="$(cat /config/S3_ENABLED)"
107 if [ true = "$S3_ENABLED" ]; then
108 setup_s3
109 if [ -s /work-space/secrets-tarball.file ]; then
110 /s5cmd cp "$(cat /work-space/secrets-tarball.file)" /work-space/secrets.tar.gz
111 fi
112 fi
113 ;;
114 # restore
115 (restore-secrets)
116 if [ -e /work-space/secrets.tar.gz ]; then
117 rm -rf /work-space/secret-restore
118 mkdir /work-space/secret-restore
119 tar -zxC /work-space/secret-restore -f /work-space/secrets.tar.gz
120 kubectl create --dry-run=client -o yaml -f /work-space/secret-restore/* | kubectl apply -f -
121 fi
122 ;;
123 esac
124
125