From c09b07327b688a6a47f523a15c1a5c29d4f476d0 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 7 May 2022 22:45:49 +0200 Subject: k8s: rename masters to control-plane nodes --- .../control-plane/templates/encryption-config.j2 | 13 ++ .../control-plane/templates/kubeadm.config.j2 | 53 +++++ .../templates/net_kube-router/config.0.4.0.yml.j2 | 235 ++++++++++++++++++++ .../templates/net_kube-router/config.1.1.1.yml.j2 | 236 +++++++++++++++++++++ .../templates/net_kube-router/config.1.4.0.yml.j2 | 236 +++++++++++++++++++++ .../net_kubeguard/kube-router.0.4.0.yml.j2 | 170 +++++++++++++++ .../net_kubeguard/kube-router.1.1.1.yml.j2 | 170 +++++++++++++++ .../control-plane/templates/node-local-dns.yml.j2 | 211 ++++++++++++++++++ 8 files changed, 1324 insertions(+) create mode 100644 roles/kubernetes/kubeadm/control-plane/templates/encryption-config.j2 create mode 100644 roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 create mode 100644 roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.0.4.0.yml.j2 create mode 100644 roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.1.1.1.yml.j2 create mode 100644 roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.1.4.0.yml.j2 create mode 100644 roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.0.4.0.yml.j2 create mode 100644 roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.1.1.1.yml.j2 create mode 100644 roles/kubernetes/kubeadm/control-plane/templates/node-local-dns.yml.j2 (limited to 'roles/kubernetes/kubeadm/control-plane/templates') diff --git a/roles/kubernetes/kubeadm/control-plane/templates/encryption-config.j2 b/roles/kubernetes/kubeadm/control-plane/templates/encryption-config.j2 new file mode 100644 index 00000000..345c9bf9 --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/templates/encryption-config.j2 @@ -0,0 +1,13 @@ +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - secretbox: + keys: +{% for key in kubernetes_secrets.encryption_config_keys %} + - name: key{{ loop.index }} + secret: {{ key }} +{% endfor %} + - identity: {} diff --git a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 new file mode 100644 index 00000000..2fa98ed6 --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 @@ -0,0 +1,53 @@ +{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2 #} +{# #} +apiVersion: kubeadm.k8s.io/v1beta2 +kind: InitConfiguration +{# TODO: this is ugly but we want to create our own token so we can #} +{# better control it's lifetime #} +bootstrapTokens: +- ttl: "1s" +localAPIEndpoint: + bindPort: 6442 +{% if kubernetes_overlay_node_ip is defined %} + advertiseAddress: {{ kubernetes_overlay_node_ip }} +{% endif %} +nodeRegistration: + criSocket: {{ kubernetes_cri_socket }} +--- +apiVersion: kubeadm.k8s.io/v1beta2 +kind: ClusterConfiguration +kubernetesVersion: {{ kubernetes_version }} +clusterName: {{ kubernetes.cluster_name }} +imageRepository: k8s.gcr.io +controlPlaneEndpoint: 127.0.0.1:6443 +networking: + dnsDomain: {{ kubernetes.dns_domain | default('cluster.local') }} + podSubnet: {{ kubernetes.pod_ip_range }} + serviceSubnet: {{ kubernetes.service_ip_range }} +apiServer: + extraArgs: + encryption-provider-config: /etc/kubernetes/encryption/config + extraVolumes: + - name: encryption-config + hostPath: /etc/kubernetes/encryption + mountPath: /etc/kubernetes/encryption + readOnly: true + pathType: Directory +{% if (kubernetes.api_extra_sans | default([]) | length) == 0 %} + certSANs: [] +{% else %} + certSANs: + {{ kubernetes.api_extra_sans | to_nice_yaml | indent(width=2) }} +{% endif %} +controllerManager: + extraArgs: + node-cidr-mask-size: "{{ kubernetes.pod_ip_range_size }}" +scheduler: {} +dns: + type: CoreDNS +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +clusterDNS: +- {{ kubernetes_nodelocal_dnscache_ip }} +cgroupDriver: systemd diff --git a/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.0.4.0.yml.j2 b/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.0.4.0.yml.j2 new file mode 100644 index 00000000..a2660db2 --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.0.4.0.yml.j2 @@ -0,0 +1,235 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-kubeconfig + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://127.0.0.1:{{ kubernetes_api_lb_port | default('6443') }} + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-cfg + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + cni-conf.json: | + { + "cniVersion":"0.3.0", + "name":"mynet", + "plugins":[ + { + "name":"kubernetes", + "type":"bridge", + "bridge":"kube-bridge", + "isDefaultGateway":true, + "hairpinMode": true, + "ipam":{ + "type":"host-local" + } + }, + { + "type":"portmap", + "capabilities":{ + "snat":true, + "portMappings":true + } + } + ] + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-router + tier: node + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + priorityClassName: system-node-critical + serviceAccountName: kube-router + serviceAccount: kube-router + containers: + - name: kube-router + image: docker.io/cloudnativelabs/kube-router:v{{ kubernetes_network_plugin_version }} + imagePullPolicy: Always + args: + - --run-router=true + - --run-firewall=true + - --run-service-proxy={{ kubernetes_network_plugin_replaces_kube_proxy | string | lower }} + - --kubeconfig=/var/lib/kube-router/kubeconfig + - --hairpin-mode + - --iptables-sync-period=10s + - --ipvs-sync-period=10s + - --routes-sync-period=10s + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBE_ROUTER_CNI_CONF_FILE + value: /etc/cni/net.d/10-kuberouter.conflist + livenessProbe: + httpGet: + path: /healthz + port: 20244 + initialDelaySeconds: 10 + periodSeconds: 3 + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + initContainers: + - name: install-cni + image: busybox + imagePullPolicy: Always + command: + - /bin/sh + - -c + - set -e -x; + if [ ! -f /etc/cni/net.d/10-kuberouter.conflist ]; then + if [ -f /etc/cni/net.d/*.conf ]; then + rm -f /etc/cni/net.d/*.conf; + fi; + TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; + cp /etc/kube-router/cni-conf.json ${TMP}; + mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; + fi + volumeMounts: + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kube-router-cfg + mountPath: /etc/kube-router + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d + - name: kube-router-cfg + configMap: + name: kube-router-cfg + - name: kubeconfig + configMap: + name: kube-router-kubeconfig + items: + - key: kubeconfig.conf + path: kubeconfig + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system diff --git a/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.1.1.1.yml.j2 b/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.1.1.1.yml.j2 new file mode 100644 index 00000000..382164cb --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.1.1.1.yml.j2 @@ -0,0 +1,236 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-kubeconfig + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://127.0.0.1:{{ kubernetes_api_lb_port | default('6443') }} + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-cfg + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + cni-conf.json: | + { + "cniVersion":"0.3.0", + "name":"mynet", + "plugins":[ + { + "name":"kubernetes", + "type":"bridge", + "bridge":"kube-bridge", + "isDefaultGateway":true, + "hairpinMode": true, + "ipam":{ + "type":"host-local" + } + }, + { + "type":"portmap", + "capabilities":{ + "snat":true, + "portMappings":true + } + } + ] + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-router + tier: node + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + priorityClassName: system-node-critical + serviceAccountName: kube-router + serviceAccount: kube-router + containers: + - name: kube-router + image: docker.io/cloudnativelabs/kube-router:v{{ kubernetes_network_plugin_version }} + imagePullPolicy: Always + args: + - --run-router=true + - --run-firewall=true + - --run-service-proxy={{ kubernetes_network_plugin_replaces_kube_proxy | string | lower }} + - --bgp-graceful-restart=true + - --kubeconfig=/var/lib/kube-router/kubeconfig + - --hairpin-mode + - --iptables-sync-period=10s + - --ipvs-sync-period=10s + - --routes-sync-period=10s + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBE_ROUTER_CNI_CONF_FILE + value: /etc/cni/net.d/10-kuberouter.conflist + livenessProbe: + httpGet: + path: /healthz + port: 20244 + initialDelaySeconds: 10 + periodSeconds: 3 + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + initContainers: + - name: install-cni + image: busybox + imagePullPolicy: Always + command: + - /bin/sh + - -c + - set -e -x; + if [ ! -f /etc/cni/net.d/10-kuberouter.conflist ]; then + if [ -f /etc/cni/net.d/*.conf ]; then + rm -f /etc/cni/net.d/*.conf; + fi; + TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; + cp /etc/kube-router/cni-conf.json ${TMP}; + mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; + fi + volumeMounts: + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kube-router-cfg + mountPath: /etc/kube-router + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d + - name: kube-router-cfg + configMap: + name: kube-router-cfg + - name: kubeconfig + configMap: + name: kube-router-kubeconfig + items: + - key: kubeconfig.conf + path: kubeconfig + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system diff --git a/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.1.4.0.yml.j2 b/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.1.4.0.yml.j2 new file mode 100644 index 00000000..382164cb --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/config.1.4.0.yml.j2 @@ -0,0 +1,236 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-kubeconfig + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://127.0.0.1:{{ kubernetes_api_lb_port | default('6443') }} + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-cfg + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + cni-conf.json: | + { + "cniVersion":"0.3.0", + "name":"mynet", + "plugins":[ + { + "name":"kubernetes", + "type":"bridge", + "bridge":"kube-bridge", + "isDefaultGateway":true, + "hairpinMode": true, + "ipam":{ + "type":"host-local" + } + }, + { + "type":"portmap", + "capabilities":{ + "snat":true, + "portMappings":true + } + } + ] + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-router + tier: node + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + priorityClassName: system-node-critical + serviceAccountName: kube-router + serviceAccount: kube-router + containers: + - name: kube-router + image: docker.io/cloudnativelabs/kube-router:v{{ kubernetes_network_plugin_version }} + imagePullPolicy: Always + args: + - --run-router=true + - --run-firewall=true + - --run-service-proxy={{ kubernetes_network_plugin_replaces_kube_proxy | string | lower }} + - --bgp-graceful-restart=true + - --kubeconfig=/var/lib/kube-router/kubeconfig + - --hairpin-mode + - --iptables-sync-period=10s + - --ipvs-sync-period=10s + - --routes-sync-period=10s + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBE_ROUTER_CNI_CONF_FILE + value: /etc/cni/net.d/10-kuberouter.conflist + livenessProbe: + httpGet: + path: /healthz + port: 20244 + initialDelaySeconds: 10 + periodSeconds: 3 + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + initContainers: + - name: install-cni + image: busybox + imagePullPolicy: Always + command: + - /bin/sh + - -c + - set -e -x; + if [ ! -f /etc/cni/net.d/10-kuberouter.conflist ]; then + if [ -f /etc/cni/net.d/*.conf ]; then + rm -f /etc/cni/net.d/*.conf; + fi; + TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; + cp /etc/kube-router/cni-conf.json ${TMP}; + mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; + fi + volumeMounts: + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kube-router-cfg + mountPath: /etc/kube-router + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d + - name: kube-router-cfg + configMap: + name: kube-router-cfg + - name: kubeconfig + configMap: + name: kube-router-kubeconfig + items: + - key: kubeconfig.conf + path: kubeconfig + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system diff --git a/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.0.4.0.yml.j2 b/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.0.4.0.yml.j2 new file mode 100644 index 00000000..e343f4a7 --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.0.4.0.yml.j2 @@ -0,0 +1,170 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-kubeconfig + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://127.0.0.1:{{ kubernetes_api_lb_port | default('6443') }} + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-router + tier: node + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + priorityClassName: system-node-critical + serviceAccountName: kube-router + serviceAccount: kube-router + containers: + - name: kube-router + image: docker.io/cloudnativelabs/kube-router:v{{ kubernetes_network_plugin_version }} + imagePullPolicy: Always + args: + - --cluster-cidr={{ kubernetes.pod_ip_range }} + - --run-router=false + - --run-firewall=true + - --run-service-proxy={{ kubernetes_network_plugin_replaces_kube_proxy | string | lower }} + - --kubeconfig=/var/lib/kube-router/kubeconfig + - --hairpin-mode + - --iptables-sync-period=10s + - --ipvs-sync-period=10s + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + livenessProbe: + httpGet: + path: /healthz + port: 20244 + initialDelaySeconds: 10 + periodSeconds: 3 + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: kubeconfig + configMap: + name: kube-router-kubeconfig + items: + - key: kubeconfig.conf + path: kubeconfig + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system diff --git a/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.1.1.1.yml.j2 b/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.1.1.1.yml.j2 new file mode 100644 index 00000000..ec30d670 --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.1.1.1.yml.j2 @@ -0,0 +1,170 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-kubeconfig + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://127.0.0.1:{{ kubernetes_api_lb_port | default('6443') }} + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-router + tier: node + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + priorityClassName: system-node-critical + serviceAccountName: kube-router + serviceAccount: kube-router + containers: + - name: kube-router + image: docker.io/cloudnativelabs/kube-router:v{{ kubernetes_network_plugin_version }} + imagePullPolicy: Always + args: + - --run-router=false + - --run-firewall=true + - --run-service-proxy={{ kubernetes_network_plugin_replaces_kube_proxy | string | lower }} + - --bgp-graceful-restart=true + - --kubeconfig=/var/lib/kube-router/kubeconfig + - --hairpin-mode + - --iptables-sync-period=10s + - --ipvs-sync-period=10s + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + livenessProbe: + httpGet: + path: /healthz + port: 20244 + initialDelaySeconds: 10 + periodSeconds: 3 + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: kubeconfig + configMap: + name: kube-router-kubeconfig + items: + - key: kubeconfig.conf + path: kubeconfig + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system diff --git a/roles/kubernetes/kubeadm/control-plane/templates/node-local-dns.yml.j2 b/roles/kubernetes/kubeadm/control-plane/templates/node-local-dns.yml.j2 new file mode 100644 index 00000000..d536d5a7 --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/templates/node-local-dns.yml.j2 @@ -0,0 +1,211 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + {{ kubernetes.dns_domain | default('cluster.local') }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ kubernetes_nodelocal_dnscache_ip }} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + health {{ kubernetes_nodelocal_dnscache_ip }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ kubernetes_nodelocal_dnscache_ip }} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ kubernetes_nodelocal_dnscache_ip }} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ kubernetes_nodelocal_dnscache_ip }} + forward . __PILLAR__UPSTREAM__SERVERS__ { + force_tcp + } + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + containers: + - name: node-cache + image: k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0 + resources: + requests: + cpu: 25m + memory: 5Mi + args: [ "-localip", "{{ kubernetes_nodelocal_dnscache_ip }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + securityContext: + privileged: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ kubernetes_nodelocal_dnscache_ip }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile.base +--- +# A headless service is a service with a service IP but instead of load-balancing it will return the IPs of our associated Pods. +# We use this to expose metrics to Prometheus. +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + labels: + k8s-app: node-local-dns + name: node-local-dns + namespace: kube-system +spec: + clusterIP: None + ports: + - name: metrics + port: 9253 + targetPort: 9253 + selector: + k8s-app: node-local-dns -- cgit v1.2.3 From 09c8120540735c22316a55593f4c56bcd6ae7e88 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 8 May 2022 01:08:36 +0200 Subject: add support for cluster with kubernetes 1.24 --- inventory/group_vars/k8s-chtest/vars.yml | 2 +- roles/kubernetes/kubeadm/control-plane/tasks/primary.yml | 6 +++--- .../kubeadm/control-plane/templates/kubeadm.config.j2 | 12 +++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) (limited to 'roles/kubernetes/kubeadm/control-plane/templates') diff --git a/inventory/group_vars/k8s-chtest/vars.yml b/inventory/group_vars/k8s-chtest/vars.yml index 66824314..939d93da 100644 --- a/inventory/group_vars/k8s-chtest/vars.yml +++ b/inventory/group_vars/k8s-chtest/vars.yml @@ -1,5 +1,5 @@ --- -kubernetes_version: 1.23.6 +kubernetes_version: 1.24.0 kubernetes_container_runtime: containerd kubernetes_network_plugin: kube-router kubernetes_network_plugin_version: 1.4.0 diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml index 22a5af42..450c3a1a 100644 --- a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml +++ b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml @@ -28,8 +28,8 @@ - name: initialize kubernetes primary control-plane node and store log block: - name: initialize kubernetes primary control-plane node - command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_network_plugin_replaces_kube_proxy %} --skip-phases addon/kube-proxy{% endif %} --skip-token-print" - # command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_network_plugin_replaces_kube_proxy %} --skip-phases addon/kube-proxy{% endif %} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print" + command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }} --skip-token-print" + # command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print" args: creates: /etc/kubernetes/pki/ca.crt register: kubeadm_init @@ -47,7 +47,7 @@ content: "{{ kubeadm_init.stderr }}\n" dest: /etc/kubernetes/kubeadm-init.errors - - name: create bootstrap token for existing cluster + - name: create bootstrap token for new cluster command: kubeadm token create --ttl 42m check_mode: no register: kubeadm_token_generate diff --git a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 index 2fa98ed6..a0f3efe7 100644 --- a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 +++ b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 @@ -1,6 +1,6 @@ -{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2 #} +{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 #} {# #} -apiVersion: kubeadm.k8s.io/v1beta2 +apiVersion: kubeadm.k8s.io/v1beta3 kind: InitConfiguration {# TODO: this is ugly but we want to create our own token so we can #} {# better control it's lifetime #} @@ -11,10 +11,14 @@ localAPIEndpoint: {% if kubernetes_overlay_node_ip is defined %} advertiseAddress: {{ kubernetes_overlay_node_ip }} {% endif %} +{% if kubernetes_network_plugin_replaces_kube_proxy %} +skipPhases: +- addon/kube-proxy +{% endif %} nodeRegistration: criSocket: {{ kubernetes_cri_socket }} --- -apiVersion: kubeadm.k8s.io/v1beta2 +apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration kubernetesVersion: {{ kubernetes_version }} clusterName: {{ kubernetes.cluster_name }} @@ -43,8 +47,6 @@ controllerManager: extraArgs: node-cidr-mask-size: "{{ kubernetes.pod_ip_range_size }}" scheduler: {} -dns: - type: CoreDNS --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration -- cgit v1.2.3