diff options
27 files changed, 457 insertions, 27 deletions
diff --git a/common/kubernetes-cleanup.yml b/common/kubernetes-cluster-cleanup.yml index 83d6945c..83d6945c 100644 --- a/common/kubernetes-cleanup.yml +++ b/common/kubernetes-cluster-cleanup.yml diff --git a/common/kubernetes-cluster-layout.yml b/common/kubernetes-cluster-layout.yml index 64856fc5..bd972a6d 100644 --- a/common/kubernetes-cluster-layout.yml +++ b/common/kubernetes-cluster-layout.yml @@ -17,7 +17,7 @@ - name: sanity check - fail on multiple masters if no primary master is configured assert: - msg: "For multiple masters to work you need to define kubernetes_cluster_layout.primary_master" + msg: "for multiple masters to work you need to define kubernetes_cluster_layout.primary_master" that: (kubernetes_cluster_layout.masters | length) == 1 or kubernetes_cluster_layout.primary_master is defined - name: create group for all kubernetes nodes diff --git a/common/kubernetes.yml b/common/kubernetes-cluster.yml index d970c1fe..31b7bbf4 100644 --- a/common/kubernetes.yml +++ b/common/kubernetes-cluster.yml @@ -2,7 +2,6 @@ - name: prepare variables and do some sanity checks hosts: _kubernetes_nodes_ gather_facts: no - run_once: yes tasks: - name: sanity checks for kubeguard when: kubernetes_network_plugin == 'kubeguard' @@ -22,17 +21,15 @@ msg: "At least one node_index is < 1 (indizes start at 1)" that: (kubeguard.node_index.values() | min) > 0 - - name: check whether overlay node io is configured > 0 + - name: check whether overlay node ip is configured assert: msg: "For kubeguard to work you need to configure kubernetes_overlay_node_ip" that: kubernetes_overlay_node_ip is defined - - name: make sure the kubernetes_cri_socket variable is configured correctly - when: kubernetes_container_runtime == 'containerd' + - name: make sure all nodes do belong to the kubernetes-cluster group assert: - msg: "The variable kubernetes_cri_socket is not configured correctly for use with containerd!" - that: - - kubernetes_cri_socket == "unix:///run/containerd/containerd.sock" + msg: "The host '{{ inventory_hostname }}' does not belong to the group 'kubernetes-cluster'" + that: "'kubernetes-cluster' in group_names" ######## - name: kubernetes base installation @@ -45,12 +42,12 @@ - role: kubernetes/base - role: kubernetes/kubeadm/base -- name: configure kubernetes primary master +- name: configure primary kubernetes master hosts: _kubernetes_primary_master_ roles: - - role: kubernetes/kubeadm/master + - role: kubernetes/kubeadm/masterg -- name: configure kubernetes secondary masters +- name: configure secondary kubernetes masters hosts: _kubernetes_masters_:!_kubernetes_primary_master_ roles: - role: kubernetes/kubeadm/master diff --git a/dan/k8s-emc.yml b/dan/k8s-emc.yml index b966460b..273b1c1b 100644 --- a/dan/k8s-emc.yml +++ b/dan/k8s-emc.yml @@ -22,5 +22,10 @@ kubernetes_secrets_cooked: "{{ kubernetes_secrets }}" external_ip_cooked: "{{ external_ip }}" -- import_playbook: ../common/kubernetes.yml -- import_playbook: ../common/kubernetes-cleanup.yml +- import_playbook: ../common/kubernetes-cluster.yml +- import_playbook: ../common/kubernetes-cluster-cleanup.yml + +- name: install addons + hosts: _kubernetes_primary_master_ + roles: + - role: kubernetes/addons/metrics-server diff --git a/inventory/group_vars/k8s-emc/main.yml b/inventory/group_vars/k8s-emc/main.yml index 367fcda9..605c7310 100644 --- a/inventory/group_vars/k8s-emc/main.yml +++ b/inventory/group_vars/k8s-emc/main.yml @@ -50,3 +50,5 @@ kubeguard: ele-helene: eno2 kubernetes_overlay_node_ip: "{{ kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) | ipaddr(1) | ipaddr('address') }}" + +kubernetes_metrics_server_version: 0.3.6 diff --git a/inventory/group_vars/k8s-lwl/main.yml b/inventory/group_vars/k8s-lwl/main.yml index 3827841a..791e393f 100644 --- a/inventory/group_vars/k8s-lwl/main.yml +++ b/inventory/group_vars/k8s-lwl/main.yml @@ -47,3 +47,5 @@ kubeguard: lw-helene: eno2 kubernetes_overlay_node_ip: "{{ kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) | ipaddr(1) | ipaddr('address') }}" + +kubernetes_metrics_server_version: 0.3.6 diff --git a/inventory/group_vars/kubernetes-cluster/vars.yml b/inventory/group_vars/kubernetes-cluster/vars.yml new file mode 100644 index 00000000..c1149988 --- /dev/null +++ b/inventory/group_vars/kubernetes-cluster/vars.yml @@ -0,0 +1,2 @@ +--- +kubernetes_nodelocal_dnscache_ip: 169.254.20.10 diff --git a/inventory/group_vars/kubernetes/vars.yml b/inventory/group_vars/kubernetes/vars.yml new file mode 100644 index 00000000..a0e08c5c --- /dev/null +++ b/inventory/group_vars/kubernetes/vars.yml @@ -0,0 +1,6 @@ +--- +kubernetes_cri_sockets: + containerd: "unix:///run/containerd/containerd.sock" + docker: "" + +kubernetes_cri_socket: "{{ kubernetes_cri_sockets[kubernetes_container_runtime] }}" diff --git a/inventory/host_vars/sk-cloudia/vars.yml b/inventory/host_vars/sk-cloudia/vars.yml index 44090907..afeee13d 100644 --- a/inventory/host_vars/sk-cloudia/vars.yml +++ b/inventory/host_vars/sk-cloudia/vars.yml @@ -37,7 +37,6 @@ kubernetes_standalone_max_pods: 42 kubernetes_standalone_resolv_conf: /var/run/systemd/resolve/resolv.conf kubernetes_standalone_pod_cidr: 192.168.255.0/24 kubernetes_standalone_cni_variant: with-localonly-portmap -kubernetes_cri_socket: "unix:///run/containerd/containerd.sock" acmetool_directory_server: "{{ acmetool_directory_server_le_live_v2 }}" diff --git a/inventory/hosts.ini b/inventory/hosts.ini index 24f2c438..89d073a8 100644 --- a/inventory/hosts.ini +++ b/inventory/hosts.ini @@ -296,6 +296,22 @@ ele-dolmetsch-ctl ele-dolmetsch-raspi + +[kubernetes-cluster:children] +k8s-emc +k8s-lwl + +[standalone-kubelet] +sk-cloudia +ele-thetys +lw-thetys + +[kubernetes:children] +kubernetes-cluster +standalone-kubelet + + + ### Kubernetes Cluster: emc [k8s-emc-encoder] #ele-dione @@ -323,6 +339,7 @@ elevate k8s-emc + ### Kubernetes Cluster: lendwirbel-live [k8s-lwl-encoder] lw-dione diff --git a/roles/apps/coturn/templates/acmetool-reload.sh.j2 b/roles/apps/coturn/templates/acmetool-reload.sh.j2 index 1eff1ad3..c90c296d 100644 --- a/roles/apps/coturn/templates/acmetool-reload.sh.j2 +++ b/roles/apps/coturn/templates/acmetool-reload.sh.j2 @@ -18,7 +18,7 @@ while read name; do install -m 0644 -o root -g coturn "$certdir/fullchain" "$SSL_D/cert.pem" install -m 0640 -o root -g coturn "$certdir/privkey" "$SSL_D/privkey.pem" -{% if kubernetes_cri_socket is defined %} +{% if kubernetes_cri_socket %} export CONTAINER_RUNTIME_ENDPOINT="{{ kubernetes_cri_socket }}" {% endif %} pod_id=$(crictl pods -q --state ready --name "^coturn-{{ coturn_realm }}-{{ ansible_nodename }}$") diff --git a/roles/apps/nextcloud/templates/nextcloud-occ.j2 b/roles/apps/nextcloud/templates/nextcloud-occ.j2 index 571aecc4..7e2a51d4 100755 --- a/roles/apps/nextcloud/templates/nextcloud-occ.j2 +++ b/roles/apps/nextcloud/templates/nextcloud-occ.j2 @@ -9,7 +9,7 @@ if [ -z "$INST_NAME" ]; then fi set -eu -{% if kubernetes_cri_socket is defined %} +{% if kubernetes_cri_socket %} export CONTAINER_RUNTIME_ENDPOINT="{{ kubernetes_cri_socket }}" {% endif %} diff --git a/roles/apps/nextcloud/templates/run-cron.sh.j2 b/roles/apps/nextcloud/templates/run-cron.sh.j2 index 9936bad1..755b7cb1 100644 --- a/roles/apps/nextcloud/templates/run-cron.sh.j2 +++ b/roles/apps/nextcloud/templates/run-cron.sh.j2 @@ -1,6 +1,6 @@ #!/bin/bash -{% if kubernetes_cri_socket is defined %} +{% if kubernetes_cri_socket %} export CONTAINER_RUNTIME_ENDPOINT="{{ kubernetes_cri_socket }}" {% endif %} diff --git a/roles/kubernetes/addons/metrics-server/tasks/main.yml b/roles/kubernetes/addons/metrics-server/tasks/main.yml new file mode 100644 index 00000000..e09106c1 --- /dev/null +++ b/roles/kubernetes/addons/metrics-server/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: copy config for metrics-server + template: + src: "components.{{ kubernetes_metrics_server_version }}.yml.j2" + dest: /etc/kubernetes/metrics-server.yml + +- name: install metrics-server onto the cluster + command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/metrics-server.yml + register: kube_metrics_server_apply_result + changed_when: (kube_metrics_server_apply_result.stdout_lines | reject("regex", " unchanged$") | list | length) > 0 diff --git a/roles/kubernetes/addons/metrics-server/templates/components.0.3.6.yml.j2 b/roles/kubernetes/addons/metrics-server/templates/components.0.3.6.yml.j2 new file mode 100644 index 00000000..1e3789bb --- /dev/null +++ b/roles/kubernetes/addons/metrics-server/templates/components.0.3.6.yml.j2 @@ -0,0 +1,156 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:aggregated-metrics-reader + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io +spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-server + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server +spec: + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + name: metrics-server + labels: + k8s-app: metrics-server + spec: + serviceAccountName: metrics-server + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} + containers: + - name: metrics-server + image: k8s.gcr.io/metrics-server-amd64:v0.3.6 + imagePullPolicy: IfNotPresent + args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-insecure-tls + - --kubelet-preferred-address-types=InternalIP,ExternalIP + ports: + - name: main-port + containerPort: 4443 + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - name: tmp-dir + mountPath: /tmp + nodeSelector: + kubernetes.io/os: linux + kubernetes.io/arch: "amd64" + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master +--- +apiVersion: v1 +kind: Service +metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/name: "Metrics-server" + kubernetes.io/cluster-service: "true" +spec: + selector: + k8s-app: metrics-server + ports: + - port: 443 + protocol: TCP + targetPort: main-port +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + - namespaces + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/roles/kubernetes/base/tasks/cri_containerd.yml b/roles/kubernetes/base/tasks/cri_containerd.yml index ff904ffc..549ccae0 100644 --- a/roles/kubernetes/base/tasks/cri_containerd.yml +++ b/roles/kubernetes/base/tasks/cri_containerd.yml @@ -1,7 +1,7 @@ --- - name: make sure the kubernetes_cri_socket variable is configured correctly assert: - msg: "The variable kubernetes_cri_socket is not configured to use containerd as container runtime." + msg: "The variable kubernetes_cri_socket is not configured correctly. You might need to move your host to the group kubernetes-cluster or standalone-kubelet!" that: - kubernetes_cri_socket == "unix:///run/containerd/containerd.sock" diff --git a/roles/kubernetes/base/tasks/cri_docker.yml b/roles/kubernetes/base/tasks/cri_docker.yml index 67196f51..50558d70 100644 --- a/roles/kubernetes/base/tasks/cri_docker.yml +++ b/roles/kubernetes/base/tasks/cri_docker.yml @@ -1,7 +1,14 @@ --- + +- name: make sure the kubernetes_cri_socket variable is configured correctly + assert: + msg: "The variable kubernetes_cri_socket is not configured correctly. You might need to move your host to the group kubernetes-cluster or standalone-kubelet!" + that: + - not kubernetes_cri_socket + - name: disable bridge and iptables in docker daemon config set_fact: - docker_daemon_config: "{{ docker_daemon_config | default({}) | combine({'bridge': 'none', 'iptables': false}) }}" + docker_daemon_config: "{{ docker_daemon_config | default({}) | combine({'exec-opts': ['native.cgroupdriver=systemd'], 'bridge': 'none', 'iptables': false}) }}" - name: install docker include_role: diff --git a/roles/kubernetes/base/tasks/main.yml b/roles/kubernetes/base/tasks/main.yml index 721bc730..602266d5 100644 --- a/roles/kubernetes/base/tasks/main.yml +++ b/roles/kubernetes/base/tasks/main.yml @@ -37,7 +37,7 @@ create: yes marker: "### {mark} ANSIBLE MANAGED BLOCK for crictl ###" content: | - {% if kubernetes_cri_socket is defined %} + {% if kubernetes_cri_socket %} alias crictl="crictl --runtime-endpoint {{ kubernetes_cri_socket }}" {% endif %} {% if item == 'zsh' %} diff --git a/roles/kubernetes/kubeadm/master/tasks/primary-master.yml b/roles/kubernetes/kubeadm/master/tasks/primary-master.yml index 9bbe9ecc..f24e9ac1 100644 --- a/roles/kubernetes/kubeadm/master/tasks/primary-master.yml +++ b/roles/kubernetes/kubeadm/master/tasks/primary-master.yml @@ -27,8 +27,8 @@ - name: initialize kubernetes master and store log block: - name: initialize kubernetes master - command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --skip-token-print" - # command: "kubeadm init --config /etc/kubernetes/kubeadm.config{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print" + command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --skip-token-print" + # command: "kubeadm init --config /etc/kubernetes/kubeadm.config{% if kubernetes_cri_socket %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print" args: creates: /etc/kubernetes/pki/ca.crt register: kubeadm_init @@ -104,6 +104,19 @@ loop: "{{ groups['_kubernetes_nodes_'] }}" +## install node-local-dns + +- name: generate node-local dns cache config + template: + src: node-local-dns.yml.j2 + dest: /etc/kubernetes/node-local-dns.yml + +- name: install node-local dns cache + command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/node-local-dns.yml + register: kube_node_local_dns_apply_result + changed_when: (kube_node_local_dns_apply_result.stdout_lines | reject("regex", " unchanged$") | list | length) > 0 + + ## Network Plugin # - name: install network plugin diff --git a/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml b/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml index c00c3203..cb135adc 100644 --- a/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml +++ b/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml @@ -28,7 +28,7 @@ - name: join kubernetes secondary master node and store log block: - name: join kubernetes secondary master node - command: "kubeadm join 127.0.0.1:6443 --node-name {{ inventory_hostname }} --apiserver-bind-port 6442{% if kubernetes_overlay_node_ip is defined %} --apiserver-advertise-address {{ kubernetes_overlay_node_ip }}{% endif %}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}' --control-plane --certificate-key {{ kubeadm_upload_certs_key }}" + command: "kubeadm join 127.0.0.1:6443 --node-name {{ inventory_hostname }} --apiserver-bind-port 6442{% if kubernetes_overlay_node_ip is defined %} --apiserver-advertise-address {{ kubernetes_overlay_node_ip }}{% endif %}{% if kubernetes_cri_socket %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}' --control-plane --certificate-key {{ kubeadm_upload_certs_key }}" args: creates: /etc/kubernetes/kubelet.conf register: kubeadm_join diff --git a/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2 b/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2 index bb7f9a96..4b8548f7 100644 --- a/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2 +++ b/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2 @@ -43,3 +43,8 @@ controllerManager: scheduler: {} dns: type: CoreDNS +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +clusterDNS: +- {{ kubernetes_nodelocal_dnscache_ip }} diff --git a/roles/kubernetes/kubeadm/master/templates/node-local-dns.yml.j2 b/roles/kubernetes/kubeadm/master/templates/node-local-dns.yml.j2 new file mode 100644 index 00000000..210c551a --- /dev/null +++ b/roles/kubernetes/kubeadm/master/templates/node-local-dns.yml.j2 @@ -0,0 +1,190 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + {{ kubernetes.dns_domain | default('cluster.local') }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ kubernetes_nodelocal_dnscache_ip }} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + health {{ kubernetes_nodelocal_dnscache_ip }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ kubernetes_nodelocal_dnscache_ip }} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ kubernetes_nodelocal_dnscache_ip }} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ kubernetes_nodelocal_dnscache_ip }} + forward . __PILLAR__UPSTREAM__SERVERS__ { + force_tcp + } + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + containers: + - name: node-cache + image: k8s.gcr.io/k8s-dns-node-cache:1.15.13 + resources: + requests: + cpu: 25m + memory: 5Mi + args: [ "-localip", "{{ kubernetes_nodelocal_dnscache_ip }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + securityContext: + privileged: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ kubernetes_nodelocal_dnscache_ip }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile.base diff --git a/roles/kubernetes/kubeadm/node/tasks/main.yml b/roles/kubernetes/kubeadm/node/tasks/main.yml index 1d5178ea..655b1b18 100644 --- a/roles/kubernetes/kubeadm/node/tasks/main.yml +++ b/roles/kubernetes/kubeadm/node/tasks/main.yml @@ -2,7 +2,7 @@ - name: join kubernetes node and store log block: - name: join kubernetes node - command: "kubeadm join 127.0.0.1:6443 --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}'" + command: "kubeadm join 127.0.0.1:6443 --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}'" args: creates: /etc/kubernetes/kubelet.conf register: kubeadm_join diff --git a/roles/kubernetes/kubeadm/reset/tasks/main.yml b/roles/kubernetes/kubeadm/reset/tasks/main.yml index 1e3539e1..c35e2bfc 100644 --- a/roles/kubernetes/kubeadm/reset/tasks/main.yml +++ b/roles/kubernetes/kubeadm/reset/tasks/main.yml @@ -12,3 +12,16 @@ file: path: "{{ item }}" state: absent + +- name: get list of all cni configs + find: + paths: /etc/cni/net.d + register: kubeadm_reset_cni + +- name: remove all cni configs + loop: "{{ kubeadm_reset_cni.files }}" + loop_control: + label: "{{ item.path }}" + file: + path: "{{ item.path }}" + state: absent diff --git a/roles/kubernetes/standalone/templates/kubelet-config.yml.j2 b/roles/kubernetes/standalone/templates/kubelet-config.yml.j2 index 4e6716eb..d6af0f24 100644 --- a/roles/kubernetes/standalone/templates/kubelet-config.yml.j2 +++ b/roles/kubernetes/standalone/templates/kubelet-config.yml.j2 @@ -19,6 +19,7 @@ maxPods: {{ kubernetes_standalone_max_pods }} makeIPTablesUtilChains: false hairpinMode: none resolvConf: {{ kubernetes_standalone_resolv_conf }} +cgroupDriver: systemd enableControllerAttachDetach: false featureGates: RuntimeClass: false diff --git a/roles/kubernetes/standalone/templates/kubelet.service.override.j2 b/roles/kubernetes/standalone/templates/kubelet.service.override.j2 index 3eb8b63d..fe8bfb4c 100644 --- a/roles/kubernetes/standalone/templates/kubelet.service.override.j2 +++ b/roles/kubernetes/standalone/templates/kubelet.service.override.j2 @@ -2,7 +2,7 @@ ExecStart= ExecStart=/usr/bin/kubelet \ --config=/etc/kubernetes/kubelet.yml \ -{% if kubernetes_cri_socket is defined %} +{% if kubernetes_cri_socket %} --container-runtime=remote \ --container-runtime-endpoint={{ kubernetes_cri_socket }} \ {% endif %} diff --git a/spreadspace/k8s-lwl.yml b/spreadspace/k8s-lwl.yml index d587c0f7..2efc4f5f 100644 --- a/spreadspace/k8s-lwl.yml +++ b/spreadspace/k8s-lwl.yml @@ -22,5 +22,10 @@ kubernetes_secrets_cooked: "{{ kubernetes_secrets }}" external_ip_cooked: "{{ external_ip }}" -- import_playbook: ../common/kubernetes.yml -- import_playbook: ../common/kubernetes-cleanup.yml +- import_playbook: ../common/kubernetes-cluster.yml +- import_playbook: ../common/kubernetes-cluster-cleanup.yml + +- name: install addons + hosts: _kubernetes_primary_master_ + roles: + - role: kubernetes/addons/metrics-server |