diff options
author | Christian Pointner <equinox@spreadspace.org> | 2021-01-03 20:23:47 +0100 |
---|---|---|
committer | Christian Pointner <equinox@spreadspace.org> | 2021-01-03 20:23:47 +0100 |
commit | a194954e4c36cc5fd04fcd0c40d9f5a332824d8c (patch) | |
tree | 8d4ba2f548c2462ee8d595a9539924cf9d0e96a6 | |
parent | infobeamer: configure video resolution via userconfig.txt and tvservice file (diff) | |
parent | kubernetes: add chaos-at-home test cluster (diff) |
Merge branch 'topic/kubernetes-upgrades'
15 files changed, 766 insertions, 32 deletions
diff --git a/chaos-at-home/k8s-chtest.yml b/chaos-at-home/k8s-chtest.yml new file mode 100644 index 00000000..f5eda5cf --- /dev/null +++ b/chaos-at-home/k8s-chtest.yml @@ -0,0 +1,35 @@ +--- +- name: Basic Node Setup + hosts: k8s-chtest + roles: + - role: apt-repo/base + - role: core/base + - role: core/sshd + - role: core/zsh + - role: core/ntp + +- import_playbook: ../common/kubernetes-cluster-layout.yml + vars: + kubernetes_cluster_layout: + nodes_group: k8s-chtest + masters: + - ch-k8s-master + +### hack hack hack... +- name: cook kubernetes secrets + hosts: _kubernetes_nodes_ + gather_facts: no + tasks: + - set_fact: + kubernetes_secrets_cooked: "{{ kubernetes_secrets }}" + - when: external_ip is defined + set_fact: + external_ip_cooked: "{{ external_ip }}" + +- import_playbook: ../common/kubernetes-cluster.yml +- import_playbook: ../common/kubernetes-cluster-cleanup.yml + +- name: install addons + hosts: _kubernetes_primary_master_ + roles: + - role: kubernetes/addons/metrics-server diff --git a/inventory/group_vars/k8s-chtest/vars.yml b/inventory/group_vars/k8s-chtest/vars.yml new file mode 100644 index 00000000..034253e7 --- /dev/null +++ b/inventory/group_vars/k8s-chtest/vars.yml @@ -0,0 +1,49 @@ +--- +docker_pkg_provider: docker-com +docker_pkg_name: docker-ce + +kubernetes_version: 1.20.1 +kubernetes_container_runtime: docker +kubernetes_network_plugin: kubeguard +# kubernetes_network_plugin_variant: with-kube-router +# kubernetes_network_plugin_version: 0.4.0 +# kubernetes_network_plugin_replaces_kube_proxy: true + + +kubernetes: + cluster_name: chtest + + dedicated_master: True + api_extra_sans: + - 192.168.32.20 + + pod_ip_range: 172.18.0.0/16 + pod_ip_range_size: 24 + service_ip_range: 172.18.192.0/18 + + +kubernetes_secrets: + encryption_config_keys: "{{ vault_kubernetes_encryption_config_keys }}" + + +kubeguard: + ## node_index must be in the range between 1 and 190 -> 189 hosts possible + ## + ## hardcoded hostnames are not nice but if we do this via host_vars + ## the info is spread over multiple files and this makes it more diffcult + ## to find mistakes, so it is nicer to keep it in one place... + node_index: + ch-dione: 111 + ch-helene: 112 + ch-k8s-master: 127 + + direct_net_zones: + encoder: + transfer_net: 172.18.191.0/24 + node_interface: + ch-dione: eno2 + ch-helene: eno2 + +kubernetes_overlay_node_ip: "{{ kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) | ipaddr(1) | ipaddr('address') }}" + +kubernetes_metrics_server_version: 0.4.1 diff --git a/inventory/host_vars/ch-dione.yml b/inventory/host_vars/ch-dione.yml index 6c241a75..84eded0c 100644 --- a/inventory/host_vars/ch-dione.yml +++ b/inventory/host_vars/ch-dione.yml @@ -46,9 +46,6 @@ admin_users_host: blackmagic_desktopvideo_version: 11.1a11 -docker_pkg_provider: docker-com -docker_pkg_name: docker-ce - docker_lvm: vg: "{{ host_name }}" lv: docker diff --git a/inventory/host_vars/ch-helene.yml b/inventory/host_vars/ch-helene.yml index 2bb0350d..6b9e4302 100644 --- a/inventory/host_vars/ch-helene.yml +++ b/inventory/host_vars/ch-helene.yml @@ -42,9 +42,6 @@ admin_users_host: blackmagic_desktopvideo_version: 11.1a11 -docker_pkg_provider: docker-com -docker_pkg_name: docker-ce - docker_lvm: vg: "{{ host_name }}" lv: docker diff --git a/inventory/host_vars/ch-k8s-master.yml b/inventory/host_vars/ch-k8s-master.yml index bb8fa845..752a6c0f 100644 --- a/inventory/host_vars/ch-k8s-master.yml +++ b/inventory/host_vars/ch-k8s-master.yml @@ -31,3 +31,16 @@ network: gateway: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets['ch-gw-lan']) | ipaddr('address') }}" interfaces: - *_network_primary_ + + +docker_lvm: + vg: "{{ host_name }}" + lv: docker + size: 7G + fs: ext4 + +kubelet_lvm: + vg: "{{ host_name }}" + lv: kubelet + size: 5G + fs: ext4 diff --git a/inventory/hosts.ini b/inventory/hosts.ini index d34219f6..cbc0d839 100644 --- a/inventory/hosts.ini +++ b/inventory/hosts.ini @@ -393,6 +393,7 @@ k8s-emc [kubernetes-cluster:children] k8s-emc k8s-lwl +k8s-chtest [standalone-kubelet] sk-cloudio @@ -448,3 +449,16 @@ k8s-lwl-master k8s-lwl-encoder k8s-lwl-distribution k8s-lwl-streamer + + +### Kubernetes Cluster: chtest +[k8s-chtest-encoder] +ch-dione +ch-helene + +[k8s-chtest-master] +ch-k8s-master + +[k8s-chtest:children] +k8s-chtest-master +k8s-chtest-encoder diff --git a/roles/kubernetes/addons/metrics-server/templates/components.0.3.7.yml.j2 b/roles/kubernetes/addons/metrics-server/templates/components.0.3.7.yml.j2 index c20dbcda..fc8d287b 100644 --- a/roles/kubernetes/addons/metrics-server/templates/components.0.3.7.yml.j2 +++ b/roles/kubernetes/addons/metrics-server/templates/components.0.3.7.yml.j2 @@ -103,7 +103,6 @@ spec: mountPath: /tmp nodeSelector: kubernetes.io/os: linux - kubernetes.io/arch: "amd64" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master diff --git a/roles/kubernetes/addons/metrics-server/templates/components.0.4.1.yml.j2 b/roles/kubernetes/addons/metrics-server/templates/components.0.4.1.yml.j2 new file mode 100644 index 00000000..6b22508d --- /dev/null +++ b/roles/kubernetes/addons/metrics-server/templates/components.0.4.1.yml.j2 @@ -0,0 +1,187 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + - namespaces + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-insecure-tls + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + image: k8s.gcr.io/metrics-server/metrics-server:v0.4.1 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + periodSeconds: 10 + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + volumes: + - emptyDir: {} + name: tmp-dir +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 diff --git a/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml b/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml index 2d706a03..ed5403d3 100644 --- a/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml +++ b/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml @@ -1,12 +1,17 @@ --- +- name: fail if kubernetes_network_plugin_variant is set to with-kube-router + run_once: yes + assert: + msg: Unfortunately using kube-router together with kubeguard does not work at the moment! + that: "kubernetes_network_plugin_variant != 'with-kube-router'" + - name: make sure kubernetes_network_plugin_replaces_kube_proxy is not set when: - - kubernetes_network_plugin_variant != 'with-kube-router' + - "kubernetes_network_plugin_variant != 'with-kube-router'" run_once: yes assert: - msg: "kubeguard variant '{{ kubernetes_network_plugin_variant }}' can not replace kube-proxy please set kubernetes_network_plugin_replaces_kube_proxy to false or configure a differnt kubernetes_network_plugin_variant." - that: - - not kubernetes_network_plugin_replaces_kube_proxy + msg: "kubeguard variant '{{ kubernetes_network_plugin_variant }}' can not replace kube-proxy, please set kubernetes_network_plugin_replaces_kube_proxy to false or configure a differnt kubernetes_network_plugin_variant." + that: not kubernetes_network_plugin_replaces_kube_proxy - name: install wireguard diff --git a/roles/kubernetes/kubeadm/master/tasks/main.yml b/roles/kubernetes/kubeadm/master/tasks/main.yml index 19037adc..04df760f 100644 --- a/roles/kubernetes/kubeadm/master/tasks/main.yml +++ b/roles/kubernetes/kubeadm/master/tasks/main.yml @@ -31,14 +31,28 @@ set_fact: kube_node_taints: "{% set node_info = kubectl_get_node.stdout | from_json %}{%if node_info.spec.taints is defined %}{{ node_info.spec.taints | map(attribute='key') | list }}{% endif %}" -- name: remove taint from master node - when: not kubernetes.dedicated_master and 'node-role.kubernetes.io/master' in kube_node_taints - command: "kubectl --kubeconfig /etc/kubernetes/admin.conf taint nodes {{ inventory_hostname }} node-role.kubernetes.io/master-" +- name: remove taint from master/control-plane node + when: not kubernetes.dedicated_master + block: + - name: remove master taint from node + when: "'node-role.kubernetes.io/master' in kube_node_taints" + command: "kubectl --kubeconfig /etc/kubernetes/admin.conf taint nodes {{ inventory_hostname }} node-role.kubernetes.io/master-" -- name: add taint for master node - when: kubernetes.dedicated_master and 'node-role.kubernetes.io/master' not in kube_node_taints - command: "kubectl --kubeconfig /etc/kubernetes/admin.conf taint nodes {{ inventory_hostname }} node-role.kubernetes.io/master='':NoSchedule" + - name: remove control-plane taint from node + when: "'node-role.kubernetes.io/control-plane' in kube_node_taints" + command: "kubectl --kubeconfig /etc/kubernetes/admin.conf taint nodes {{ inventory_hostname }} node-role.kubernetes.io/control-plane-" +- name: add taint from master/control-plane node + when: kubernetes.dedicated_master + block: + - name: add master taint from node + when: "'node-role.kubernetes.io/master' not in kube_node_taints" + command: "kubectl --kubeconfig /etc/kubernetes/admin.conf taint nodes {{ inventory_hostname }} node-role.kubernetes.io/master='':NoSchedule" + + ## TODO: enable this once all needed addons and workloads have tolerations set accordingly + # - name: add control-plane taint from node + # when: "'node-role.kubernetes.io/control-plane' not in kube_node_taints" + # command: "kubectl --kubeconfig /etc/kubernetes/admin.conf taint nodes {{ inventory_hostname }} node-role.kubernetes.io/control-plane='':NoSchedule" - name: prepare kubectl (1/2) file: diff --git a/roles/kubernetes/kubeadm/master/templates/net_kube-router/config.0.4.0.yml.j2 b/roles/kubernetes/kubeadm/master/templates/net_kube-router/config.0.4.0.yml.j2 index b06687d5..a2660db2 100644 --- a/roles/kubernetes/kubeadm/master/templates/net_kube-router/config.0.4.0.yml.j2 +++ b/roles/kubernetes/kubeadm/master/templates/net_kube-router/config.0.4.0.yml.j2 @@ -153,13 +153,11 @@ spec: mountPath: /etc/kube-router hostNetwork: true tolerations: - - key: CriticalAddonsOnly - operator: Exists - effect: NoSchedule - key: node-role.kubernetes.io/master operator: Exists - - effect: NoSchedule - key: node.kubernetes.io/not-ready + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute operator: Exists volumes: - name: lib-modules @@ -189,7 +187,7 @@ metadata: namespace: kube-system --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: kube-router namespace: kube-system @@ -224,7 +222,7 @@ rules: - watch --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: kube-router roleRef: diff --git a/roles/kubernetes/kubeadm/master/templates/net_kube-router/config.1.1.1.yml.j2 b/roles/kubernetes/kubeadm/master/templates/net_kube-router/config.1.1.1.yml.j2 new file mode 100644 index 00000000..382164cb --- /dev/null +++ b/roles/kubernetes/kubeadm/master/templates/net_kube-router/config.1.1.1.yml.j2 @@ -0,0 +1,236 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-kubeconfig + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://127.0.0.1:{{ kubernetes_api_lb_port | default('6443') }} + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-cfg + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + cni-conf.json: | + { + "cniVersion":"0.3.0", + "name":"mynet", + "plugins":[ + { + "name":"kubernetes", + "type":"bridge", + "bridge":"kube-bridge", + "isDefaultGateway":true, + "hairpinMode": true, + "ipam":{ + "type":"host-local" + } + }, + { + "type":"portmap", + "capabilities":{ + "snat":true, + "portMappings":true + } + } + ] + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-router + tier: node + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + priorityClassName: system-node-critical + serviceAccountName: kube-router + serviceAccount: kube-router + containers: + - name: kube-router + image: docker.io/cloudnativelabs/kube-router:v{{ kubernetes_network_plugin_version }} + imagePullPolicy: Always + args: + - --run-router=true + - --run-firewall=true + - --run-service-proxy={{ kubernetes_network_plugin_replaces_kube_proxy | string | lower }} + - --bgp-graceful-restart=true + - --kubeconfig=/var/lib/kube-router/kubeconfig + - --hairpin-mode + - --iptables-sync-period=10s + - --ipvs-sync-period=10s + - --routes-sync-period=10s + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBE_ROUTER_CNI_CONF_FILE + value: /etc/cni/net.d/10-kuberouter.conflist + livenessProbe: + httpGet: + path: /healthz + port: 20244 + initialDelaySeconds: 10 + periodSeconds: 3 + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + initContainers: + - name: install-cni + image: busybox + imagePullPolicy: Always + command: + - /bin/sh + - -c + - set -e -x; + if [ ! -f /etc/cni/net.d/10-kuberouter.conflist ]; then + if [ -f /etc/cni/net.d/*.conf ]; then + rm -f /etc/cni/net.d/*.conf; + fi; + TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; + cp /etc/kube-router/cni-conf.json ${TMP}; + mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; + fi + volumeMounts: + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kube-router-cfg + mountPath: /etc/kube-router + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d + - name: kube-router-cfg + configMap: + name: kube-router-cfg + - name: kubeconfig + configMap: + name: kube-router-kubeconfig + items: + - key: kubeconfig.conf + path: kubeconfig + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system diff --git a/roles/kubernetes/kubeadm/master/templates/net_kubeguard/kube-router.0.4.0.yml.j2 b/roles/kubernetes/kubeadm/master/templates/net_kubeguard/kube-router.0.4.0.yml.j2 index 51bfdaae..e343f4a7 100644 --- a/roles/kubernetes/kubeadm/master/templates/net_kubeguard/kube-router.0.4.0.yml.j2 +++ b/roles/kubernetes/kubeadm/master/templates/net_kubeguard/kube-router.0.4.0.yml.j2 @@ -57,6 +57,7 @@ spec: image: docker.io/cloudnativelabs/kube-router:v{{ kubernetes_network_plugin_version }} imagePullPolicy: Always args: + - --cluster-cidr={{ kubernetes.pod_ip_range }} - --run-router=false - --run-firewall=true - --run-service-proxy={{ kubernetes_network_plugin_replaces_kube_proxy | string | lower }} @@ -93,13 +94,11 @@ spec: readOnly: false hostNetwork: true tolerations: - - key: CriticalAddonsOnly - operator: Exists - effect: NoSchedule - key: node-role.kubernetes.io/master operator: Exists - - effect: NoSchedule - key: node.kubernetes.io/not-ready + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute operator: Exists volumes: - name: lib-modules @@ -123,7 +122,7 @@ metadata: namespace: kube-system --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: kube-router namespace: kube-system @@ -158,7 +157,7 @@ rules: - watch --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: kube-router roleRef: diff --git a/roles/kubernetes/kubeadm/master/templates/net_kubeguard/kube-router.1.1.1.yml.j2 b/roles/kubernetes/kubeadm/master/templates/net_kubeguard/kube-router.1.1.1.yml.j2 new file mode 100644 index 00000000..ec30d670 --- /dev/null +++ b/roles/kubernetes/kubeadm/master/templates/net_kubeguard/kube-router.1.1.1.yml.j2 @@ -0,0 +1,170 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-kubeconfig + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://127.0.0.1:{{ kubernetes_api_lb_port | default('6443') }} + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-router + tier: node + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + priorityClassName: system-node-critical + serviceAccountName: kube-router + serviceAccount: kube-router + containers: + - name: kube-router + image: docker.io/cloudnativelabs/kube-router:v{{ kubernetes_network_plugin_version }} + imagePullPolicy: Always + args: + - --run-router=false + - --run-firewall=true + - --run-service-proxy={{ kubernetes_network_plugin_replaces_kube_proxy | string | lower }} + - --bgp-graceful-restart=true + - --kubeconfig=/var/lib/kube-router/kubeconfig + - --hairpin-mode + - --iptables-sync-period=10s + - --ipvs-sync-period=10s + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + livenessProbe: + httpGet: + path: /healthz + port: 20244 + initialDelaySeconds: 10 + periodSeconds: 3 + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: kubeconfig + configMap: + name: kube-router-kubeconfig + items: + - key: kubeconfig.conf + path: kubeconfig + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system diff --git a/roles/kubernetes/kubeadm/master/templates/node-local-dns.yml.j2 b/roles/kubernetes/kubeadm/master/templates/node-local-dns.yml.j2 index 210c551a..d536d5a7 100644 --- a/roles/kubernetes/kubeadm/master/templates/node-local-dns.yml.j2 +++ b/roles/kubernetes/kubeadm/master/templates/node-local-dns.yml.j2 @@ -140,7 +140,7 @@ spec: operator: "Exists" containers: - name: node-cache - image: k8s.gcr.io/k8s-dns-node-cache:1.15.13 + image: k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0 resources: requests: cpu: 25m @@ -188,3 +188,24 @@ spec: items: - key: Corefile path: Corefile.base +--- +# A headless service is a service with a service IP but instead of load-balancing it will return the IPs of our associated Pods. +# We use this to expose metrics to Prometheus. +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + labels: + k8s-app: node-local-dns + name: node-local-dns + namespace: kube-system +spec: + clusterIP: None + ports: + - name: metrics + port: 9253 + targetPort: 9253 + selector: + k8s-app: node-local-dns |