kubernetes/kubeadm: revamp handling of dns-cache and prepare for net-plugin dependant variations

11 files changed, 269 insertions, 367 deletions

diff --git a/roles/kubernetes/kubeadm/control-plane/defaults/main.yml b/roles/kubernetes/kubeadm/control-plane/defaults/main.yml
new file mode 100644
index 00000000..c1149988
--- /dev/null
+++ b/roles/kubernetes/kubeadm/control-plane/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+kubernetes_nodelocal_dnscache_ip: 169.254.20.10
diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/net_kube-router.yml b/roles/kubernetes/kubeadm/control-plane/tasks/net_kube-router.yml
index 4584e583..aad6467b 100644
--- a/roles/kubernetes/kubeadm/control-plane/tasks/net_kube-router.yml
+++ b/roles/kubernetes/kubeadm/control-plane/tasks/net_kube-router.yml
@@ -2,15 +2,35 @@
 - name: generate kube-router configuration
   template:
     src: "net_kube-router/config.{{ kubernetes_network_plugin_version }}.yml.j2"
-    dest: /etc/kubernetes/network-plugin.yml
+    dest: /etc/kubernetes/network-plugin/config.yml
 
 - name: check if kube-router is already installed
   check_mode: no
-  command: kubectl --kubeconfig /etc/kubernetes/admin.conf diff -f /etc/kubernetes/network-plugin.yml
+  command: kubectl --kubeconfig /etc/kubernetes/admin.conf diff -f /etc/kubernetes/network-plugin/config.yml
   failed_when: false
   changed_when: false
   register: kube_router_diff_result
 
 - name: install kube-router on to the cluster
   when: kube_router_diff_result.rc != 0
-  command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/network-plugin.yml
+  command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/network-plugin/config.yml
+
+
+- name: install node-local dns cache
+  when: kubernetes_enable_nodelocal_dnscache
+  block:
+  - name: generate node-local dns cache config
+    template:
+      src: net_kube-router/node-local-dns.yml.j2
+      dest: /etc/kubernetes/network-plugin/node-local-dns.yml
+
+  - name: check if node-local dns cache is already installed
+    check_mode: no
+    command: kubectl --kubeconfig /etc/kubernetes/admin.conf diff -f /etc/kubernetes/network-plugin/node-local-dns.yml
+    failed_when: false
+    changed_when: false
+    register: kube_node_local_dns_diff_result
+
+  - name: install node-local dns cache
+    when: kube_node_local_dns_diff_result.rc != 0
+    command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/network-plugin/node-local-dns.yml
diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/net_kubeguard.yml b/roles/kubernetes/kubeadm/control-plane/tasks/net_kubeguard.yml
index 94832c38..05428ea1 100644
--- a/roles/kubernetes/kubeadm/control-plane/tasks/net_kubeguard.yml
+++ b/roles/kubernetes/kubeadm/control-plane/tasks/net_kubeguard.yml
@@ -1,2 +1,19 @@
 ---
-## nothing to do here
+- name: install node-local dns cache
+  when: kubernetes_enable_nodelocal_dnscache
+  block:
+  - name: generate node-local dns cache config
+    template:
+      src: net_kubeguard/node-local-dns.yml.j2
+      dest: /etc/kubernetes/network-plugin/node-local-dns.yml
+
+  - name: check if node-local dns cache is already installed
+    check_mode: no
+    command: kubectl --kubeconfig /etc/kubernetes/admin.conf diff -f /etc/kubernetes/network-plugin/node-local-dns.yml
+    failed_when: false
+    changed_when: false
+    register: kube_node_local_dns_diff_result
+
+  - name: install node-local dns cache
+    when: kube_node_local_dns_diff_result.rc != 0
+    command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/network-plugin/node-local-dns.yml
diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/net_none.yml b/roles/kubernetes/kubeadm/control-plane/tasks/net_none.yml
index bf1a16d5..4b5824d4 100644
--- a/roles/kubernetes/kubeadm/control-plane/tasks/net_none.yml
+++ b/roles/kubernetes/kubeadm/control-plane/tasks/net_none.yml
@@ -1,2 +1,5 @@
 ---
-## this "plugin" is for testing purposes only
+- name: install node-local dns cache
+  when: kubernetes_enable_nodelocal_dnscache
+  debug:
+    msg: "the 'none' network-plugin is intended to manually install/test network-plugins and since the exact deployment variant for the node-local dns cache is highly dependent on what the network plugin supports we do not install anything here. If the network plugin does not support nodelocal dns caches at all you may want to set kubernetes_enable_nodelocal_dnscache to false."
diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml
index 36195235..c0591032 100644
--- a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml
+++ b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml
@@ -101,26 +101,12 @@
     kube_bootstrap_ca_cert_hash: "sha256:{{ kube_ca_openssl.stdout }}"
 
 
-## install node-local-dns
-
-- name: generate node-local dns cache config
-  template:
-    src: node-local-dns.yml.j2
-    dest: /etc/kubernetes/node-local-dns.yml
-
-- name: check if node-local dns cache is already installed
-  check_mode: no
-  command: kubectl --kubeconfig /etc/kubernetes/admin.conf diff -f /etc/kubernetes/node-local-dns.yml
-  failed_when: false
-  changed_when: false
-  register: kube_node_local_dns_diff_result
-
-- name: install node-local dns cache
-  when: kube_node_local_dns_diff_result.rc != 0
-  command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/node-local-dns.yml
-
-
 ## Network Plugin
 
+- name: create network plugin config directory
+  file:
+    path: /etc/kubernetes/network-plugin
+    state: directory
+
 - name: install network plugin
   include_tasks: "net_{{ kubernetes_network_plugin }}.yml"
diff --git a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm-init.config.j2 b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm-init.config.j2
index 9dfd6825..3a82a8e1 100644
--- a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm-init.config.j2
+++ b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm-init.config.j2
@@ -56,6 +56,8 @@ scheduler: {}
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
+{% if kubernetes_enable_nodelocal_dnscache %}
 clusterDNS:
 - "{{ kubernetes_nodelocal_dnscache_ip }}"
-cgroupDriver: systemd
+{% endif %}
diff --git a/roles/kubernetes/kubeadm/control-plane/templates/node-local-dns.yml.j2 b/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/node-local-dns.yml.j2
index 22ae6c22..22ae6c22 100644
--- a/roles/kubernetes/kubeadm/control-plane/templates/node-local-dns.yml.j2
+++ b/roles/kubernetes/kubeadm/control-plane/templates/net_kube-router/node-local-dns.yml.j2
diff --git a/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.0.4.0.yml.j2 b/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.0.4.0.yml.j2
deleted file mode 100644
index e343f4a7..00000000
--- a/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.0.4.0.yml.j2
+++ /dev/null
@@ -1,170 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-router-kubeconfig
-  namespace: kube-system
-  labels:
-    tier: node
-    k8s-app: kube-router
-data:
-  kubeconfig.conf: |
-    apiVersion: v1
-    kind: Config
-    clusters:
-    - cluster:
-        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        server: https://127.0.0.1:{{ kubernetes_api_lb_port | default('6443') }}
-      name: default
-    contexts:
-    - context:
-        cluster: default
-        namespace: default
-        user: default
-      name: default
-    current-context: default
-    users:
-    - name: default
-      user:
-        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  labels:
-    k8s-app: kube-router
-    tier: node
-  name: kube-router
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kube-router
-      tier: node
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-router
-        tier: node
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-    spec:
-      priorityClassName: system-node-critical
-      serviceAccountName: kube-router
-      serviceAccount: kube-router
-      containers:
-      - name: kube-router
-        image: docker.io/cloudnativelabs/kube-router:v{{ kubernetes_network_plugin_version }}
-        imagePullPolicy: Always
-        args:
-        - --cluster-cidr={{ kubernetes.pod_ip_range }}
-        - --run-router=false
-        - --run-firewall=true
-        - --run-service-proxy={{ kubernetes_network_plugin_replaces_kube_proxy | string | lower }}
-        - --kubeconfig=/var/lib/kube-router/kubeconfig
-        - --hairpin-mode
-        - --iptables-sync-period=10s
-        - --ipvs-sync-period=10s
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 20244
-          initialDelaySeconds: 10
-          periodSeconds: 3
-        resources:
-          requests:
-            cpu: 250m
-            memory: 250Mi
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: lib-modules
-          mountPath: /lib/modules
-          readOnly: true
-        - name: kubeconfig
-          mountPath: /var/lib/kube-router
-          readOnly: true
-        - name: xtables-lock
-          mountPath: /run/xtables.lock
-          readOnly: false
-      hostNetwork: true
-      tolerations:
-      - effect: NoSchedule
-        operator: Exists
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - effect: NoExecute
-        operator: Exists
-      volumes:
-      - name: lib-modules
-        hostPath:
-          path: /lib/modules
-      - name: kubeconfig
-        configMap:
-          name: kube-router-kubeconfig
-          items:
-          - key: kubeconfig.conf
-            path: kubeconfig
-      - name: xtables-lock
-        hostPath:
-          path: /run/xtables.lock
-          type: FileOrCreate
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-router
-  namespace: kube-system
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: kube-router
-  namespace: kube-system
-rules:
-  - apiGroups:
-    - ""
-    resources:
-      - namespaces
-      - pods
-      - services
-      - nodes
-      - endpoints
-    verbs:
-      - list
-      - get
-      - watch
-  - apiGroups:
-    - "networking.k8s.io"
-    resources:
-      - networkpolicies
-    verbs:
-      - list
-      - get
-      - watch
-  - apiGroups:
-    - extensions
-    resources:
-      - networkpolicies
-    verbs:
-      - get
-      - list
-      - watch
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: kube-router
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kube-router
-subjects:
-- kind: ServiceAccount
-  name: kube-router
-  namespace: kube-system
diff --git a/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.1.1.1.yml.j2 b/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.1.1.1.yml.j2
deleted file mode 100644
index ec30d670..00000000
--- a/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/kube-router.1.1.1.yml.j2
+++ /dev/null
@@ -1,170 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-router-kubeconfig
-  namespace: kube-system
-  labels:
-    tier: node
-    k8s-app: kube-router
-data:
-  kubeconfig.conf: |
-    apiVersion: v1
-    kind: Config
-    clusters:
-    - cluster:
-        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        server: https://127.0.0.1:{{ kubernetes_api_lb_port | default('6443') }}
-      name: default
-    contexts:
-    - context:
-        cluster: default
-        namespace: default
-        user: default
-      name: default
-    current-context: default
-    users:
-    - name: default
-      user:
-        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  labels:
-    k8s-app: kube-router
-    tier: node
-  name: kube-router
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kube-router
-      tier: node
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-router
-        tier: node
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-    spec:
-      priorityClassName: system-node-critical
-      serviceAccountName: kube-router
-      serviceAccount: kube-router
-      containers:
-      - name: kube-router
-        image: docker.io/cloudnativelabs/kube-router:v{{ kubernetes_network_plugin_version }}
-        imagePullPolicy: Always
-        args:
-        - --run-router=false
-        - --run-firewall=true
-        - --run-service-proxy={{ kubernetes_network_plugin_replaces_kube_proxy | string | lower }}
-        - --bgp-graceful-restart=true
-        - --kubeconfig=/var/lib/kube-router/kubeconfig
-        - --hairpin-mode
-        - --iptables-sync-period=10s
-        - --ipvs-sync-period=10s
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 20244
-          initialDelaySeconds: 10
-          periodSeconds: 3
-        resources:
-          requests:
-            cpu: 250m
-            memory: 250Mi
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: lib-modules
-          mountPath: /lib/modules
-          readOnly: true
-        - name: kubeconfig
-          mountPath: /var/lib/kube-router
-          readOnly: true
-        - name: xtables-lock
-          mountPath: /run/xtables.lock
-          readOnly: false
-      hostNetwork: true
-      tolerations:
-      - effect: NoSchedule
-        operator: Exists
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - effect: NoExecute
-        operator: Exists
-      volumes:
-      - name: lib-modules
-        hostPath:
-          path: /lib/modules
-      - name: kubeconfig
-        configMap:
-          name: kube-router-kubeconfig
-          items:
-          - key: kubeconfig.conf
-            path: kubeconfig
-      - name: xtables-lock
-        hostPath:
-          path: /run/xtables.lock
-          type: FileOrCreate
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-router
-  namespace: kube-system
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: kube-router
-  namespace: kube-system
-rules:
-  - apiGroups:
-    - ""
-    resources:
-      - namespaces
-      - pods
-      - services
-      - nodes
-      - endpoints
-    verbs:
-      - list
-      - get
-      - watch
-  - apiGroups:
-    - "networking.k8s.io"
-    resources:
-      - networkpolicies
-    verbs:
-      - list
-      - get
-      - watch
-  - apiGroups:
-    - extensions
-    resources:
-      - networkpolicies
-    verbs:
-      - get
-      - list
-      - watch
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: kube-router
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kube-router
-subjects:
-- kind: ServiceAccount
-  name: kube-router
-  namespace: kube-system
diff --git a/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/node-local-dns.yml.j2 b/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/node-local-dns.yml.j2
new file mode 100644
index 00000000..22ae6c22
--- /dev/null
+++ b/roles/kubernetes/kubeadm/control-plane/templates/net_kubeguard/node-local-dns.yml.j2
@@ -0,0 +1,213 @@
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: node-local-dns
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns-upstream
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "KubeDNSUpstream"
+spec:
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+    targetPort: 53
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+    targetPort: 53
+  selector:
+    k8s-app: kube-dns
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: node-local-dns
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+data:
+  Corefile: |
+    {{ kubernetes.dns_domain | default('cluster.local') }}:53 {
+        errors
+        cache {
+                success 9984 30
+                denial 9984 5
+        }
+        reload
+        loop
+        bind {{ kubernetes_nodelocal_dnscache_ip }}
+        forward . __PILLAR__CLUSTER__DNS__ {
+                force_tcp
+        }
+        prometheus :9253
+        health {{ kubernetes_nodelocal_dnscache_ip }}:8080
+        }
+    in-addr.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ kubernetes_nodelocal_dnscache_ip }}
+        forward . __PILLAR__CLUSTER__DNS__ {
+                force_tcp
+        }
+        prometheus :9253
+        }
+    ip6.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ kubernetes_nodelocal_dnscache_ip }}
+        forward . __PILLAR__CLUSTER__DNS__ {
+                force_tcp
+        }
+        prometheus :9253
+        }
+    .:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ kubernetes_nodelocal_dnscache_ip }}
+        forward . __PILLAR__UPSTREAM__SERVERS__ {
+                force_tcp
+        }
+        prometheus :9253
+        }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-local-dns
+  namespace: kube-system
+  labels:
+    k8s-app: node-local-dns
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 10%
+  selector:
+    matchLabels:
+      k8s-app: node-local-dns
+  template:
+    metadata:
+      labels:
+        k8s-app: node-local-dns
+      annotations:
+        prometheus.io/port: "9253"
+        prometheus.io/scrape: "true"
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: node-local-dns
+      hostNetwork: true
+      dnsPolicy: Default  # Don't use cluster DNS.
+      tolerations:
+      - key: "CriticalAddonsOnly"
+        operator: "Exists"
+      - effect: "NoExecute"
+        operator: "Exists"
+      - effect: "NoSchedule"
+        operator: "Exists"
+      containers:
+      - name: node-cache
+        image: registry.k8s.io/dns/k8s-dns-node-cache:1.22.20
+        resources:
+          requests:
+            cpu: 25m
+            memory: 5Mi
+        args: [ "-localip", "{{ kubernetes_nodelocal_dnscache_ip }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ]
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9253
+          name: metrics
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            host: {{ kubernetes_nodelocal_dnscache_ip }}
+            path: /health
+            port: 8080
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+        volumeMounts:
+        - mountPath: /run/xtables.lock
+          name: xtables-lock
+          readOnly: false
+        - name: config-volume
+          mountPath: /etc/coredns
+        - name: kube-dns-config
+          mountPath: /etc/kube-dns
+      volumes:
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: kube-dns-config
+        configMap:
+          name: kube-dns
+          optional: true
+      - name: config-volume
+        configMap:
+          name: node-local-dns
+          items:
+            - key: Corefile
+              path: Corefile.base
+---
+# A headless service is a service with a service IP but instead of load-balancing it will return the IPs of our associated Pods.
+# We use this to expose metrics to Prometheus.
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    prometheus.io/port: "9253"
+    prometheus.io/scrape: "true"
+  labels:
+    k8s-app: node-local-dns
+  name: node-local-dns
+  namespace: kube-system
+spec:
+  clusterIP: None
+  ports:
+    - name: metrics
+      port: 9253
+      targetPort: 9253
+  selector:
+    k8s-app: node-local-dns
diff --git a/roles/kubernetes/kubeadm/reset/tasks/main.yml b/roles/kubernetes/kubeadm/reset/tasks/main.yml
index d89dd32f..29b88847 100644
--- a/roles/kubernetes/kubeadm/reset/tasks/main.yml
+++ b/roles/kubernetes/kubeadm/reset/tasks/main.yml
@@ -11,8 +11,7 @@
   - /etc/kubernetes/kubeadm-join.errors
   - /etc/kubernetes/pki
   - /etc/kubernetes/encryption
-  - /etc/kubernetes/network-plugin.yml
-  - /etc/kubernetes/node-local-dns.yml
+  - /etc/kubernetes/network-plugin
   - /etc/kubernetes/addons
   - /etc/kubernetes/decorations
   - /etc/default/kubelet