kubernetes: multi master cluster works now

7 files changed, 60 insertions, 42 deletions

diff --git a/inventory/group_vars/k8s-test/main.yml b/inventory/group_vars/k8s-test/main.yml
index 0d4d0857..b5863ad1 100644
--- a/inventory/group_vars/k8s-test/main.yml
+++ b/inventory/group_vars/k8s-test/main.yml
@@ -14,6 +14,7 @@ kubernetes:
 
   dedicated_master: False
   api_extra_sans:
+  - 89.106.215.23
   - k8s-test.spreadspace.org
 
   pod_ip_range: 172.18.0.0/16
@@ -25,8 +26,6 @@ kubernetes:
 
 
 kubeguard:
-  kube_router_version: 0.4.0-rc1
-
   ## node_index must be in the range between 1 and 190 -> 189 hosts possible
   ##
   ## hardcoded hostnames are not nice but if we do this via host_vars
diff --git a/roles/kubernetes/kubeadm/master/tasks/primary-master.yml b/roles/kubernetes/kubeadm/master/tasks/primary-master.yml
index e814e847..115c8616 100644
--- a/roles/kubernetes/kubeadm/master/tasks/primary-master.yml
+++ b/roles/kubernetes/kubeadm/master/tasks/primary-master.yml
@@ -24,35 +24,39 @@
   #   check_mode: no
   #   register: kubeadm_token_generate
 
-  - name: initialize kubernetes master
-    command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --skip-token-print"
-#    command: "kubeadm init --config /etc/kubernetes/kubeadm.config{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print"
-    args:
-      creates: /etc/kubernetes/pki/ca.crt
-    register: kubeadm_init
-
-  - name: dump output of kubeadm init to log file
-    when: kubeadm_init.changed
-    copy:
-      content: "{{ kubeadm_init.stdout }}\n"
-      dest: /etc/kubernetes/kubeadm-init.log
+  - name: initialize kubernetes master and store log
+    block:
+    - name: initialize kubernetes master
+      command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --skip-token-print"
+  #    command: "kubeadm init --config /etc/kubernetes/kubeadm.config{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print"
+      args:
+        creates: /etc/kubernetes/pki/ca.crt
+      register: kubeadm_init
+
+    always:
+    - name: dump output of kubeadm init to log file
+      when: kubeadm_init.changed
+      copy:
+        content: "{{ kubeadm_init.stdout }}\n"
+        dest: /etc/kubernetes/kubeadm-init.log
 
   - name: create bootstrap token for existing cluster
     command: kubeadm token create --ttl 42m
     check_mode: no
     register: kubeadm_token_generate
 
+
 ### cluster is already initialized but config has changed
 
 - name: upgrade cluster config
   when: kubeconfig_kubelet_stats.stat.exists and kubeadm_config is changed
   block:
 
-
   - name: fail for cluster upgrades
     fail:
       msg: "upgrading cluster config is currently not supported!"
 
+
 ### cluster is already initialized
 
 - name: prepare cluster for new nodes
diff --git a/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml b/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml
index 7025ace0..ffe1b4b2 100644
--- a/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml
+++ b/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml
@@ -25,18 +25,21 @@
   set_fact:
     kubeadm_upload_certs_key: "{% if kubeadm_upload_certs.stdout is defined %}{{ kubeadm_upload_certs.stdout_lines | last }}{% endif %}"
 
-- name: join kubernetes secondary master node
-  command: "kubeadm join {{ host_vars[groups['_kubernetes_primary_master_']].kubernetes_kubelet_node_ip }}:6443 --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}' --control-plane --certificate-key {{ kubeadm_upload_certs_key }}"
-  args:
-    creates: /etc/kubernetes/kubelet.conf
-  register: kubeadm_join
-
-- name: dump output of kubeadm join to log file
-  when: kubeadm_join is changed
-  # This is not a handler by design to make sure this action runs at this point of the play.
-  copy: # noqa 503
-    content: "{{ kubeadm_join.stdout }}\n"
-    dest: /etc/kubernetes/kubeadm-join.log
+- name: join kubernetes secondary master node and store log
+  block:
+  - name: join kubernetes secondary master node
+    command: "kubeadm join {{ hostvars[groups['_kubernetes_primary_master_'][0]].kubernetes_kubelet_node_ip }}:6443 --node-name {{ inventory_hostname }}{% if kubernetes_kubelet_node_ip is defined %} --apiserver-advertise-address {{ kubernetes_kubelet_node_ip }}{% endif %}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}' --control-plane --certificate-key {{ kubeadm_upload_certs_key }}"
+    args:
+      creates: /etc/kubernetes/kubelet.conf
+    register: kubeadm_join
+
+  always:
+  - name: dump output of kubeadm join to log file
+    when: kubeadm_join is changed
+    # This is not a handler by design to make sure this action runs at this point of the play.
+    copy: # noqa 503
+      content: "{{ kubeadm_join.stdout }}\n"
+      dest: /etc/kubernetes/kubeadm-join.log
 
   # TODO: acutally check if node has registered
 - name: give the new master(s) a moment to register
diff --git a/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2 b/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2
index 3c10e59b..869c809f 100644
--- a/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2
+++ b/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2
@@ -1,4 +1,4 @@
-{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1 #}
+{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2 #}
 {#  #}
 apiVersion: kubeadm.k8s.io/v1beta2
 kind: InitConfiguration
@@ -6,20 +6,25 @@ kind: InitConfiguration
 {# better control it's lifetime #}
 bootstrapTokens:
 - ttl: "1s"
+{% if kubernetes_kubelet_node_ip is defined %}
+localAPIEndpoint:
+  advertiseAddress: {{ kubernetes_kubelet_node_ip }}
+{% endif %}
 ---
 apiVersion: kubeadm.k8s.io/v1beta2
 kind: ClusterConfiguration
 kubernetesVersion: {{ kubernetes_version }}
 clusterName: {{ kubernetes.cluster_name }}
 imageRepository: k8s.gcr.io
+{% if kubernetes_kubelet_node_ip is defined %}
 controlPlaneEndpoint: "{{ kubernetes_kubelet_node_ip }}:6443"
+{% endif %}
 networking:
   dnsDomain: {{ kubernetes.dns_domain | default('cluster.local') }}
   podSubnet: {{ kubernetes.pod_ip_range }}
   serviceSubnet: {{ kubernetes.service_ip_range }}
 apiServer:
-  extraArgs:
-    advertise-address: {{ kubernetes_kubelet_node_ip }}
+  #extraArgs:
   #   encryption-provider-config: /etc/kubernetes/encryption/config
   # extraVolumes:
   # - name: encryption-config
diff --git a/roles/kubernetes/kubeadm/node/tasks/main.yml b/roles/kubernetes/kubeadm/node/tasks/main.yml
index f7efdd81..61d47111 100644
--- a/roles/kubernetes/kubeadm/node/tasks/main.yml
+++ b/roles/kubernetes/kubeadm/node/tasks/main.yml
@@ -1,13 +1,16 @@
 ---
-- name: join kubernetes node
-  command: "kubeadm join {{ hostvars[groups['_kubernetes_primary_master_'][0]].kubernetes_kubelet_node_ip }}:6443 --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}'"
-  args:
-    creates: /etc/kubernetes/kubelet.conf
-  register: kubeadm_join
+- name: join kubernetes node and store log
+  block:
+  - name: join kubernetes node
+    command: "kubeadm join {{ hostvars[groups['_kubernetes_primary_master_'][0]].kubernetes_kubelet_node_ip }}:6443 --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}'"
+    args:
+      creates: /etc/kubernetes/kubelet.conf
+    register: kubeadm_join
 
-- name: dump output of kubeadm join to log file
-  when: kubeadm_join is changed
-  # This is not a handler by design to make sure this action runs at this point of the play.
-  copy: # noqa 503
-    content: "{{ kubeadm_join.stdout }}\n"
-    dest: /etc/kubernetes/kubeadm-join.log
+  always:
+  - name: dump output of kubeadm join to log file
+    when: kubeadm_join is changed
+    # This is not a handler by design to make sure this action runs at this point of the play.
+    copy: # noqa 503
+      content: "{{ kubeadm_join.stdout }}\n"
+      dest: /etc/kubernetes/kubeadm-join.log
diff --git a/roles/kubernetes/net/kubeguard/templates/kubeguard-peer.service.j2 b/roles/kubernetes/net/kubeguard/templates/kubeguard-peer.service.j2
index 6f36b571..9ca444e8 100644
--- a/roles/kubernetes/net/kubeguard/templates/kubeguard-peer.service.j2
+++ b/roles/kubernetes/net/kubeguard/templates/kubeguard-peer.service.j2
@@ -4,6 +4,7 @@ After=network.target
 Requires=kubeguard-interfaces.service
 After=kubeguard-interfaces.service
 
+{% set pod_ip_self = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) | ipaddr(1) | ipaddr('address') -%}
 {% set pod_net_peer = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[peer]) -%}
 {% set direct_zone = kubeguard.direct_net_zones | direct_net_zone(inventory_hostname, peer) -%}
 {% if direct_zone %}
@@ -22,7 +23,7 @@ Type=oneshot
 {% if direct_zone %}
 ExecStart=/sbin/ip addr add {{ direct_ip }} dev {{ direct_interface }}
 ExecStart=/sbin/ip link set up dev {{ direct_interface }}
-ExecStart=/sbin/ip route add {{ pod_net_peer }} via {{ direct_ip_peer | ipaddr('address') }}
+ExecStart=/sbin/ip route add {{ pod_net_peer }} via {{ direct_ip_peer | ipaddr('address') }} src {{ pod_ip_self }}
 ExecStop=/sbin/ip route del {{ pod_net_peer }}
 ExecStop=/sbin/ip link set down dev {{ direct_interface }}
 ExecStop=/sbin/ip addr del {{ direct_ip }} dev {{ direct_interface }}
diff --git a/spreadspace/k8s-test.yml b/spreadspace/k8s-test.yml
index 97daa5b0..f21b3fae 100644
--- a/spreadspace/k8s-test.yml
+++ b/spreadspace/k8s-test.yml
@@ -12,6 +12,9 @@
       nodes_group: k8s-test
       masters:
       - s2-k8s-test0
+      - s2-k8s-test1
+      - s2-k8s-test2
+      primary_master: s2-k8s-test0
 
 - import_playbook: ../common/kubernetes.yml
 - import_playbook: ../common/kubernetes-cleanup.yml