summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristian Pointner <equinox@spreadspace.org>2020-01-17 22:24:09 +0100
committerChristian Pointner <equinox@spreadspace.org>2020-01-31 22:31:22 +0100
commit8010f57a73885f7abb5c98c1f77c49baa59a7d16 (patch)
tree334f5c6c0af02fe3fce098feb398808101a066d9
parentkubernetes: node cleanup works now (diff)
kubernetes: multi master cluster works now
-rw-r--r--inventory/group_vars/k8s-test/main.yml3
-rw-r--r--roles/kubernetes/kubeadm/master/tasks/primary-master.yml30
-rw-r--r--roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml27
-rw-r--r--roles/kubernetes/kubeadm/master/templates/kubeadm.config.j211
-rw-r--r--roles/kubernetes/kubeadm/node/tasks/main.yml25
-rw-r--r--roles/kubernetes/net/kubeguard/templates/kubeguard-peer.service.j23
-rw-r--r--spreadspace/k8s-test.yml3
7 files changed, 60 insertions, 42 deletions
diff --git a/inventory/group_vars/k8s-test/main.yml b/inventory/group_vars/k8s-test/main.yml
index 0d4d0857..b5863ad1 100644
--- a/inventory/group_vars/k8s-test/main.yml
+++ b/inventory/group_vars/k8s-test/main.yml
@@ -14,6 +14,7 @@ kubernetes:
dedicated_master: False
api_extra_sans:
+ - 89.106.215.23
- k8s-test.spreadspace.org
pod_ip_range: 172.18.0.0/16
@@ -25,8 +26,6 @@ kubernetes:
kubeguard:
- kube_router_version: 0.4.0-rc1
-
## node_index must be in the range between 1 and 190 -> 189 hosts possible
##
## hardcoded hostnames are not nice but if we do this via host_vars
diff --git a/roles/kubernetes/kubeadm/master/tasks/primary-master.yml b/roles/kubernetes/kubeadm/master/tasks/primary-master.yml
index e814e847..115c8616 100644
--- a/roles/kubernetes/kubeadm/master/tasks/primary-master.yml
+++ b/roles/kubernetes/kubeadm/master/tasks/primary-master.yml
@@ -24,35 +24,39 @@
# check_mode: no
# register: kubeadm_token_generate
- - name: initialize kubernetes master
- command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --skip-token-print"
-# command: "kubeadm init --config /etc/kubernetes/kubeadm.config{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print"
- args:
- creates: /etc/kubernetes/pki/ca.crt
- register: kubeadm_init
-
- - name: dump output of kubeadm init to log file
- when: kubeadm_init.changed
- copy:
- content: "{{ kubeadm_init.stdout }}\n"
- dest: /etc/kubernetes/kubeadm-init.log
+ - name: initialize kubernetes master and store log
+ block:
+ - name: initialize kubernetes master
+ command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --skip-token-print"
+ # command: "kubeadm init --config /etc/kubernetes/kubeadm.config{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %}{% if kubernetes_network_plugin == 'kube-router' %} --skip-phases addon/kube-proxy{% endif %} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print"
+ args:
+ creates: /etc/kubernetes/pki/ca.crt
+ register: kubeadm_init
+
+ always:
+ - name: dump output of kubeadm init to log file
+ when: kubeadm_init.changed
+ copy:
+ content: "{{ kubeadm_init.stdout }}\n"
+ dest: /etc/kubernetes/kubeadm-init.log
- name: create bootstrap token for existing cluster
command: kubeadm token create --ttl 42m
check_mode: no
register: kubeadm_token_generate
+
### cluster is already initialized but config has changed
- name: upgrade cluster config
when: kubeconfig_kubelet_stats.stat.exists and kubeadm_config is changed
block:
-
- name: fail for cluster upgrades
fail:
msg: "upgrading cluster config is currently not supported!"
+
### cluster is already initialized
- name: prepare cluster for new nodes
diff --git a/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml b/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml
index 7025ace0..ffe1b4b2 100644
--- a/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml
+++ b/roles/kubernetes/kubeadm/master/tasks/secondary-masters.yml
@@ -25,18 +25,21 @@
set_fact:
kubeadm_upload_certs_key: "{% if kubeadm_upload_certs.stdout is defined %}{{ kubeadm_upload_certs.stdout_lines | last }}{% endif %}"
-- name: join kubernetes secondary master node
- command: "kubeadm join {{ host_vars[groups['_kubernetes_primary_master_']].kubernetes_kubelet_node_ip }}:6443 --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}' --control-plane --certificate-key {{ kubeadm_upload_certs_key }}"
- args:
- creates: /etc/kubernetes/kubelet.conf
- register: kubeadm_join
-
-- name: dump output of kubeadm join to log file
- when: kubeadm_join is changed
- # This is not a handler by design to make sure this action runs at this point of the play.
- copy: # noqa 503
- content: "{{ kubeadm_join.stdout }}\n"
- dest: /etc/kubernetes/kubeadm-join.log
+- name: join kubernetes secondary master node and store log
+ block:
+ - name: join kubernetes secondary master node
+ command: "kubeadm join {{ hostvars[groups['_kubernetes_primary_master_'][0]].kubernetes_kubelet_node_ip }}:6443 --node-name {{ inventory_hostname }}{% if kubernetes_kubelet_node_ip is defined %} --apiserver-advertise-address {{ kubernetes_kubelet_node_ip }}{% endif %}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}' --control-plane --certificate-key {{ kubeadm_upload_certs_key }}"
+ args:
+ creates: /etc/kubernetes/kubelet.conf
+ register: kubeadm_join
+
+ always:
+ - name: dump output of kubeadm join to log file
+ when: kubeadm_join is changed
+ # This is not a handler by design to make sure this action runs at this point of the play.
+ copy: # noqa 503
+ content: "{{ kubeadm_join.stdout }}\n"
+ dest: /etc/kubernetes/kubeadm-join.log
# TODO: acutally check if node has registered
- name: give the new master(s) a moment to register
diff --git a/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2 b/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2
index 3c10e59b..869c809f 100644
--- a/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2
+++ b/roles/kubernetes/kubeadm/master/templates/kubeadm.config.j2
@@ -1,4 +1,4 @@
-{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1 #}
+{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2 #}
{# #}
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
@@ -6,20 +6,25 @@ kind: InitConfiguration
{# better control it's lifetime #}
bootstrapTokens:
- ttl: "1s"
+{% if kubernetes_kubelet_node_ip is defined %}
+localAPIEndpoint:
+ advertiseAddress: {{ kubernetes_kubelet_node_ip }}
+{% endif %}
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: {{ kubernetes_version }}
clusterName: {{ kubernetes.cluster_name }}
imageRepository: k8s.gcr.io
+{% if kubernetes_kubelet_node_ip is defined %}
controlPlaneEndpoint: "{{ kubernetes_kubelet_node_ip }}:6443"
+{% endif %}
networking:
dnsDomain: {{ kubernetes.dns_domain | default('cluster.local') }}
podSubnet: {{ kubernetes.pod_ip_range }}
serviceSubnet: {{ kubernetes.service_ip_range }}
apiServer:
- extraArgs:
- advertise-address: {{ kubernetes_kubelet_node_ip }}
+ #extraArgs:
# encryption-provider-config: /etc/kubernetes/encryption/config
# extraVolumes:
# - name: encryption-config
diff --git a/roles/kubernetes/kubeadm/node/tasks/main.yml b/roles/kubernetes/kubeadm/node/tasks/main.yml
index f7efdd81..61d47111 100644
--- a/roles/kubernetes/kubeadm/node/tasks/main.yml
+++ b/roles/kubernetes/kubeadm/node/tasks/main.yml
@@ -1,13 +1,16 @@
---
-- name: join kubernetes node
- command: "kubeadm join {{ hostvars[groups['_kubernetes_primary_master_'][0]].kubernetes_kubelet_node_ip }}:6443 --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}'"
- args:
- creates: /etc/kubernetes/kubelet.conf
- register: kubeadm_join
+- name: join kubernetes node and store log
+ block:
+ - name: join kubernetes node
+ command: "kubeadm join {{ hostvars[groups['_kubernetes_primary_master_'][0]].kubernetes_kubelet_node_ip }}:6443 --node-name {{ inventory_hostname }}{% if kubernetes_cri_socket is defined %} --cri-socket {{ kubernetes_cri_socket }}{% endif %} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}'"
+ args:
+ creates: /etc/kubernetes/kubelet.conf
+ register: kubeadm_join
-- name: dump output of kubeadm join to log file
- when: kubeadm_join is changed
- # This is not a handler by design to make sure this action runs at this point of the play.
- copy: # noqa 503
- content: "{{ kubeadm_join.stdout }}\n"
- dest: /etc/kubernetes/kubeadm-join.log
+ always:
+ - name: dump output of kubeadm join to log file
+ when: kubeadm_join is changed
+ # This is not a handler by design to make sure this action runs at this point of the play.
+ copy: # noqa 503
+ content: "{{ kubeadm_join.stdout }}\n"
+ dest: /etc/kubernetes/kubeadm-join.log
diff --git a/roles/kubernetes/net/kubeguard/templates/kubeguard-peer.service.j2 b/roles/kubernetes/net/kubeguard/templates/kubeguard-peer.service.j2
index 6f36b571..9ca444e8 100644
--- a/roles/kubernetes/net/kubeguard/templates/kubeguard-peer.service.j2
+++ b/roles/kubernetes/net/kubeguard/templates/kubeguard-peer.service.j2
@@ -4,6 +4,7 @@ After=network.target
Requires=kubeguard-interfaces.service
After=kubeguard-interfaces.service
+{% set pod_ip_self = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) | ipaddr(1) | ipaddr('address') -%}
{% set pod_net_peer = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[peer]) -%}
{% set direct_zone = kubeguard.direct_net_zones | direct_net_zone(inventory_hostname, peer) -%}
{% if direct_zone %}
@@ -22,7 +23,7 @@ Type=oneshot
{% if direct_zone %}
ExecStart=/sbin/ip addr add {{ direct_ip }} dev {{ direct_interface }}
ExecStart=/sbin/ip link set up dev {{ direct_interface }}
-ExecStart=/sbin/ip route add {{ pod_net_peer }} via {{ direct_ip_peer | ipaddr('address') }}
+ExecStart=/sbin/ip route add {{ pod_net_peer }} via {{ direct_ip_peer | ipaddr('address') }} src {{ pod_ip_self }}
ExecStop=/sbin/ip route del {{ pod_net_peer }}
ExecStop=/sbin/ip link set down dev {{ direct_interface }}
ExecStop=/sbin/ip addr del {{ direct_ip }} dev {{ direct_interface }}
diff --git a/spreadspace/k8s-test.yml b/spreadspace/k8s-test.yml
index 97daa5b0..f21b3fae 100644
--- a/spreadspace/k8s-test.yml
+++ b/spreadspace/k8s-test.yml
@@ -12,6 +12,9 @@
nodes_group: k8s-test
masters:
- s2-k8s-test0
+ - s2-k8s-test1
+ - s2-k8s-test2
+ primary_master: s2-k8s-test0
- import_playbook: ../common/kubernetes.yml
- import_playbook: ../common/kubernetes-cleanup.yml