From c2500036c7b67d1d94f32a2702a2f251b05ac457 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 9 Dec 2018 17:10:52 +0100 Subject: refactored kubernetes playbook --- common/kubernetes.yml | 101 +++++++++++++++++++++++++++++++++++ roles/kubernetes/net/meta/main.yml | 2 +- roles/kubernetes/net/tasks/main.yml | 4 +- roles/kubernetes/node/tasks/add.yml | 6 +-- roles/kubernetes/node/tasks/main.yml | 4 +- spreadspace/k8s-emc.yml | 97 +++++---------------------------- 6 files changed, 121 insertions(+), 93 deletions(-) create mode 100644 common/kubernetes.yml diff --git a/common/kubernetes.yml b/common/kubernetes.yml new file mode 100644 index 00000000..1ad583af --- /dev/null +++ b/common/kubernetes.yml @@ -0,0 +1,101 @@ +--- +- name: prepare variables and do some sanity checks + hosts: _kubernetes_nodes_ + gather_facts: no + run_once: yes + tasks: + - name: check if master group contains only one node + fail: + msg: "There must be exactly one master node defined" + failed_when: (groups['_kubernetes_masters_'] | length) != 1 + + - name: setup variables + set_fact: + kubernetes_nodes: "{{ groups['_kubernetes_nodes_'] }}" + kubernetes_master: "{{ groups['_kubernetes_masters_'] | first }}" + + - name: check whether every node has a net_index assigned + fail: + msg: "There are nodes without an assigned net-index: {{ kubernetes_nodes | difference(kubernetes.net_index.keys()) | join(', ') }}" + failed_when: kubernetes_nodes | difference(kubernetes.net_index.keys()) | length > 0 + + - name: check whether net indizes are unique + fail: + msg: "There are duplicate entries in the net_index table, every net-index is only allowed once" + failed_when: (kubernetes.net_index.keys() | length) != (kubernetes.net_index.values() | unique | length) + + - name: check whether net indizes are all > 0 + fail: + msg: "At least one net-index is < 1 (indizes start at 1)" + failed_when: (kubernetes.net_index.values() | min) < 1 + +######## +- name: install kubernetes and overlay network + hosts: _kubernetes_nodes_ + roles: + ## Since `base` has a dependency for docker it would install and start the daemon + ## without the docker daemon config file generated by `net`. + ## This means that the docker daemon will create a bridge and install iptables rules + ## upon first startup (the first time this playbook runs on a specific host). + ## Since it is a tedious task to remove the interface and the firewall rules it is much + ## easier to just run `net` before `base` as `net` does not need anything from `base`. + - role: kubernetes/net + - role: kubernetes/base + +- name: configure kubernetes master + hosts: _kubernetes_masters_ + roles: + - role: kubernetes/master + +- name: configure kubernetes non-master nodes + hosts: _kubernetes_nodes_:!_kubernetes_masters_ + roles: + - role: kubernetes/node + +######## +- name: check for nodes to be removed + hosts: _kubernetes_masters_ + tasks: + - name: fetch list of current nodes + command: kubectl get nodes -o name + changed_when: False + check_mode: no + register: kubectl_node_list + + - name: generate list of nodes to be removed + with_items: "{{ kubectl_node_list.stdout_lines | map('replace', 'nodes/', '') | list | difference(kubernetes_nodes) }}" + add_host: + name: "{{ item }}" + inventory_dir: "{{ inventory_dir }}" + group: _kubernetes_nodes_remove_ + changed_when: False + + - name: drain superflous nodes + with_items: "{{ groups['_kubernetes_nodes_remove_'] | default([]) }}" + command: "kubectl drain {{ item }} --delete-local-data --force --ignore-daemonsets" + +- name: try to clean superflous nodes + hosts: _kubernetes_nodes_remove_ + vars: + kubernetes_remove_node: yes + roles: + - role: kubernetes/node + - role: kubernetes/net + +- name: remove node from api server + hosts: _kubernetes_masters_ + tasks: + - name: remove superflous nodes + with_items: "{{ groups['_kubernetes_nodes_remove_'] | default([]) }}" + command: "kubectl delete node {{ item }}" + + - name: wait a litte before removing bootstrap-token so new nodes have time to generate certificates for themselves + when: kube_bootstrap_token != "" + pause: + seconds: 42 + + - name: remove bootstrap-token + when: kube_bootstrap_token != "" + command: "kubectl --namespace kube-system delete secret bootstrap-token-{{ kube_bootstrap_token.split('.') | first }}" + +### TODO: add node labels (ie. for ingress daeomnset) diff --git a/roles/kubernetes/net/meta/main.yml b/roles/kubernetes/net/meta/main.yml index 03dfcb23..a3d4d97b 100644 --- a/roles/kubernetes/net/meta/main.yml +++ b/roles/kubernetes/net/meta/main.yml @@ -1,4 +1,4 @@ --- dependencies: - role: wireguard - when: k8s_remove_node is not defined + when: kubernetes_remove_node is not defined diff --git a/roles/kubernetes/net/tasks/main.yml b/roles/kubernetes/net/tasks/main.yml index 8c94292e..8aa7221e 100644 --- a/roles/kubernetes/net/tasks/main.yml +++ b/roles/kubernetes/net/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: add node to overlay network include_tasks: add.yml - when: k8s_remove_node is not defined + when: kubernetes_remove_node is not defined - name: remove node from overlay network include_tasks: remove.yml - when: k8s_remove_node is defined + when: kubernetes_remove_node is defined diff --git a/roles/kubernetes/node/tasks/add.yml b/roles/kubernetes/node/tasks/add.yml index dd784b35..9f0057f9 100644 --- a/roles/kubernetes/node/tasks/add.yml +++ b/roles/kubernetes/node/tasks/add.yml @@ -1,9 +1,9 @@ --- - name: get master vars set_fact: - kube_bootstrap_token: "{{ hostvars[kubernetes_nodes_master].kube_bootstrap_token }}" - kube_bootstrap_ca_cert_hash: "{{ hostvars[kubernetes_nodes_master].kube_bootstrap_ca_cert_hash }}" - kube_master_addr: "{{ kubernetes.api_advertise_ip | default(hostvars[kubernetes_nodes_master].ansible_default_ipv4.address) }}" + kube_bootstrap_token: "{{ hostvars[kubernetes_master].kube_bootstrap_token }}" + kube_bootstrap_ca_cert_hash: "{{ hostvars[kubernetes_master].kube_bootstrap_ca_cert_hash }}" + kube_master_addr: "{{ kubernetes.api_advertise_ip | default(hostvars[kubernetes_master].ansible_default_ipv4.address) }}" - name: join kubernetes node command: "kubeadm join --token {{ kube_bootstrap_token }} {{ kube_master_addr }}:6443 --discovery-token-ca-cert-hash {{ kube_bootstrap_ca_cert_hash }}" diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index d078d2fb..e29fbc29 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: add node cluster include_tasks: add.yml - when: k8s_remove_node is not defined + when: kubernetes_remove_node is not defined - name: remove node from cluster include_tasks: remove.yml - when: k8s_remove_node is defined + when: kubernetes_remove_node is defined diff --git a/spreadspace/k8s-emc.yml b/spreadspace/k8s-emc.yml index b6f09808..cd4e8e2e 100644 --- a/spreadspace/k8s-emc.yml +++ b/spreadspace/k8s-emc.yml @@ -1,96 +1,23 @@ --- -- name: prepare variables and do some sanity checks +- name: setup cluster config hosts: k8s-emc gather_facts: no run_once: yes tasks: - - name: setup variables - set_fact: - kubernetes_nodes: "{{ groups['k8s-emc'] }}" - kubernetes_nodes_master: "{{ groups['k8s-emc-master'] | first }}" - - - name: check whether every node has a net_index assigned - fail: - msg: "There are nodes without an assigned net-index: {{ kubernetes_nodes | difference(kubernetes.net_index.keys()) | join(', ') }}" - failed_when: kubernetes_nodes | difference(kubernetes.net_index.keys()) | length > 0 - - - name: check whether net indizes are unique - fail: - msg: "There are duplicate entries in the net_index table, every net-index is only allowed once" - failed_when: (kubernetes.net_index.keys() | length) != (kubernetes.net_index.values() | unique | length) - - - name: check whether net indizes are all > 0 - fail: - msg: "At least one net-index is < 1 (indizes start at 1)" - failed_when: (kubernetes.net_index.values() | min) < 1 - -######## -- name: install kubernetes and overlay network - hosts: k8s-emc - roles: - ## Since `base` has a dependency for docker it would install and start the daemon - ## without the docker daemon config file generated by `net`. - ## This means that the docker daemon will create a bridge and install iptables rules - ## upon first startup (the first time this playbook runs on a specific host). - ## Since it is a tedious task to remove the interface and the firewall rules it is much - ## easier to just run `net` before `base` as `net` does not need anything from `base`. - - role: kubernetes/net - - role: kubernetes/base - -- name: configure kubernetes master - hosts: k8s-emc-master - roles: - - role: kubernetes/master - -- name: configure kubernetes nodes - hosts: k8s-emc:!k8s-emc-master - roles: - - role: kubernetes/node - -######## -- name: check for nodes to be removed - hosts: k8s-emc-master - tasks: - - name: fetch list of current nodes - command: kubectl get nodes -o name + - name: create group for all kubernetes nodes + with_items: "{{ groups['k8s-emc'] }}" + add_host: + name: "{{ item }}" + inventory_dir: "{{ inventory_dir }}" + group: _kubernetes_nodes_ changed_when: False - check_mode: no - register: kubectl_node_list - - name: generate list of nodes to be removed - with_items: "{{ kubectl_node_list.stdout_lines | map('replace', 'nodes/', '') | list | difference(kubernetes_nodes) }}" + - name: create group for kubernetes master nodes + with_items: "{{ groups['k8s-emc-master'] }}" add_host: name: "{{ item }}" - inventory_dir: "{{inventory_dir}}" - group: _k8s-emc-remove_ + inventory_dir: "{{ inventory_dir }}" + group: _kubernetes_masters_ changed_when: False - - name: drain superflous nodes - with_items: "{{ groups['_k8s-emc-remove_'] | default([]) }}" - command: "kubectl drain {{ item }} --delete-local-data --force --ignore-daemonsets" - -- name: try to clean superflous nodes - hosts: _k8s-emc-remove_ - vars: - k8s_remove_node: yes - roles: - - role: kubernetes/node - - role: kubernetes/net - -- name: remove node from api server - hosts: k8s-emc-master - tasks: - - name: remove superflous nodes - with_items: "{{ groups['_k8s-emc-remove_'] | default([]) }}" - command: "kubectl delete node {{ item }}" - - - name: wait a litte before removing bootstrap-token so new nodes have time to generate certificates for themselves - when: kube_bootstrap_token != "" - pause: - seconds: 42 - - - name: remove bootstrap-token - when: kube_bootstrap_token != "" - command: "kubectl --namespace kube-system delete secret bootstrap-token-{{ kube_bootstrap_token.split('.') | first }}" - -### TODO: add node labels (ie. for ingress daeomnset) +- import_playbook: ../common/kubernetes.yml -- cgit v1.2.3