From 266d0b0f06184e2865edf17b00a015a6aa8f050f Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 21 Jan 2018 21:32:09 +0100 Subject: adding kubernetes nodes after first run works now --- group_vars/k8s-stream/vars.yml | 8 +-- hosts.ini | 3 + playbooks/k8s-stream.yaml | 17 +++++ roles/blackmagic-desktopvideo/tasks/main.yaml | 4 +- roles/kubernetes-master/tasks/main.yml | 89 +++++++++++++++++++-------- roles/kubernetes-net/tasks/main.yaml | 2 +- roles/kubernetes-node/tasks/main.yml | 2 +- 7 files changed, 89 insertions(+), 36 deletions(-) diff --git a/group_vars/k8s-stream/vars.yml b/group_vars/k8s-stream/vars.yml index 898bd91f..ef333f72 100644 --- a/group_vars/k8s-stream/vars.yml +++ b/group_vars/k8s-stream/vars.yml @@ -3,11 +3,9 @@ docker_pkg_version: 17.03.2~ce-0~debian-stretch kubernetes: pkg_version: 1.9.2-00 - # currentley only one master is supported - master: "{{ groups['k8s-stream-master'] | first }}" + dedicated_master: False api_extra_sans: - dione.spreadspace.org - dedicated_master: False pod_ip_range: 172.18.0.0/16 pod_ip_range_size: 24 @@ -22,8 +20,8 @@ kubernetes: emc-02: 2 emc-03: 3 emc-04: 4 -# emc-test: 99 -# emc-master: 100 + emc-test: 99 + emc-master: 100 dione: 101 helene: 102 kube2016: 120 diff --git a/hosts.ini b/hosts.ini index 0df7bfb8..d414aba8 100644 --- a/hosts.ini +++ b/hosts.ini @@ -63,6 +63,9 @@ emc-xx dione helene +# [k8s-stream-streamer] +# emc-test + [k8s-stream-streamer:children] emc-xx diff --git a/playbooks/k8s-stream.yaml b/playbooks/k8s-stream.yaml index ff369435..ec2d2748 100644 --- a/playbooks/k8s-stream.yaml +++ b/playbooks/k8s-stream.yaml @@ -1,4 +1,19 @@ --- +- name: prepare cluster config + hosts: k8s-stream + gather_facts: no + run_once: yes + tasks: + - name: configure node lists + set_fact: + kubernetes_nodes: "{{ groups['k8s-stream'] }}" + kubernetes_nodes_master: "{{ groups['k8s-stream-master'] | first }}" + + - name: check if every node has a net_index assigned + debug: + msg: "nodes without net-index: {{ kubernetes_nodes | difference(kubernetes.net_index.keys()) | join(', ') }}" + failed_when: kubernetes_nodes | difference(kubernetes.net_index.keys()) | length > 0 + - name: install kubernetes and overlay network hosts: k8s-stream roles: @@ -20,3 +35,5 @@ hosts: k8s-stream:!k8s-stream-master roles: - role: kubernetes-node + +## TODO: cleanup (remove superflous nodes and delete bootstrap token) diff --git a/roles/blackmagic-desktopvideo/tasks/main.yaml b/roles/blackmagic-desktopvideo/tasks/main.yaml index 4190bb8f..fde4de37 100644 --- a/roles/blackmagic-desktopvideo/tasks/main.yaml +++ b/roles/blackmagic-desktopvideo/tasks/main.yaml @@ -16,12 +16,12 @@ name: desktopvideo state: present -- name: install improved kill mode for DesktopVideoHelper (1/2 +- name: install improved kill mode for DesktopVideoHelper (1/2) file: name: /etc/systemd/system/DesktopVideoHelper.service.d state: directory -- name: install improved kill mode for DesktopVideoHelper (1/2 +- name: install improved kill mode for DesktopVideoHelper (1/2) copy: src: DesktopVideoHelper-killmode.conf dest: /etc/systemd/system/DesktopVideoHelper.service.d/killmode.conf diff --git a/roles/kubernetes-master/tasks/main.yml b/roles/kubernetes-master/tasks/main.yml index e83bd749..4d6bdd6b 100644 --- a/roles/kubernetes-master/tasks/main.yml +++ b/roles/kubernetes-master/tasks/main.yml @@ -1,36 +1,56 @@ --- -- name: generate bootstrap token (1/2) - command: kubeadm token generate - changed_when: False - check_mode: no - register: kubeadm_token_generate +- name: check if kubeconfig admin.conf already exists + stat: + path: /etc/kubernetes/admin.conf + register: kubeconfig_admin_stats -- name: generate bootstrap token (2/2) - set_fact: - kube_bootstrap_token: "{{ kubeadm_token_generate.stdout }}" +### cluster not yet initialized + +- name: create new cluster + when: kubeconfig_admin_stats.stat.exists == False + block: + + - name: generate bootstrap token for new cluster + command: kubeadm token generate + changed_when: False + check_mode: no + register: kubeadm_token_generate -################ -### check if master ist already initialized + - name: set up kubernetes master + command: "kubeadm init --pod-network-cidr {{ kubernetes.pod_ip_range }} --service-cidr {{ kubernetes.service_ip_range }} {% if kubernetes.api_extra_sans | length > 0 %}--apiserver-cert-extra-sans {{ kubernetes.api_extra_sans | join(',') }}{% endif %} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print" + args: + creates: /etc/kubernetes/pki/ca.crt + register: kubeadm_init -# if not -- name: set up kubernetes master - command: "kubeadm init --pod-network-cidr {{ kubernetes.pod_ip_range }} --service-cidr {{ kubernetes.service_ip_range }} {% if kubernetes.api_extra_sans | length > 0 %}--apiserver-cert-extra-sans {{ kubernetes.api_extra_sans | join(',') }}{% endif %} --token '{{ kube_bootstrap_token }}' --token-ttl 42m --skip-token-print" - args: - creates: /etc/kubernetes/pki/ca.crt - register: kubeadm_init + - name: dump output of kubeadm init to log file + when: kubeadm_init.changed + copy: + content: "{{ kubeadm_init.stdout }}\n" + dest: /etc/kubernetes/kubeadm-init.log -# if it is initialized +### cluster is already initialized -## TODO: add bootstrap token (with expiry) +- name: prepare cluster for new nodes + when: kubeconfig_admin_stats.stat.exists == True + block: -### -################ + - name: fetch list of current nodes + command: kubectl get nodes -o name + changed_when: False + check_mode: no + register: kubectl_node_list -- name: dump output of kubeadm init to log file - when: kubeadm_init.changed - copy: - content: "{{ kubeadm_init.stdout }}\n" - dest: /etc/kubernetes/kubeadm-init.log + - name: save list of current nodes + set_fact: + kubernetes_current_nodes: "{{ kubectl_node_list.stdout_lines | map('replace', 'nodes/', '') | list }}" + + - name: create bootstrap token for existing cluster + when: kubernetes_nodes | difference(kubernetes_current_nodes) | length > 0 + command: kubeadm token create --ttl 42m + check_mode: no + register: kubeadm_token_create + +## - name: check if master is tainted (1/2) command: "kubectl --kubeconfig /etc/kubernetes/admin.conf get node {{ inventory_hostname }} -o json" @@ -55,11 +75,26 @@ name: openssl state: present -- name: get ca certificate digest (1/2) +- name: get ca certificate digest shell: "openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'" register: kube_ca_openssl changed_when: False -- name: get ca certificate digest (2/2) +- name: set variables needed kubernetes-nodes to join the cluster set_fact: + kube_bootstrap_token: "{% if kubeadm_token_generate.stdout is defined %}{{ kubeadm_token_generate.stdout }}{% elif kubeadm_token_create.stdout is defined %}{{ kubeadm_token_create.stdout }}{% endif %}" kube_bootstrap_ca_cert_hash: "sha256:{{ kube_ca_openssl.stdout }}" + +- debug: + var: kube_bootstrap_token + +- name: prepare kubectl (1/2) + file: + name: /root/.kube + state: directory + +- name: prepare kubectl (2/2) + file: + dest: /root/.kube/config + src: /etc/kubernetes/admin.conf + state: link diff --git a/roles/kubernetes-net/tasks/main.yaml b/roles/kubernetes-net/tasks/main.yaml index 88f50fd8..ba8b2089 100644 --- a/roles/kubernetes-net/tasks/main.yaml +++ b/roles/kubernetes-net/tasks/main.yaml @@ -59,7 +59,7 @@ - name: compute list of peers to be added set_fact: - kubenet_peers_to_add: "{{ kubernetes.net_index.keys() | difference(inventory_hostname) }}" + kubenet_peers_to_add: "{{ kubernetes_nodes | difference(inventory_hostname) }}" - name: compute list of peers to be removed set_fact: diff --git a/roles/kubernetes-node/tasks/main.yml b/roles/kubernetes-node/tasks/main.yml index c6b24fc6..5b2eb9bc 100644 --- a/roles/kubernetes-node/tasks/main.yml +++ b/roles/kubernetes-node/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: get master vars set_fact: - kube_master_vars: "{{ hostvars[kubernetes.master] }}" + kube_master_vars: "{{ hostvars[kubernetes_nodes_master] }}" - name: set up kubernetes master command: "kubeadm join --token {{ kube_master_vars.kube_bootstrap_token }} {{ kube_master_vars.ansible_default_ipv4.address }}:6443 --discovery-token-ca-cert-hash {{ kube_master_vars.kube_bootstrap_ca_cert_hash }}" -- cgit v1.2.3