From c09b07327b688a6a47f523a15c1a5c29d4f476d0 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 7 May 2022 22:45:49 +0200 Subject: k8s: rename masters to control-plane nodes --- .../kubeadm/control-plane/tasks/primary.yml | 131 +++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 roles/kubernetes/kubeadm/control-plane/tasks/primary.yml (limited to 'roles/kubernetes/kubeadm/control-plane/tasks/primary.yml') diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml new file mode 100644 index 00000000..22a5af42 --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml @@ -0,0 +1,131 @@ +--- +- name: check if kubeconfig kubelet.conf already exists + stat: + path: /etc/kubernetes/kubelet.conf + register: kubeconfig_kubelet_stats + + ## TODO: switch to kubeadm config version v1beta3 (available since 1.22) +- name: generate kubeadm.config + template: + src: kubeadm.config.j2 + dest: /etc/kubernetes/kubeadm.config + register: kubeadm_config + +### cluster not yet initialized + +- name: create new cluster + when: not kubeconfig_kubelet_stats.stat.exists + block: + + #### kubeadm wants token to come from --config if --config is used + #### i think this is stupid -> TODO: send bug report + # - name: generate bootstrap token for new cluster + # command: kubeadm token generate + # changed_when: False + # check_mode: no + # register: kubeadm_token_generate + + - name: initialize kubernetes primary control-plane node and store log + block: + - name: initialize kubernetes primary control-plane node + command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_network_plugin_replaces_kube_proxy %} --skip-phases addon/kube-proxy{% endif %} --skip-token-print" + # command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_network_plugin_replaces_kube_proxy %} --skip-phases addon/kube-proxy{% endif %} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print" + args: + creates: /etc/kubernetes/pki/ca.crt + register: kubeadm_init + + always: + - name: dump output of kubeadm init to log file + when: kubeadm_init.changed + copy: + content: "{{ kubeadm_init.stdout }}\n" + dest: /etc/kubernetes/kubeadm-init.log + + - name: dump error output of kubeadm init to log file + when: kubeadm_init.changed and kubeadm_init.stderr + copy: + content: "{{ kubeadm_init.stderr }}\n" + dest: /etc/kubernetes/kubeadm-init.errors + + - name: create bootstrap token for existing cluster + command: kubeadm token create --ttl 42m + check_mode: no + register: kubeadm_token_generate + + +### cluster is already initialized but config has changed + +- name: upgrade cluster config + when: kubeconfig_kubelet_stats.stat.exists and kubeadm_config is changed + block: + + - name: fail for cluster upgrades + fail: + msg: "upgrading cluster config is currently not supported!" + + +### cluster is already initialized + +- name: prepare cluster for new nodes + when: kubeconfig_kubelet_stats.stat.exists and kubeadm_config is not changed + block: + + - name: fetch list of current nodes + command: kubectl --kubeconfig /etc/kubernetes/admin.conf get nodes -o name + changed_when: False + check_mode: no + register: kubectl_node_list + + - name: save list of current nodes + set_fact: + kubernetes_current_nodes: "{{ kubectl_node_list.stdout_lines | map('replace', 'node/', '') | list }}" + + - name: create bootstrap token for existing cluster + when: "groups['_kubernetes_nodes_'] | difference(kubernetes_current_nodes) | length > 0" + command: kubeadm token create --ttl 42m + check_mode: no + register: kubeadm_token_create + + +## calculate certificate digest + +- name: install openssl + apt: + name: openssl + state: present + +- name: get ca certificate digest + shell: "set -o pipefail && openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'" + args: + executable: /bin/bash + check_mode: no + register: kube_ca_openssl + changed_when: False + +- name: set variables needed by kubernetes/nodes to join the cluster + set_fact: + kube_bootstrap_token: "{% if kubeadm_token_generate.stdout is defined %}{{ kubeadm_token_generate.stdout }}{% elif kubeadm_token_create.stdout is defined %}{{ kubeadm_token_create.stdout }}{% endif %}" + kube_bootstrap_ca_cert_hash: "sha256:{{ kube_ca_openssl.stdout }}" + delegate_to: "{{ item }}" + delegate_facts: True + loop: "{{ groups['_kubernetes_nodes_'] }}" + + +## install node-local-dns + +- name: generate node-local dns cache config + template: + src: node-local-dns.yml.j2 + dest: /etc/kubernetes/node-local-dns.yml + + ## TODO: move to server-side apply (GA since 1.22) +- name: install node-local dns cache + command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/node-local-dns.yml + register: kube_node_local_dns_apply_result + changed_when: (kube_node_local_dns_apply_result.stdout_lines | reject("regex", " unchanged$") | list | length) > 0 + + +## Network Plugin + +- name: install network plugin + include_tasks: "net_{{ kubernetes_network_plugin }}.yml" -- cgit v1.2.3 From 09c8120540735c22316a55593f4c56bcd6ae7e88 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 8 May 2022 01:08:36 +0200 Subject: add support for cluster with kubernetes 1.24 --- inventory/group_vars/k8s-chtest/vars.yml | 2 +- roles/kubernetes/kubeadm/control-plane/tasks/primary.yml | 6 +++--- .../kubeadm/control-plane/templates/kubeadm.config.j2 | 12 +++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) (limited to 'roles/kubernetes/kubeadm/control-plane/tasks/primary.yml') diff --git a/inventory/group_vars/k8s-chtest/vars.yml b/inventory/group_vars/k8s-chtest/vars.yml index 66824314..939d93da 100644 --- a/inventory/group_vars/k8s-chtest/vars.yml +++ b/inventory/group_vars/k8s-chtest/vars.yml @@ -1,5 +1,5 @@ --- -kubernetes_version: 1.23.6 +kubernetes_version: 1.24.0 kubernetes_container_runtime: containerd kubernetes_network_plugin: kube-router kubernetes_network_plugin_version: 1.4.0 diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml index 22a5af42..450c3a1a 100644 --- a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml +++ b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml @@ -28,8 +28,8 @@ - name: initialize kubernetes primary control-plane node and store log block: - name: initialize kubernetes primary control-plane node - command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_network_plugin_replaces_kube_proxy %} --skip-phases addon/kube-proxy{% endif %} --skip-token-print" - # command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }}{% if kubernetes_network_plugin_replaces_kube_proxy %} --skip-phases addon/kube-proxy{% endif %} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print" + command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }} --skip-token-print" + # command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print" args: creates: /etc/kubernetes/pki/ca.crt register: kubeadm_init @@ -47,7 +47,7 @@ content: "{{ kubeadm_init.stderr }}\n" dest: /etc/kubernetes/kubeadm-init.errors - - name: create bootstrap token for existing cluster + - name: create bootstrap token for new cluster command: kubeadm token create --ttl 42m check_mode: no register: kubeadm_token_generate diff --git a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 index 2fa98ed6..a0f3efe7 100644 --- a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 +++ b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 @@ -1,6 +1,6 @@ -{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2 #} +{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 #} {# #} -apiVersion: kubeadm.k8s.io/v1beta2 +apiVersion: kubeadm.k8s.io/v1beta3 kind: InitConfiguration {# TODO: this is ugly but we want to create our own token so we can #} {# better control it's lifetime #} @@ -11,10 +11,14 @@ localAPIEndpoint: {% if kubernetes_overlay_node_ip is defined %} advertiseAddress: {{ kubernetes_overlay_node_ip }} {% endif %} +{% if kubernetes_network_plugin_replaces_kube_proxy %} +skipPhases: +- addon/kube-proxy +{% endif %} nodeRegistration: criSocket: {{ kubernetes_cri_socket }} --- -apiVersion: kubeadm.k8s.io/v1beta2 +apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration kubernetesVersion: {{ kubernetes_version }} clusterName: {{ kubernetes.cluster_name }} @@ -43,8 +47,6 @@ controllerManager: extraArgs: node-cidr-mask-size: "{{ kubernetes.pod_ip_range_size }}" scheduler: {} -dns: - type: CoreDNS --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration -- cgit v1.2.3 From 05e65f43df9c502eb764b184a66dd1ef5a76685c Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 8 May 2022 01:55:09 +0200 Subject: k8s/kubeadm: fix some minor TODOs --- roles/kubernetes/addons/metrics-server/tasks/main.yml | 11 ++++++++--- .../kubeadm/control-plane/tasks/net_kube-router.yml | 11 ++++++++--- .../kubeadm/control-plane/tasks/net_kubeguard.yml | 11 ++++++++--- roles/kubernetes/kubeadm/control-plane/tasks/primary.yml | 14 +++++++++----- 4 files changed, 33 insertions(+), 14 deletions(-) (limited to 'roles/kubernetes/kubeadm/control-plane/tasks/primary.yml') diff --git a/roles/kubernetes/addons/metrics-server/tasks/main.yml b/roles/kubernetes/addons/metrics-server/tasks/main.yml index 5236e4e3..87c57346 100644 --- a/roles/kubernetes/addons/metrics-server/tasks/main.yml +++ b/roles/kubernetes/addons/metrics-server/tasks/main.yml @@ -9,8 +9,13 @@ src: "components.{{ kubernetes_metrics_server_version }}.yml.j2" dest: /etc/kubernetes/addons/metrics-server/config.yml - ## TODO: move to server-side apply (GA since 1.22) +- name: check if metrics-server is already installed + check_mode: no + command: kubectl --kubeconfig /etc/kubernetes/admin.conf diff -f /etc/kubernetes/addons/metrics-server/config.yml + failed_when: false + changed_when: false + register: kube_metrics_server_diff_result + - name: install metrics-server onto the cluster + when: kube_metrics_server_diff_result.rc != 0 command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/addons/metrics-server/config.yml - register: kube_metrics_server_apply_result - changed_when: (kube_metrics_server_apply_result.stdout_lines | reject("regex", " unchanged$") | list | length) > 0 diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/net_kube-router.yml b/roles/kubernetes/kubeadm/control-plane/tasks/net_kube-router.yml index 0a216414..4584e583 100644 --- a/roles/kubernetes/kubeadm/control-plane/tasks/net_kube-router.yml +++ b/roles/kubernetes/kubeadm/control-plane/tasks/net_kube-router.yml @@ -4,8 +4,13 @@ src: "net_kube-router/config.{{ kubernetes_network_plugin_version }}.yml.j2" dest: /etc/kubernetes/network-plugin.yml - ## TODO: move to server-side apply (GA since 1.22) +- name: check if kube-router is already installed + check_mode: no + command: kubectl --kubeconfig /etc/kubernetes/admin.conf diff -f /etc/kubernetes/network-plugin.yml + failed_when: false + changed_when: false + register: kube_router_diff_result + - name: install kube-router on to the cluster + when: kube_router_diff_result.rc != 0 command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/network-plugin.yml - register: kube_router_apply_result - changed_when: (kube_router_apply_result.stdout_lines | reject("regex", " unchanged$") | list | length) > 0 diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/net_kubeguard.yml b/roles/kubernetes/kubeadm/control-plane/tasks/net_kubeguard.yml index a572ca89..66dac49b 100644 --- a/roles/kubernetes/kubeadm/control-plane/tasks/net_kubeguard.yml +++ b/roles/kubernetes/kubeadm/control-plane/tasks/net_kubeguard.yml @@ -7,8 +7,13 @@ src: "net_kubeguard/kube-router.{{ kubernetes_network_plugin_version }}.yml.j2" dest: /etc/kubernetes/network-plugin.yml - ## TODO: move to server-side apply (GA since 1.22) + - name: check if kubeguard (kube-router) is already installed + check_mode: no + command: kubectl --kubeconfig /etc/kubernetes/admin.conf diff -f /etc/kubernetes/network-plugin.yml + failed_when: false + changed_when: false + register: kubeguard_diff_result + - name: install kubeguard (kube-router) on to the cluster + when: kubeguard_diff_result.rc != 0 command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/network-plugin.yml - register: kubeguard_apply_result - changed_when: (kubeguard_apply_result.stdout_lines | reject("regex", " unchanged$") | list | length) > 0 diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml index 450c3a1a..65a6f7c8 100644 --- a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml +++ b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml @@ -4,7 +4,6 @@ path: /etc/kubernetes/kubelet.conf register: kubeconfig_kubelet_stats - ## TODO: switch to kubeadm config version v1beta3 (available since 1.22) - name: generate kubeadm.config template: src: kubeadm.config.j2 @@ -118,11 +117,16 @@ src: node-local-dns.yml.j2 dest: /etc/kubernetes/node-local-dns.yml - ## TODO: move to server-side apply (GA since 1.22) -- name: install node-local dns cache +- name: check if node-local dns cache is already installed + check_mode: no + command: kubectl --kubeconfig /etc/kubernetes/admin.conf diff -f /etc/kubernetes/node-local-dns.yml + failed_when: false + changed_when: false + register: kube_node_local_dns_diff_result + +- name: install node-local dns cache + when: kube_node_local_dns_diff_result.rc != 0 command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f /etc/kubernetes/node-local-dns.yml - register: kube_node_local_dns_apply_result - changed_when: (kube_node_local_dns_apply_result.stdout_lines | reject("regex", " unchanged$") | list | length) > 0 ## Network Plugin -- cgit v1.2.3