From e13a8fec52694d16da2066f4f4d13942a203a601 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 15 Aug 2022 19:24:03 +0200 Subject: kubernetes/kubeadm: only use config files for init and join --- .../kubeadm/control-plane/tasks/primary.yml | 15 ++---- .../kubeadm/control-plane/tasks/secondary.yml | 8 +++- .../control-plane/templates/kubeadm-init.config.j2 | 56 ++++++++++++++++++++++ .../control-plane/templates/kubeadm-join.config.j2 | 20 ++++++++ .../control-plane/templates/kubeadm.config.j2 | 55 --------------------- roles/kubernetes/kubeadm/reset/tasks/main.yml | 22 ++++----- roles/kubernetes/kubeadm/worker/tasks/main.yml | 7 ++- .../kubeadm/worker/templates/kubeadm.config.j2 | 13 +++++ 8 files changed, 116 insertions(+), 80 deletions(-) create mode 100644 roles/kubernetes/kubeadm/control-plane/templates/kubeadm-init.config.j2 create mode 100644 roles/kubernetes/kubeadm/control-plane/templates/kubeadm-join.config.j2 delete mode 100644 roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 create mode 100644 roles/kubernetes/kubeadm/worker/templates/kubeadm.config.j2 (limited to 'roles/kubernetes') diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml index 65a6f7c8..4204c07d 100644 --- a/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml +++ b/roles/kubernetes/kubeadm/control-plane/tasks/primary.yml @@ -6,7 +6,7 @@ - name: generate kubeadm.config template: - src: kubeadm.config.j2 + src: kubeadm-init.config.j2 dest: /etc/kubernetes/kubeadm.config register: kubeadm_config @@ -16,19 +16,10 @@ when: not kubeconfig_kubelet_stats.stat.exists block: - #### kubeadm wants token to come from --config if --config is used - #### i think this is stupid -> TODO: send bug report - # - name: generate bootstrap token for new cluster - # command: kubeadm token generate - # changed_when: False - # check_mode: no - # register: kubeadm_token_generate - - name: initialize kubernetes primary control-plane node and store log block: - - name: initialize kubernetes primary control-plane node - command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }} --skip-token-print" - # command: "kubeadm init --config /etc/kubernetes/kubeadm.config --node-name {{ inventory_hostname }} --token '{{ kubeadm_token_generate.stdout }}' --token-ttl 42m --skip-token-print" + - name: initialize kubernetes primary control-plane node + command: "kubeadm init --config /etc/kubernetes/kubeadm.config --skip-token-print" args: creates: /etc/kubernetes/pki/ca.crt register: kubeadm_init diff --git a/roles/kubernetes/kubeadm/control-plane/tasks/secondary.yml b/roles/kubernetes/kubeadm/control-plane/tasks/secondary.yml index a2dbe081..965fb03e 100644 --- a/roles/kubernetes/kubeadm/control-plane/tasks/secondary.yml +++ b/roles/kubernetes/kubeadm/control-plane/tasks/secondary.yml @@ -25,11 +25,17 @@ set_fact: kubeadm_upload_certs_key: "{% if kubeadm_upload_certs.stdout is defined %}{{ kubeadm_upload_certs.stdout_lines | last }}{% endif %}" +- name: generate kubeadm.config + template: + src: kubeadm-join.config.j2 + dest: /etc/kubernetes/kubeadm.config + register: kubeadm_config + - name: join kubernetes secondary control-plane node and store log block: - name: join kubernetes secondary control-plane node throttle: 1 - command: "kubeadm join 127.0.0.1:6443 --node-name {{ inventory_hostname }} --apiserver-bind-port 6442{% if kubernetes_overlay_node_ip is defined %} --apiserver-advertise-address {{ kubernetes_overlay_node_ip }}{% endif %} --cri-socket {{ kubernetes_cri_socket }} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}' --control-plane --certificate-key {{ kubeadm_upload_certs_key }}" + command: "kubeadm join --config /etc/kubernetes/kubeadm.config" args: creates: /etc/kubernetes/kubelet.conf register: kubeadm_join diff --git a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm-init.config.j2 b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm-init.config.j2 new file mode 100644 index 00000000..d4fb26cf --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm-init.config.j2 @@ -0,0 +1,56 @@ +{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 #} +{# #} +apiVersion: kubeadm.k8s.io/v1beta3 +kind: InitConfiguration +{# it's easier to extract the bootstap token from separate `kubeadm token create` call #} +{# so make sure the token created by init expires fast #} +bootstrapTokens: +- ttl: "1s" +localAPIEndpoint: + bindPort: 6442 +{% if kubernetes_overlay_node_ip is defined %} + advertiseAddress: "{{ kubernetes_overlay_node_ip }}" +{% endif %} +{% if kubernetes_network_plugin_replaces_kube_proxy %} +skipPhases: +- addon/kube-proxy +{% endif %} +nodeRegistration: + name: "{{ inventory_hostname }}" + criSocket: "{{ kubernetes_cri_socket }}" +--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: ClusterConfiguration +kubernetesVersion: {{ kubernetes_version }} +clusterName: "{{ kubernetes.cluster_name }}" +imageRepository: k8s.gcr.io +controlPlaneEndpoint: 127.0.0.1:6443 +networking: + dnsDomain: "{{ kubernetes.dns_domain | default('cluster.local') }}" + podSubnet: "{{ kubernetes.pod_ip_range }}" + serviceSubnet: "{{ kubernetes.service_ip_range }}" +apiServer: + extraArgs: + encryption-provider-config: /etc/kubernetes/encryption/config + extraVolumes: + - name: encryption-config + hostPath: /etc/kubernetes/encryption + mountPath: /etc/kubernetes/encryption + readOnly: true + pathType: Directory +{% if (kubernetes.api_extra_sans | default([]) | length) == 0 %} + certSANs: [] +{% else %} + certSANs: + {{ kubernetes.api_extra_sans | to_nice_yaml | indent(width=2) }} +{% endif %} +controllerManager: + extraArgs: + node-cidr-mask-size: "{{ kubernetes.pod_ip_range_size }}" +scheduler: {} +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +clusterDNS: +- "{{ kubernetes_nodelocal_dnscache_ip }}" +cgroupDriver: systemd diff --git a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm-join.config.j2 b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm-join.config.j2 new file mode 100644 index 00000000..553463bb --- /dev/null +++ b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm-join.config.j2 @@ -0,0 +1,20 @@ +{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 #} +{# #} +apiVersion: kubeadm.k8s.io/v1beta3 +kind: JoinConfiguration +discovery: + bootstrapToken: + apiServerEndpoint: "127.0.0.1:6443" + token: "{{ kube_bootstrap_token }}" + caCertHashes: + - "{{ kube_bootstrap_ca_cert_hash }}" +controlPlane: + certificateKey: "{{ kubeadm_upload_certs_key }}" + localAPIEndpoint: + bindPort: 6442 +{% if kubernetes_overlay_node_ip is defined %} + advertiseAddress: "{{ kubernetes_overlay_node_ip }}" +{% endif %} +nodeRegistration: + name: "{{ inventory_hostname }}" + criSocket: "{{ kubernetes_cri_socket }}" diff --git a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 b/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 deleted file mode 100644 index a0f3efe7..00000000 --- a/roles/kubernetes/kubeadm/control-plane/templates/kubeadm.config.j2 +++ /dev/null @@ -1,55 +0,0 @@ -{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 #} -{# #} -apiVersion: kubeadm.k8s.io/v1beta3 -kind: InitConfiguration -{# TODO: this is ugly but we want to create our own token so we can #} -{# better control it's lifetime #} -bootstrapTokens: -- ttl: "1s" -localAPIEndpoint: - bindPort: 6442 -{% if kubernetes_overlay_node_ip is defined %} - advertiseAddress: {{ kubernetes_overlay_node_ip }} -{% endif %} -{% if kubernetes_network_plugin_replaces_kube_proxy %} -skipPhases: -- addon/kube-proxy -{% endif %} -nodeRegistration: - criSocket: {{ kubernetes_cri_socket }} ---- -apiVersion: kubeadm.k8s.io/v1beta3 -kind: ClusterConfiguration -kubernetesVersion: {{ kubernetes_version }} -clusterName: {{ kubernetes.cluster_name }} -imageRepository: k8s.gcr.io -controlPlaneEndpoint: 127.0.0.1:6443 -networking: - dnsDomain: {{ kubernetes.dns_domain | default('cluster.local') }} - podSubnet: {{ kubernetes.pod_ip_range }} - serviceSubnet: {{ kubernetes.service_ip_range }} -apiServer: - extraArgs: - encryption-provider-config: /etc/kubernetes/encryption/config - extraVolumes: - - name: encryption-config - hostPath: /etc/kubernetes/encryption - mountPath: /etc/kubernetes/encryption - readOnly: true - pathType: Directory -{% if (kubernetes.api_extra_sans | default([]) | length) == 0 %} - certSANs: [] -{% else %} - certSANs: - {{ kubernetes.api_extra_sans | to_nice_yaml | indent(width=2) }} -{% endif %} -controllerManager: - extraArgs: - node-cidr-mask-size: "{{ kubernetes.pod_ip_range_size }}" -scheduler: {} ---- -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -clusterDNS: -- {{ kubernetes_nodelocal_dnscache_ip }} -cgroupDriver: systemd diff --git a/roles/kubernetes/kubeadm/reset/tasks/main.yml b/roles/kubernetes/kubeadm/reset/tasks/main.yml index 8a21fbd5..bc38ce81 100644 --- a/roles/kubernetes/kubeadm/reset/tasks/main.yml +++ b/roles/kubernetes/kubeadm/reset/tasks/main.yml @@ -4,17 +4,17 @@ - name: clean up extra configs and logs loop: - - /etc/kubernetes/kubeadm.config - - /etc/kubernetes/kubeadm-init.log - - /etc/kubernetes/kubeadm-init.errors - - /etc/kubernetes/kubeadm-join.log - - /etc/kubernetes/kubeadm-join.errors - - /etc/kubernetes/pki - - /etc/kubernetes/encryption - - /etc/kubernetes/network-plugin.yml - - /etc/kubernetes/node-local-dns.yml - - /etc/kubernetes/addons - - /etc/default/kubelet + - /etc/kubernetes/kubeadm.config + - /etc/kubernetes/kubeadm-init.log + - /etc/kubernetes/kubeadm-init.errors + - /etc/kubernetes/kubeadm-join.log + - /etc/kubernetes/kubeadm-join.errors + - /etc/kubernetes/pki + - /etc/kubernetes/encryption + - /etc/kubernetes/network-plugin.yml + - /etc/kubernetes/node-local-dns.yml + - /etc/kubernetes/addons + - /etc/default/kubelet file: path: "{{ item }}" state: absent diff --git a/roles/kubernetes/kubeadm/worker/tasks/main.yml b/roles/kubernetes/kubeadm/worker/tasks/main.yml index eabb7a1f..efd14238 100644 --- a/roles/kubernetes/kubeadm/worker/tasks/main.yml +++ b/roles/kubernetes/kubeadm/worker/tasks/main.yml @@ -1,8 +1,13 @@ --- +- name: generate kubeadm.config + template: + src: kubeadm.config.j2 + dest: /etc/kubernetes/kubeadm.config + - name: join kubernetes worker node and store log block: - name: join kubernetes worker node - command: "kubeadm join 127.0.0.1:6443 --node-name {{ inventory_hostname }} --cri-socket {{ kubernetes_cri_socket }} --token '{{ kube_bootstrap_token }}' --discovery-token-ca-cert-hash '{{ kube_bootstrap_ca_cert_hash }}'" + command: "kubeadm join --config /etc/kubernetes/kubeadm.config" args: creates: /etc/kubernetes/kubelet.conf register: kubeadm_join diff --git a/roles/kubernetes/kubeadm/worker/templates/kubeadm.config.j2 b/roles/kubernetes/kubeadm/worker/templates/kubeadm.config.j2 new file mode 100644 index 00000000..664d31f1 --- /dev/null +++ b/roles/kubernetes/kubeadm/worker/templates/kubeadm.config.j2 @@ -0,0 +1,13 @@ +{# https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 #} +{# #} +apiVersion: kubeadm.k8s.io/v1beta3 +kind: JoinConfiguration +discovery: + bootstrapToken: + apiServerEndpoint: "127.0.0.1:6443" + token: "{{ kube_bootstrap_token }}" + caCertHashes: + - "{{ kube_bootstrap_ca_cert_hash }}" +nodeRegistration: + name: "{{ inventory_hostname }}" + criSocket: "{{ kubernetes_cri_socket }}" -- cgit v1.2.3