From b39c3b91269a8482207863234acc298f623deae6 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 20 Jun 2020 05:20:46 +0200 Subject: kubernetes: add node pruning role --- .../kubeadm/base/tasks/net_kubeguard.yml | 20 ++++++------ .../base/templates/net_kubeguard/cni.json.j2 | 12 +++++++ .../base/templates/net_kubeguard/ifupdown.sh.j2 | 4 +-- .../templates/net_kubeguard/interface.service.j2 | 12 +++++++ .../base/templates/net_kubeguard/k8s.json.j2 | 12 ------- .../net_kubeguard/kubeguard-interfaces.service.j2 | 12 ------- .../net_kubeguard/kubeguard-peer.service.j2 | 37 ---------------------- .../base/templates/net_kubeguard/peer.service.j2 | 37 ++++++++++++++++++++++ roles/kubernetes/kubeadm/prune/tasks/main.yml | 9 ++++++ .../kubeadm/prune/tasks/net_kubeguard.yml | 14 ++++++++ roles/kubernetes/kubeadm/prune/tasks/net_none.yml | 2 ++ .../kubeadm/reset/tasks/net_kubeguard.yml | 6 ++-- .../net/kubeguard/cleanup/tasks/main.yml | 14 -------- 13 files changed, 101 insertions(+), 90 deletions(-) create mode 100644 roles/kubernetes/kubeadm/base/templates/net_kubeguard/cni.json.j2 create mode 100644 roles/kubernetes/kubeadm/base/templates/net_kubeguard/interface.service.j2 delete mode 100644 roles/kubernetes/kubeadm/base/templates/net_kubeguard/k8s.json.j2 delete mode 100644 roles/kubernetes/kubeadm/base/templates/net_kubeguard/kubeguard-interfaces.service.j2 delete mode 100644 roles/kubernetes/kubeadm/base/templates/net_kubeguard/kubeguard-peer.service.j2 create mode 100644 roles/kubernetes/kubeadm/base/templates/net_kubeguard/peer.service.j2 create mode 100644 roles/kubernetes/kubeadm/prune/tasks/main.yml create mode 100644 roles/kubernetes/kubeadm/prune/tasks/net_kubeguard.yml create mode 100644 roles/kubernetes/kubeadm/prune/tasks/net_none.yml delete mode 100644 roles/kubernetes/net/kubeguard/cleanup/tasks/main.yml (limited to 'roles') diff --git a/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml b/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml index 8c5f5065..37b5030d 100644 --- a/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml +++ b/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml @@ -25,26 +25,26 @@ # it must probably be brought down by the old version of the script - name: generate wireguard private key - shell: "umask 077; wg genkey > /var/lib/kubeguard/kube-wg0.privatekey" + shell: "umask 077; wg genkey > /var/lib/kubeguard/kubeguard-wg0.privatekey" args: - creates: /var/lib/kubeguard/kube-wg0.privatekey + creates: /var/lib/kubeguard/kubeguard-wg0.privatekey - name: fetch wireguard public key - shell: "wg pubkey < /var/lib/kubeguard/kube-wg0.privatekey" + shell: "wg pubkey < /var/lib/kubeguard/kubeguard-wg0.privatekey" register: kubeguard_wireguard_pubkey changed_when: false check_mode: no -- name: install systemd service unit for network interfaces +- name: install systemd service unit for network interface template: - src: net_kubeguard/kubeguard-interfaces.service.j2 - dest: /etc/systemd/system/kubeguard-interfaces.service + src: net_kubeguard/interface.service.j2 + dest: /etc/systemd/system/kubeguard-interface.service # TODO: notify: reload??? -- name: make sure kubeguard interfaces service is started and enabled +- name: make sure kubeguard interface service is started and enabled systemd: daemon_reload: yes - name: kubeguard-interfaces.service + name: kubeguard-interface.service state: started enabled: yes @@ -53,7 +53,7 @@ loop_control: loop_var: peer template: - src: net_kubeguard/kubeguard-peer.service.j2 + src: net_kubeguard/peer.service.j2 dest: "/etc/systemd/system/kubeguard-peer-{{ peer }}.service" # TODO: notify restart for peers that change... @@ -80,5 +80,5 @@ - name: install cni config template: - src: net_kubeguard/k8s.json.j2 + src: net_kubeguard/cni.json.j2 dest: /etc/cni/net.d/kubeguard.json diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/cni.json.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/cni.json.j2 new file mode 100644 index 00000000..eb9e3d61 --- /dev/null +++ b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/cni.json.j2 @@ -0,0 +1,12 @@ +{ + "cniVersion": "0.3.1", + "name": "kubeguard", + "type": "bridge", + "bridge": "kubeguard-br0", + "isDefaultGateway": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "subnet": "{{ kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) }}" + } +} diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/ifupdown.sh.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/ifupdown.sh.j2 index d8153102..f940d413 100644 --- a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/ifupdown.sh.j2 +++ b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/ifupdown.sh.j2 @@ -9,12 +9,12 @@ INET_IF="{{ ansible_default_ipv4.interface }}" POD_NET_CIDR="{{ kubernetes.pod_ip_range }}" {% set br_net = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) -%} -BR_IF="kube-br0" +BR_IF="kubeguard-br0" BR_IP="{{ br_net | ipaddr(1) | ipaddr('address') }}" BR_IP_CIDR="{{ br_net | ipaddr(1) }}" BR_NET_CIDR="{{ br_net }}" -TUN_IF="kube-wg0" +TUN_IF="kubeguard-wg0" TUN_IP_CIDR="{{ kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, 0) | ipaddr(kubeguard.node_index[inventory_hostname]) }}" diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/interface.service.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/interface.service.j2 new file mode 100644 index 00000000..35fc8f90 --- /dev/null +++ b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/interface.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Kubeguard Network Setup +After=network.target + +[Service] +Type=oneshot +ExecStart=/var/lib/kubeguard/ifupdown.sh up +ExecStop=/var/lib/kubeguard/ifupdown.sh down +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/k8s.json.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/k8s.json.j2 deleted file mode 100644 index 65b1357a..00000000 --- a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/k8s.json.j2 +++ /dev/null @@ -1,12 +0,0 @@ -{ - "cniVersion": "0.3.1", - "name": "k8s", - "type": "bridge", - "bridge": "kube-br0", - "isDefaultGateway": true, - "hairpinMode": true, - "ipam": { - "type": "host-local", - "subnet": "{{ kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) }}" - } -} diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/kubeguard-interfaces.service.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/kubeguard-interfaces.service.j2 deleted file mode 100644 index 35fc8f90..00000000 --- a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/kubeguard-interfaces.service.j2 +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Kubeguard Network Setup -After=network.target - -[Service] -Type=oneshot -ExecStart=/var/lib/kubeguard/ifupdown.sh up -ExecStop=/var/lib/kubeguard/ifupdown.sh down -RemainAfterExit=yes - -[Install] -WantedBy=multi-user.target diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/kubeguard-peer.service.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/kubeguard-peer.service.j2 deleted file mode 100644 index 92300253..00000000 --- a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/kubeguard-peer.service.j2 +++ /dev/null @@ -1,37 +0,0 @@ -[Unit] -Description=Kubernetes Network Peer {{ peer }} -After=network.target -Requires=kubeguard-interfaces.service -After=kubeguard-interfaces.service - -{% set pod_ip_self = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) | ipaddr(1) | ipaddr('address') -%} -{% set pod_net_peer = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[peer]) -%} -{% set direct_zone = kubeguard.direct_net_zones | default({}) | kubeguard_direct_net_zone(inventory_hostname, peer) -%} -{% if direct_zone %} -{% set direct_ip = kubeguard.direct_net_zones[direct_zone].transfer_net | ipaddr(kubeguard.node_index[inventory_hostname]) %} -{% set direct_interface = kubeguard.direct_net_zones[direct_zone].node_interface[inventory_hostname] %} -{% set direct_ip_peer = kubeguard.direct_net_zones[direct_zone].transfer_net | ipaddr(kubeguard.node_index[peer]) %} -{% else %} -{% set tun_ip = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, 0) | ipaddr(kubeguard.node_index[peer]) -%} -{% set wg_pubkey = hostvars[peer].kubeguard_wireguard_pubkey.stdout -%} -{% set wg_host = hostvars[peer].external_ip_cooked | default(hostvars[peer].ansible_default_ipv4.address) -%} -{% set wg_port = hostvars[peer].kubeguard_wireguard_port | default(51820) -%} -{% set wg_allowedips = (tun_ip | ipaddr('address')) + "/32," + pod_net_peer %} -{% endif %} -[Service] -Type=oneshot -{% if direct_zone %} -ExecStart=/sbin/ip addr add {{ direct_ip }} dev {{ direct_interface }} -ExecStart=/sbin/ip link set up dev {{ direct_interface }} -ExecStart=/sbin/ip route add {{ pod_net_peer }} via {{ direct_ip_peer | ipaddr('address') }} src {{ pod_ip_self }} -ExecStop=/sbin/ip route del {{ pod_net_peer }} -ExecStop=/sbin/ip link set down dev {{ direct_interface }} -ExecStop=/sbin/ip addr del {{ direct_ip }} dev {{ direct_interface }} -{% else %} -ExecStart=/usr/bin/wg set kube-wg0 peer {{ wg_pubkey }} allowed-ips {{ wg_allowedips }} endpoint {{ wg_host }}:{{ wg_port }} persistent-keepalive 10 -ExecStop=/usr/bin/wg set kube-wg0 peer {{ wg_pubkey }} remove -{% endif %} -RemainAfterExit=yes - -[Install] -WantedBy=multi-user.target diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/peer.service.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/peer.service.j2 new file mode 100644 index 00000000..c9d96a5a --- /dev/null +++ b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/peer.service.j2 @@ -0,0 +1,37 @@ +[Unit] +Description=Kubernetes Network Peer {{ peer }} +After=network.target +Requires=kubeguard-interface.service +After=kubeguard-interface.service + +{% set pod_ip_self = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) | ipaddr(1) | ipaddr('address') -%} +{% set pod_net_peer = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[peer]) -%} +{% set direct_zone = kubeguard.direct_net_zones | default({}) | kubeguard_direct_net_zone(inventory_hostname, peer) -%} +{% if direct_zone %} +{% set direct_ip = kubeguard.direct_net_zones[direct_zone].transfer_net | ipaddr(kubeguard.node_index[inventory_hostname]) %} +{% set direct_interface = kubeguard.direct_net_zones[direct_zone].node_interface[inventory_hostname] %} +{% set direct_ip_peer = kubeguard.direct_net_zones[direct_zone].transfer_net | ipaddr(kubeguard.node_index[peer]) %} +{% else %} +{% set tun_ip = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, 0) | ipaddr(kubeguard.node_index[peer]) -%} +{% set wg_pubkey = hostvars[peer].kubeguard_wireguard_pubkey.stdout -%} +{% set wg_host = hostvars[peer].external_ip_cooked | default(hostvars[peer].ansible_default_ipv4.address) -%} +{% set wg_port = hostvars[peer].kubeguard_wireguard_port | default(51820) -%} +{% set wg_allowedips = (tun_ip | ipaddr('address')) + "/32," + pod_net_peer %} +{% endif %} +[Service] +Type=oneshot +{% if direct_zone %} +ExecStart=/sbin/ip addr add {{ direct_ip }} dev {{ direct_interface }} +ExecStart=/sbin/ip link set up dev {{ direct_interface }} +ExecStart=/sbin/ip route add {{ pod_net_peer }} via {{ direct_ip_peer | ipaddr('address') }} src {{ pod_ip_self }} +ExecStop=/sbin/ip route del {{ pod_net_peer }} +ExecStop=/sbin/ip link set down dev {{ direct_interface }} +ExecStop=/sbin/ip addr del {{ direct_ip }} dev {{ direct_interface }} +{% else %} +ExecStart=/usr/bin/wg set kubeguard-wg0 peer {{ wg_pubkey }} allowed-ips {{ wg_allowedips }} endpoint {{ wg_host }}:{{ wg_port }} persistent-keepalive 10 +ExecStop=/usr/bin/wg set kubeguard-wg0 peer {{ wg_pubkey }} remove +{% endif %} +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/roles/kubernetes/kubeadm/prune/tasks/main.yml b/roles/kubernetes/kubeadm/prune/tasks/main.yml new file mode 100644 index 00000000..71ed0d04 --- /dev/null +++ b/roles/kubernetes/kubeadm/prune/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: remove nodes from api server + run_once: true + delegate_to: "{{ groups['_kubernetes_primary_master_'] | first }}" + loop: "{{ groups['_kubernetes_nodes_prune_'] | default([]) }}" + command: "kubectl delete node {{ item }}" + +- name: prune network plugin + include_tasks: "net_{{ kubernetes_network_plugin }}.yml" diff --git a/roles/kubernetes/kubeadm/prune/tasks/net_kubeguard.yml b/roles/kubernetes/kubeadm/prune/tasks/net_kubeguard.yml new file mode 100644 index 00000000..8a8c7752 --- /dev/null +++ b/roles/kubernetes/kubeadm/prune/tasks/net_kubeguard.yml @@ -0,0 +1,14 @@ +--- +- name: stop/disable systemd units for stale kubeguard peers + loop: "{{ groups['_kubernetes_nodes_prune_'] | default([]) }}" + systemd: + name: "kubeguard-peer-{{ item }}.service" + state: stopped + enabled: no + failed_when: false + +- name: remove systemd units for stale kubeguard peers + loop: "{{ groups['_kubernetes_nodes_prune_'] | default([]) }}" + file: + name: "/etc/systemd/system/kubeguard-peer-{{ item }}.service" + state: absent diff --git a/roles/kubernetes/kubeadm/prune/tasks/net_none.yml b/roles/kubernetes/kubeadm/prune/tasks/net_none.yml new file mode 100644 index 00000000..94832c38 --- /dev/null +++ b/roles/kubernetes/kubeadm/prune/tasks/net_none.yml @@ -0,0 +1,2 @@ +--- +## nothing to do here diff --git a/roles/kubernetes/kubeadm/reset/tasks/net_kubeguard.yml b/roles/kubernetes/kubeadm/reset/tasks/net_kubeguard.yml index 03b3f205..bcb48960 100644 --- a/roles/kubernetes/kubeadm/reset/tasks/net_kubeguard.yml +++ b/roles/kubernetes/kubeadm/reset/tasks/net_kubeguard.yml @@ -1,13 +1,13 @@ --- - name: check if kubeguard interface service unit exists stat: - path: /etc/systemd/system/kubeguard-interfaces.service + path: /etc/systemd/system/kubeguard-interface.service register: kubeguard_interface_unit - name: bring down kubeguard interface when: kubeguard_interface_unit.stat.exists systemd: - name: kubeguard-interfaces.service + name: kubeguard-interface.service state: stopped - name: gather list of all kubeguard related service units @@ -15,7 +15,7 @@ path: /etc/systemd/system/ patterns: - "kubeguard-peer-*.service" - - kubeguard-interfaces.service + - kubeguard-interface.service register: kubeguard_units_installed - name: remove all kubeguard related files and directories diff --git a/roles/kubernetes/net/kubeguard/cleanup/tasks/main.yml b/roles/kubernetes/net/kubeguard/cleanup/tasks/main.yml deleted file mode 100644 index f15058d2..00000000 --- a/roles/kubernetes/net/kubeguard/cleanup/tasks/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: stop/disable systemd units for stale kubeguard peers - loop: "{{ groups['_kubernetes_nodes_remove_'] | default([]) }}" - systemd: - name: "kubeguard-peer-{{ item }}.service" - state: stopped - enabled: no - failed_when: false - -- name: remove systemd units for stale kubeguard peers - loop: "{{ groups['_kubernetes_nodes_remove_'] | default([]) }}" - file: - name: "/etc/systemd/system/kubeguard-peer-{{ item }}.service" - state: absent -- cgit v1.2.3