diff options
Diffstat (limited to 'roles/kubernetes/kubeadm/base')
9 files changed, 269 insertions, 13 deletions
diff --git a/roles/kubernetes/kubeadm/base/filter_plugins/net_kubeguard.py b/roles/kubernetes/kubeadm/base/filter_plugins/net_kubeguard.py new file mode 100644 index 00000000..2220e545 --- /dev/null +++ b/roles/kubernetes/kubeadm/base/filter_plugins/net_kubeguard.py @@ -0,0 +1,33 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible import errors + + +def kubeguard_direct_net_zone(data, myname, peer): + try: + zones = [] + for zone in data: + if myname in data[zone]['node_interface'] and peer in data[zone]['node_interface']: + zones.append(zone) + + if not zones: + return "" + if len(zones) > 1: + raise errors.AnsibleFilterError("host '%s' and '%s' have multiple direct net zones in common: %s" % + (myname, peer, zones.join(','))) + return zones[0] + + except Exception as e: + raise errors.AnsibleFilterError("kubeguard_direct_net_zones(): %s" % str(e)) + + +class FilterModule(object): + + ''' Kubeguard Network Filters ''' + filter_map = { + 'kubeguard_direct_net_zone': kubeguard_direct_net_zone, + } + + def filters(self): + return self.filter_map diff --git a/roles/kubernetes/kubeadm/base/tasks/main.yml b/roles/kubernetes/kubeadm/base/tasks/main.yml index 2d2bd324..7d882f31 100644 --- a/roles/kubernetes/kubeadm/base/tasks/main.yml +++ b/roles/kubernetes/kubeadm/base/tasks/main.yml @@ -3,7 +3,7 @@ apt: name: - haproxy - - hatop + - haproxyctl - "kubeadm={{ kubernetes_version }}-00" - "kubectl={{ kubernetes_version }}-00" state: present @@ -48,16 +48,13 @@ state: "{% if haproxy_config is changed %}restarted{% else %}started{% endif %}" enabled: yes -- name: add hatop config for shells - loop: - - zsh - - bash - blockinfile: - path: "/root/.{{ item }}rc" - create: yes - marker: "### {mark} ANSIBLE MANAGED BLOCK for hatop ###" - content: | - alias hatop="hatop -s /var/run/haproxy/admin.sock" +## loading the modules temporarly because kubeadm will complain if they are not there +# but i don't think it is necessary to make this persistent, also ignoring changes here +- name: load module br_netfilter to satisfy kubeadm init/join + modprobe: + name: br_netfilter + state: present + changed_when: false -# - name: prepare network plugin -# include_tasks: "net_{{ kubernetes_network_plugin }}.yml" +- name: prepare network plugin + include_tasks: "net_{{ kubernetes_network_plugin }}.yml" diff --git a/roles/kubernetes/kubeadm/base/tasks/net_kube-router.yml b/roles/kubernetes/kubeadm/base/tasks/net_kube-router.yml new file mode 100644 index 00000000..246b20bc --- /dev/null +++ b/roles/kubernetes/kubeadm/base/tasks/net_kube-router.yml @@ -0,0 +1,8 @@ +--- +- name: install packages needed for debugging kube-router + apt: + name: + - iptables + - ipvsadm + - ipset + state: present diff --git a/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml b/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml new file mode 100644 index 00000000..2d706a03 --- /dev/null +++ b/roles/kubernetes/kubeadm/base/tasks/net_kubeguard.yml @@ -0,0 +1,95 @@ +--- +- name: make sure kubernetes_network_plugin_replaces_kube_proxy is not set + when: + - kubernetes_network_plugin_variant != 'with-kube-router' + run_once: yes + assert: + msg: "kubeguard variant '{{ kubernetes_network_plugin_variant }}' can not replace kube-proxy please set kubernetes_network_plugin_replaces_kube_proxy to false or configure a differnt kubernetes_network_plugin_variant." + that: + - not kubernetes_network_plugin_replaces_kube_proxy + + +- name: install wireguard + import_role: + name: wireguard/base + +- name: create network config directory + file: + name: /var/lib/kubeguard/ + state: directory + +- name: install ifupdown script + template: + src: net_kubeguard/ifupdown.sh.j2 + dest: /var/lib/kubeguard/ifupdown.sh + mode: 0755 + # TODO: notify reload... this is unfortunately already to late because + # it must probably be brought down by the old version of the script + +- name: generate wireguard private key + shell: "umask 077; wg genkey > /var/lib/kubeguard/kubeguard-wg0.privatekey" + args: + creates: /var/lib/kubeguard/kubeguard-wg0.privatekey + +- name: fetch wireguard public key + shell: "wg pubkey < /var/lib/kubeguard/kubeguard-wg0.privatekey" + register: kubeguard_wireguard_pubkey + changed_when: false + check_mode: no + +- name: install systemd service unit for network interface + template: + src: net_kubeguard/interface.service.j2 + dest: /etc/systemd/system/kubeguard-interface.service + # TODO: notify: reload??? + +- name: make sure kubeguard interface service is started and enabled + systemd: + daemon_reload: yes + name: kubeguard-interface.service + state: started + enabled: yes + +- name: install systemd units for every kubeguard peer + loop: "{{ groups['_kubernetes_nodes_'] | difference(inventory_hostname) }}" + loop_control: + loop_var: peer + template: + src: net_kubeguard/peer.service.j2 + dest: "/etc/systemd/system/kubeguard-peer-{{ peer }}.service" + # TODO: notify restart for peers that change... + +- name: make sure kubeguard peer services are started and enabled + loop: "{{ groups['_kubernetes_nodes_'] | difference(inventory_hostname) }}" + systemd: + daemon_reload: yes + name: "kubeguard-peer-{{ item }}.service" + state: started + enabled: yes + +- name: enable IPv4 forwarding + sysctl: + name: net.ipv4.ip_forward + value: '1' + sysctl_set: yes + state: present + reload: yes + +- name: create cni config directory + file: + name: /etc/cni/net.d + state: directory + +- name: install cni config + template: + src: net_kubeguard/cni.json.j2 + dest: /etc/cni/net.d/kubeguard.json + +- name: install packages needed for debugging kube-router + when: kubernetes_network_plugin_variant == 'with-kube-router' + apt: + name: + - iptables + - ipvsadm + - ipset + state: present diff --git a/roles/kubernetes/kubeadm/base/tasks/net_none.yml b/roles/kubernetes/kubeadm/base/tasks/net_none.yml new file mode 100644 index 00000000..0924c458 --- /dev/null +++ b/roles/kubernetes/kubeadm/base/tasks/net_none.yml @@ -0,0 +1,7 @@ +--- +- name: make sure kubernetes_network_plugin_replaces_kube_proxy is not set + run_once: yes + assert: + msg: "this network plugin can not replace kube-proxy please set kubernetes_network_plugin_replaces_kube_proxy to false." + that: + - not kubernetes_network_plugin_replaces_kube_proxy diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/cni.json.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/cni.json.j2 new file mode 100644 index 00000000..eb9e3d61 --- /dev/null +++ b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/cni.json.j2 @@ -0,0 +1,12 @@ +{ + "cniVersion": "0.3.1", + "name": "kubeguard", + "type": "bridge", + "bridge": "kubeguard-br0", + "isDefaultGateway": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "subnet": "{{ kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) }}" + } +} diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/ifupdown.sh.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/ifupdown.sh.j2 new file mode 100644 index 00000000..f940d413 --- /dev/null +++ b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/ifupdown.sh.j2 @@ -0,0 +1,55 @@ +#!/bin/bash + +set -e + +CONF_D="/var/lib/kubeguard/" + +INET_IF="{{ ansible_default_ipv4.interface }}" + +POD_NET_CIDR="{{ kubernetes.pod_ip_range }}" + +{% set br_net = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) -%} +BR_IF="kubeguard-br0" +BR_IP="{{ br_net | ipaddr(1) | ipaddr('address') }}" +BR_IP_CIDR="{{ br_net | ipaddr(1) }}" +BR_NET_CIDR="{{ br_net }}" + +TUN_IF="kubeguard-wg0" +TUN_IP_CIDR="{{ kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, 0) | ipaddr(kubeguard.node_index[inventory_hostname]) }}" + + +case "$1" in + up) + # bring up bridge for local pods + ip link add dev "$BR_IF" type bridge + ip addr add dev "$BR_IF" "$BR_IP_CIDR" + ip link set up dev "$BR_IF" + iptables -t nat -A POSTROUTING -s "$BR_NET_CIDR" -o "$INET_IF" -j MASQUERADE + modprobe br_netfilter + + # bring up wireguard tunnel to other nodes + ip link add dev "$TUN_IF" type wireguard + ip addr add dev "$TUN_IF" "$TUN_IP_CIDR" + wg set "$TUN_IF" listen-port {{ kubeguard_wireguard_port | default(51820) }} private-key "$CONF_D/$TUN_IF.privatekey" + ip link set up dev "$TUN_IF" + + # make pods and service IPs reachable + # !!! use IP of bridge as source so we don't produce martians if direct-zones are involved!!! + ip route add "$POD_NET_CIDR" dev "$TUN_IF" src "$BR_IP" + ;; + down) + # bring down wireguard tunnel to other nodes + ip route del "$POD_NET_CIDR" dev "$TUN_IF" + ip link del dev "$TUN_IF" + + # bring down bridge for local pods + iptables -t nat -D POSTROUTING -s "$BR_NET_CIDR" -o "$INET_IF" -j MASQUERADE + ip link del dev "$BR_IF" + ;; + *) + echo "usage: $0 (up|down)" + exit 1 + ;; +esac + +exit 0 diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/interface.service.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/interface.service.j2 new file mode 100644 index 00000000..35fc8f90 --- /dev/null +++ b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/interface.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Kubeguard Network Setup +After=network.target + +[Service] +Type=oneshot +ExecStart=/var/lib/kubeguard/ifupdown.sh up +ExecStop=/var/lib/kubeguard/ifupdown.sh down +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/roles/kubernetes/kubeadm/base/templates/net_kubeguard/peer.service.j2 b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/peer.service.j2 new file mode 100644 index 00000000..c9d96a5a --- /dev/null +++ b/roles/kubernetes/kubeadm/base/templates/net_kubeguard/peer.service.j2 @@ -0,0 +1,37 @@ +[Unit] +Description=Kubernetes Network Peer {{ peer }} +After=network.target +Requires=kubeguard-interface.service +After=kubeguard-interface.service + +{% set pod_ip_self = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) | ipaddr(1) | ipaddr('address') -%} +{% set pod_net_peer = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[peer]) -%} +{% set direct_zone = kubeguard.direct_net_zones | default({}) | kubeguard_direct_net_zone(inventory_hostname, peer) -%} +{% if direct_zone %} +{% set direct_ip = kubeguard.direct_net_zones[direct_zone].transfer_net | ipaddr(kubeguard.node_index[inventory_hostname]) %} +{% set direct_interface = kubeguard.direct_net_zones[direct_zone].node_interface[inventory_hostname] %} +{% set direct_ip_peer = kubeguard.direct_net_zones[direct_zone].transfer_net | ipaddr(kubeguard.node_index[peer]) %} +{% else %} +{% set tun_ip = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, 0) | ipaddr(kubeguard.node_index[peer]) -%} +{% set wg_pubkey = hostvars[peer].kubeguard_wireguard_pubkey.stdout -%} +{% set wg_host = hostvars[peer].external_ip_cooked | default(hostvars[peer].ansible_default_ipv4.address) -%} +{% set wg_port = hostvars[peer].kubeguard_wireguard_port | default(51820) -%} +{% set wg_allowedips = (tun_ip | ipaddr('address')) + "/32," + pod_net_peer %} +{% endif %} +[Service] +Type=oneshot +{% if direct_zone %} +ExecStart=/sbin/ip addr add {{ direct_ip }} dev {{ direct_interface }} +ExecStart=/sbin/ip link set up dev {{ direct_interface }} +ExecStart=/sbin/ip route add {{ pod_net_peer }} via {{ direct_ip_peer | ipaddr('address') }} src {{ pod_ip_self }} +ExecStop=/sbin/ip route del {{ pod_net_peer }} +ExecStop=/sbin/ip link set down dev {{ direct_interface }} +ExecStop=/sbin/ip addr del {{ direct_ip }} dev {{ direct_interface }} +{% else %} +ExecStart=/usr/bin/wg set kubeguard-wg0 peer {{ wg_pubkey }} allowed-ips {{ wg_allowedips }} endpoint {{ wg_host }}:{{ wg_port }} persistent-keepalive 10 +ExecStop=/usr/bin/wg set kubeguard-wg0 peer {{ wg_pubkey }} remove +{% endif %} +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target |