From 75372fa6a0e84ca8f3a6d6da4a9b389da4ba0b86 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 17 Jun 2020 11:57:05 +0200 Subject: kubernetes: add chaos-at-home test cluster and fix some bugs --- chaos-at-home/group_vars/k8s-chtest.yml | 10 +++++ chaos-at-home/k8s-chtest.yml | 37 +++++++++++++++++ dan/k8s-emc.yml | 4 +- inventory/group_vars/k8s-chtest/main.yml | 41 +++++++++++++++++++ inventory/host_vars/ch-hroottest.yml | 3 ++ inventory/host_vars/ch-k8s-m0.yml | 47 ++++++++++++++++++++++ inventory/host_vars/ch-k8s-m1.yml | 47 ++++++++++++++++++++++ inventory/host_vars/ch-k8s-m2.yml | 44 ++++++++++++++++++++ inventory/host_vars/ch-k8s-w0.yml | 44 ++++++++++++++++++++ inventory/host_vars/ch-k8s-w1.yml | 44 ++++++++++++++++++++ inventory/host_vars/sk-2019vm.yml | 4 ++ inventory/hosts.ini | 38 +++++++++++++---- .../node/templates/kubeguard-peer.service.j2 | 2 +- spreadspace/k8s-lwl.yml | 4 +- 14 files changed, 359 insertions(+), 10 deletions(-) create mode 100644 chaos-at-home/group_vars/k8s-chtest.yml create mode 100644 chaos-at-home/k8s-chtest.yml create mode 100644 inventory/group_vars/k8s-chtest/main.yml create mode 100644 inventory/host_vars/ch-k8s-m0.yml create mode 100644 inventory/host_vars/ch-k8s-m1.yml create mode 100644 inventory/host_vars/ch-k8s-m2.yml create mode 100644 inventory/host_vars/ch-k8s-w0.yml create mode 100644 inventory/host_vars/ch-k8s-w1.yml diff --git a/chaos-at-home/group_vars/k8s-chtest.yml b/chaos-at-home/group_vars/k8s-chtest.yml new file mode 100644 index 00000000..b824f9dd --- /dev/null +++ b/chaos-at-home/group_vars/k8s-chtest.yml @@ -0,0 +1,10 @@ +$ANSIBLE_VAULT;1.2;AES256;chaos-at-home +64343538336637373635323961366666663233376166326663316362346135353465363432616462 +6530623534623435366466656163343436333064316434650a333232643966653634663531396138 +66643633656133396139353565313834653165353331386637316664383237393237633232393337 +3363626365306538380a333361613761343263356639656632633030626265653730393232653165 +32303034393934303538386664616366613339316265653734656562303232396234623733316532 +32313837623163633663633635396664313732323939663633613238303436656534336432363433 +32623863373239326133303932336361366164383462633730653934333830346636616630356666 +37636638666332393639353738623135313331336166333435363063373733313437613264323138 +39373564363637323034373636323430323437623636623935396237323263383362 diff --git a/chaos-at-home/k8s-chtest.yml b/chaos-at-home/k8s-chtest.yml new file mode 100644 index 00000000..e3daf681 --- /dev/null +++ b/chaos-at-home/k8s-chtest.yml @@ -0,0 +1,37 @@ +--- +- name: Basic Node Setup + hosts: k8s-chtest + roles: + - role: apt-repo/base + - role: core/base + - role: core/sshd + - role: core/zsh + +- import_playbook: ../common/kubernetes-cluster-layout.yml + vars: + kubernetes_cluster_layout: + nodes_group: k8s-chtest + masters: + - ch-k8s-m0 + - ch-k8s-m1 + - ch-k8s-m2 + primary_master: ch-k8s-m0 + +### hack hack hack... +- name: cook kubernetes secrets + hosts: _kubernetes_nodes_ + gather_facts: no + tasks: + - set_fact: + kubernetes_secrets_cooked: "{{ kubernetes_secrets }}" + - when: external_ip is defined + set_fact: + external_ip_cooked: "{{ external_ip }}" + +- import_playbook: ../common/kubernetes-cluster.yml +- import_playbook: ../common/kubernetes-cluster-cleanup.yml + +- name: install addons + hosts: _kubernetes_primary_master_ + roles: + - role: kubernetes/addons/metrics-server diff --git a/dan/k8s-emc.yml b/dan/k8s-emc.yml index 7b4e9158..d81d8358 100644 --- a/dan/k8s-emc.yml +++ b/dan/k8s-emc.yml @@ -16,11 +16,13 @@ ### hack hack hack... - name: cook kubernetes secrets - hosts: _kubernetes_masters_ + hosts: _kubernetes_nodes_ gather_facts: no tasks: - set_fact: kubernetes_secrets_cooked: "{{ kubernetes_secrets }}" + - when: external_ip is defined + set_fact: external_ip_cooked: "{{ external_ip }}" - import_playbook: ../common/kubernetes-cluster.yml diff --git a/inventory/group_vars/k8s-chtest/main.yml b/inventory/group_vars/k8s-chtest/main.yml new file mode 100644 index 00000000..2e5d56d3 --- /dev/null +++ b/inventory/group_vars/k8s-chtest/main.yml @@ -0,0 +1,41 @@ +--- +docker_pkg_provider: docker-com +docker_pkg_name: docker-ce + +kubernetes_version: 1.18.3 +kubernetes_container_runtime: docker +kubernetes_network_plugin: kubeguard + +kubernetes: + cluster_name: chtest + + dedicated_master: True + api_extra_sans: + - 178.63.180.139 + - 178.63.180.140 + + pod_ip_range: 172.18.0.0/16 + pod_ip_range_size: 24 + service_ip_range: 172.18.192.0/18 + + +kubernetes_secrets: + encryption_config_keys: "{{ vault_kubernetes_encryption_config_keys }}" + + +kubeguard: + ## node_index must be in the range between 1 and 190 -> 189 hosts possible + ## + ## hardcoded hostnames are not nice but if we do this via host_vars + ## the info is spread over multiple files and this makes it more diffcult + ## to find mistakes, so it is nicer to keep it in one place... + node_index: + ch-k8s-w0: 1 + ch-k8s-w1: 2 + ch-k8s-m0: 100 + ch-k8s-m1: 101 + ch-k8s-m2: 102 + +kubernetes_overlay_node_ip: "{{ kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) | ipaddr(1) | ipaddr('address') }}" + +kubernetes_metrics_server_version: 0.3.6 diff --git a/inventory/host_vars/ch-hroottest.yml b/inventory/host_vars/ch-hroottest.yml index c56845fa..555791ca 100644 --- a/inventory/host_vars/ch-hroottest.yml +++ b/inventory/host_vars/ch-hroottest.yml @@ -44,6 +44,9 @@ vm_host: offsets: ch-hroottest-vm1: 100 ch-hroottest-obsd: 101 + ch-k8s-m2: 200 + ch-k8s-w0: 210 + ch-k8s-w1: 211 nat: yes zfs: default: diff --git a/inventory/host_vars/ch-k8s-m0.yml b/inventory/host_vars/ch-k8s-m0.yml new file mode 100644 index 00000000..30239ab3 --- /dev/null +++ b/inventory/host_vars/ch-k8s-m0.yml @@ -0,0 +1,47 @@ +--- +_vm_host_: sk-2019vm + +install: + vm: + host: "{{ _vm_host_ }}" + mem: 4096 + numcpu: 2 + autostart: True + disks: + primary: /dev/sda + scsi: + sda: + type: zfs + name: root + size: 20g + properties: + refreservation: none + interfaces: + - bridge: br-public + name: primary0 + +network: + nameservers: "{{ hostvars[_vm_host_].vm_host.network.dns }}" + domain: "{{ host_domain }}" + systemd_link: + interfaces: "{{ install.interfaces }}" + primary: + interface: primary0 + ip: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr(hostvars[_vm_host_].vm_host.network.bridges.public.offsets[inventory_hostname]) | ipaddr('address') }}" + mask: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr('netmask') }}" + gateway: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr('address') }}" + overlay: "{{ (hostvars[_vm_host_].vm_host.network.bridges.public.overlay.prefix | ipaddr(hostvars[_vm_host_].vm_host.network.bridges.public.overlay.offsets[inventory_hostname])).split('/')[0] }}" + +external_ip: "{{ network.primary.overlay }}" + +docker_lvm: + vg: "{{ host_name }}" + lv: docker + size: 7G + fs: ext4 + +kubelet_lvm: + vg: "{{ host_name }}" + lv: kubelet + size: 5G + fs: ext4 diff --git a/inventory/host_vars/ch-k8s-m1.yml b/inventory/host_vars/ch-k8s-m1.yml new file mode 100644 index 00000000..30239ab3 --- /dev/null +++ b/inventory/host_vars/ch-k8s-m1.yml @@ -0,0 +1,47 @@ +--- +_vm_host_: sk-2019vm + +install: + vm: + host: "{{ _vm_host_ }}" + mem: 4096 + numcpu: 2 + autostart: True + disks: + primary: /dev/sda + scsi: + sda: + type: zfs + name: root + size: 20g + properties: + refreservation: none + interfaces: + - bridge: br-public + name: primary0 + +network: + nameservers: "{{ hostvars[_vm_host_].vm_host.network.dns }}" + domain: "{{ host_domain }}" + systemd_link: + interfaces: "{{ install.interfaces }}" + primary: + interface: primary0 + ip: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr(hostvars[_vm_host_].vm_host.network.bridges.public.offsets[inventory_hostname]) | ipaddr('address') }}" + mask: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr('netmask') }}" + gateway: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr('address') }}" + overlay: "{{ (hostvars[_vm_host_].vm_host.network.bridges.public.overlay.prefix | ipaddr(hostvars[_vm_host_].vm_host.network.bridges.public.overlay.offsets[inventory_hostname])).split('/')[0] }}" + +external_ip: "{{ network.primary.overlay }}" + +docker_lvm: + vg: "{{ host_name }}" + lv: docker + size: 7G + fs: ext4 + +kubelet_lvm: + vg: "{{ host_name }}" + lv: kubelet + size: 5G + fs: ext4 diff --git a/inventory/host_vars/ch-k8s-m2.yml b/inventory/host_vars/ch-k8s-m2.yml new file mode 100644 index 00000000..a41c97a8 --- /dev/null +++ b/inventory/host_vars/ch-k8s-m2.yml @@ -0,0 +1,44 @@ +--- +_vm_host_: ch-hroottest + +install: + vm: + host: "{{ _vm_host_ }}" + mem: 4096 + numcpu: 2 + autostart: True + disks: + primary: /dev/sda + scsi: + sda: + type: zfs + name: root + size: 20g + properties: + refreservation: none + interfaces: + - bridge: br-public + name: primary0 + +network: + nameservers: "{{ hostvars[_vm_host_].vm_host.network.dns }}" + domain: "{{ host_domain }}" + systemd_link: + interfaces: "{{ install.interfaces }}" + primary: + interface: primary0 + ip: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr(hostvars[_vm_host_].vm_host.network.bridges.public.offsets[inventory_hostname]) | ipaddr('address') }}" + mask: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr('netmask') }}" + gateway: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr('address') }}" + +docker_lvm: + vg: "{{ host_name }}" + lv: docker + size: 7G + fs: ext4 + +kubelet_lvm: + vg: "{{ host_name }}" + lv: kubelet + size: 5G + fs: ext4 diff --git a/inventory/host_vars/ch-k8s-w0.yml b/inventory/host_vars/ch-k8s-w0.yml new file mode 100644 index 00000000..5cf6d444 --- /dev/null +++ b/inventory/host_vars/ch-k8s-w0.yml @@ -0,0 +1,44 @@ +--- +_vm_host_: ch-hroottest + +install: + vm: + host: "{{ _vm_host_ }}" + mem: 4096 + numcpu: 4 + autostart: True + disks: + primary: /dev/sda + scsi: + sda: + type: zfs + name: root + size: 100g + properties: + refreservation: none + interfaces: + - bridge: br-public + name: primary0 + +network: + nameservers: "{{ hostvars[_vm_host_].vm_host.network.dns }}" + domain: "{{ host_domain }}" + systemd_link: + interfaces: "{{ install.interfaces }}" + primary: + interface: primary0 + ip: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr(hostvars[_vm_host_].vm_host.network.bridges.public.offsets[inventory_hostname]) | ipaddr('address') }}" + mask: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr('netmask') }}" + gateway: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr('address') }}" + +docker_lvm: + vg: "{{ host_name }}" + lv: docker + size: 15G + fs: ext4 + +kubelet_lvm: + vg: "{{ host_name }}" + lv: kubelet + size: 15G + fs: ext4 diff --git a/inventory/host_vars/ch-k8s-w1.yml b/inventory/host_vars/ch-k8s-w1.yml new file mode 100644 index 00000000..5cf6d444 --- /dev/null +++ b/inventory/host_vars/ch-k8s-w1.yml @@ -0,0 +1,44 @@ +--- +_vm_host_: ch-hroottest + +install: + vm: + host: "{{ _vm_host_ }}" + mem: 4096 + numcpu: 4 + autostart: True + disks: + primary: /dev/sda + scsi: + sda: + type: zfs + name: root + size: 100g + properties: + refreservation: none + interfaces: + - bridge: br-public + name: primary0 + +network: + nameservers: "{{ hostvars[_vm_host_].vm_host.network.dns }}" + domain: "{{ host_domain }}" + systemd_link: + interfaces: "{{ install.interfaces }}" + primary: + interface: primary0 + ip: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr(hostvars[_vm_host_].vm_host.network.bridges.public.offsets[inventory_hostname]) | ipaddr('address') }}" + mask: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr('netmask') }}" + gateway: "{{ hostvars[_vm_host_].vm_host.network.bridges.public.prefix | ipaddr('address') }}" + +docker_lvm: + vg: "{{ host_name }}" + lv: docker + size: 15G + fs: ext4 + +kubelet_lvm: + vg: "{{ host_name }}" + lv: kubelet + size: 15G + fs: ext4 diff --git a/inventory/host_vars/sk-2019vm.yml b/inventory/host_vars/sk-2019vm.yml index 503995ee..de162712 100644 --- a/inventory/host_vars/sk-2019vm.yml +++ b/inventory/host_vars/sk-2019vm.yml @@ -62,6 +62,8 @@ vm_host: # emc-master: 137 lw-master: 137 ele-gwhetzner: 138 + ch-k8s-m0: 139 + ch-k8s-m1: 140 ch-mimas: 142 sk-testvm: 253 nat: yes @@ -72,6 +74,8 @@ vm_host: # emc-master: 1 lw-master: 1 ele-gwhetzner: 2 + ch-k8s-m0: 3 + ch-k8s-m1: 4 ch-mimas: 6 sk-testvm: 7 zfs: diff --git a/inventory/hosts.ini b/inventory/hosts.ini index ad231374..9add78d3 100644 --- a/inventory/hosts.ini +++ b/inventory/hosts.ini @@ -32,6 +32,7 @@ ch-hroottest-obsd host_name=hroot-test-obsd mz-chaos-at-home chaos-at-home-switches chaos-at-home-ap +chaos-at-home-k8s [mz-chaos-at-home] mz-router ansible_host=chmz-router @@ -53,6 +54,13 @@ ch-ap1 host_name=ap1 ch-router ch-pan +[chaos-at-home-k8s] +ch-k8s-m0 host_name=k8s-master0 +ch-k8s-m1 host_name=k8s-master1 +ch-k8s-m2 host_name=k8s-master2 +ch-k8s-w0 host_name=k8s-worker0 +ch-k8s-w1 host_name=k8s-worker1 + [realraum:vars] host_domain=realraum.at @@ -261,6 +269,8 @@ sk-tomnext-nc sk-tomnext-hp ch-hroottest-vm1 ch-hroottest-obsd +ch-k8s-m[0:2] +ch-k8s-w[0:1] [hroot] sk-2019 @@ -313,9 +323,18 @@ ele-dolmetsch-raspi +### Elevate Festival +[elevate-festival:children] +elevate +k8s-emc + + + +## Kubernetes [kubernetes-cluster:children] k8s-emc k8s-lwl +k8s-chtest [standalone-kubelet] sk-cloudia @@ -350,13 +369,6 @@ k8s-emc-distribution k8s-emc-streamer -### Elevate Festival -[elevate-festival:children] -elevate -k8s-emc - - - ### Kubernetes Cluster: lendwirbel-live [k8s-lwl-encoder] lw-dione @@ -376,3 +388,15 @@ k8s-lwl-master k8s-lwl-encoder k8s-lwl-distribution k8s-lwl-streamer + + +### Kubernetes Cluster: ch-test +[k8s-chtest-master] +ch-k8s-m[0:2] + +[k8s-chtest-worker] +ch-k8s-w[0:1] + +[k8s-chtest:children] +k8s-chtest-master +k8s-chtest-worker diff --git a/roles/kubernetes/net/kubeguard/node/templates/kubeguard-peer.service.j2 b/roles/kubernetes/net/kubeguard/node/templates/kubeguard-peer.service.j2 index 72b39c3f..0503ca03 100644 --- a/roles/kubernetes/net/kubeguard/node/templates/kubeguard-peer.service.j2 +++ b/roles/kubernetes/net/kubeguard/node/templates/kubeguard-peer.service.j2 @@ -6,7 +6,7 @@ After=kubeguard-interfaces.service {% set pod_ip_self = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[inventory_hostname]) | ipaddr(1) | ipaddr('address') -%} {% set pod_net_peer = kubernetes.pod_ip_range | ipsubnet(kubernetes.pod_ip_range_size, kubeguard.node_index[peer]) -%} -{% set direct_zone = kubeguard.direct_net_zones | direct_net_zone(inventory_hostname, peer) -%} +{% set direct_zone = kubeguard.direct_net_zones | default({}) | direct_net_zone(inventory_hostname, peer) -%} {% if direct_zone %} {% set direct_ip = kubeguard.direct_net_zones[direct_zone].transfer_net | ipaddr(kubeguard.node_index[inventory_hostname]) %} {% set direct_interface = kubeguard.direct_net_zones[direct_zone].node_interface[inventory_hostname] %} diff --git a/spreadspace/k8s-lwl.yml b/spreadspace/k8s-lwl.yml index 17d21601..902d833d 100644 --- a/spreadspace/k8s-lwl.yml +++ b/spreadspace/k8s-lwl.yml @@ -16,11 +16,13 @@ ### hack hack hack... - name: cook kubernetes secrets - hosts: _kubernetes_masters_ + hosts: _kubernetes_nodes_ gather_facts: no tasks: - set_fact: kubernetes_secrets_cooked: "{{ kubernetes_secrets }}" + - when: external_ip is defined + set_fact: external_ip_cooked: "{{ external_ip }}" - import_playbook: ../common/kubernetes-cluster.yml -- cgit v1.2.3