From 863837f090f55928a9555d831bf523fb635d2dbe Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 11 Mar 2023 23:14:56 +0100 Subject: kubernetes/addons: add ingress-nginx --- dan/k8s-emc.yml | 1 + inventory/group_vars/k8s-emc/vars.yml | 5 + .../addons/ingress-nginx/defaults/main.yml | 8 + .../addons/ingress-nginx/files/deploy.1.6.4.yml | 641 +++++++++++++++++++++ .../kubernetes/addons/ingress-nginx/tasks/main.yml | 32 + ...kustomization.daemonset-with-hostnetwork.yml.j2 | 34 ++ .../kustomization.deployment-with-nodeport.yml.j2 | 16 + 7 files changed, 737 insertions(+) create mode 100644 roles/kubernetes/addons/ingress-nginx/defaults/main.yml create mode 100644 roles/kubernetes/addons/ingress-nginx/files/deploy.1.6.4.yml create mode 100644 roles/kubernetes/addons/ingress-nginx/tasks/main.yml create mode 100644 roles/kubernetes/addons/ingress-nginx/templates/kustomization.daemonset-with-hostnetwork.yml.j2 create mode 100644 roles/kubernetes/addons/ingress-nginx/templates/kustomization.deployment-with-nodeport.yml.j2 diff --git a/dan/k8s-emc.yml b/dan/k8s-emc.yml index e35707ed..86fea6c5 100644 --- a/dan/k8s-emc.yml +++ b/dan/k8s-emc.yml @@ -59,5 +59,6 @@ - role: kubernetes/addons/metrics-server #- role: kubernetes/addons/openebs-zfs - role: kubernetes/addons/cert-manager + - role: kubernetes/addons/ingress-nginx #- role: kubernetes/addons/node-feature-discovery #- role: kubernetes/addons/intel-gpu-device-plugin diff --git a/inventory/group_vars/k8s-emc/vars.yml b/inventory/group_vars/k8s-emc/vars.yml index 29e8f251..83708b7e 100644 --- a/inventory/group_vars/k8s-emc/vars.yml +++ b/inventory/group_vars/k8s-emc/vars.yml @@ -67,6 +67,11 @@ kubernetes_openebs_zfs_storage_classes: kubernetes_cert_manager_version: 1.9.2 +kubernetes_ingress_nginx_version: 1.6.4 +kubernetes_ingress_nginx_variant: daemonset-with-hostnetwork +kubernetes_ingress_nginx_node_selector: + streaming.spreadspace.org/lb: "" + kubernetes_node_feature_discovery_version: 0.11.2 kubernetes_node_feature_discovery_worker_config: | core: diff --git a/roles/kubernetes/addons/ingress-nginx/defaults/main.yml b/roles/kubernetes/addons/ingress-nginx/defaults/main.yml new file mode 100644 index 00000000..dbbbfdb7 --- /dev/null +++ b/roles/kubernetes/addons/ingress-nginx/defaults/main.yml @@ -0,0 +1,8 @@ +--- +# kubernetes_ingress_nginx_version: 1.6.4 + +kubernetes_ingress_nginx_variant: deployment-with-nodeport +#kubernetes_ingress_nginx_variant: daemonset-with-hostnetwork + +# kubernetes_ingress_nginx_node_selector: +# node.spreadspace.org/ingress: "nginx" diff --git a/roles/kubernetes/addons/ingress-nginx/files/deploy.1.6.4.yml b/roles/kubernetes/addons/ingress-nginx/files/deploy.1.6.4.yml new file mode 100644 index 00000000..a642d67d --- /dev/null +++ b/roles/kubernetes/addons/ingress-nginx/files/deploy.1.6.4.yml @@ -0,0 +1,641 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx +--- +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx + namespace: ingress-nginx +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-admission + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx + namespace: ingress-nginx +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resourceNames: + - ingress-nginx-leader + resources: + - leases + verbs: + - get + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-admission + namespace: ingress-nginx +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-admission +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: +- kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-admission + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: +- kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: +- kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-admission +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: +- kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +apiVersion: v1 +data: + allow-snippet-annotations: "true" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-controller + namespace: ingress-nginx +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - appProtocol: http + name: http + port: 80 + protocol: TCP + targetPort: http + - appProtocol: https + name: https + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + ports: + - appProtocol: https + name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + minReadySeconds: 0 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + spec: + containers: + - args: + - /nginx-ingress-controller + - --election-id=ingress-nginx-leader + - --controller-class=k8s.io/ingress-nginx + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + image: registry.k8s.io/ingress-nginx/controller:v1.6.4@sha256:15be4666c53052484dd2992efacf2f50ea77a78ae8aa21ccd91af6baaa7ea22f + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 80 + name: http + protocol: TCP + - containerPort: 443 + name: https + protocol: TCP + - containerPort: 8443 + name: webhook + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 90Mi + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsUser: 101 + volumeMounts: + - mountPath: /usr/local/certificates/ + name: webhook-cert + readOnly: true + dnsPolicy: ClusterFirst + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: ingress-nginx + terminationGracePeriodSeconds: 300 + volumes: + - name: webhook-cert + secret: + secretName: ingress-nginx-admission +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-admission-create + namespace: ingress-nginx +spec: + template: + metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-admission-create + spec: + containers: + - args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f + imagePullPolicy: IfNotPresent + name: create + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + serviceAccountName: ingress-nginx-admission +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-admission-patch + namespace: ingress-nginx +spec: + template: + metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-admission-patch + spec: + containers: + - args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f + imagePullPolicy: IfNotPresent + name: patch + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + serviceAccountName: ingress-nginx-admission +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: nginx +spec: + controller: k8s.io/ingress-nginx +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.6.4 + name: ingress-nginx-admission +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: ingress-nginx-controller-admission + namespace: ingress-nginx + path: /networking/v1/ingresses + failurePolicy: Fail + matchPolicy: Equivalent + name: validate.nginx.ingress.kubernetes.io + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + sideEffects: None diff --git a/roles/kubernetes/addons/ingress-nginx/tasks/main.yml b/roles/kubernetes/addons/ingress-nginx/tasks/main.yml new file mode 100644 index 00000000..db918a4d --- /dev/null +++ b/roles/kubernetes/addons/ingress-nginx/tasks/main.yml @@ -0,0 +1,32 @@ +--- +- name: deploy ingress-nginx addon + run_once: true + delegate_to: "{{ groups['_kubernetes_primary_controlplane_node_'] | first }}" + block: + - name: create base directory for ingress-nginx addon + file: + path: /etc/kubernetes/addons/ingress-nginx + state: directory + + ## you may download these using the following command: + # wget -O deploy.{{ kubernetes_ingress_nginx_version }}.yml https://github.com/kubernetes/ingress-nginx/raw/controller-v{{ kubernetes_ingress_nginx_version }}/deploy/static/provider/baremetal/deploy.yaml + - name: copy base config for ingress-nginx + copy: + src: "deploy.{{ kubernetes_ingress_nginx_version }}.yml" + dest: /etc/kubernetes/addons/ingress-nginx/upstream.yml + + - name: generate kustomization for ingress-nginx + template: + src: "kustomization.{{ kubernetes_ingress_nginx_variant }}.yml.j2" + dest: /etc/kubernetes/addons/ingress-nginx/kustomization.yml + + - name: check if ingress-nginx is already installed + check_mode: no + command: kubectl --kubeconfig /etc/kubernetes/admin.conf diff -k /etc/kubernetes/addons/ingress-nginx + failed_when: false + changed_when: false + register: kube_ingress_nginx_diff_result + + - name: install ingress-nginx onto the cluster + when: kube_ingress_nginx_diff_result.rc != 0 + command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -k /etc/kubernetes/addons/ingress-nginx diff --git a/roles/kubernetes/addons/ingress-nginx/templates/kustomization.daemonset-with-hostnetwork.yml.j2 b/roles/kubernetes/addons/ingress-nginx/templates/kustomization.daemonset-with-hostnetwork.yml.j2 new file mode 100644 index 00000000..bb421735 --- /dev/null +++ b/roles/kubernetes/addons/ingress-nginx/templates/kustomization.daemonset-with-hostnetwork.yml.j2 @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - upstream.yml + +patches: + - target: + kind: Deployment + namespace: ingress-nginx + name: ingress-nginx-controller + patch: |- + - op: replace + path: /kind + value: DaemonSet + - op: replace + path: /spec/template/spec/hostNetwork + value: true + - op: replace + path: /spec/template/spec/dnsPolicy + value: ClusterFirstWithHostNet + - op: replace + path: /spec/template/spec/nodeSelector + value: + {{ kubernetes_ingress_nginx_node_selector | to_nice_yaml(indent=2) | indent(10) }} + options: + allowKindChange: true + - patch: |- + apiVersion: v1 + kind: Service + metadata: + name: ingress-nginx-controller + namespace: ingress-nginx + $patch: delete diff --git a/roles/kubernetes/addons/ingress-nginx/templates/kustomization.deployment-with-nodeport.yml.j2 b/roles/kubernetes/addons/ingress-nginx/templates/kustomization.deployment-with-nodeport.yml.j2 new file mode 100644 index 00000000..f2b5591d --- /dev/null +++ b/roles/kubernetes/addons/ingress-nginx/templates/kustomization.deployment-with-nodeport.yml.j2 @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - upstream.yml + +patches: + - target: + kind: Deployment + namespace: ingress-nginx + name: ingress-nginx-controller + patch: |- + - op: replace + path: /spec/template/spec/nodeSelector + value: + {{ kubernetes_ingress_nginx_node_selector | to_nice_yaml(indent=2) | indent(10) }} -- cgit v1.2.3