From 1f3ad3812836ed0f8f4f290346014821e6601c4d Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 4 Apr 2021 00:51:06 +0200 Subject: move zfs roles to new storage subdir --- roles/storage/zfs/info.txt | 1 + roles/storage/zfs/pools/defaults/main.yml | 19 +++++++ roles/storage/zfs/pools/tasks/Debian.yml | 42 +++++++++++++++ roles/storage/zfs/pools/tasks/Ubuntu.yml | 2 + .../pools/tasks/enable-systemd-mount-generator.yml | 31 +++++++++++ roles/storage/zfs/pools/tasks/main.yml | 61 ++++++++++++++++++++++ roles/storage/zfs/sanoid/defaults/main.yml | 28 ++++++++++ roles/storage/zfs/sanoid/tasks/main.yml | 15 ++++++ roles/storage/zfs/sanoid/templates/sanoid.conf.j2 | 23 ++++++++ roles/zfs/base/defaults/main.yml | 19 ------- roles/zfs/base/tasks/Debian.yml | 42 --------------- roles/zfs/base/tasks/Ubuntu.yml | 2 - .../base/tasks/enable-systemd-mount-generator.yml | 31 ----------- roles/zfs/base/tasks/main.yml | 61 ---------------------- roles/zfs/sanoid/defaults/main.yml | 28 ---------- roles/zfs/sanoid/tasks/main.yml | 15 ------ roles/zfs/sanoid/templates/sanoid.conf.j2 | 23 -------- 17 files changed, 222 insertions(+), 221 deletions(-) create mode 100644 roles/storage/zfs/info.txt create mode 100644 roles/storage/zfs/pools/defaults/main.yml create mode 100644 roles/storage/zfs/pools/tasks/Debian.yml create mode 100644 roles/storage/zfs/pools/tasks/Ubuntu.yml create mode 100644 roles/storage/zfs/pools/tasks/enable-systemd-mount-generator.yml create mode 100644 roles/storage/zfs/pools/tasks/main.yml create mode 100644 roles/storage/zfs/sanoid/defaults/main.yml create mode 100644 roles/storage/zfs/sanoid/tasks/main.yml create mode 100644 roles/storage/zfs/sanoid/templates/sanoid.conf.j2 delete mode 100644 roles/zfs/base/defaults/main.yml delete mode 100644 roles/zfs/base/tasks/Debian.yml delete mode 100644 roles/zfs/base/tasks/Ubuntu.yml delete mode 100644 roles/zfs/base/tasks/enable-systemd-mount-generator.yml delete mode 100644 roles/zfs/base/tasks/main.yml delete mode 100644 roles/zfs/sanoid/defaults/main.yml delete mode 100644 roles/zfs/sanoid/tasks/main.yml delete mode 100644 roles/zfs/sanoid/templates/sanoid.conf.j2 (limited to 'roles') diff --git a/roles/storage/zfs/info.txt b/roles/storage/zfs/info.txt new file mode 100644 index 00000000..6a838617 --- /dev/null +++ b/roles/storage/zfs/info.txt @@ -0,0 +1 @@ +https://jrs-s.net/2018/08/17/zfs-tuning-cheat-sheet/ diff --git a/roles/storage/zfs/pools/defaults/main.yml b/roles/storage/zfs/pools/defaults/main.yml new file mode 100644 index 00000000..dcef7628 --- /dev/null +++ b/roles/storage/zfs/pools/defaults/main.yml @@ -0,0 +1,19 @@ +--- +zfs_use_systemd_mount_generator: yes +#zfs_arc_size: +# min: 2GB +# max: 8GB + +zfs_pool_properties: + ashift: 12 + +# zfs_pools: +# storage: +# mountpoint: /srv/storage +# create_vdevs: mirror /dev/sda /dev/sdb mirror /dev/sdc /dev/sdd log mirror /dev/nvme0n1p3 /dev/nvme1n1p3 cache /dev/nvme0n1p4 /dev/nvme1n1p4 +# test: +# mountpoint: /srv/storage +# create_vdevs: raidz /dev/sda /dev/sdb /dev/sdc /dev/sdd +# properties: +# ashift: 12 +# prop: value diff --git a/roles/storage/zfs/pools/tasks/Debian.yml b/roles/storage/zfs/pools/tasks/Debian.yml new file mode 100644 index 00000000..a1ed0387 --- /dev/null +++ b/roles/storage/zfs/pools/tasks/Debian.yml @@ -0,0 +1,42 @@ +--- +- name: install dkms + import_role: + name: prepare-dkms + +- name: check if contrib apt component is enabled + assert: + msg: "Debian zfs packages are in contrib - please enable it using 'apt_repo_components'" + that: + - apt_repo_components is defined + - "'contrib' in apt_repo_components" + +- name: enable backports and force ZFS packages from backports for buster + when: (ansible_distribution_major_version | int) == 10 + block: + - name: add backports repo + include_role: + name: apt-repo/backports + + - name: pin zfs packages to buster-backports + copy: + content: | + Package: libnvpair1linux libuutil1linux libzfs2linux libzpool2linux spl-dkms zfs-dkms zfs-test zfsutils-linux zfsutils-linux-dev zfs-zed + Pin: release n=buster-backports + Pin-Priority: 990 + dest: /etc/apt/preferences.d/zfs-from-buster-backports + +- name: install zfs modules via dkms + apt: + name: zfs-dkms + state: present + +- name: check if module is available for the currently running kernel + command: modprobe --dry-run zfs + check_mode: no + register: zfs_module_available + failed_when: false + changed_when: false + +- name: rebuild zfs module + when: zfs_module_available.rc != 0 + command: dpkg-reconfigure zfs-dkms diff --git a/roles/storage/zfs/pools/tasks/Ubuntu.yml b/roles/storage/zfs/pools/tasks/Ubuntu.yml new file mode 100644 index 00000000..9745d716 --- /dev/null +++ b/roles/storage/zfs/pools/tasks/Ubuntu.yml @@ -0,0 +1,2 @@ +--- +## nothing to do here - zfs modules are part of ubuntu core already diff --git a/roles/storage/zfs/pools/tasks/enable-systemd-mount-generator.yml b/roles/storage/zfs/pools/tasks/enable-systemd-mount-generator.yml new file mode 100644 index 00000000..eaa3cebe --- /dev/null +++ b/roles/storage/zfs/pools/tasks/enable-systemd-mount-generator.yml @@ -0,0 +1,31 @@ +--- +- name: enable zfs-list-cacher zlet + file: + src: /usr/lib/zfs-linux/zed.d/history_event-zfs-list-cacher.sh + dest: /etc/zfs/zed.d/history_event-zfs-list-cacher.sh + state: link + +- name: create base-directory for zfs-list.cache + file: + path: /etc/zfs/zfs-list.cache/ + state: directory + +- name: create zfs-list.cache file for all pools + loop: "{{ zfs_pools | dict2items }}" + loop_control: + label: "{{ item.key }}" + copy: + content: "" + dest: "/etc/zfs/zfs-list.cache/{{ item.key }}" + force: no + +## TODO: if this is installed after the zpool has already been created zed needs to be triggered +## using someing like: zfs set canmount=on DATASET + +### HACK HACK HACK (see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=966565) +- name: patch zfs mount generator (Debian Bug 966565) + when: ansible_distribution == 'Debian' + lineinfile: + path: /usr/lib/systemd/system-generators/zfs-mount-generator + regexp: '^pools=\$\(zpool list -H -o name\)$' + line: "pools=$(zpool list -H -o name || true)" diff --git a/roles/storage/zfs/pools/tasks/main.yml b/roles/storage/zfs/pools/tasks/main.yml new file mode 100644 index 00000000..1a6b911d --- /dev/null +++ b/roles/storage/zfs/pools/tasks/main.yml @@ -0,0 +1,61 @@ +--- +- name: configure arc size + when: zfs_arc_size is defined + copy: + content: | + options zfs zfs_arc_min={{ zfs_arc_size.min | human_to_bytes }} + options zfs zfs_arc_max={{ zfs_arc_size.max | human_to_bytes }} + dest: /etc/modprobe.d/zfs.conf + +- name: install zfs modules + include_tasks: "{{ ansible_distribution }}.yml" + +- name: load zfs kernel module + modprobe: + name: zfs + state: present + +- name: install zed and zfs file system utilities + apt: + name: + - zfsutils-linux + - zfs-zed + state: present + +- name: enable systemd mount-generator + when: zfs_use_systemd_mount_generator + import_tasks: enable-systemd-mount-generator.yml + +- name: gather zpool facts + zpool_facts: + +- name: generate list of existing pools + set_fact: + zfs_existing_pools: "{{ ansible_zfs_pools | map(attribute='name') | list }}" + +- name: try to import existing pools + loop: "{{ zfs_pools | dict2items }}" + loop_control: + label: "{{ item.key }}" + when: item.key not in zfs_existing_pools + command: "zpool import{% for dev in (item.value.create_vdevs.split(' ') | select('match', '^/dev/') ) %} -d {{ dev }}{% endfor %} -f {{ item.key }}" + register: zfs_import_result + failed_when: + - zfs_import_result.rc == 0 + - (zfs_import_result.stderr_lines | reject("regex", " no such pool available$") | list | length) > 0 + changed_when: + - (zfs_import_result.stderr_lines | length) == 0 + +- name: gather zpool facts (again) + zpool_facts: + +- name: generate list of existing pools (again) + set_fact: + zfs_existing_pools: "{{ ansible_zfs_pools | map(attribute='name') | list }}" + +- name: create pools + loop: "{{ zfs_pools | dict2items }}" + loop_control: + label: "{{ item.key }}" + when: item.key not in zfs_existing_pools + command: "zpool create -m {{ item.value.mountpoint }} {% for prop, value in (item.value.properties | default(zfs_pool_properties)).items() %}-o {{ prop }}={{ value }} {% endfor %}{{ item.key }} {{ item.value.create_vdevs }}" diff --git a/roles/storage/zfs/sanoid/defaults/main.yml b/roles/storage/zfs/sanoid/defaults/main.yml new file mode 100644 index 00000000..f48e9fd2 --- /dev/null +++ b/roles/storage/zfs/sanoid/defaults/main.yml @@ -0,0 +1,28 @@ +--- +zfs_sanoid_modules: {} + +zfs_sanoid_templates: + production: + frequently: 0 + hourly: 36 + daily: 7 + monthly: 0 + yearly: 0 + autosnap: yes + autoprune: yes + + backup: + frequently: 0 + hourly: 0 + daily: 60 + monthly: 6 + yearly: 0 + autosnap: no + autoprune: yes + + ignore: + autoprune: no + autosnap: no + monitor: no + +zfs_sanoid_templates_extra: {} diff --git a/roles/storage/zfs/sanoid/tasks/main.yml b/roles/storage/zfs/sanoid/tasks/main.yml new file mode 100644 index 00000000..e35190e9 --- /dev/null +++ b/roles/storage/zfs/sanoid/tasks/main.yml @@ -0,0 +1,15 @@ +--- +- name: install sanoid + apt: + name: sanoid + state: present + +- name: create sanoid config directory + file: + path: /etc/sanoid + state: directory + +- name: genarate sanoid config + template: + src: sanoid.conf.j2 + dest: /etc/sanoid/sanoid.conf diff --git a/roles/storage/zfs/sanoid/templates/sanoid.conf.j2 b/roles/storage/zfs/sanoid/templates/sanoid.conf.j2 new file mode 100644 index 00000000..82eb27a9 --- /dev/null +++ b/roles/storage/zfs/sanoid/templates/sanoid.conf.j2 @@ -0,0 +1,23 @@ +############################### +## modules +############################### +{% for name,options in zfs_sanoid_modules.items() %} + +[{{ name }}] +{% for option,value in options.items() %} + {{ option }} = {{ value }} +{% endfor %} +{% endfor %} + + +############################### +## templates +############################### +{% set templates = zfs_sanoid_templates | combine(zfs_sanoid_templates_extra) %} +{% for name,options in templates.items() %} + +[template_{{ name }}] +{% for option,value in options.items() %} + {{ option }} = {{ value }} +{% endfor %} +{% endfor %} diff --git a/roles/zfs/base/defaults/main.yml b/roles/zfs/base/defaults/main.yml deleted file mode 100644 index dcef7628..00000000 --- a/roles/zfs/base/defaults/main.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -zfs_use_systemd_mount_generator: yes -#zfs_arc_size: -# min: 2GB -# max: 8GB - -zfs_pool_properties: - ashift: 12 - -# zfs_pools: -# storage: -# mountpoint: /srv/storage -# create_vdevs: mirror /dev/sda /dev/sdb mirror /dev/sdc /dev/sdd log mirror /dev/nvme0n1p3 /dev/nvme1n1p3 cache /dev/nvme0n1p4 /dev/nvme1n1p4 -# test: -# mountpoint: /srv/storage -# create_vdevs: raidz /dev/sda /dev/sdb /dev/sdc /dev/sdd -# properties: -# ashift: 12 -# prop: value diff --git a/roles/zfs/base/tasks/Debian.yml b/roles/zfs/base/tasks/Debian.yml deleted file mode 100644 index a1ed0387..00000000 --- a/roles/zfs/base/tasks/Debian.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- name: install dkms - import_role: - name: prepare-dkms - -- name: check if contrib apt component is enabled - assert: - msg: "Debian zfs packages are in contrib - please enable it using 'apt_repo_components'" - that: - - apt_repo_components is defined - - "'contrib' in apt_repo_components" - -- name: enable backports and force ZFS packages from backports for buster - when: (ansible_distribution_major_version | int) == 10 - block: - - name: add backports repo - include_role: - name: apt-repo/backports - - - name: pin zfs packages to buster-backports - copy: - content: | - Package: libnvpair1linux libuutil1linux libzfs2linux libzpool2linux spl-dkms zfs-dkms zfs-test zfsutils-linux zfsutils-linux-dev zfs-zed - Pin: release n=buster-backports - Pin-Priority: 990 - dest: /etc/apt/preferences.d/zfs-from-buster-backports - -- name: install zfs modules via dkms - apt: - name: zfs-dkms - state: present - -- name: check if module is available for the currently running kernel - command: modprobe --dry-run zfs - check_mode: no - register: zfs_module_available - failed_when: false - changed_when: false - -- name: rebuild zfs module - when: zfs_module_available.rc != 0 - command: dpkg-reconfigure zfs-dkms diff --git a/roles/zfs/base/tasks/Ubuntu.yml b/roles/zfs/base/tasks/Ubuntu.yml deleted file mode 100644 index 9745d716..00000000 --- a/roles/zfs/base/tasks/Ubuntu.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -## nothing to do here - zfs modules are part of ubuntu core already diff --git a/roles/zfs/base/tasks/enable-systemd-mount-generator.yml b/roles/zfs/base/tasks/enable-systemd-mount-generator.yml deleted file mode 100644 index eaa3cebe..00000000 --- a/roles/zfs/base/tasks/enable-systemd-mount-generator.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- name: enable zfs-list-cacher zlet - file: - src: /usr/lib/zfs-linux/zed.d/history_event-zfs-list-cacher.sh - dest: /etc/zfs/zed.d/history_event-zfs-list-cacher.sh - state: link - -- name: create base-directory for zfs-list.cache - file: - path: /etc/zfs/zfs-list.cache/ - state: directory - -- name: create zfs-list.cache file for all pools - loop: "{{ zfs_pools | dict2items }}" - loop_control: - label: "{{ item.key }}" - copy: - content: "" - dest: "/etc/zfs/zfs-list.cache/{{ item.key }}" - force: no - -## TODO: if this is installed after the zpool has already been created zed needs to be triggered -## using someing like: zfs set canmount=on DATASET - -### HACK HACK HACK (see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=966565) -- name: patch zfs mount generator (Debian Bug 966565) - when: ansible_distribution == 'Debian' - lineinfile: - path: /usr/lib/systemd/system-generators/zfs-mount-generator - regexp: '^pools=\$\(zpool list -H -o name\)$' - line: "pools=$(zpool list -H -o name || true)" diff --git a/roles/zfs/base/tasks/main.yml b/roles/zfs/base/tasks/main.yml deleted file mode 100644 index 732c3346..00000000 --- a/roles/zfs/base/tasks/main.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- -- name: configure arc size - when: zfs_arc_size is defined - copy: - content: | - options zfs zfs_arc_min={{ zfs_arc_size.min | human_to_bytes }} - options zfs zfs_arc_max={{ zfs_arc_size.max | human_to_bytes }} - dest: /etc/modprobe.d/zfs.conf - -- name: install zfs modules - include_tasks: "{{ ansible_distribution }}.yml" - -- name: load zfs kernel module - modprobe: - name: zfs - state: present - -- name: install zed and zfs file system utilities - apt: - name: - - zfsutils-linux - - zfs-zed - state: present - -- name: enable systemd mount-generator - when: zfs_use_systemd_mount_generator - import_tasks: enable-systemd-mount-generator.yml - -- name: gather zpool facts - zpool_facts: - -- name: generate list of existing pools - set_fact: - zfs_existing_pools: "{{ ansible_zfs_pools | map(attribute='name') | list }}" - -- name: try to import existing pools - loop: "{{ zfs_pools | dict2items }}" - loop_control: - label: "{{ item.key }}" - when: item.key not in zfs_existing_pools - command: "zpool import{% for dev in (item.value.create_vdevs.split(' ') | select('match', '^/dev/') ) %} -d {{ dev }}{% endfor %} -f {{ item.key }}" - register: zfs_import_result - failed_when: - - zfs_import_result.rc == 0 - - (zfs_import_result.stderr_lines | reject("regex", " no such pool available$") | list | length) > 0 - changed_when: - - (zfs_import_result.stderr_lines | length) == 0 - -- name: gather zpool facts (again) - zpool_facts: - -- name: generate list of existing pools (again) - set_fact: - zfs_existing_pools: "{{ ansible_zfs_pools | map(attribute='name') | list }}" - -- name: create pools - loop: "{{ zfs_pools | dict2items }}" - loop_control: - label: "{{ item.key }}" - when: item.key not in zfs_existing_pools - command: "zpool create -m {{ item.value.mountpoint }} {% for prop, value in (item.value.properties | default(zfs_pool_properties)).items() %}-o {{ prop }}={{ value }} {% endfor %}{{ item.key }} {{ item.value.create_vdevs }}" diff --git a/roles/zfs/sanoid/defaults/main.yml b/roles/zfs/sanoid/defaults/main.yml deleted file mode 100644 index f48e9fd2..00000000 --- a/roles/zfs/sanoid/defaults/main.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -zfs_sanoid_modules: {} - -zfs_sanoid_templates: - production: - frequently: 0 - hourly: 36 - daily: 7 - monthly: 0 - yearly: 0 - autosnap: yes - autoprune: yes - - backup: - frequently: 0 - hourly: 0 - daily: 60 - monthly: 6 - yearly: 0 - autosnap: no - autoprune: yes - - ignore: - autoprune: no - autosnap: no - monitor: no - -zfs_sanoid_templates_extra: {} diff --git a/roles/zfs/sanoid/tasks/main.yml b/roles/zfs/sanoid/tasks/main.yml deleted file mode 100644 index e35190e9..00000000 --- a/roles/zfs/sanoid/tasks/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: install sanoid - apt: - name: sanoid - state: present - -- name: create sanoid config directory - file: - path: /etc/sanoid - state: directory - -- name: genarate sanoid config - template: - src: sanoid.conf.j2 - dest: /etc/sanoid/sanoid.conf diff --git a/roles/zfs/sanoid/templates/sanoid.conf.j2 b/roles/zfs/sanoid/templates/sanoid.conf.j2 deleted file mode 100644 index 82eb27a9..00000000 --- a/roles/zfs/sanoid/templates/sanoid.conf.j2 +++ /dev/null @@ -1,23 +0,0 @@ -############################### -## modules -############################### -{% for name,options in zfs_sanoid_modules.items() %} - -[{{ name }}] -{% for option,value in options.items() %} - {{ option }} = {{ value }} -{% endfor %} -{% endfor %} - - -############################### -## templates -############################### -{% set templates = zfs_sanoid_templates | combine(zfs_sanoid_templates_extra) %} -{% for name,options in templates.items() %} - -[template_{{ name }}] -{% for option,value in options.items() %} - {{ option }} = {{ value }} -{% endfor %} -{% endfor %} -- cgit v1.2.3