From 5f3230aad459d06cfd257616564bddc4cd9ad7cc Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 11 Jan 2022 00:04:35 +0100 Subject: zfs/syncoid: add error handling to pull --- roles/storage/zfs/syncoid/tasks/main.yml | 31 ++++--------- .../storage/zfs/syncoid/templates/pull.service.j2 | 7 ++- .../zfs/syncoid/templates/syncoid_finalize.j2 | 53 ++++++++++++++++++++++ .../storage/zfs/syncoid/templates/syncoid_pull.j2 | 29 ++++++++++++ 4 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 roles/storage/zfs/syncoid/templates/syncoid_finalize.j2 create mode 100644 roles/storage/zfs/syncoid/templates/syncoid_pull.j2 (limited to 'roles/storage') diff --git a/roles/storage/zfs/syncoid/tasks/main.yml b/roles/storage/zfs/syncoid/tasks/main.yml index 738b7041..9e573db7 100644 --- a/roles/storage/zfs/syncoid/tasks/main.yml +++ b/roles/storage/zfs/syncoid/tasks/main.yml @@ -16,12 +16,10 @@ type: ed25519 comment: ZFS Backup syncoid@{{ host_name }} -- name: generate syncoid ssh config wrapper - copy: - content: | - #!/bin/bash - exec /usr/sbin/syncoid --sshoption "UserKnownHostsFile=/var/lib/syncoid/ssh.knownhosts" --sshoption "HashKnownHosts=no" --sshkey "/var/lib/syncoid/id_ssh_ed25519" --no-sync-snap --compress zstd-fast "$@" - dest: /var/lib/syncoid/syncoid_wrapper +- name: generate syncoid pull script + template: + src: syncoid_pull.j2 + dest: /var/lib/syncoid/syncoid_pull mode: 0755 - name: configure lvm to ignore zfs volumes @@ -81,21 +79,6 @@ Persistent=false dest: /etc/systemd/system/sanoid.timer.d/no-persistence.conf - - name: generate syncoid post script - copy: - content: | - #!/bin/bash - - echo "running sanoid --prune to cleanup old snapshots" - sleep 5 - systemctl start --wait sanoid-prune.service - - echo "wait 30s (autosuspend cooldown period)" - sleep 30 - echo "done." - dest: /var/lib/syncoid/syncoid_post - mode: 0755 - - name: install python deps apt: name: "{{ python_basename }}-dbus" @@ -121,3 +104,9 @@ name: syncoid-autosuspend.timer enabled: yes state: started + +- name: generate syncoid finalize script + template: + src: syncoid_finalize.j2 + dest: /var/lib/syncoid/syncoid_finalize + mode: 0755 diff --git a/roles/storage/zfs/syncoid/templates/pull.service.j2 b/roles/storage/zfs/syncoid/templates/pull.service.j2 index 955161aa..cf8f61ae 100644 --- a/roles/storage/zfs/syncoid/templates/pull.service.j2 +++ b/roles/storage/zfs/syncoid/templates/pull.service.j2 @@ -2,13 +2,12 @@ Description=syncoid-based backup for {{ item.key }} [Service] +RuntimeDirectory=syncoid-pull-{{ item.key }} Type=oneshot {% for path,config in item.value.paths.items() %} -ExecStart=/var/lib/syncoid/syncoid_wrapper{% if 'periodic' not in item.value %} --create-bookmark{% endif %} --dumpsnaps --quiet {{ config.recursive | default(false) | ternary('-r ', '') }}{{ config.skip_parent | default(false) | ternary('--skip-parent ', '') }}{% for re in config.exclude | default([]) %}--exclude='{{ re }}' {% endfor %}{% if 'ssh_port' in item.value %}--sshport {{ item.value.ssh_port }} {% endif %}root@{{ item.value.ssh_hostname }}:{{ path }} {{ zfs_syncoid_target_pool }}/{{ item.key }}/{{ path }} +ExecStart=-/var/lib/syncoid/syncoid_pull "root@{{ item.value.ssh_hostname }}" "{{ path }}" "{{ zfs_syncoid_target_pool }}/{{ item.key }}/{{ path }}" --{% if 'periodic' not in item.value %} --create-bookmark{% endif %} --dumpsnaps --quiet {{ config.recursive | default(false) | ternary('-r ', '') }}{{ config.skip_parent | default(false) | ternary('--skip-parent ', '') }}{% for re in config.exclude | default([]) %}--exclude='{{ re }}' {% endfor %}{% if 'ssh_port' in item.value %}--sshport {{ item.value.ssh_port }} {% endif %}{{ '' }} {% endfor %} -{% if zfs_syncoid_autosuspend %} -ExecStart=/var/lib/syncoid/syncoid_post -{% endif %} +ExecStart=/var/lib/syncoid/syncoid_finalize{% if 'report_prometheus_textfile_path' in item.value %} "root@{{ item.value.ssh_hostname }}" "{{ item.value.report_prometheus_textfile_path }}" --{% if 'ssh_port' in item.value %} -p {{ item.value.ssh_port }}{% endif %}{% endif %}{{ '' }} {% if 'periodic' in item.value %} TimeoutStartSec={{ item.value.periodic.timeout }} {% endif %} diff --git a/roles/storage/zfs/syncoid/templates/syncoid_finalize.j2 b/roles/storage/zfs/syncoid/templates/syncoid_finalize.j2 new file mode 100644 index 00000000..5ddc691b --- /dev/null +++ b/roles/storage/zfs/syncoid/templates/syncoid_finalize.j2 @@ -0,0 +1,53 @@ +#!/bin/bash +{% if zfs_syncoid_autosuspend %} + +echo "running sanoid --prune to cleanup old snapshots" +sleep 5 +echo systemctl start --wait sanoid-prune.service + +echo "wait 30s (autosuspend cooldown period)" +sleep 30 +echo "done." +{% endif %} + +if [ -z "$RUNTIME_DIRECTORY" ]; then + exit 0 +fi + +ret=0 +for line in $(cat "$RUNTIME_DIRECTORY/exit_codes"); do + if [[ $line =~ [^[:digit:]] ]]; then + ret=-1 + echo "invalid exit_code from pull script: $line" + else + ret=$line + fi + if [ $ret -ne 0 ]; then + break + fi +done + +cat <> "$RUNTIME_DIRECTORY/metrics" +syncoid_pull_last_run{backup_server="{{ inventory_hostname }}"} $(date +"%s") +EOF + + +if [ -z "$1" ] || [ -z "$2" ] || [ "$1" == "--" ] || [ "$2" == "--" ]; then + echo "$0 [ -- ]" + exit -2 +fi +report_to=$1 +textfile_collector_path=$2 +shift 2 +if [ -n "$1" ]; then + if [ "$1" != "--" ]; then + echo "additional arguments must be separated using '--'" + exit -2 + fi + shift +fi + +echo "transfering prometheus metrics to: $report_to:$textfile_collector_path" +cat "$RUNTIME_DIRECTORY/metrics" | ssh -o "UserKnownHostsFile=/var/lib/syncoid/ssh.knownhosts" -o "HashKnownHosts=no" -i "/var/lib/syncoid/id_ssh_ed25519" "$@" "$report_to" "cat > '$textfile_collector_path/syncoid-pull-{{ inventory_hostname }}.prom'" + +exit $ret diff --git a/roles/storage/zfs/syncoid/templates/syncoid_pull.j2 b/roles/storage/zfs/syncoid/templates/syncoid_pull.j2 new file mode 100644 index 00000000..ab6c3116 --- /dev/null +++ b/roles/storage/zfs/syncoid/templates/syncoid_pull.j2 @@ -0,0 +1,29 @@ +#!/bin/bash + +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ "$1" == "--" ] || [ "$2" == "--" ] || [ "$3" == "--" ]; then + echo "$0 [ -- ]" + exit 1 +fi +src_host=$1 +src_path=$2 +dest_path=$3 +shift 3 +if [ -n "$1" ]; then + if [ "$1" != "--" ]; then + echo "additional arguments must be separated using '--'" + exit 1 + fi + shift +fi + +/usr/sbin/syncoid --sshoption "UserKnownHostsFile=/var/lib/syncoid/ssh.knownhosts" --sshoption "HashKnownHosts=no" --sshkey "/var/lib/syncoid/id_ssh_ed25519" --no-sync-snap --compress zstd-fast "$@" "$src_host:$src_path" "$dest_path" +ret=$? + +if [ -n "$RUNTIME_DIRECTORY" ]; then + cat <> "$RUNTIME_DIRECTORY/metrics" +syncoid_pull_exit_code{backup_server="{{ inventory_hostname }}",source="$src_path",destination="$dest_path"} $ret +EOF + echo $ret >> "$RUNTIME_DIRECTORY/exit_codes" +fi + +exit $ret -- cgit v1.2.3