zfs/syncoid: add error handling to pull

4 files changed, 95 insertions, 25 deletions

diff --git a/roles/storage/zfs/syncoid/tasks/main.yml b/roles/storage/zfs/syncoid/tasks/main.yml
index 738b7041..9e573db7 100644
--- a/roles/storage/zfs/syncoid/tasks/main.yml
+++ b/roles/storage/zfs/syncoid/tasks/main.yml
@@ -16,12 +16,10 @@
     type: ed25519
     comment: ZFS Backup syncoid@{{ host_name }}
 
-- name: generate syncoid ssh config wrapper
-  copy:
-    content: |
-      #!/bin/bash
-      exec /usr/sbin/syncoid --sshoption "UserKnownHostsFile=/var/lib/syncoid/ssh.knownhosts" --sshoption "HashKnownHosts=no" --sshkey "/var/lib/syncoid/id_ssh_ed25519" --no-sync-snap --compress zstd-fast "$@"
-    dest: /var/lib/syncoid/syncoid_wrapper
+- name: generate syncoid pull script
+  template:
+    src: syncoid_pull.j2
+    dest: /var/lib/syncoid/syncoid_pull
     mode: 0755
 
 - name: configure lvm to ignore zfs volumes
@@ -81,21 +79,6 @@
         Persistent=false
       dest: /etc/systemd/system/sanoid.timer.d/no-persistence.conf
 
-  - name: generate syncoid post script
-    copy:
-      content: |
-        #!/bin/bash
-
-        echo "running sanoid --prune to cleanup old snapshots"
-        sleep 5
-        systemctl start --wait sanoid-prune.service
-
-        echo "wait 30s (autosuspend cooldown period)"
-        sleep 30
-        echo "done."
-      dest: /var/lib/syncoid/syncoid_post
-      mode: 0755
-
   - name: install python deps
     apt:
       name: "{{ python_basename }}-dbus"
@@ -121,3 +104,9 @@
       name: syncoid-autosuspend.timer
       enabled: yes
       state: started
+
+- name: generate syncoid finalize script
+  template:
+    src: syncoid_finalize.j2
+    dest: /var/lib/syncoid/syncoid_finalize
+    mode: 0755
diff --git a/roles/storage/zfs/syncoid/templates/pull.service.j2 b/roles/storage/zfs/syncoid/templates/pull.service.j2
index 955161aa..cf8f61ae 100644
--- a/roles/storage/zfs/syncoid/templates/pull.service.j2
+++ b/roles/storage/zfs/syncoid/templates/pull.service.j2
@@ -2,13 +2,12 @@
 Description=syncoid-based backup for {{ item.key }}
 
 [Service]
+RuntimeDirectory=syncoid-pull-{{ item.key }}
 Type=oneshot
 {% for path,config in item.value.paths.items() %}
-ExecStart=/var/lib/syncoid/syncoid_wrapper{% if 'periodic' not in item.value %} --create-bookmark{% endif %} --dumpsnaps --quiet {{ config.recursive | default(false) | ternary('-r ', '') }}{{ config.skip_parent | default(false) | ternary('--skip-parent ', '') }}{% for re in config.exclude | default([]) %}--exclude='{{ re }}' {% endfor %}{% if 'ssh_port' in item.value %}--sshport {{ item.value.ssh_port }} {% endif %}root@{{ item.value.ssh_hostname }}:{{ path }} {{ zfs_syncoid_target_pool }}/{{ item.key }}/{{ path }}
+ExecStart=-/var/lib/syncoid/syncoid_pull "root@{{ item.value.ssh_hostname }}" "{{ path }}" "{{ zfs_syncoid_target_pool }}/{{ item.key }}/{{ path }}" --{% if 'periodic' not in item.value %} --create-bookmark{% endif %} --dumpsnaps --quiet {{ config.recursive | default(false) | ternary('-r ', '') }}{{ config.skip_parent | default(false) | ternary('--skip-parent ', '') }}{% for re in config.exclude | default([]) %}--exclude='{{ re }}' {% endfor %}{% if 'ssh_port' in item.value %}--sshport {{ item.value.ssh_port }} {% endif %}{{ '' }}
 {% endfor %}
-{% if zfs_syncoid_autosuspend %}
-ExecStart=/var/lib/syncoid/syncoid_post
-{% endif %}
+ExecStart=/var/lib/syncoid/syncoid_finalize{% if 'report_prometheus_textfile_path' in item.value %} "root@{{ item.value.ssh_hostname }}" "{{ item.value.report_prometheus_textfile_path }}" --{% if 'ssh_port' in item.value %} -p {{ item.value.ssh_port }}{% endif %}{% endif %}{{ '' }}
 {% if 'periodic' in item.value %}
 TimeoutStartSec={{ item.value.periodic.timeout }}
 {% endif %}
diff --git a/roles/storage/zfs/syncoid/templates/syncoid_finalize.j2 b/roles/storage/zfs/syncoid/templates/syncoid_finalize.j2
new file mode 100644
index 00000000..5ddc691b
--- /dev/null
+++ b/roles/storage/zfs/syncoid/templates/syncoid_finalize.j2
@@ -0,0 +1,53 @@
+#!/bin/bash
+{% if zfs_syncoid_autosuspend %}
+
+echo "running sanoid --prune to cleanup old snapshots"
+sleep 5
+echo systemctl start --wait sanoid-prune.service
+
+echo "wait 30s (autosuspend cooldown period)"
+sleep 30
+echo "done."
+{% endif %}
+
+if [ -z "$RUNTIME_DIRECTORY" ]; then
+  exit 0
+fi
+
+ret=0
+for line in $(cat "$RUNTIME_DIRECTORY/exit_codes"); do
+  if [[ $line =~ [^[:digit:]] ]]; then
+    ret=-1
+    echo "invalid exit_code from pull script: $line"
+  else
+    ret=$line
+  fi
+  if [ $ret -ne 0 ]; then
+    break
+  fi
+done
+
+cat <<EOF >> "$RUNTIME_DIRECTORY/metrics"
+syncoid_pull_last_run{backup_server="{{ inventory_hostname }}"} $(date +"%s")
+EOF
+
+
+if [ -z "$1" ] || [ -z "$2" ] || [ "$1" == "--" ] || [ "$2" == "--" ]; then
+  echo "$0 <host> <textfile-collector-path> [ -- <additional ssh arguments .. > ]"
+  exit -2
+fi
+report_to=$1
+textfile_collector_path=$2
+shift 2
+if [ -n "$1" ]; then
+  if [ "$1" != "--" ]; then
+    echo "additional arguments must be separated using '--'"
+    exit -2
+  fi
+  shift
+fi
+
+echo "transfering prometheus metrics to: $report_to:$textfile_collector_path"
+cat "$RUNTIME_DIRECTORY/metrics" | ssh -o "UserKnownHostsFile=/var/lib/syncoid/ssh.knownhosts" -o "HashKnownHosts=no" -i "/var/lib/syncoid/id_ssh_ed25519" "$@" "$report_to" "cat > '$textfile_collector_path/syncoid-pull-{{ inventory_hostname }}.prom'"
+
+exit $ret
diff --git a/roles/storage/zfs/syncoid/templates/syncoid_pull.j2 b/roles/storage/zfs/syncoid/templates/syncoid_pull.j2
new file mode 100644
index 00000000..ab6c3116
--- /dev/null
+++ b/roles/storage/zfs/syncoid/templates/syncoid_pull.j2
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ "$1" == "--" ] || [ "$2" == "--" ] || [ "$3" == "--" ]; then
+  echo "$0 <src-host> <src-path> <dest-path> [ -- <additional syncoid arguments .. > ]"
+  exit 1
+fi
+src_host=$1
+src_path=$2
+dest_path=$3
+shift 3
+if [ -n "$1" ]; then
+  if [ "$1" != "--" ]; then
+    echo "additional arguments must be separated using '--'"
+    exit 1
+  fi
+  shift
+fi
+
+/usr/sbin/syncoid --sshoption "UserKnownHostsFile=/var/lib/syncoid/ssh.knownhosts" --sshoption "HashKnownHosts=no" --sshkey "/var/lib/syncoid/id_ssh_ed25519" --no-sync-snap --compress zstd-fast "$@" "$src_host:$src_path" "$dest_path"
+ret=$?
+
+if [ -n "$RUNTIME_DIRECTORY" ]; then
+  cat <<EOF >> "$RUNTIME_DIRECTORY/metrics"
+syncoid_pull_exit_code{backup_server="{{ inventory_hostname }}",source="$src_path",destination="$dest_path"} $ret
+EOF
+  echo $ret >> "$RUNTIME_DIRECTORY/exit_codes"
+fi
+
+exit $ret