raspios: major refactoring for image fetching, TODO: customization

5 files changed, 107 insertions, 29 deletions

diff --git a/roles/raspios/image/defaults/main.yml b/roles/raspios/image/defaults/main.yml
index 197d1f7f..28dbd5fe 100644
--- a/roles/raspios/image/defaults/main.yml
+++ b/roles/raspios/image/defaults/main.yml
@@ -1,6 +1,5 @@
 ---
 raspios_variant: lite ## (lite|desktop|full)
-# raspios_release_date:
 raspios_codename: "{{ install_codename }}"
 # raspios_arch: (arm64|armhf)
 raspios_download_dir: "{{ global_cache_dir }}/raspios"
diff --git a/roles/raspios/image/filter_plugins/main.py b/roles/raspios/image/filter_plugins/main.py
new file mode 100644
index 00000000..a8086f66
--- /dev/null
+++ b/roles/raspios/image/filter_plugins/main.py
@@ -0,0 +1,62 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+from ansible import errors
+from urllib.parse import urlparse, urlunparse
+
+
+def raspios_extract_download_base_url(os_list, variant, codename, arch):
+    try:
+        os_name = 'Raspberry Pi OS '
+        if variant != 'desktop':
+            os_name += '%s ' % variant.capitalize()
+        os_name += {'armhf': '(32-bit)', 'arm64': '(64-bit)'}[arch]
+
+        for item in os_list:
+            if item['os_name'] != os_name:
+                continue
+            if item['version'] != codename:
+                continue
+            result = urlparse(item['os_info'])
+            result = result._replace(path=os.path.dirname(result.path))
+            return urlunparse(result)
+    except Exception as e:
+        raise errors.AnsibleFilterError("raspios_extract_download_base_url(): %s" % str(e))
+
+    raise errors.AnsibleFilterError("unable to find base url for: %s / %s / %s" % (variant, codename, arch))
+
+
+def _raspios_extract_latest_image_download_url_recursive(items, base_url):
+    base_url = base_url.replace('http://', 'https://')
+    for item in items:
+        if 'url' in item and item['url'].replace('http://', 'https://').startswith(base_url):
+            return item['url']
+        if 'subitems' in item:
+            result = _raspios_extract_latest_image_download_url_recursive(item['subitems'], base_url)
+            if result:
+                return result
+    return None
+
+
+def raspios_extract_latest_image_download_url(os_list, base_url):
+    try:
+        result = _raspios_extract_latest_image_download_url_recursive(os_list, base_url)
+        if result:
+            return result
+
+    except Exception as e:
+        raise errors.AnsibleFilterError("raspios_extract_latest_image_download_url: %s" % str(e))
+
+    raise errors.AnsibleFilterError("unable to find latest image url for: %s" % (base_url))
+
+
+class FilterModule(object):
+
+    filter_map = {
+        'raspios_extract_download_base_url': raspios_extract_download_base_url,
+        'raspios_extract_latest_image_download_url': raspios_extract_latest_image_download_url,
+    }
+
+    def filters(self):
+        return self.filter_map
diff --git a/roles/raspios/image/tasks/fetch.yml b/roles/raspios/image/tasks/fetch.yml
index c95f1dea..5060bdfe 100644
--- a/roles/raspios/image/tasks/fetch.yml
+++ b/roles/raspios/image/tasks/fetch.yml
@@ -4,33 +4,41 @@
     dest: "{{ raspios_download_dir }}"
     state: directory
 
+- name: fetch imageutility os list from download server
+  check_mode: no
+  uri:
+    url: "{{ raspios_download_url }}/os_list_imagingutility_v3.json"
+    body_format: json
+  register: raspios_os_list_imagingutility
+
+- set_fact:
+    raspios_download_url_image: "{{ raspios_os_list_imagingutility.json.os_list | raspios_extract_latest_image_download_url(raspios_download_base_url) }}"
+
 - name: download the raspios image
   block:
-  - name: download sha256sum
-    get_url:
-      url: "{{ raspios_download_base_url }}/{{ raspios_download_image_base_name }}.zip.sha256"
-      dest: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sha256"
-
-  - name: download signature
+  - name: download sha256sum and signature
+    loop:
+    - sha256
+    - sig
     get_url:
-      url: "{{ raspios_download_base_url }}/{{ raspios_download_image_base_name }}.zip.sig"
-      dest: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sig"
+      url: "{{ raspios_download_url_image }}.{{ item }}"
+      dest: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.{{ item }}"
 
   - name: extract SHA256 hash of the image archive
-    command: grep '{{ raspios_download_image_base_name }}.zip' "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sha256"
+    command: grep '{{ raspios_download_image_base_name }}' "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.sha256"
     changed_when: False
     register: sha256
 
   - name: download image
     get_url:
-      url: "{{ raspios_download_base_url }}/{{ raspios_download_image_base_name }}.zip"
-      dest: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip"
+      url: "{{ raspios_download_url_image }}"
+      dest: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}"
       checksum: sha256:{{ sha256.stdout.split(' ') | first }}
 
   - name: check OpenPGP signature
     command: >-
       gpgv --keyring "{{ global_files_dir }}/common/keyrings/raspios.gpg"
-          "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sig" "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip"
+          "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.sig" "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}"
     changed_when: False
     register: raspios_image_gpg_result
 
@@ -40,9 +48,9 @@
   rescue:
   - name: delete downloaded artifacts
     loop:
-      - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sha256"
-      - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sig"
-      - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip"
+      - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}-sha256"
+      - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.sig"
+      - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}"
     file:
       path: "{{ item }}"
       state: absent
diff --git a/roles/raspios/image/tasks/main.yml b/roles/raspios/image/tasks/main.yml
index 86373da6..24c7c821 100644
--- a/roles/raspios/image/tasks/main.yml
+++ b/roles/raspios/image/tasks/main.yml
@@ -1,4 +1,14 @@
 ---
+- name: fetch os list from download server
+  check_mode: no
+  uri:
+    url: "{{ raspios_download_url }}/os_list_v3.json"
+    body_format: json
+  register: raspios_os_list
+
+- set_fact:
+    raspios_download_base_url: "{{ raspios_os_list.json.os_list | raspios_extract_download_base_url(raspios_variant, raspios_codename, raspios_arch) }}"
+
 - name: fetch base image
   run_once: true
   import_tasks: fetch.yml
@@ -16,15 +26,16 @@
     register: tmpdir
 
   - name: extract image
-    environment: ### TODO: remove once this lands in ansible: https://github.com/ansible/ansible/pull/76542
-      LANGUAGE: en_US.utf8
-    unarchive:
-      src: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip"
-      remote_src: yes
+    decompress:
+      src: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}"
       dest: "{{ tmpdir.path }}"
+    register: raspios_image_extract_result
+
+  - set_fact:
+      raspios_output_image_base_name: "{{ raspios_image_extract_result.files | first | basename }}"
 
   - name: read partition layout from image
-    command: "sfdisk -q -r -J '{{ tmpdir.path }}/{{ raspios_download_image_base_name }}.img'"
+    command: "sfdisk -q -r -J '{{ tmpdir.path }}/{{ raspios_output_image_base_name }}'"
     register: sfdisk_result
 
   - debug:
@@ -38,13 +49,13 @@
 
   - name: copy newly built raspios image
     copy:
-      src: "{{ tmpdir.path }}/{{ raspios_download_image_base_name }}.img"
-      dest: "{{ raspios_output_dir }}/{{ raspios_output_image_base_name }}.img"
+      src: "{{ tmpdir.path }}/{{ raspios_output_image_base_name }}"
+      dest: "{{ raspios_output_dir }}/{{ raspios_output_image_base_name }}"
 
   - name: set output image names
     set_fact:
       output_images:
-      - "{{ raspios_output_dir }}/{{ raspios_output_image_base_name }}.img"
+      - "{{ (raspios_output_dir, raspios_output_image_base_name) | path_join | realpath }}"
 
   always:
   - name: save stdout build-log to output directory
diff --git a/roles/raspios/image/vars/main.yml b/roles/raspios/image/vars/main.yml
index f04f9eba..7c547ce2 100644
--- a/roles/raspios/image/vars/main.yml
+++ b/roles/raspios/image/vars/main.yml
@@ -1,6 +1,4 @@
 ---
-raspios_download_base_path: "raspios{{ (raspios_variant == 'desktop') | ternary('', '_'+raspios_variant) }}_{{ raspios_arch }}"
+raspios_download_url: "https://downloads.raspberrypi.org"
 
-raspios_download_base_url: "https://downloads.raspberrypi.org/{{ raspios_download_base_path }}/images/{{ raspios_download_base_path }}-{{ raspios_release_date }}"
-raspios_download_image_base_name: "{{ raspios_release_date }}-raspios-{{ raspios_codename }}-{{ raspios_arch }}{{ (raspios_variant == 'desktop') | ternary('', '-'+raspios_variant) }}"
-raspios_output_image_base_name: "raspios-{{ raspios_codename }}-{{ raspios_arch }}{{ (raspios_variant == 'desktop') | ternary('', '-'+raspios_variant) }}"
+raspios_download_image_base_name: "{{ raspios_download_url_image | urlsplit('path') | basename }}"