diff options
-rw-r--r-- | inventory/group_vars/chaos-at-home/network.yml | 1 | ||||
-rw-r--r-- | inventory/host_vars/ch-cm4-test.yml | 20 | ||||
-rw-r--r-- | inventory/host_vars/ch-equinox-t450s.yml | 1 | ||||
-rw-r--r-- | inventory/host_vars/ch-equinox-ws.yml | 1 | ||||
-rw-r--r-- | inventory/host_vars/ch-mc-pi.yml | 2 | ||||
-rw-r--r-- | inventory/hosts.ini | 1 | ||||
-rw-r--r-- | library/decompress.py | 226 | ||||
-rw-r--r-- | roles/raspios/image/defaults/main.yml | 1 | ||||
-rw-r--r-- | roles/raspios/image/filter_plugins/main.py | 62 | ||||
-rw-r--r-- | roles/raspios/image/tasks/fetch.yml | 38 | ||||
-rw-r--r-- | roles/raspios/image/tasks/main.yml | 29 | ||||
-rw-r--r-- | roles/raspios/image/vars/main.yml | 6 |
12 files changed, 357 insertions, 31 deletions
diff --git a/inventory/group_vars/chaos-at-home/network.yml b/inventory/group_vars/chaos-at-home/network.yml index fa4109d5..779915d2 100644 --- a/inventory/group_vars/chaos-at-home/network.yml +++ b/inventory/group_vars/chaos-at-home/network.yml @@ -24,6 +24,7 @@ network_zones: ch-hpws-mini1: 31 ch-alix1d: 32 ch-raspi-ntp: 33 + ch-cm4-test: 34 ele-media: 99 ch-prometheus: 200 ch-epimetheus: 201 diff --git a/inventory/host_vars/ch-cm4-test.yml b/inventory/host_vars/ch-cm4-test.yml new file mode 100644 index 00000000..11218262 --- /dev/null +++ b/inventory/host_vars/ch-cm4-test.yml @@ -0,0 +1,20 @@ +--- +raspios_variant: lite +raspios_arch: arm64 + +network: + nameservers: "{{ network_zones.lan.dns }}" + domain: "{{ host_domain }}" + primary: &_network_primary_ + name: eth0 + address: "{{ network_zones.lan.prefix | ipaddr(network_zones.lan.offsets[inventory_hostname]) | ipaddr('address/prefix') }}" + gateway: "{{ network_zones.lan.gateway }}" + interfaces: + - *_network_primary_ + +### +# [all] +# dtparam=i2c_vc=on +# dtoverlay=i2c-rtc,pcf85063a,i2c_csi_dsi +# dtoverlay=i2c-fan,emc2301,i2c_csi_dsi + diff --git a/inventory/host_vars/ch-equinox-t450s.yml b/inventory/host_vars/ch-equinox-t450s.yml index 9e184373..07e593d3 100644 --- a/inventory/host_vars/ch-equinox-t450s.yml +++ b/inventory/host_vars/ch-equinox-t450s.yml @@ -199,6 +199,7 @@ ws_base_extra_packages: - python3-sphinx - python3-sphinx-rtd-theme - python3-toml + - python3-xopen - qemu-kvm - qemu-utils - quilt diff --git a/inventory/host_vars/ch-equinox-ws.yml b/inventory/host_vars/ch-equinox-ws.yml index 0acb29d3..8d1b1dc7 100644 --- a/inventory/host_vars/ch-equinox-ws.yml +++ b/inventory/host_vars/ch-equinox-ws.yml @@ -200,6 +200,7 @@ ws_base_extra_packages: - python3-sphinx - python3-sphinx-rtd-theme - python3-toml + - python3-xopen - qemu-kvm - qemu-utils - quilt diff --git a/inventory/host_vars/ch-mc-pi.yml b/inventory/host_vars/ch-mc-pi.yml index a8701c54..064ad560 100644 --- a/inventory/host_vars/ch-mc-pi.yml +++ b/inventory/host_vars/ch-mc-pi.yml @@ -1,6 +1,4 @@ --- -## TODO: remove once autodetection works... -raspios_release_date: "2022-01-28" #raspios_variant: desktop raspios_arch: arm64 diff --git a/inventory/hosts.ini b/inventory/hosts.ini index 3ebe5680..1ac1d45a 100644 --- a/inventory/hosts.ini +++ b/inventory/hosts.ini @@ -56,6 +56,7 @@ ch-hpws-maxi ch-hpws-mini1 ch-alix1d ch-raspi-ntp +ch-cm4-test [chaos-at-home:children] mz-chaos-at-home diff --git a/library/decompress.py b/library/decompress.py new file mode 100644 index 00000000..e07eb949 --- /dev/null +++ b/library/decompress.py @@ -0,0 +1,226 @@ +#!/usr/bin/python +# this is based on https://github.com/socratesx/Ansible-Decompress + +import zipfile +import os +import posixpath +import shutil +import xopen +from ansible.module_utils.basic import AnsibleModule +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = ''' +--- +module: decompress + +short_description: This is a simple module for decompressing unarchived files + +version_added: "2.4" + +description: + - "The module gets either a gz,bz2 or zip compressed file and just uncompress its content. Its purpose is to cover the case where a single file is just compressed such as a bootimage.iso.gz. but not archived. In case the file is archived, e.g. bootimage.tar.gz then the core module unarchive can handle it." + +options: + src: + description: + - This is the absolute path of the compressed file + required: true + dest: + description: + - The destination of the uncompressed file. This can be an absolute file path where the content will be saved as the specified filename, a directory where the filename will match the src filename without the (.gz|.bz2|.xz|.zip) extention or undefined. In the last case the file will be uncompressed in the same directory of the src with the same filename without the compress extention. + required: false + force: + description: + - Set this option to True to overwrite the extracted file in case it exists on destination. + required: false + default: false + update: + description: + - Set this to True to overwrite the file only if it has different size/ + required: false + default: true + +extends_documentation_fragment: + - files + +author: + - Socrates Chouridis + - Christian Pointner +''' + +EXAMPLES = ''' +# This will decompress the bootimage.iso.gz and move it to ~/isos/bootimage.iso +- name: Decompress a gzipped iso image downloaded from the Internet + decompress: + src: '/tmp/bootimage.iso.gz' + dest: '~/isos/' + +# Setting the extracted file name explicitly +- name: Decompress a bz2 iso image downloaded from the Internet + decompress: + src: '/tmp/bootimage.iso.bz2' + dest: '~/isos/newname_image.iso' + +# Extract multiple images using with_items +- name: decompress multiple images + decompress: + src: "{{ item }}" + dest: '/my-images/' + force: true + with_items: "{{ compressed_isos_list }}" + +# Setting just the src will use the same name ommiting the extention, the result will be /tmp/bootimage.iso +- name: Decompress in the same folder using the same name + decompress: + src: '/tmp/bootimage.iso.zip' + + +''' + +RETURN = ''': +message: + description: An information message regarding the decompression result. + returned: 'always' + type: 'str' +files: + description: A list containing the files in the compressed file. + returned: success + type: list +''' + + +def decompress_file(data={}): + try: + destination = data['dest'] + force = data['force'] + update = data['update'] + original_file_path = str(posixpath.abspath(data['src'])) # Original File Absolute Path + original_file_dir = os.path.dirname(original_file_path) + path_list = os.path.splitext(original_file_path) # List + ext = path_list[1] + + if not destination: + extracted_filename = path_list[0] + elif not os.path.dirname(destination): + extracted_filename = os.path.join(original_file_dir, destination) + elif os.path.isdir(destination): + extracted_filename = os.path.join(destination, os.path.basename(path_list[0])) + + if ext in [".gz", ".bz2", ".xz"]: + result = use_xopen(original_file_path, extracted_filename, force) + elif ext == ".zip": + result = use_unzip(original_file_path, os.path.dirname(extracted_filename), force, update) + else: + message = "The file type " + "\"" + ext + "\"" + " is not supported by this module. Supported File Formats: .gz, .bz2, .xz, .zip" + return True, False, message, [original_file_path] + + return result[0], result[1], result[2], result[3] + + except Exception as e: + message = "An error occured. Decompress Failed: " + str(e) + return True, False, message, [] + + +def use_xopen(src, dst, force): + dst_exists = False + if os.path.exists(dst): + dst_exists = True + + with xopen.xopen(src, 'rb') as f_in, open(dst, 'wb') as f_out: + if not dst_exists: + shutil.copyfileobj(f_in, f_out) + message = "File Extracted Successfully: " + dst + return False, True, message, [dst] + else: + if force: + shutil.copyfileobj(f_in, f_out) + message = "File Extracted Successfully and replaced file (Force = True): " + dst + return False, True, message, [dst] + else: + message = "File Exists: skipping extraction (Use Force=True to Overwrite): " + dst + return False, False, message, [dst] + + +def use_unzip(src, dst, force=False, update=True): + filelist = [] + for root, dirs, files in os.walk(dst): + for d in dirs: + filelist.append(os.path.relpath(os.path.join(root, d), dst)) + for f in files: + filelist.append(os.path.relpath(os.path.join(root, f), dst)) + + extracted_files = [] + excluded_files = [] + with zipfile.ZipFile(src, 'r') as zip_file: + for f in zip_file.namelist(): + if not os.path.relpath(f) in filelist: + zip_file.extract(f, dst) + extracted_files.append(f) + else: + if force: + zip_file.extract(f, dst) + extracted_files.append(f) + else: + info = zip_file.getinfo(f) + if info.file_size != os.stat(dst + "/" + f).st_size: + if update: + zip_file.extract(f, dst) + extracted_files.append(f) + else: + excluded_files.append(f) + else: + excluded_files.append(f) + if not excluded_files: + message = " All files were extracted successfully" + for file in extracted_files: + if not force and update and file in filelist: + message = "Files with the same name on destination were replaced as they had different size (Update=True)" + elif force: + message = "All files were extracted successfully replacing any files on destination with the same name (Force=True)" + return False, True, message, [os.path.join(dst, file) for file in extracted_files] + else: + if not extracted_files: + message = "All Files Exist on Destination, Skipping Extraction... Use Force=True to Overwrite" + return False, False, message, [os.path.join(dst, file) for file in filelist] + else: + message = "Some Files Skipped Extraction as they Existed on destination. Use Force=True to Overwrite " + return False, True, message, [os.path.join(dst, file) for file in filelist] + + +def run_module(): + module_args = dict( + src=dict(type='str', required=True), + dest=dict(type='str', required=False), + force=dict(type='bool', required=False, default=False), + update=dict(type='bool', required=False, default=True), + ) + + result = dict( + changed=False, + message='', + failed=False, + files=[], + ) + + module = AnsibleModule(argument_spec=module_args) + + (error, is_changed, message, files) = decompress_file(module.params) + + result['changed'] = is_changed + result['message'] = message + result['failed'] = error + result['files'] = files + + module.exit_json(**result) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/roles/raspios/image/defaults/main.yml b/roles/raspios/image/defaults/main.yml index 197d1f7f..28dbd5fe 100644 --- a/roles/raspios/image/defaults/main.yml +++ b/roles/raspios/image/defaults/main.yml @@ -1,6 +1,5 @@ --- raspios_variant: lite ## (lite|desktop|full) -# raspios_release_date: raspios_codename: "{{ install_codename }}" # raspios_arch: (arm64|armhf) raspios_download_dir: "{{ global_cache_dir }}/raspios" diff --git a/roles/raspios/image/filter_plugins/main.py b/roles/raspios/image/filter_plugins/main.py new file mode 100644 index 00000000..a8086f66 --- /dev/null +++ b/roles/raspios/image/filter_plugins/main.py @@ -0,0 +1,62 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +from ansible import errors +from urllib.parse import urlparse, urlunparse + + +def raspios_extract_download_base_url(os_list, variant, codename, arch): + try: + os_name = 'Raspberry Pi OS ' + if variant != 'desktop': + os_name += '%s ' % variant.capitalize() + os_name += {'armhf': '(32-bit)', 'arm64': '(64-bit)'}[arch] + + for item in os_list: + if item['os_name'] != os_name: + continue + if item['version'] != codename: + continue + result = urlparse(item['os_info']) + result = result._replace(path=os.path.dirname(result.path)) + return urlunparse(result) + except Exception as e: + raise errors.AnsibleFilterError("raspios_extract_download_base_url(): %s" % str(e)) + + raise errors.AnsibleFilterError("unable to find base url for: %s / %s / %s" % (variant, codename, arch)) + + +def _raspios_extract_latest_image_download_url_recursive(items, base_url): + base_url = base_url.replace('http://', 'https://') + for item in items: + if 'url' in item and item['url'].replace('http://', 'https://').startswith(base_url): + return item['url'] + if 'subitems' in item: + result = _raspios_extract_latest_image_download_url_recursive(item['subitems'], base_url) + if result: + return result + return None + + +def raspios_extract_latest_image_download_url(os_list, base_url): + try: + result = _raspios_extract_latest_image_download_url_recursive(os_list, base_url) + if result: + return result + + except Exception as e: + raise errors.AnsibleFilterError("raspios_extract_latest_image_download_url: %s" % str(e)) + + raise errors.AnsibleFilterError("unable to find latest image url for: %s" % (base_url)) + + +class FilterModule(object): + + filter_map = { + 'raspios_extract_download_base_url': raspios_extract_download_base_url, + 'raspios_extract_latest_image_download_url': raspios_extract_latest_image_download_url, + } + + def filters(self): + return self.filter_map diff --git a/roles/raspios/image/tasks/fetch.yml b/roles/raspios/image/tasks/fetch.yml index c95f1dea..5060bdfe 100644 --- a/roles/raspios/image/tasks/fetch.yml +++ b/roles/raspios/image/tasks/fetch.yml @@ -4,33 +4,41 @@ dest: "{{ raspios_download_dir }}" state: directory +- name: fetch imageutility os list from download server + check_mode: no + uri: + url: "{{ raspios_download_url }}/os_list_imagingutility_v3.json" + body_format: json + register: raspios_os_list_imagingutility + +- set_fact: + raspios_download_url_image: "{{ raspios_os_list_imagingutility.json.os_list | raspios_extract_latest_image_download_url(raspios_download_base_url) }}" + - name: download the raspios image block: - - name: download sha256sum - get_url: - url: "{{ raspios_download_base_url }}/{{ raspios_download_image_base_name }}.zip.sha256" - dest: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sha256" - - - name: download signature + - name: download sha256sum and signature + loop: + - sha256 + - sig get_url: - url: "{{ raspios_download_base_url }}/{{ raspios_download_image_base_name }}.zip.sig" - dest: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sig" + url: "{{ raspios_download_url_image }}.{{ item }}" + dest: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.{{ item }}" - name: extract SHA256 hash of the image archive - command: grep '{{ raspios_download_image_base_name }}.zip' "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sha256" + command: grep '{{ raspios_download_image_base_name }}' "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.sha256" changed_when: False register: sha256 - name: download image get_url: - url: "{{ raspios_download_base_url }}/{{ raspios_download_image_base_name }}.zip" - dest: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip" + url: "{{ raspios_download_url_image }}" + dest: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}" checksum: sha256:{{ sha256.stdout.split(' ') | first }} - name: check OpenPGP signature command: >- gpgv --keyring "{{ global_files_dir }}/common/keyrings/raspios.gpg" - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sig" "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip" + "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.sig" "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}" changed_when: False register: raspios_image_gpg_result @@ -40,9 +48,9 @@ rescue: - name: delete downloaded artifacts loop: - - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sha256" - - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip.sig" - - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip" + - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}-sha256" + - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.sig" + - "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}" file: path: "{{ item }}" state: absent diff --git a/roles/raspios/image/tasks/main.yml b/roles/raspios/image/tasks/main.yml index 86373da6..24c7c821 100644 --- a/roles/raspios/image/tasks/main.yml +++ b/roles/raspios/image/tasks/main.yml @@ -1,4 +1,14 @@ --- +- name: fetch os list from download server + check_mode: no + uri: + url: "{{ raspios_download_url }}/os_list_v3.json" + body_format: json + register: raspios_os_list + +- set_fact: + raspios_download_base_url: "{{ raspios_os_list.json.os_list | raspios_extract_download_base_url(raspios_variant, raspios_codename, raspios_arch) }}" + - name: fetch base image run_once: true import_tasks: fetch.yml @@ -16,15 +26,16 @@ register: tmpdir - name: extract image - environment: ### TODO: remove once this lands in ansible: https://github.com/ansible/ansible/pull/76542 - LANGUAGE: en_US.utf8 - unarchive: - src: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}.zip" - remote_src: yes + decompress: + src: "{{ raspios_download_dir }}/{{ raspios_download_image_base_name }}" dest: "{{ tmpdir.path }}" + register: raspios_image_extract_result + + - set_fact: + raspios_output_image_base_name: "{{ raspios_image_extract_result.files | first | basename }}" - name: read partition layout from image - command: "sfdisk -q -r -J '{{ tmpdir.path }}/{{ raspios_download_image_base_name }}.img'" + command: "sfdisk -q -r -J '{{ tmpdir.path }}/{{ raspios_output_image_base_name }}'" register: sfdisk_result - debug: @@ -38,13 +49,13 @@ - name: copy newly built raspios image copy: - src: "{{ tmpdir.path }}/{{ raspios_download_image_base_name }}.img" - dest: "{{ raspios_output_dir }}/{{ raspios_output_image_base_name }}.img" + src: "{{ tmpdir.path }}/{{ raspios_output_image_base_name }}" + dest: "{{ raspios_output_dir }}/{{ raspios_output_image_base_name }}" - name: set output image names set_fact: output_images: - - "{{ raspios_output_dir }}/{{ raspios_output_image_base_name }}.img" + - "{{ (raspios_output_dir, raspios_output_image_base_name) | path_join | realpath }}" always: - name: save stdout build-log to output directory diff --git a/roles/raspios/image/vars/main.yml b/roles/raspios/image/vars/main.yml index f04f9eba..7c547ce2 100644 --- a/roles/raspios/image/vars/main.yml +++ b/roles/raspios/image/vars/main.yml @@ -1,6 +1,4 @@ --- -raspios_download_base_path: "raspios{{ (raspios_variant == 'desktop') | ternary('', '_'+raspios_variant) }}_{{ raspios_arch }}" +raspios_download_url: "https://downloads.raspberrypi.org" -raspios_download_base_url: "https://downloads.raspberrypi.org/{{ raspios_download_base_path }}/images/{{ raspios_download_base_path }}-{{ raspios_release_date }}" -raspios_download_image_base_name: "{{ raspios_release_date }}-raspios-{{ raspios_codename }}-{{ raspios_arch }}{{ (raspios_variant == 'desktop') | ternary('', '-'+raspios_variant) }}" -raspios_output_image_base_name: "raspios-{{ raspios_codename }}-{{ raspios_arch }}{{ (raspios_variant == 'desktop') | ternary('', '-'+raspios_variant) }}" +raspios_download_image_base_name: "{{ raspios_download_url_image | urlsplit('path') | basename }}" |