From ef90875a1c2460fb40643d7152f5fd0458623ddb Mon Sep 17 00:00:00 2001 From: Noel Miller <4983138+noelmiller@users.noreply.github.com> Date: Tue, 5 Mar 2024 12:11:52 -0600 Subject: [PATCH 01/20] Add Support for Secure Boot (#37) --- .dockerignore | 5 ++ .github/workflows/build-and-test.yml | 5 ++ Containerfile | 7 +-- Makefile | 58 ++++++++++++++----- action.yml | 9 +++ .../scripts/post/enroll-secureboot-key | 26 +++++++++ xorriso/gen_input.sh.in | 14 +++-- 7 files changed, 100 insertions(+), 24 deletions(-) create mode 100644 .dockerignore create mode 100755 lorax_templates/scripts/post/enroll-secureboot-key diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..06aa2e2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.devcontainer +.github +.gitignore +action.yml +Containerfile diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a3b3650..942c763 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -15,6 +15,9 @@ env: IMAGE_REPO: 'quay.io/fedora-ostree-desktops' VERSION: '39' VARIANT: 'Server' + SECURE_BOOT_KEY_URL: 'https://github.com/ublue-os/akmods/raw/main/certs/public_key.der' + ENROLLMENT_PASSWORD: 'container-installer' + jobs: build-container: @@ -87,6 +90,8 @@ jobs: image_repo: ${{ env.IMAGE_REPO}} version: ${{ env.VERSION }} variant: ${{ env.VARIANT }} + secure_boot_key_url: ${{ env.SECURE_BOOT_KEY_URL }} + enrollment_password: ${{ env.ENROLLMENT_PASSWORD }} - name: Rename ISO run: | diff --git a/Containerfile b/Containerfile index 196a151..6ca8af4 100644 --- a/Containerfile +++ b/Containerfile @@ -11,10 +11,7 @@ ENV VERSION="${VERSION}" ENV WEB_UI="false" RUN mkdir /build-container-installer -COPY /lorax_templates /build-container-installer/lorax_templates -COPY /xorriso /build-container-installer/xorriso -COPY /Makefile /build-container-installer -COPY /entrypoint.sh / +COPY / /build-container-installer/ WORKDIR /build-container-installer @@ -22,5 +19,5 @@ RUN dnf install -y make && make install-deps VOLUME /build-container-installer/build -ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] +ENTRYPOINT ["/bin/bash", "/build-container-installer/entrypoint.sh"] diff --git a/Makefile b/Makefile index 0362032..6064d01 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,8 @@ IMAGE_TAG = $(VERSION) VARIANT = Server WEB_UI = false REPOS = /etc/yum.repos.d/fedora.repo /etc/yum.repos.d/fedora-updates.repo +ENROLLMENT_PASSWORD = +SECURE_BOOT_KEY_URL = ADDITIONAL_TEMPLATES = "" ROOTFS_SIZE = 4 @@ -19,7 +21,7 @@ _IMAGE_REPO_DOUBLE_ESCAPED = $(subst \,\\\,$(_IMAGE_REPO_ESCAPED)) _VOLID = $(firstword $(subst -, ,$(IMAGE_NAME)))-$(ARCH)-$(IMAGE_TAG) _REPO_FILES = $(subst /etc/yum.repos.d,repos,$(REPOS)) _LORAX_TEMPLATES = $(subst .in,,$(shell ls lorax_templates/*.tmpl.in)) $(foreach file,$(shell ls lorax_templates/scripts/post),lorax_templates/post_$(file).tmpl) -_TEMPLATE_VARS = ARCH VERSION IMAGE_REPO IMAGE_NAME IMAGE_TAG VARIANT WEB_UI REPOS _IMAGE_REPO_ESCAPED _IMAGE_REPO_DOUBLE_ESCAPED +_TEMPLATE_VARS = ARCH VERSION IMAGE_REPO IMAGE_NAME IMAGE_TAG VARIANT WEB_UI REPOS _IMAGE_REPO_ESCAPED _IMAGE_REPO_DOUBLE_ESCAPED ENROLLMENT_PASSWORD ifeq ($(VARIANT),Server) _LORAX_ARGS = --macboot --noupgrade @@ -44,19 +46,30 @@ lorax_templates/post_%.tmpl: lorax_templates/scripts/post/% $(eval _ISO_FILE = usr/share/anaconda/interactive-defaults.ks) header=0; \ + skip=0; \ while read -r line; \ do \ - if [[ $$line =~ ^\<\% ]]; \ - then \ + if [[ $$line =~ ^\<\% ]]; \ + then \ echo $$line >> lorax_templates/post_$*.tmpl; \ echo >> lorax_templates/post_$*.tmpl; \ - else \ - if [[ $$header == 0 ]]; \ + else \ + if [[ $$header == 0 ]]; \ then \ - echo "append $(_ISO_FILE) \"%post --erroronfail\"" >> lorax_templates/post_$*.tmpl; \ + if [[ $$line =~ ^##\ (.*)$$ ]]; \ + then \ + echo "append $(_ISO_FILE) \"%post --erroronfail $${BASH_REMATCH[1]}\"" >> lorax_templates/post_$*.tmpl; \ + skip=1; \ + else \ + echo "append $(_ISO_FILE) \"%post --erroronfail\"" >> lorax_templates/post_$*.tmpl; \ + fi; \ header=1; \ fi; \ - echo "append $(_ISO_FILE) \"$$line\"" >> lorax_templates/post_$*.tmpl; \ + if [[ $$skip == 0 ]]; \ + then \ + echo "append $(_ISO_FILE) \"$${line//\"/\\\"}\"" >> lorax_templates/post_$*.tmpl; \ + fi; \ + skip=0; \ fi; \ done < lorax_templates/scripts/post/$* echo "append $(_ISO_FILE) \"%end\"" >> lorax_templates/post_$*.tmpl @@ -65,18 +78,30 @@ lorax_templates/post_%.tmpl: lorax_templates/scripts/post/% $(eval _ISO_FILE = usr/share/anaconda/post-scripts/configure_upgrades.ks) header=0; \ + skip=0; \ while read -r line; \ do \ - if [[ $$line =~ ^\<\% ]]; \ - then \ + if [[ $$line =~ ^\<\% ]]; \ + then \ + echo $$line >> lorax_templates/post_$*.tmpl; \ echo >> lorax_templates/post_$*.tmpl; \ - else \ - if [[ $$header == 0 ]]; \ + else \ + if [[ $$header == 0 ]]; \ then \ - echo "append $(_ISO_FILE) \"%post --erroronfail\"" >> lorax_templates/post_$*.tmpl; \ + if [[ $$line =~ ^##\ (.*)$$ ]]; \ + then \ + echo "append $(_ISO_FILE) \"%post --erroronfail $${BASH_REMATCH[1]}\"" >> lorax_templates/post_$*.tmpl; \ + skip=1; \ + else \ + echo "append $(_ISO_FILE) \"%post --erroronfail\"" >> lorax_templates/post_$*.tmpl; \ + fi; \ header=1; \ fi; \ - echo "append $(_ISO_FILE) \"$$line\"" >> lorax_templates/post_$*.tmpl; \ + if [[ $$skip == 0 ]]; \ + then \ + echo "append $(_ISO_FILE) \"$${line//\"/\\\"}\"" >> lorax_templates/post_$*.tmpl; \ + fi; \ + skip=0; \ fi; \ done < lorax_templates/scripts/post/$* echo "append $(_ISO_FILE) \"%end\"" >> lorax_templates/post_$*.tmpl @@ -100,6 +125,13 @@ repos/%.repo: /etc/yum.repos.d/%.repo boot.iso: $(_LORAX_TEMPLATES) $(_REPO_FILES) rm -Rf $(_BASE_DIR)/results || true rm /etc/rpm/macros.image-language-conf || true + + # Download the secure boot key + if [ -n "$(SECURE_BOOT_KEY_URL)" ]; \ + then \ + curl --fail -L -o $(_BASE_DIR)/sb_pubkey.der $(SECURE_BOOT_KEY_URL); \ + fi + lorax -p $(IMAGE_NAME) -v $(VERSION) -r $(VERSION) -t $(VARIANT) \ --isfinal --squashfs-only --buildarch=$(ARCH) --volid=$(_VOLID) \ $(_LORAX_ARGS) \ diff --git a/action.yml b/action.yml index 219cc6b..22914f1 100644 --- a/action.yml +++ b/action.yml @@ -29,6 +29,13 @@ inputs: description: Enable Anaconda WebUI required: false default: "false" + enrollment_password: + description: Used for supporting secure boot (requires SECURE_BOOT_KEY_URL to be defined) + required: false + default: "container-installer" + secure_boot_key_url: + description: Secure boot key that is installed from URL location + required: false action_version: description: Version of the action container to run deprecationMessage: No longer used. github.action_ref replaces the need for this. Will be removed in a future version. @@ -54,5 +61,7 @@ runs: VERSION=${{ inputs.version }} \ IMAGE_TAG=${{ inputs.image_tag || inputs.version }} \ WEB_UI=${{ inputs.web_ui }} \ + ENROLLMENT_PASSWORD=${{ inputs.enrollment_password }} \ + SECURE_BOOT_KEY_URL=${{ inputs.secure_boot_key_url }} \ "ADDITIONAL_TEMPLATES=${{ inputs.additional_templates }}" diff --git a/lorax_templates/scripts/post/enroll-secureboot-key b/lorax_templates/scripts/post/enroll-secureboot-key new file mode 100755 index 0000000..622a1ec --- /dev/null +++ b/lorax_templates/scripts/post/enroll-secureboot-key @@ -0,0 +1,26 @@ +<%page args="enrollment_password "/> +## --nochroot + +set -oue pipefail + +readonly ENROLLMENT_PASSWORD=${enrollment_password} +readonly SECUREBOOT_KEY="/run/install/repo/sb_pubkey.der" + +if [[ ! -d "/sys/firmware/efi" ]]; then + echo "EFI mode not detected. Skipping key enrollment." + exit 0 +fi + +if [[ ! -f "$SECUREBOOT_KEY" ]]; then + echo "Secure boot key not provided: $SECUREBOOT_KEY" + exit 0 +fi + +SYS_ID="$(cat /sys/devices/virtual/dmi/id/product_name)" +if [[ ":Jupiter:Galileo:" =~ ":$SYS_ID:" ]]; then + echo "Steam Deck hardware detected. Skipping key enrollment." + exit 0 +fi + +mokutil --timeout -1 || : +echo -e "$ENROLLMENT_PASSWORD\n$ENROLLMENT_PASSWORD" | mokutil --import "$SECUREBOOT_KEY" || : diff --git a/xorriso/gen_input.sh.in b/xorriso/gen_input.sh.in index b660e1b..00acd91 100644 --- a/xorriso/gen_input.sh.in +++ b/xorriso/gen_input.sh.in @@ -5,12 +5,14 @@ echo "-outdev $(pwd)/build/deploy.iso" echo "-boot_image any replay" echo "-joliet on" echo "-compliance joliet_long_names" -pushd container > /dev/null -for file in $(find ${IMAGE_NAME}-${IMAGE_TAG}) -do - echo "-map $(pwd)/${file} ${file}" - echo "-chmod 0444 ${file}" +if [ -f $(pwd)/sb_pubkey.der ]; then + echo "-map $(pwd)/sb_pubkey.der sb_pubkey.der" + echo "-chmod 0444 /sb_pubkey.der" +fi +pushd container >/dev/null +for file in $(find ${IMAGE_NAME}-${IMAGE_TAG}); do + echo "-map $(pwd)/${file} ${file}" + echo "-chmod 0444 ${file}" done popd > /dev/null echo "-end" - From 8d9ceeb5799eaf33f538797fa9b4bc7f932c4113 Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Tue, 5 Mar 2024 16:10:08 -0500 Subject: [PATCH 02/20] ci: Test resulting test ISO (#26) --- .github/workflows/build-and-test.yml | 110 ++++++++++++++++++++++++++- Makefile | 7 +- tests/iso/README.md | 1 + tests/iso/os-release.sh | 31 ++++++++ tests/vm/README.md | 1 + 5 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 tests/iso/README.md create mode 100644 tests/iso/os-release.sh create mode 100644 tests/vm/README.md diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 942c763..2981e79 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -82,7 +82,7 @@ jobs: type=ref,event=branch type=ref,event=pr - - name: Build ISO + - name: Build ISO with new container uses: ./ with: arch: ${{ env.ARCH}} @@ -98,6 +98,7 @@ jobs: mv build/deploy.iso build/${{ env.IMAGE_NAME }}-${{ env.VERSION }}.iso - name: Upload ISO as artifact + id: upload uses: actions/upload-artifact@v4 with: name: ${{ env.IMAGE_NAME }}-${{ env.VERSION }}.iso @@ -106,3 +107,110 @@ jobs: retention-days: 0 compression-level: 0 overwrite: true + + test-qemu: + runs-on: ubuntu-latest + needs: + - build-and-push-iso + permissions: + contents: read + packages: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Ensure qemu is installed + run: | + sudo apt-get update + sudo apt-get install -y qemu qemu-utils xorriso unzip qemu-system-x86 netcat socat jq isomd5sum ansible make + + - name: Create disk + run: | + qemu-img create -f qcow2 disk.qcow2 50G + + - name: Download generated ISO + uses: actions/download-artifact@v4 + with: + name: ${{ env.IMAGE_NAME }}-${{ env.VERSION }}.iso + + - name: Verify ISO + run: checkisomd5 ${{ env.IMAGE_NAME }}-${{ env.VERSION }}.iso + + - name: Run ISO checks + run: | + mv ${{ env.IMAGE_NAME }}-${{ env.VERSION }}.iso deploy.iso + make test-iso + + - name: Add Kickstart and Grub options to ISO + run: | + sudo mkdir /mnt/iso || true + sudo mount -o loop deploy.iso /mnt/iso + cp /mnt/iso/boot/grub2/grub.cfg grub.cfg + sudo umount /mnt/iso + sed -i 's/quiet/console=ttyS0,115200n8 inst.ks=cdrom:\/ks.cfg/' grub.cfg + sed -i 's/set default="1"/set default="0"/' grub.cfg + sed -i 's/set timeout=60/set timeout=1/' grub.cfg + cat << EOF > ks.cfg + lang en_US.UTF-8 + keyboard us + timezone Americas/New_York + zerombr + clearpart --all --initlabel + autopart + poweroff + user --name=core --groups=wheel --password=foobar + %include /usr/share/anaconda/interactive-defaults.ks + EOF + xorriso -dialog on << EOF + -indev deploy.iso + -outdev test.iso + -boot_image any replay + -map ks.cfg ks.cfg + -chmod 0444 ks.cfg + -map grub.cfg boot/grub2/grub.cfg + -end + EOF + + - name: Install the test VM + run: | + timeout 1h qemu-system-x86_64 -name "Anaconda" -boot d -m 4096 -cpu qemu64 -display none -cdrom test.iso -smp 2 -hda disk.qcow2 -serial telnet:localhost:4321,server,nowait & QEMU_PID=$! + echo "PID: $QEMU_PID" + timeout 1m bash -c "while ! (echo > /dev/tcp/127.0.0.1/4321); do sleep 0.1; done" + (nc localhost 4321 | tee vm.stdout) & + wait $QEMU_PID + + - name: Start the test VM + env: + VM_USER: core + VM_PASS: foobar + run: | + mkfifo vm.stdin + qemu-system-x86_64 -name "Anaconda" \ + -m 4096 -cpu qemu64 -display none -smp 2 \ + -chardev socket,path=/tmp/qga.sock,server,nowait,id=qga0 \ + -device virtio-serial \ + -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \ + -boot c -hda disk.qcow2 -serial telnet:localhost:4321,server,nowait & export QEMU_PID=$! + echo "PID: $QEMU_PID" + + timeout 1m bash -c "while ! (echo > /dev/tcp/127.0.0.1/4321); do sleep 0.1; done" + (tail -f vm.stdin | nc localhost 4321 | tee vm.stdout) & + + timeout 30m bash -c "while ! (grep 'login:' vm.stdout); do sleep 1; done" + + export VM_IP=$({ echo '{"execute": "guest-network-get-interfaces"}'; sleep 2; } | socat unix-connect:/tmp/qga.sock - | jq -r '.return[1]["ip-addresses"][] | select(."ip-address-type"=="ipv4") | .["ip-address"]') + if ! (echo > /dev/tcp/${VM_IP}/22) + then + echo "SSH must be installed and enabled inside the container" + fi + + echo "VM ready for tests at IP ${VM_IP}" + for check in $(ls tests/vm) + do + if [[ -f $check && $check != "README.md" ]] + then + chmod +x $check + ./${check} + fi + done + diff --git a/Makefile b/Makefile index 6064d01..81db881 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ WEB_UI = false REPOS = /etc/yum.repos.d/fedora.repo /etc/yum.repos.d/fedora-updates.repo ENROLLMENT_PASSWORD = SECURE_BOOT_KEY_URL = -ADDITIONAL_TEMPLATES = "" +ADDITIONAL_TEMPLATES = ROOTFS_SIZE = 4 # Generated vars @@ -175,6 +175,11 @@ clean: install-deps: dnf install -y lorax xorriso skopeo + +test-iso: + $(eval _TESTS = $(filter-out README.md,$(shell ls tests/iso))) + $(foreach test,$(_TESTS),chmod +x tests/iso/$(test)) + $(foreach test,$(_TESTS),./tests/iso/$(test) deploy.iso) .PHONY: clean install-deps diff --git a/tests/iso/README.md b/tests/iso/README.md new file mode 100644 index 0000000..bd511d9 --- /dev/null +++ b/tests/iso/README.md @@ -0,0 +1 @@ +Place scripts that will test the ISO. The ISO file will be passed as the first argument \ No newline at end of file diff --git a/tests/iso/os-release.sh b/tests/iso/os-release.sh new file mode 100644 index 0000000..ea066bc --- /dev/null +++ b/tests/iso/os-release.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +iso=$1 + +sudo apt-get update +sudo apt-get install -y squashfs-tools + +sudo mkdir /mnt/{iso,install} + +# Mount ISO +sudo mount -o loop $iso /mnt/iso + +# Mount squashfs +sudo mount -t squashfs -o loop /mnt/iso/images/install.img /mnt/install + +FOUND_VERSION=$(cat /mnt/install/etc/os-release | grep VERSION_ID | cut -d= -f2) + +# Cleanup +sudo umount /mnt/install +sudo umount /mnt/iso + +if [[ ${FOUND_VERSION} != ${VERSION} ]] +then + echo "Version mismatch" + echo "Expected: ${VERSION}" + echo "Found: ${FOUND_VERSION}" + exit 1 +else + echo "Correct version found" + exit 0 +fi \ No newline at end of file diff --git a/tests/vm/README.md b/tests/vm/README.md new file mode 100644 index 0000000..39840b4 --- /dev/null +++ b/tests/vm/README.md @@ -0,0 +1 @@ +Place scripts that will test the VM. The VM will be available at ${VM_IP} using username ${VM_USER} and password ${VM_PASS} \ No newline at end of file From 8746fd7f0dec28f7f101e14ac2e83a2d256eef8e Mon Sep 17 00:00:00 2001 From: Noel Miller <4983138+noelmiller@users.noreply.github.com> Date: Wed, 6 Mar 2024 16:45:09 -0600 Subject: [PATCH 03/20] Fix bootc command (#40) --- lorax_templates/scripts/post/configure_upgrades | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lorax_templates/scripts/post/configure_upgrades b/lorax_templates/scripts/post/configure_upgrades index 5b56ef2..761764d 100644 --- a/lorax_templates/scripts/post/configure_upgrades +++ b/lorax_templates/scripts/post/configure_upgrades @@ -1,7 +1,7 @@ <%page args="image_repo, _image_repo_double_escaped, image_name, image_tag"/> if (which bootc &> /dev/null) then - bootc switch ${image_repo}/${image_name}:${image_tag} + bootc switch --mutate-in-place --enforce-container-sigpolicy --transport registry ${image_repo}/${image_name}:${image_tag} else sed -i 's/container-image-reference=.*/container-image-reference=ostree-image-signed:docker:\/\/${_image_repo_double_escaped}\/${image_name}:${image_tag}/' /ostree/deploy/default/deploy/*.origin fi From 5ea604d4056acb94a6fad36562dcbeba1c1ee991 Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:14:09 -0600 Subject: [PATCH 04/20] Add support for extra boot options (#35) --- Makefile | 5 ++++- xorriso/gen_input.sh.in | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 81db881..c8f8c01 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,8 @@ WEB_UI = false REPOS = /etc/yum.repos.d/fedora.repo /etc/yum.repos.d/fedora-updates.repo ENROLLMENT_PASSWORD = SECURE_BOOT_KEY_URL = -ADDITIONAL_TEMPLATES = +ADDITIONAL_TEMPLATES = +EXTRA_BOOT_PARAMS = ROOTFS_SIZE = 4 # Generated vars @@ -150,6 +151,8 @@ container/$(IMAGE_NAME)-$(IMAGE_TAG): # Step 5: Generate xorriso script xorriso/%.sh: xorriso/%.sh.in + sed -i 's/quiet/quiet $(EXTRA_BOOT_PARAMS)/g' results/boot/grub2/grub.cfg + sed -i 's/quiet/quiet $(EXTRA_BOOT_PARAMS)/g' results/EFI/BOOT/grub.cfg $(eval _VARS = IMAGE_NAME IMAGE_TAG ARCH VERSION) $(foreach var,$(_VARS),$(var)=$($(var))) envsubst '$(foreach var,$(_VARS),$$$(var))' < $(_BASE_DIR)/xorriso/$*.sh.in > $(_BASE_DIR)/xorriso/$*.sh diff --git a/xorriso/gen_input.sh.in b/xorriso/gen_input.sh.in index 00acd91..f2f678c 100644 --- a/xorriso/gen_input.sh.in +++ b/xorriso/gen_input.sh.in @@ -5,14 +5,22 @@ echo "-outdev $(pwd)/build/deploy.iso" echo "-boot_image any replay" echo "-joliet on" echo "-compliance joliet_long_names" -if [ -f $(pwd)/sb_pubkey.der ]; then +echo "-map $(pwd)/results/boot/grub2/grub.cfg boot/grub2/grub.cfg" +echo "-chmod 0444 boot/grub2/grub.cfg" +echo "-map $(pwd)/results/EFI/BOOT/grub.cfg EFI/BOOT/grub.cfg" +echo "-chmod 0444 EFI/BOOT/grub.cfg" + +if [ -f $(pwd)/sb_pubkey.der ] +then echo "-map $(pwd)/sb_pubkey.der sb_pubkey.der" echo "-chmod 0444 /sb_pubkey.der" fi -pushd container >/dev/null -for file in $(find ${IMAGE_NAME}-${IMAGE_TAG}); do - echo "-map $(pwd)/${file} ${file}" - echo "-chmod 0444 ${file}" + +pushd container > /dev/null +for file in $(find ${IMAGE_NAME}-${IMAGE_TAG}) +do + echo "-map $(pwd)/${file} ${file}" + echo "-chmod 0444 ${file}" done popd > /dev/null echo "-end" From 0b68c00dcfcaf0c54cceda8e80e39035687d1ddb Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Thu, 7 Mar 2024 10:42:05 -0600 Subject: [PATCH 05/20] Run tests using multiple installer versions (#47) --- .github/workflows/build-and-test.yml | 27 +++++++++++++++++++-------- external/fedora-lorax-templates | 1 + 2 files changed, 20 insertions(+), 8 deletions(-) create mode 160000 external/fedora-lorax-templates diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 2981e79..74801ed 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -13,7 +13,7 @@ env: ARCH: 'x86_64' IMAGE_NAME: 'base' IMAGE_REPO: 'quay.io/fedora-ostree-desktops' - VERSION: '39' + IMAGE_TAG: '39' VARIANT: 'Server' SECURE_BOOT_KEY_URL: 'https://github.com/ublue-os/akmods/raw/main/certs/public_key.der' ENROLLMENT_PASSWORD: 'container-installer' @@ -64,6 +64,11 @@ jobs: permissions: contents: read packages: write + strategy: + matrix: + version: + - 38 + - 39 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -88,20 +93,21 @@ jobs: arch: ${{ env.ARCH}} image_name: ${{ env.IMAGE_NAME}} image_repo: ${{ env.IMAGE_REPO}} - version: ${{ env.VERSION }} + image_tag: ${{ env.IMAGE_TAG }} + version: ${{ matrix.version }} variant: ${{ env.VARIANT }} secure_boot_key_url: ${{ env.SECURE_BOOT_KEY_URL }} enrollment_password: ${{ env.ENROLLMENT_PASSWORD }} - name: Rename ISO run: | - mv build/deploy.iso build/${{ env.IMAGE_NAME }}-${{ env.VERSION }}.iso + mv build/deploy.iso build/${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso - name: Upload ISO as artifact id: upload uses: actions/upload-artifact@v4 with: - name: ${{ env.IMAGE_NAME }}-${{ env.VERSION }}.iso + name: ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso path: build/*.iso if-no-files-found: error retention-days: 0 @@ -115,6 +121,11 @@ jobs: permissions: contents: read packages: write + strategy: + matrix: + version: + - 38 + - 39 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -131,15 +142,15 @@ jobs: - name: Download generated ISO uses: actions/download-artifact@v4 with: - name: ${{ env.IMAGE_NAME }}-${{ env.VERSION }}.iso + name: ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso - name: Verify ISO - run: checkisomd5 ${{ env.IMAGE_NAME }}-${{ env.VERSION }}.iso + run: checkisomd5 ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso - name: Run ISO checks run: | - mv ${{ env.IMAGE_NAME }}-${{ env.VERSION }}.iso deploy.iso - make test-iso + mv ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso deploy.iso + make test-iso VERSION=${{ matrix.version }} - name: Add Kickstart and Grub options to ISO run: | diff --git a/external/fedora-lorax-templates b/external/fedora-lorax-templates new file mode 160000 index 0000000..cc11553 --- /dev/null +++ b/external/fedora-lorax-templates @@ -0,0 +1 @@ +Subproject commit cc1155372046baa58f9d2cc27a9e5473bf05a3fb From 54d89036fb481af6a26106e1fc760a7ace93bfd3 Mon Sep 17 00:00:00 2001 From: Noel Miller <4983138+noelmiller@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:05:20 -0600 Subject: [PATCH 06/20] Fix Container Copying to Directory (#41) --- .github/workflows/build-and-test.yml | 32 +++++++----- Containerfile | 1 + Makefile | 4 +- README.md | 64 +++++++++++++++++------- action.yml | 74 +++++++++++++++++++++------- entrypoint.sh | 3 +- 6 files changed, 127 insertions(+), 51 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 74801ed..ad1a839 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -18,7 +18,6 @@ env: SECURE_BOOT_KEY_URL: 'https://github.com/ublue-os/akmods/raw/main/certs/public_key.der' ENROLLMENT_PASSWORD: 'container-installer' - jobs: build-container: runs-on: ubuntu-latest @@ -47,7 +46,7 @@ jobs: with: containerfiles: Containerfile tags: ${{ steps.meta.outputs.tags }} - + - name: Push image uses: redhat-actions/push-to-registry@v2 with: @@ -56,7 +55,6 @@ jobs: username: ${{ github.actor }} password: ${{ github.token }} - build-and-push-iso: runs-on: ubuntu-latest needs: @@ -69,6 +67,9 @@ jobs: version: - 38 - 39 + outputs: + iso_name-38: ${{ steps.save_output.outputs.iso_name-38 }} + iso_name-39: ${{ steps.save_output.outputs.iso_name-39 }} steps: - name: Checkout repo uses: actions/checkout@v4 @@ -89,6 +90,7 @@ jobs: - name: Build ISO with new container uses: ./ + id: build with: arch: ${{ env.ARCH}} image_name: ${{ env.IMAGE_NAME}} @@ -98,17 +100,22 @@ jobs: variant: ${{ env.VARIANT }} secure_boot_key_url: ${{ env.SECURE_BOOT_KEY_URL }} enrollment_password: ${{ env.ENROLLMENT_PASSWORD }} + iso_name: ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso - - name: Rename ISO + - name: Save output + id: save_output + shell: bash run: | - mv build/deploy.iso build/${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso + echo "iso_name-${{ matrix.version }}=${{ steps.build.outputs.iso_name}}" >> $GITHUB_OUTPUT - name: Upload ISO as artifact id: upload uses: actions/upload-artifact@v4 with: - name: ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso - path: build/*.iso + name: ${{ steps.build.outputs.iso_name }} + path: | + ${{ steps.build.outputs.iso_path }} + ${{ steps.build.outputs.iso_path }}-CHECKSUM if-no-files-found: error retention-days: 0 compression-level: 0 @@ -133,7 +140,7 @@ jobs: - name: Ensure qemu is installed run: | sudo apt-get update - sudo apt-get install -y qemu qemu-utils xorriso unzip qemu-system-x86 netcat socat jq isomd5sum ansible make + sudo apt-get install -y qemu qemu-utils xorriso unzip qemu-system-x86 netcat socat jq isomd5sum ansible make coreutils - name: Create disk run: | @@ -142,14 +149,16 @@ jobs: - name: Download generated ISO uses: actions/download-artifact@v4 with: - name: ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso + name: ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }} - name: Verify ISO - run: checkisomd5 ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso + run: | + checkisomd5 ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }} + sha256sum -c ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }}-CHECKSUM - name: Run ISO checks run: | - mv ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso deploy.iso + mv ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }} deploy.iso make test-iso VERSION=${{ matrix.version }} - name: Add Kickstart and Grub options to ISO @@ -224,4 +233,3 @@ jobs: ./${check} fi done - diff --git a/Containerfile b/Containerfile index 6ca8af4..7488139 100644 --- a/Containerfile +++ b/Containerfile @@ -11,6 +11,7 @@ ENV VERSION="${VERSION}" ENV WEB_UI="false" RUN mkdir /build-container-installer + COPY / /build-container-installer/ WORKDIR /build-container-installer diff --git a/Makefile b/Makefile index c8f8c01..a21d1be 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ ifeq ($(WEB_UI),true) _LORAX_ARGS += -i anaconda-webui endif -# Step 7: Buid end ISO +# Step 7: Build end ISO ## Default action build/deploy.iso: boot.iso container/$(IMAGE_NAME)-$(IMAGE_TAG) xorriso/input.txt mkdir $(_BASE_DIR)/build || true @@ -177,7 +177,7 @@ clean: rm -f $(_BASE_DIR)/*.log || true install-deps: - dnf install -y lorax xorriso skopeo + dnf install -y lorax xorriso skopeo coreutils test-iso: $(eval _TESTS = $(filter-out README.md,$(shell ls tests/iso))) diff --git a/README.md b/README.md index 82217f9..cdc4119 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,28 @@ This action is designed to be called from a GitHub workflow using the following ```yaml - name: Build ISO uses: jasonn3/build-container-installer/v1.0.0 + id: build with: arch: ${{ env.ARCH}} image_name: ${{ env.IMAGE_NAME}} image_repo: ${{ env.IMAGE_REPO}} + image_tag: ${{ env.IMAGE_TAG }} version: ${{ env.VERSION }} variant: ${{ env.VARIANT }} + iso_name: ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ env.VERSION }}.iso + +# This example is for uploading your ISO as a Github artifact. You can do something similar using any cloud storage, so long as you copy the output +- name: Upload ISO as artifact + id: upload + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.build.outputs.iso_name }} + path: | + ${{ steps.build.outputs.iso_path }} + ${{ steps.build.outputs.iso_path }}-CHECKSUM + if-no-files-found: error + retention-days: 0 + compression-level: 0 ``` See [Customizing](#customizing) for information about customizing the ISO that gets created using `with` @@ -21,46 +37,61 @@ See [Customizing](#customizing) for information about customizing the ISO that g ## Customizing The following variables can be used to customize the created ISO. -| Variable | Description | Default Value | -| ----------------- | -------------------------------------------------------- | ------------------------------ | -| ARCH | Architecture for image to build | x86_64 | -| VERSION | Fedora version of installer to build | 39 | -| IMAGE_REPO | Repository containing the source container image | quay.io/fedora-ostree-desktops | -| IMAGE_NAME | Name of the source container image | base | -| IMAGE_TAG | Tag of the source container image | *VERSION* | -| EXTRA_BOOT_PARAMS | Extra params used by grub to boot the anaconda installer | \[empty\] | -| VARIANT | Source container variant\* | Server | -| WEB_UI | Enable Anaconda WebUI (experimental) | false | +### Inputs +| Variable | Description | Default Value | +| -------------------- | ---------------------------------------------------------------------------- | ------------------------------ | +| additional_templates | Space delimited list of additional Lorax templates to include | \[empty\] | +| arch | Architecture for image to build | x86_64 | +| enrollment_password | Used for supporting secure boot (requires SECURE_BOOT_KEY_URL to be defined) | container-installer | +| extra_boot_params | Extra params used by grub to boot the anaconda installer | \[empty\] | +| image_name | Name of the source container image | base | +| image_repo | Repository containing the source container image | quay.io/fedora-ostree-desktops | +| image_tag | Tag of the source container image | *VERSION* | +| iso_name | Name of the ISO you wish to output when completed | build/deploy.iso | +| secure_boot_key_url | Secure boot key that is installed from URL location\*\* | \[empty\] | +| variant | Source container variant\* | Server | +| version | Fedora version of installer to build | 39 | +| web_ui | Enable Anaconda WebUI (experimental) | false | -Available options for VARIANT can be found by running `dnf provides system-release`. +\*Available options for VARIANT can be found by running `dnf provides system-release`. Variant will be the third item in the package name. Example: `fedora-release-kinoite-39-34.noarch` will be kinoite +\*\* If you need to reference a local file, you can use `file://*path*` + +### Outputs +| Variable | Description | Usage | +| -------- | ----------------------------------------| ------------------------------------------------ | +| iso_name | The name of the resulting .iso | ${{ steps.YOUR_ID_FOR_ACTION.outputs.iso_name }} | +| iso_path | The name and path of the resulting .iso | ${{ steps.YOUR_ID_FOR_ACTION.outputs.iso_name }} | + +For outputs, see example above. + ## Development ### Makefile The Makefile contains all of the commands that are run in the action. There are separate targets for each file generated, however `make` can be used to generate the final image and `make clean` can be used to clean up the workspace. The resulting ISO will be stored in the `build` directory. `make install-deps` can be used to install the necessary packages -See [Customizing](#customizing) for information about customizing the ISO that gets created. +See [Customizing](#customizing) for information about customizing the ISO that gets created. All variable should be specified CAPITALIZED. ### Container A container with `make install-deps` already run is provided at `ghcr.io/jasonn3/build-container-installer:latest` To use the container file, run `docker run --privileged --volume .:/build-container-installer/build ghcr.io/jasonn3/build-container-installer:latest`. -This will create an ISO with the baked in defaults of the container image. +This will create an ISO with the baked in defaults of the container image. The resulting file will be called `deploy.iso` -See [Customizing](#customizing) for information about customizing the ISO that gets created. The variable can either be defined as environment variables. +See [Customizing](#customizing) for information about customizing the ISO that gets created. The variable can either be defined as environment variables. All variable should be specified CAPITALIZED. Examples: Building an ISO to install Fedora 38 ```bash -docker run --rm --privileged --volume .:/build-container-installer/build -e VERSION=38 -e IMAGE_NAME=base -e IMAGE_TAG=38 -e VARIANT=Server ghcr.io/jasonn3/build-container-installer:latest +docker run --rm --privileged --volume .:/github/workspace/build ghcr.io/jasonn3/build-container-installer:latest VERSION=38 IMAGE_NAME=base IMAGE_TAG=38 VARIANT=Server ``` Building an ISO to install Fedora 39 ```bash -docker run --rm --privileged --volume .:/build-container-installer/build -e VERSION=39 -e IMAGE_NAME=base -e IMAGE_TAG=39 -e VARIANT=Server ghcr.io/jasonn3/build-container-installer:latest +docker run --rm --privileged --volume .:/github/workspace/build ghcr.io/jasonn3/build-container-installer:latest VERSION=39 IMAGE_NAME=base IMAGE_TAG=39 VARIANT=Server ``` ### VSCode Dev Container @@ -105,4 +136,3 @@ Build a new container image: "privileged": true } ``` - diff --git a/action.yml b/action.yml index 22914f1..75c7fb5 100644 --- a/action.yml +++ b/action.yml @@ -2,10 +2,24 @@ name: Build Container Installer description: Generates an ISO for installing an OSTree stored in a container image inputs: + action_version: + description: Version of the action container to run + deprecationMessage: No longer used. github.action_ref replaces the need for this. Will be removed in a future version. + required: false + additional_templates: + description: Space delimited list of additional Lorax templates to include + required: false arch: description: Architecture for image to build required: true default: x86_64 + enrollment_password: + description: Used for supporting secure boot (requires secure_boot_key_url to be defined) + required: false + default: "container-installer" + extra_boot_params: + description: Extra params used by grub to boot the anaconda installer + required: false image_name: description: Name of the source container image required: true @@ -14,6 +28,16 @@ inputs: description: Repository containing the source container image required: true default: quay.io/fedora-ostree-desktops + image_tag: + description: Tag of the source container image. Defaults to the installer version + required: false + iso_name: + description: "Name of the resulting ISO. Relative paths are relative to github.workspace" + required: false + default: build/deploy.iso + secure_boot_key_url: + description: Secure boot key that is installed from URL location + required: false variant: description: "Source container variant. Available options can be found by running `dnf provides system-release`. Variant will be the third item in the package name. Example: `fedora-release-kinoite-39-34.noarch` will be kinonite" required: true @@ -22,27 +46,18 @@ inputs: description: Fedora version of installer to build required: true default: "39" - image_tag: - description: Tag of the source container image. Defaults to the installer version - required: false web_ui: description: Enable Anaconda WebUI required: false default: "false" - enrollment_password: - description: Used for supporting secure boot (requires SECURE_BOOT_KEY_URL to be defined) - required: false - default: "container-installer" - secure_boot_key_url: - description: Secure boot key that is installed from URL location - required: false - action_version: - description: Version of the action container to run - deprecationMessage: No longer used. github.action_ref replaces the need for this. Will be removed in a future version. - required: false - additional_templates: - description: Space delimetered list of additional Lorax templates to include - required: false + +outputs: + iso_name: + value: ${{ steps.rename_iso.outputs.iso_name }} + description: The name of the resulting .iso + iso_path: + value: ${{ steps.rename_iso.outputs.iso_path }} + description: The name and path of the resulting .iso runs: using: composite @@ -53,7 +68,7 @@ runs: # Check if running inside of the action repo if [[ -z "${{ github.action_ref }}" ]]; then if [[ "${{ github.ref_name }}" =~ (.*)/merge ]]; then tag=pr-${BASH_REMATCH[1]}; else tag=${{ github.ref_name }}; fi; fi if [[ -z "${tag}" ]]; then tag=${{ github.action_ref }}; fi - docker run --privileged --volume ${{ github.workspace }}:/github/workspace/ ghcr.io/jasonn3/build-container-installer:${tag} \ + docker run --privileged --volume ${{ github.workspace }}:/github/workspace ghcr.io/jasonn3/build-container-installer:${tag} \ ARCH=${{ inputs.arch }} \ IMAGE_NAME=${{ inputs.image_name }} \ IMAGE_REPO=${{ inputs.image_repo }} \ @@ -61,7 +76,30 @@ runs: VERSION=${{ inputs.version }} \ IMAGE_TAG=${{ inputs.image_tag || inputs.version }} \ WEB_UI=${{ inputs.web_ui }} \ + ISO_NAME=${{ inputs.iso_name }} \ ENROLLMENT_PASSWORD=${{ inputs.enrollment_password }} \ SECURE_BOOT_KEY_URL=${{ inputs.secure_boot_key_url }} \ "ADDITIONAL_TEMPLATES=${{ inputs.additional_templates }}" + - name: Rename ISO file + id: rename_iso + shell: bash + run: | + if [[ ! ( "${{ inputs.iso_name }}" =~ \.iso$ ) ]] + then + iso_name="${{ inputs.iso_name }}.iso" + else + iso_name="${{ inputs.iso_name }}" + fi + if [[ "${{ inputs.iso_name }}" =~ ^/ ]] + then + full_path="${iso_name}" + else + full_path="${{ github.workspace }}/${iso_name}" + fi + mv ${{ github.workspace }}/build/deploy.iso ${full_path} || true + cd $(dirname ${full_path}) + iso_fn=$(basename ${iso_name}) + sha256sum ${iso_fn} > ${iso_fn}-CHECKSUM + echo "iso_path=${full_path}" >> $GITHUB_OUTPUT + echo "iso_name=${iso_fn}" >> $GITHUB_OUTPUT diff --git a/entrypoint.sh b/entrypoint.sh index 5005737..3f78761 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -21,5 +21,4 @@ mkdir /github/workspace/build || true # Copy resulting iso to github workspace and fix permissions cp build/deploy.iso /github/workspace/build -chmod -R ugo=rwX /github/workspace/build - +chmod -R ugo=rwx /github/workspace/build From bfa150ceebf096e1e23c652a1e437a98ed4f6085 Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Sun, 10 Mar 2024 23:39:22 -0400 Subject: [PATCH 07/20] Add flatpaks (#32) --- .github/workflows/build-and-test.yml | 43 +++++++++++----- .gitmodules | 4 ++ Makefile | 49 ++++++++++++++++--- README.md | 2 +- action.yml | 26 +++++++--- entrypoint.sh | 15 ++++-- .../scripts/post/configure_flatpak | 11 +++++ lorax_templates/set_flatpak_repo.tmpl.in | 4 ++ tests/iso/os-release.sh | 17 ------- tests/iso/verify_flatpakrepo_updated.sh | 37 ++++++++++++++ tests/vm/flatpak.yml | 10 ++++ 11 files changed, 169 insertions(+), 49 deletions(-) create mode 100644 .gitmodules create mode 100644 lorax_templates/scripts/post/configure_flatpak create mode 100644 lorax_templates/set_flatpak_repo.tmpl.in create mode 100644 tests/iso/verify_flatpakrepo_updated.sh create mode 100644 tests/vm/flatpak.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ad1a839..e82b395 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -15,6 +15,7 @@ env: IMAGE_REPO: 'quay.io/fedora-ostree-desktops' IMAGE_TAG: '39' VARIANT: 'Server' + FLATPAK_REMOTE_REFS: app/org.videolan.VLC/x86_64/stable runtime/org.kde.Platform/x86_64/5.15-23.08 SECURE_BOOT_KEY_URL: 'https://github.com/ublue-os/akmods/raw/main/certs/public_key.der' ENROLLMENT_PASSWORD: 'container-installer' @@ -27,6 +28,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v4 + with: + submodules: recursive - name: Docker meta id: meta @@ -62,7 +65,9 @@ jobs: permissions: contents: read packages: write + continue-on-error: false strategy: + fail-fast: false matrix: version: - 38 @@ -98,6 +103,7 @@ jobs: image_tag: ${{ env.IMAGE_TAG }} version: ${{ matrix.version }} variant: ${{ env.VARIANT }} + flatpak_remote_refs: ${{ env.FLATPAK_REMOTE_REFS }} secure_boot_key_url: ${{ env.SECURE_BOOT_KEY_URL }} enrollment_password: ${{ env.ENROLLMENT_PASSWORD }} iso_name: ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso @@ -128,7 +134,9 @@ jobs: permissions: contents: read packages: write + continue-on-error: false strategy: + fail-fast: false matrix: version: - 38 @@ -136,6 +144,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v4 + with: + submodules: recursive - name: Ensure qemu is installed run: | @@ -159,7 +169,7 @@ jobs: - name: Run ISO checks run: | mv ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }} deploy.iso - make test-iso VERSION=${{ matrix.version }} + make test-iso VERSION=${{ matrix.version }} FLATPAK_REMOTE_REFS="${{ env.FLATPAK_REMOTE_REFS}}" - name: Add Kickstart and Grub options to ISO run: | @@ -203,11 +213,15 @@ jobs: env: VM_USER: core VM_PASS: foobar + VM_IP: "127.0.0.1" + VM_PORT: "5555" run: | mkfifo vm.stdin qemu-system-x86_64 -name "Anaconda" \ -m 4096 -cpu qemu64 -display none -smp 2 \ -chardev socket,path=/tmp/qga.sock,server,nowait,id=qga0 \ + -device e1000,netdev=net0 \ + -netdev user,id=net0,hostfwd=tcp::${VM_PORT}-:22 \ -device virtio-serial \ -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \ -boot c -hda disk.qcow2 -serial telnet:localhost:4321,server,nowait & export QEMU_PID=$! @@ -218,18 +232,23 @@ jobs: timeout 30m bash -c "while ! (grep 'login:' vm.stdout); do sleep 1; done" - export VM_IP=$({ echo '{"execute": "guest-network-get-interfaces"}'; sleep 2; } | socat unix-connect:/tmp/qga.sock - | jq -r '.return[1]["ip-addresses"][] | select(."ip-address-type"=="ipv4") | .["ip-address"]') - if ! (echo > /dev/tcp/${VM_IP}/22) + if ! (echo > /dev/tcp/${VM_IP}/${VM_PORT}) then echo "SSH must be installed and enabled inside the container" fi - echo "VM ready for tests at IP ${VM_IP}" - for check in $(ls tests/vm) - do - if [[ -f $check && $check != "README.md" ]] - then - chmod +x $check - ./${check} - fi - done + echo "VM ready for tests at IP ${VM_IP}:${VM_PORT}" + echo "Creating Ansible inventory" + cat << EOF > ansible_inventory + ungrouped: + hosts: + vm: + ansible_host: ${VM_IP} + ansible_port: ${VM_PORT} + ansible_user: ${VM_USER} + ansible_password: ${VM_PASS} + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + EOF + + #make test-vm + kill $QEMU_PID diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..fe51240 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "external/fedora-lorax-templates"] + path = external/fedora-lorax-templates + url = https://pagure.io/fedora-lorax-templates.git + branch = f39 diff --git a/Makefile b/Makefile index a21d1be..4849aca 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,10 @@ IMAGE_TAG = $(VERSION) VARIANT = Server WEB_UI = false REPOS = /etc/yum.repos.d/fedora.repo /etc/yum.repos.d/fedora-updates.repo +ADDITIONAL_TEMPLATES = +FLATPAK_REMOTE_NAME = flathub +FLATPAK_REMOTE_URL = https://flathub.org/repo/flathub.flatpakrepo +FLATPAK_REMOTE_REFS = ENROLLMENT_PASSWORD = SECURE_BOOT_KEY_URL = ADDITIONAL_TEMPLATES = @@ -22,7 +26,10 @@ _IMAGE_REPO_DOUBLE_ESCAPED = $(subst \,\\\,$(_IMAGE_REPO_ESCAPED)) _VOLID = $(firstword $(subst -, ,$(IMAGE_NAME)))-$(ARCH)-$(IMAGE_TAG) _REPO_FILES = $(subst /etc/yum.repos.d,repos,$(REPOS)) _LORAX_TEMPLATES = $(subst .in,,$(shell ls lorax_templates/*.tmpl.in)) $(foreach file,$(shell ls lorax_templates/scripts/post),lorax_templates/post_$(file).tmpl) -_TEMPLATE_VARS = ARCH VERSION IMAGE_REPO IMAGE_NAME IMAGE_TAG VARIANT WEB_UI REPOS _IMAGE_REPO_ESCAPED _IMAGE_REPO_DOUBLE_ESCAPED ENROLLMENT_PASSWORD +_EXTERNAL_TEMPLATES = fedora-lorax-templates/ostree-based-installer/lorax-embed-flatpaks.tmpl +_FLATPAK_REPO_URL = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^URL=' | cut -d= -f2) +_FLATPAK_REPO_GPG = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^GPGKey=' | cut -d= -f2) +_TEMPLATE_VARS = ARCH VERSION IMAGE_REPO IMAGE_NAME IMAGE_TAG VARIANT WEB_UI REPOS _IMAGE_REPO_ESCAPED _IMAGE_REPO_DOUBLE_ESCAPED FLATPAK_REMOTE_NAME FLATPAK_REMOTE_URL FLATPAK_REMOTE_REFS _FLATPAK_REPO_URL _FLATPAK_REPO_GPG ENROLLMENT_PASSWORD ifeq ($(VARIANT),Server) _LORAX_ARGS = --macboot --noupgrade @@ -34,6 +41,10 @@ ifeq ($(WEB_UI),true) _LORAX_ARGS += -i anaconda-webui endif +ifneq ($(FLATPAK_REMOTE_REFS),) +_LORAX_ARGS += -i flatpak-libs +endif + # Step 7: Build end ISO ## Default action build/deploy.iso: boot.iso container/$(IMAGE_NAME)-$(IMAGE_TAG) xorriso/input.txt @@ -84,7 +95,6 @@ lorax_templates/post_%.tmpl: lorax_templates/scripts/post/% do \ if [[ $$line =~ ^\<\% ]]; \ then \ - echo $$line >> lorax_templates/post_$*.tmpl; \ echo >> lorax_templates/post_$*.tmpl; \ else \ if [[ $$header == 0 ]]; \ @@ -111,7 +121,6 @@ lorax_templates/%.tmpl: lorax_templates/%.tmpl.in $(eval _VARS = IMAGE_NAME IMAGE_TAG _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED) $(foreach var,$(_VARS),$(var)=$($(var))) envsubst '$(foreach var,$(_VARS),$$$(var))' < $(_BASE_DIR)/lorax_templates/$*.tmpl.in > $(_BASE_DIR)/lorax_templates/$*.tmpl - # Step 2: Replace vars in repo files repos/%.repo: /etc/yum.repos.d/%.repo mkdir repos || true @@ -139,6 +148,8 @@ boot.iso: $(_LORAX_TEMPLATES) $(_REPO_FILES) $(foreach file,$(_REPO_FILES),--repo $(_BASE_DIR)/$(file)) \ $(foreach file,$(_LORAX_TEMPLATES),--add-template $(_BASE_DIR)/$(file)) \ $(foreach file,$(ADDITIONAL_TEMPLATES),--add-template $(file)) \ + $(foreach file,$(_FLATPAK_TEMPLATES),--add-template $(file)) \ + $(foreach file,$(_EXTERNAL_TEMPLATES),--add-template $(_BASE_DIR)/external/$(file)) \ --rootfs-size $(ROOTFS_SIZE) \ $(foreach var,$(_TEMPLATE_VARS),--add-template-var "$(shell echo $(var) | tr '[:upper:]' '[:lower:]')=$($(var))") \ $(_BASE_DIR)/results/ @@ -177,12 +188,34 @@ clean: rm -f $(_BASE_DIR)/*.log || true install-deps: - dnf install -y lorax xorriso skopeo coreutils + dnf install -y lorax xorriso skopeo flatpak dbus-daemon ostree coreutils + +test: test-iso test-vm test-iso: $(eval _TESTS = $(filter-out README.md,$(shell ls tests/iso))) - $(foreach test,$(_TESTS),chmod +x tests/iso/$(test)) - $(foreach test,$(_TESTS),./tests/iso/$(test) deploy.iso) - -.PHONY: clean install-deps + $(eval _VARS = VERSION FLATPAK_REMOTE_NAME _FLATPAK_REPO_URL) + sudo apt-get update + sudo apt-get install -y squashfs-tools + sudo modprobe loop + sudo mkdir /mnt/iso /mnt/install + sudo mount -o loop deploy.iso /mnt/iso + sudo mount -t squashfs -o loop /mnt/iso/images/install.img /mnt/install + + chmod +x $(foreach test,$(_TESTS),tests/iso/$(test)) + for test in $(_TESTS); \ + do \ + $(foreach var,$(_VARS),$(var)=$($(var))) ./tests/iso/$${test}; \ + done + + # Cleanup + sudo umount /mnt/install + sudo umount /mnt/iso + +test-vm: + $(eval _TESTS = $(filter-out README.md,$(shell ls tests/vm))) + chmod +x $(foreach test,$(_TESTS),tests/vm/$(test)) + for test in $(_TESTS); do ./tests/vm/$${test} deploy.iso; done + +.PHONY: clean install-deps test test-iso test-vm diff --git a/README.md b/README.md index cdc4119..b50912f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![Build status](https://github.com/jasonn3/build-container-installer/actions/workflows/build-and-test.yml/badge.svg??event=push) +![Build status](https://github.com/jasonn3/build-container-installer/actions/workflows/build-and-test.yml/badge.svg?event=push) # Build Container Installer Action This action is used to enerate an ISO for installing an OSTree stored in a container image. This utilizes the anaconda command `ostreecontainer` diff --git a/action.yml b/action.yml index 75c7fb5..d4e7861 100644 --- a/action.yml +++ b/action.yml @@ -20,6 +20,18 @@ inputs: extra_boot_params: description: Extra params used by grub to boot the anaconda installer required: false + flatpak_remote_name: + description: Name of the Flatpak remote repo + required: false + default: "flathub" + flatpak_remote_refs: + description: Space delimited list of refs to the flatpak packages to install + required: false + default: "" + flatpak_remote_url: + description: The URL of the Flatpak remote flatpakrepo file + required: false + default: https://flathub.org/repo/flathub.flatpakrepo image_name: description: Name of the source container image required: true @@ -69,17 +81,19 @@ runs: if [[ -z "${{ github.action_ref }}" ]]; then if [[ "${{ github.ref_name }}" =~ (.*)/merge ]]; then tag=pr-${BASH_REMATCH[1]}; else tag=${{ github.ref_name }}; fi; fi if [[ -z "${tag}" ]]; then tag=${{ github.action_ref }}; fi docker run --privileged --volume ${{ github.workspace }}:/github/workspace ghcr.io/jasonn3/build-container-installer:${tag} \ + ADDITIONAL_TEMPLATES="${{ inputs.additional_templates }}" \ ARCH=${{ inputs.arch }} \ + ENROLLMENT_PASSWORD=${{ inputs.enrollment_password }} \ + FLATPAK_REMOTE_NAME=${{ inputs.flatpak_remote_name }} \ + FLATPAK_REMOTE_REFS="${{ inputs.flatpak_remote_refs }}" \ + FLATPAK_REMOTE_URL=${{ inputs.flatpak_remote_url }} \ IMAGE_NAME=${{ inputs.image_name }} \ IMAGE_REPO=${{ inputs.image_repo }} \ + IMAGE_TAG=${{ inputs.image_tag || inputs.version }} \ + SECURE_BOOT_KEY_URL=${{ inputs.secure_boot_key_url }} \ VARIANT=${{ inputs.variant }} \ VERSION=${{ inputs.version }} \ - IMAGE_TAG=${{ inputs.image_tag || inputs.version }} \ - WEB_UI=${{ inputs.web_ui }} \ - ISO_NAME=${{ inputs.iso_name }} \ - ENROLLMENT_PASSWORD=${{ inputs.enrollment_password }} \ - SECURE_BOOT_KEY_URL=${{ inputs.secure_boot_key_url }} \ - "ADDITIONAL_TEMPLATES=${{ inputs.additional_templates }}" + WEB_UI=${{ inputs.web_ui }} - name: Rename ISO file id: rename_iso diff --git a/entrypoint.sh b/entrypoint.sh index 3f78761..c32743b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,19 +2,24 @@ set -ex -for entry in $@ +# Create /dev/loop0 if it doesn't already exist. `losetup` has an issue creating it during the first run +mknod -m 0660 /dev/loop0 b 7 0 2>/dev/null || true + +for i do - export $entry + key=$(echo ${i} | cut -d= -f1) + value=$(echo ${i} | cut -d= -f2-) + export ${key}="${value}" done # Pull container -make container/${IMAGE_NAME}-${IMAGE_TAG} $@ +make container/${IMAGE_NAME}-${IMAGE_TAG} "$@" # Build base ISO -make boot.iso $@ +make boot.iso "$@" # Add container to ISO -make build/deploy.iso $@ +make build/deploy.iso "$@" # Make output dir in github workspace mkdir /github/workspace/build || true diff --git a/lorax_templates/scripts/post/configure_flatpak b/lorax_templates/scripts/post/configure_flatpak new file mode 100644 index 0000000..1c54a7f --- /dev/null +++ b/lorax_templates/scripts/post/configure_flatpak @@ -0,0 +1,11 @@ +<%page args="_flatpak_repo_gpg"/> + +if [[ -d /ostree/deploy/default/var/lib/flatpak/repo ]] +then + echo ${_flatpak_repo_gpg} | base64 -d > /ostree/deploy/default/var/lib/flatpak/repo/flathub.trustedkeys.gpg +elif [[ -d /var/lib/flatpak/repo ]] +then + echo ${_flatpak_repo_gpg} | base64 -d > /var/lib/flatpak/repo/flathub.trustedkeys.gpg +else + echo "Could not find Flatpaks repo" +fi diff --git a/lorax_templates/set_flatpak_repo.tmpl.in b/lorax_templates/set_flatpak_repo.tmpl.in new file mode 100644 index 0000000..622e9a1 --- /dev/null +++ b/lorax_templates/set_flatpak_repo.tmpl.in @@ -0,0 +1,4 @@ +<%page args="flatpak_remote_name, _flatpak_repo_url"/> + +replace "flatpak_manager\.add_remote\(\".*\", \".*\"\)" "flatpak_manager.add_remote(\"${flatpak_remote_name}\", \"${_flatpak_repo_url}\")" /usr/lib64/python*/site-packages/pyanaconda/modules/payloads/payload/rpm_ostree/flatpak_installation.py +replace "flatpak_manager\.replace_installed_refs_remote\(\".*\"\)" "flatpak_manager.replace_installed_refs_remote(\"${flatpak_remote_name}\")" /usr/lib64/python*/site-packages/pyanaconda/modules/payloads/payload/rpm_ostree/flatpak_installation.py diff --git a/tests/iso/os-release.sh b/tests/iso/os-release.sh index ea066bc..ccef46c 100644 --- a/tests/iso/os-release.sh +++ b/tests/iso/os-release.sh @@ -1,24 +1,7 @@ #!/bin/bash -iso=$1 - -sudo apt-get update -sudo apt-get install -y squashfs-tools - -sudo mkdir /mnt/{iso,install} - -# Mount ISO -sudo mount -o loop $iso /mnt/iso - -# Mount squashfs -sudo mount -t squashfs -o loop /mnt/iso/images/install.img /mnt/install - FOUND_VERSION=$(cat /mnt/install/etc/os-release | grep VERSION_ID | cut -d= -f2) -# Cleanup -sudo umount /mnt/install -sudo umount /mnt/iso - if [[ ${FOUND_VERSION} != ${VERSION} ]] then echo "Version mismatch" diff --git a/tests/iso/verify_flatpakrepo_updated.sh b/tests/iso/verify_flatpakrepo_updated.sh new file mode 100644 index 0000000..b5ee240 --- /dev/null +++ b/tests/iso/verify_flatpakrepo_updated.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +add_line=$(grep flatpak_manager.add_remote /mnt/install/usr/lib64/python*/site-packages/pyanaconda/modules/payloads/payload/rpm_ostree/flatpak_installation.py) + +add_line_repo=$(echo ${add_line} | grep ${FLATPAK_REMOTE_NAME}) +add_line_url=$(echo ${add_line} | grep ${_FLATPAK_REPO_URL}) + +result=0 +if [ -z "${add_line_repo}" ] +then + echo "Repo name not updated on add_remote line" + result=1 +else + echo "Repo name found on add_remote line" +fi + +if [ -z "${add_line_url}" ] +then + echo "Repo url not updated on add_remote line" + result=1 +else + echo "Repo url found on add_remote line" +fi + +replace_line=$(grep flatpak_manager.replace_installed_refs_remote /mnt/install/usr/lib64/python*/site-packages/pyanaconda/modules/payloads/payload/rpm_ostree/flatpak_installation.py) + +replace_line_repo=$(echo ${replace_line} | grep ${FLATPAK_REMOTE_NAME}) + +if [ -z "${replace_line_repo}" ] +then + echo "Repo name not updated on replace_installed_refs line" + result=1 +else + echo "Repo name found on replace_installed_refs line" +fi + +exit ${result} \ No newline at end of file diff --git a/tests/vm/flatpak.yml b/tests/vm/flatpak.yml new file mode 100644 index 0000000..6f46263 --- /dev/null +++ b/tests/vm/flatpak.yml @@ -0,0 +1,10 @@ +#!/usr/bin/env -S ansible-playbook -i ./ansible_inventory +--- +- name: Test for flatpaks + hosts: vm + gather_facts: no + + tasks: + - name: Test updating flatpak packages + ansible.builtin.command: + cmd: /usr/bin/flatpak update -y From c3dfff5c5b3fdf473a1813c6d5be15cd459b700d Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:29:45 -0400 Subject: [PATCH 08/20] Allow caching of dnf (#46) --- .gitignore | 1 - Containerfile | 5 +- Makefile | 76 ++++++++++----- README.md | 2 +- action.yml | 93 ++++++++++++++++++- entrypoint.sh | 10 ++ lorax_templates/cache_copy_dnf.tmpl | 3 + ...pak_repo.tmpl.in => flatpak_set_repo.tmpl} | 0 lorax_templates/install_set_installer.tmpl | 4 + .../{configure_flatpak => flatpak_configure} | 0 ...re_upgrades => install_configure_upgrades} | 0 ...l-secureboot-key => secureboot_enroll_key} | 2 +- lorax_templates/set_installer.tmpl.in | 2 - 13 files changed, 166 insertions(+), 32 deletions(-) create mode 100644 lorax_templates/cache_copy_dnf.tmpl rename lorax_templates/{set_flatpak_repo.tmpl.in => flatpak_set_repo.tmpl} (100%) create mode 100644 lorax_templates/install_set_installer.tmpl rename lorax_templates/scripts/post/{configure_flatpak => flatpak_configure} (100%) rename lorax_templates/scripts/post/{configure_upgrades => install_configure_upgrades} (100%) rename lorax_templates/scripts/post/{enroll-secureboot-key => secureboot_enroll_key} (94%) delete mode 100644 lorax_templates/set_installer.tmpl.in diff --git a/.gitignore b/.gitignore index 8f65d56..ff22e19 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ /pkglists /repos /results -/lorax_templates/*.tmpl /xorriso/input.txt /xorriso/*.sh /original-pkgsizes.txt diff --git a/Containerfile b/Containerfile index 7488139..f0b2174 100644 --- a/Containerfile +++ b/Containerfile @@ -15,10 +15,11 @@ RUN mkdir /build-container-installer COPY / /build-container-installer/ WORKDIR /build-container-installer +VOLUME /build-container-installer/build +VOLUME /build-container-installer/repos +VOLUME /cache RUN dnf install -y make && make install-deps -VOLUME /build-container-installer/build - ENTRYPOINT ["/bin/bash", "/build-container-installer/entrypoint.sh"] diff --git a/Makefile b/Makefile index 4849aca..0c46523 100644 --- a/Makefile +++ b/Makefile @@ -1,48 +1,77 @@ # Configuration vars ## Formatting = UPPERCASE +# General +ADDITIONAL_TEMPLATES = ARCH = x86_64 -VERSION = 39 -IMAGE_REPO = quay.io/fedora-ostree-desktops +EXTRA_BOOT_PARAMS = IMAGE_NAME = base +IMAGE_REPO = quay.io/fedora-ostree-desktops IMAGE_TAG = $(VERSION) +REPOS = $(subst :,\:,$(shell ls /etc/yum.repos.d/*.repo)) +ROOTFS_SIZE = 4 VARIANT = Server +VERSION = 39 WEB_UI = false -REPOS = /etc/yum.repos.d/fedora.repo /etc/yum.repos.d/fedora-updates.repo -ADDITIONAL_TEMPLATES = +# Flatpak FLATPAK_REMOTE_NAME = flathub FLATPAK_REMOTE_URL = https://flathub.org/repo/flathub.flatpakrepo FLATPAK_REMOTE_REFS = +# Secure boot ENROLLMENT_PASSWORD = SECURE_BOOT_KEY_URL = -ADDITIONAL_TEMPLATES = -EXTRA_BOOT_PARAMS = -ROOTFS_SIZE = 4 +# Cache +DNF_CACHE = -# Generated vars +# Generated/internal vars ## Formatting = _UPPERCASE _BASE_DIR = $(shell pwd) _IMAGE_REPO_ESCAPED = $(subst /,\/,$(IMAGE_REPO)) _IMAGE_REPO_DOUBLE_ESCAPED = $(subst \,\\\,$(_IMAGE_REPO_ESCAPED)) _VOLID = $(firstword $(subst -, ,$(IMAGE_NAME)))-$(ARCH)-$(IMAGE_TAG) _REPO_FILES = $(subst /etc/yum.repos.d,repos,$(REPOS)) -_LORAX_TEMPLATES = $(subst .in,,$(shell ls lorax_templates/*.tmpl.in)) $(foreach file,$(shell ls lorax_templates/scripts/post),lorax_templates/post_$(file).tmpl) -_EXTERNAL_TEMPLATES = fedora-lorax-templates/ostree-based-installer/lorax-embed-flatpaks.tmpl +_LORAX_TEMPLATES = $(shell ls lorax_templates/install_*.tmpl) $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/install_*)),lorax_templates/post_$(file).tmpl) +_LORAX_TEMPLATES_FLATPAKS = $(shell ls lorax_templates/flatpak_*.tmpl) $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/flatpak_*)),lorax_templates/post_$(file).tmpl) external/fedora-lorax-templates/ostree-based-installer/lorax-embed-flatpaks.tmpl +_LORAX_TEMPLATES_SECUREBOOT = $(shell ls lorax_templates/secureboot_*.tmpl) $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/secureboot_*)),lorax_templates/post_$(file).tmpl) +_LORAX_TEMPLATES_CACHE = $(shell ls lorax_templates/cache_*.tmpl) $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/cache_*)),lorax_templates/post_$(file).tmpl) +_LORAX_ARGS = _FLATPAK_REPO_URL = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^URL=' | cut -d= -f2) _FLATPAK_REPO_GPG = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^GPGKey=' | cut -d= -f2) -_TEMPLATE_VARS = ARCH VERSION IMAGE_REPO IMAGE_NAME IMAGE_TAG VARIANT WEB_UI REPOS _IMAGE_REPO_ESCAPED _IMAGE_REPO_DOUBLE_ESCAPED FLATPAK_REMOTE_NAME FLATPAK_REMOTE_URL FLATPAK_REMOTE_REFS _FLATPAK_REPO_URL _FLATPAK_REPO_GPG ENROLLMENT_PASSWORD +_TEMPLATE_VARS = ARCH IMAGE_NAME IMAGE_REPO _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED IMAGE_TAG REPOS VARIANT VERSION WEB_UI -ifeq ($(VARIANT),Server) -_LORAX_ARGS = --macboot --noupgrade + +ifeq ($(findstring redhat.repo,$(REPOS)),redhat.repo) +_LORAX_ARGS += --nomacboot --noupgrade +else ifeq ($(VARIANT),Server) +_LORAX_ARGS += --macboot --noupgrade else -_LORAX_ARGS = --nomacboot +_LORAX_ARGS += --nomacboot endif ifeq ($(WEB_UI),true) _LORAX_ARGS += -i anaconda-webui endif +ifneq ($(DNF_CACHE),) +_LORAX_ARGS += --cachedir $(DNF_CACHE) +_LORAX_TEMPLATES += $(_LORAX_TEMPLATES_CACHE) +_TEMPLATE_VARS += DNF_CACHE +endif + +ifeq ($(findstring redhat.repo,$(REPOS)),redhat.repo) +_PLATFORM_ID = platform:el$(VERSION) +else +_PLATFORM_ID = platform:f$(VERSION) +endif + ifneq ($(FLATPAK_REMOTE_REFS),) -_LORAX_ARGS += -i flatpak-libs +_LORAX_ARGS += -i flatpak-libs +_LORAX_TEMPLATES += $(_LORAX_TEMPLATES_FLATPAKS) +_TEMPLATE_VARS += FLATPAK_REMOTE_NAME FLATPAK_REMOTE_REFS FLATPAK_REMOTE_URL _FLATPAK_REPO_GPG _FLATPAK_REPO_URL +endif + +ifneq ($(SECURE_BOOT_KEY_URL),) +_LORAX_TEMPLATES += $(_LORAX_TEMPLATES_SECUREBOOT) +_TEMPLATE_VARS += ENROLLMENT_PASSWORD endif # Step 7: Build end ISO @@ -117,9 +146,8 @@ lorax_templates/post_%.tmpl: lorax_templates/scripts/post/% done < lorax_templates/scripts/post/$* echo "append $(_ISO_FILE) \"%end\"" >> lorax_templates/post_$*.tmpl -lorax_templates/%.tmpl: lorax_templates/%.tmpl.in - $(eval _VARS = IMAGE_NAME IMAGE_TAG _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED) - $(foreach var,$(_VARS),$(var)=$($(var))) envsubst '$(foreach var,$(_VARS),$$$(var))' < $(_BASE_DIR)/lorax_templates/$*.tmpl.in > $(_BASE_DIR)/lorax_templates/$*.tmpl + +repos: $(_REPO_FILES) # Step 2: Replace vars in repo files repos/%.repo: /etc/yum.repos.d/%.repo @@ -132,9 +160,11 @@ repos/%.repo: /etc/yum.repos.d/%.repo %.repo: # Step 3: Build boot.iso using Lorax -boot.iso: $(_LORAX_TEMPLATES) $(_REPO_FILES) +boot.iso: $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES) rm -Rf $(_BASE_DIR)/results || true - rm /etc/rpm/macros.image-language-conf || true + mv /etc/rpm/macros.image-language-conf /etc/rpm/macros.image-language-conf.orig || true + cp /etc/os-release /etc/os-release.orig || true + sed -i 's/PLATFORM_ID=.*/PLATFORM_ID="$(_PLATFORM_ID)"/' /etc/os-release # Download the secure boot key if [ -n "$(SECURE_BOOT_KEY_URL)" ]; \ @@ -154,6 +184,8 @@ boot.iso: $(_LORAX_TEMPLATES) $(_REPO_FILES) $(foreach var,$(_TEMPLATE_VARS),--add-template-var "$(shell echo $(var) | tr '[:upper:]' '[:lower:]')=$($(var))") \ $(_BASE_DIR)/results/ mv $(_BASE_DIR)/results/images/boot.iso $(_BASE_DIR)/ + mv -f /etc/rpm/macros.image-language-conf.orig /etc/rpm/macros.image-language-conf || true + mv -f /etc/os-release.orig /etc/os-release || true # Step 4: Download container image container/$(IMAGE_NAME)-$(IMAGE_TAG): @@ -188,7 +220,7 @@ clean: rm -f $(_BASE_DIR)/*.log || true install-deps: - dnf install -y lorax xorriso skopeo flatpak dbus-daemon ostree coreutils + dnf install -y lorax xorriso skopeo flatpak dbus-daemon ostree coreutils gettext test: test-iso test-vm @@ -218,4 +250,4 @@ test-vm: chmod +x $(foreach test,$(_TESTS),tests/vm/$(test)) for test in $(_TESTS); do ./tests/vm/$${test} deploy.iso; done -.PHONY: clean install-deps test test-iso test-vm +.PHONY: clean install-deps test test-iso test-vm container/$(IMAGE_NAME)-$(IMAGE_TAG) diff --git a/README.md b/README.md index b50912f..54be4c1 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This action is used to enerate an ISO for installing an OSTree stored in a conta This action is designed to be called from a GitHub workflow using the following format ```yaml - name: Build ISO - uses: jasonn3/build-container-installer/v1.0.0 + uses: jasonn3/build-container-installer@main id: build with: arch: ${{ env.ARCH}} diff --git a/action.yml b/action.yml index d4e7861..bc6c0bf 100644 --- a/action.yml +++ b/action.yml @@ -13,6 +13,17 @@ inputs: description: Architecture for image to build required: true default: x86_64 + dnf_cache_key: + description: Overrides the dnf cache key + required: false + enable_cache_dnf: + description: Whether to enable caching for dnf + required: false + default: "true" + enable_cache_skopeo: + description: Whether to enable caching for skopeo + required: false + default: "true" enrollment_password: description: Used for supporting secure boot (requires secure_boot_key_url to be defined) required: false @@ -50,6 +61,9 @@ inputs: secure_boot_key_url: description: Secure boot key that is installed from URL location required: false + skopeo_cache_key: + description: Overrides the skopeo cache key + required: false variant: description: "Source container variant. Available options can be found by running `dnf provides system-release`. Variant will be the third item in the package name. Example: `fedora-release-kinoite-39-34.noarch` will be kinonite" required: true @@ -74,15 +88,69 @@ outputs: runs: using: composite steps: + - name: Make cache directory + shell: bash + run: | + sudo mkdir /cache + sudo chmod 777 /cache + + - name: Load dnf cache + id: load_dnf_cache + env: + dnf_cache_key: dnf-${{ inputs.version }} + if: inputs.enable_cache_dnf == 'true' + uses: actions/cache/restore@v4 + with: + path: /cache/dnf + key: ${{ inputs.dnf_cache_key || env.dnf_cache_key }} + + - name: Load skopeo cache + id: load_skopeo_cache + env: + skopeo_cache_key: skopeo-${{ inputs.image_name }}-${{ inputs.version || inputs.image_tag }} + if: inputs.enable_cache_skopeo == 'true' + uses: actions/cache/restore@v4 + with: + path: /cache/skopeo + key: ${{ inputs.skopeo_cache_key || env.skopeo_cache_key }} + + - name: Ensure cache directories exist + shell: bash + run: | + mkdir /cache/dnf || true + mkdir /cache/dnf_new || true + mkdir /cache/skopeo || true + - name: Run docker image shell: bash run: | # Check if running inside of the action repo - if [[ -z "${{ github.action_ref }}" ]]; then if [[ "${{ github.ref_name }}" =~ (.*)/merge ]]; then tag=pr-${BASH_REMATCH[1]}; else tag=${{ github.ref_name }}; fi; fi - if [[ -z "${tag}" ]]; then tag=${{ github.action_ref }}; fi - docker run --privileged --volume ${{ github.workspace }}:/github/workspace ghcr.io/jasonn3/build-container-installer:${tag} \ + if [[ -z "${{ github.action_repository }}" ]] + then + if [[ "${{ github.ref_name }}" =~ (.*)/merge ]] + then tag=pr-${BASH_REMATCH[1]} + else + tag=${{ github.ref_name }} + fi + else + tag=${{ github.action_ref }} + fi + if [[ "${{ inputs.enable_cache_dnf }}" == "true" ]] + then + cache="${cache} -v /cache/dnf:/cache/dnf" + fi + if [[ "${{ inputs.enable_cache_skopeo }}" == "true" ]] + then + cache="${cache} -v /cache/skopeo:/cache/skopeo" + fi + if [[ "${{ steps.load_dnf_cache.outputs.cache-hit }}" != "true" ]] + then + cache="${cache} -v /cache/dnf_new:/cache/dnf_new" + fi + docker run --privileged --volume ${{ github.workspace }}:/github/workspace/ ${cache} ghcr.io/jasonn3/build-container-installer:${tag} \ ADDITIONAL_TEMPLATES="${{ inputs.additional_templates }}" \ ARCH=${{ inputs.arch }} \ + DNF_CACHE=/cache/dnf \ ENROLLMENT_PASSWORD=${{ inputs.enrollment_password }} \ FLATPAK_REMOTE_NAME=${{ inputs.flatpak_remote_name }} \ FLATPAK_REMOTE_REFS="${{ inputs.flatpak_remote_refs }}" \ @@ -95,6 +163,25 @@ runs: VERSION=${{ inputs.version }} \ WEB_UI=${{ inputs.web_ui }} + - name: Save dnf cache + env: + dnf_cache_key: dnf-${{ inputs.version }} + if: inputs.enable_cache_dnf == 'true' && steps.load_dnf_cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: /cache/dnf_new + key: ${{ inputs.dnf_cache_key || env.dnf_cache_key }} + + - name: Save skopeo cache + env: + skopeo_cache_key: skopeo-${{ inputs.image_name }}-${{ inputs.version || inputs.image_tag }} + if: inputs.enable_cache_skopeo == 'true' && steps.load_dnf_cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: /cache/skopeo + key: ${{ inputs.skopeo_cache_key || env.skopeo_cache_key }} + + - name: Rename ISO file id: rename_iso shell: bash diff --git a/entrypoint.sh b/entrypoint.sh index c32743b..672d924 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -12,6 +12,16 @@ do export ${key}="${value}" done +if [[ -d /cache/skopeo ]] +then + ln -s /cache/skopeo /build-container-installer/container +fi + +if [[ ! -d /cache/dnf ]] +then + mkdir /cache/dnf +fi + # Pull container make container/${IMAGE_NAME}-${IMAGE_TAG} "$@" diff --git a/lorax_templates/cache_copy_dnf.tmpl b/lorax_templates/cache_copy_dnf.tmpl new file mode 100644 index 0000000..43d4354 --- /dev/null +++ b/lorax_templates/cache_copy_dnf.tmpl @@ -0,0 +1,3 @@ +<%page args="dnf_cache"/> + +runcmd bash -c "if [[ -e ${dnf_cache}_new ]]; then cp -R ${dnf_cache}/* ${dnf_cache}_new/; fi" \ No newline at end of file diff --git a/lorax_templates/set_flatpak_repo.tmpl.in b/lorax_templates/flatpak_set_repo.tmpl similarity index 100% rename from lorax_templates/set_flatpak_repo.tmpl.in rename to lorax_templates/flatpak_set_repo.tmpl diff --git a/lorax_templates/install_set_installer.tmpl b/lorax_templates/install_set_installer.tmpl new file mode 100644 index 0000000..fafd61e --- /dev/null +++ b/lorax_templates/install_set_installer.tmpl @@ -0,0 +1,4 @@ +<%page args="image_name, image_tag"/> + +append usr/share/anaconda/interactive-defaults.ks "ostreecontainer --url=/run/install/repo/${image_name}-${image_tag} --transport=oci --no-signature-verification" + diff --git a/lorax_templates/scripts/post/configure_flatpak b/lorax_templates/scripts/post/flatpak_configure similarity index 100% rename from lorax_templates/scripts/post/configure_flatpak rename to lorax_templates/scripts/post/flatpak_configure diff --git a/lorax_templates/scripts/post/configure_upgrades b/lorax_templates/scripts/post/install_configure_upgrades similarity index 100% rename from lorax_templates/scripts/post/configure_upgrades rename to lorax_templates/scripts/post/install_configure_upgrades diff --git a/lorax_templates/scripts/post/enroll-secureboot-key b/lorax_templates/scripts/post/secureboot_enroll_key similarity index 94% rename from lorax_templates/scripts/post/enroll-secureboot-key rename to lorax_templates/scripts/post/secureboot_enroll_key index 622a1ec..21def9e 100755 --- a/lorax_templates/scripts/post/enroll-secureboot-key +++ b/lorax_templates/scripts/post/secureboot_enroll_key @@ -1,4 +1,4 @@ -<%page args="enrollment_password "/> +<%page args="enrollment_password"/> ## --nochroot set -oue pipefail diff --git a/lorax_templates/set_installer.tmpl.in b/lorax_templates/set_installer.tmpl.in deleted file mode 100644 index 66f01e0..0000000 --- a/lorax_templates/set_installer.tmpl.in +++ /dev/null @@ -1,2 +0,0 @@ -append usr/share/anaconda/interactive-defaults.ks "ostreecontainer --url=/run/install/repo/${IMAGE_NAME}-${IMAGE_TAG} --transport=oci --no-signature-verification" - From df936e3365f994db4e5e0683d7e3cd97b9f37768 Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:08:54 -0400 Subject: [PATCH 09/20] Fix var reference (#56) --- action.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index bc6c0bf..09c4b84 100644 --- a/action.yml +++ b/action.yml @@ -122,18 +122,21 @@ runs: mkdir /cache/skopeo || true - name: Run docker image + env: + ACTION_REPO: ${{ github.action_repository }} + ACTION_REF: ${{ github.action_ref }} shell: bash run: | # Check if running inside of the action repo - if [[ -z "${{ github.action_repository }}" ]] + if [[ -z "${ACTION_REPO}" || "${ACTION_REPO}" == "${{ github.repository }}" ]] then if [[ "${{ github.ref_name }}" =~ (.*)/merge ]] then tag=pr-${BASH_REMATCH[1]} else - tag=${{ github.ref_name }} + tag=${ACTION_REF} fi else - tag=${{ github.action_ref }} + tag=${ACTION_REF} fi if [[ "${{ inputs.enable_cache_dnf }}" == "true" ]] then From 13bbd2a24d47867ec2c5d1f23f5bd8e251c94e1a Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Tue, 12 Mar 2024 09:14:57 -0400 Subject: [PATCH 10/20] Fixes macros still being readable (#61) --- Makefile | 9 +++++---- action.yml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 0c46523..9017ccd 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ _LORAX_ARGS = _FLATPAK_REPO_URL = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^URL=' | cut -d= -f2) _FLATPAK_REPO_GPG = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^GPGKey=' | cut -d= -f2) _TEMPLATE_VARS = ARCH IMAGE_NAME IMAGE_REPO _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED IMAGE_TAG REPOS VARIANT VERSION WEB_UI +_TEMP_DIR = $(shell mktemp -d) ifeq ($(findstring redhat.repo,$(REPOS)),redhat.repo) @@ -162,8 +163,8 @@ repos/%.repo: /etc/yum.repos.d/%.repo # Step 3: Build boot.iso using Lorax boot.iso: $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES) rm -Rf $(_BASE_DIR)/results || true - mv /etc/rpm/macros.image-language-conf /etc/rpm/macros.image-language-conf.orig || true - cp /etc/os-release /etc/os-release.orig || true + mv /etc/rpm/macros.image-language-conf $(_TEMP_DIR)/macros.image-language-conf || true + cp /etc/os-release $(_TEMP_DIR)/os-release || true sed -i 's/PLATFORM_ID=.*/PLATFORM_ID="$(_PLATFORM_ID)"/' /etc/os-release # Download the secure boot key @@ -184,8 +185,8 @@ boot.iso: $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES) $(foreach var,$(_TEMPLATE_VARS),--add-template-var "$(shell echo $(var) | tr '[:upper:]' '[:lower:]')=$($(var))") \ $(_BASE_DIR)/results/ mv $(_BASE_DIR)/results/images/boot.iso $(_BASE_DIR)/ - mv -f /etc/rpm/macros.image-language-conf.orig /etc/rpm/macros.image-language-conf || true - mv -f /etc/os-release.orig /etc/os-release || true + mv -f $(_TEMP_DIR)/macros.image-language-conf /etc/rpm/macros.image-language-conf || true + mv -f $(_TEMP_DIR)/os-release /etc/os-release || true # Step 4: Download container image container/$(IMAGE_NAME)-$(IMAGE_TAG): diff --git a/action.yml b/action.yml index 09c4b84..16fb2a8 100644 --- a/action.yml +++ b/action.yml @@ -23,7 +23,7 @@ inputs: enable_cache_skopeo: description: Whether to enable caching for skopeo required: false - default: "true" + default: "false" enrollment_password: description: Used for supporting secure boot (requires secure_boot_key_url to be defined) required: false From 2d28aec7401ca91dbbb5be46533a29d92443ed6f Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Tue, 12 Mar 2024 09:20:00 -0400 Subject: [PATCH 11/20] Fix builds on main (#63) --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 16fb2a8..a31dd46 100644 --- a/action.yml +++ b/action.yml @@ -133,7 +133,7 @@ runs: if [[ "${{ github.ref_name }}" =~ (.*)/merge ]] then tag=pr-${BASH_REMATCH[1]} else - tag=${ACTION_REF} + tag=${{ github.ref_name }} fi else tag=${ACTION_REF} From 4710f3002df6994d2a0f2b332228b24416c9e631 Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Tue, 12 Mar 2024 10:43:14 -0400 Subject: [PATCH 12/20] Prep for Anaconda 41 (#62) --- Makefile | 28 ++++++++++++++------------- lorax_templates/flatpak_set_repo.tmpl | 7 +++++-- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 9017ccd..5398b9e 100644 --- a/Makefile +++ b/Makefile @@ -27,18 +27,13 @@ DNF_CACHE = _BASE_DIR = $(shell pwd) _IMAGE_REPO_ESCAPED = $(subst /,\/,$(IMAGE_REPO)) _IMAGE_REPO_DOUBLE_ESCAPED = $(subst \,\\\,$(_IMAGE_REPO_ESCAPED)) -_VOLID = $(firstword $(subst -, ,$(IMAGE_NAME)))-$(ARCH)-$(IMAGE_TAG) -_REPO_FILES = $(subst /etc/yum.repos.d,repos,$(REPOS)) -_LORAX_TEMPLATES = $(shell ls lorax_templates/install_*.tmpl) $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/install_*)),lorax_templates/post_$(file).tmpl) -_LORAX_TEMPLATES_FLATPAKS = $(shell ls lorax_templates/flatpak_*.tmpl) $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/flatpak_*)),lorax_templates/post_$(file).tmpl) external/fedora-lorax-templates/ostree-based-installer/lorax-embed-flatpaks.tmpl -_LORAX_TEMPLATES_SECUREBOOT = $(shell ls lorax_templates/secureboot_*.tmpl) $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/secureboot_*)),lorax_templates/post_$(file).tmpl) -_LORAX_TEMPLATES_CACHE = $(shell ls lorax_templates/cache_*.tmpl) $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/cache_*)),lorax_templates/post_$(file).tmpl) _LORAX_ARGS = -_FLATPAK_REPO_URL = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^URL=' | cut -d= -f2) -_FLATPAK_REPO_GPG = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^GPGKey=' | cut -d= -f2) -_TEMPLATE_VARS = ARCH IMAGE_NAME IMAGE_REPO _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED IMAGE_TAG REPOS VARIANT VERSION WEB_UI +_LORAX_TEMPLATES = $(shell ls lorax_templates/install_*.tmpl) \ + $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/install_*)),lorax_templates/post_$(file).tmpl) +_REPO_FILES = $(subst /etc/yum.repos.d,repos,$(REPOS)) _TEMP_DIR = $(shell mktemp -d) - +_TEMPLATE_VARS = ARCH IMAGE_NAME IMAGE_REPO _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED IMAGE_TAG REPOS VARIANT VERSION WEB_UI +_VOLID = $(firstword $(subst -, ,$(IMAGE_NAME)))-$(ARCH)-$(IMAGE_TAG) ifeq ($(findstring redhat.repo,$(REPOS)),redhat.repo) _LORAX_ARGS += --nomacboot --noupgrade @@ -54,7 +49,8 @@ endif ifneq ($(DNF_CACHE),) _LORAX_ARGS += --cachedir $(DNF_CACHE) -_LORAX_TEMPLATES += $(_LORAX_TEMPLATES_CACHE) +_LORAX_TEMPLATES += $(shell ls lorax_templates/cache_*.tmpl) \ + $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/cache_*)),lorax_templates/post_$(file).tmpl) _TEMPLATE_VARS += DNF_CACHE endif @@ -65,13 +61,19 @@ _PLATFORM_ID = platform:f$(VERSION) endif ifneq ($(FLATPAK_REMOTE_REFS),) +_FLATPAK_REPO_GPG = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^GPGKey=' | cut -d= -f2) +_FLATPAK_REPO_URL = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^URL=' | cut -d= -f2) _LORAX_ARGS += -i flatpak-libs -_LORAX_TEMPLATES += $(_LORAX_TEMPLATES_FLATPAKS) +_LORAX_TEMPLATES += $(shell ls lorax_templates/flatpak_*.tmpl) \ + $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/flatpak_*)),lorax_templates/post_$(file).tmpl) \ + external/fedora-lorax-templates/ostree-based-installer/lorax-embed-flatpaks.tmpl _TEMPLATE_VARS += FLATPAK_REMOTE_NAME FLATPAK_REMOTE_REFS FLATPAK_REMOTE_URL _FLATPAK_REPO_GPG _FLATPAK_REPO_URL + endif ifneq ($(SECURE_BOOT_KEY_URL),) -_LORAX_TEMPLATES += $(_LORAX_TEMPLATES_SECUREBOOT) +_LORAX_TEMPLATES += $(shell ls lorax_templates/secureboot_*.tmpl) \ + $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/secureboot_*)),lorax_templates/post_$(file).tmpl) _TEMPLATE_VARS += ENROLLMENT_PASSWORD endif diff --git a/lorax_templates/flatpak_set_repo.tmpl b/lorax_templates/flatpak_set_repo.tmpl index 622e9a1..9cd006b 100644 --- a/lorax_templates/flatpak_set_repo.tmpl +++ b/lorax_templates/flatpak_set_repo.tmpl @@ -1,4 +1,7 @@ -<%page args="flatpak_remote_name, _flatpak_repo_url"/> - +<%page args="flatpak_remote_name, _flatpak_repo_url, version"/> +% if int(version) >= 41: +append etc/anaconda/conf.d/anaconda.conf "flatpak_remote = ${flatpak_remote_name} ${_flatpak_repo_url}" +% else: replace "flatpak_manager\.add_remote\(\".*\", \".*\"\)" "flatpak_manager.add_remote(\"${flatpak_remote_name}\", \"${_flatpak_repo_url}\")" /usr/lib64/python*/site-packages/pyanaconda/modules/payloads/payload/rpm_ostree/flatpak_installation.py replace "flatpak_manager\.replace_installed_refs_remote\(\".*\"\)" "flatpak_manager.replace_installed_refs_remote(\"${flatpak_remote_name}\")" /usr/lib64/python*/site-packages/pyanaconda/modules/payloads/payload/rpm_ostree/flatpak_installation.py +% endif \ No newline at end of file From f4fd87855a472f41f80ae8e2ed8e78807925761d Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:24:02 -0400 Subject: [PATCH 13/20] Use the Lorax templates for the OS version (#64) --- .github/workflows/build-and-test.yml | 6 ++++++ .gitmodules | 3 +++ Containerfile | 4 ++-- Makefile | 15 ++++++++------- external/lorax | 1 + 5 files changed, 20 insertions(+), 9 deletions(-) create mode 160000 external/lorax diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e82b395..640f37d 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -30,6 +30,8 @@ jobs: uses: actions/checkout@v4 with: submodules: recursive + fetch-depth: 0 + fetch-tags: 'true' - name: Docker meta id: meta @@ -49,6 +51,7 @@ jobs: with: containerfiles: Containerfile tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} - name: Push image uses: redhat-actions/push-to-registry@v2 @@ -72,9 +75,11 @@ jobs: version: - 38 - 39 + - 40 outputs: iso_name-38: ${{ steps.save_output.outputs.iso_name-38 }} iso_name-39: ${{ steps.save_output.outputs.iso_name-39 }} + iso_name-40: ${{ steps.save_output.outputs.iso_name-40 }} steps: - name: Checkout repo uses: actions/checkout@v4 @@ -141,6 +146,7 @@ jobs: version: - 38 - 39 + - 40 steps: - name: Checkout repo uses: actions/checkout@v4 diff --git a/.gitmodules b/.gitmodules index fe51240..e05748a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = external/fedora-lorax-templates url = https://pagure.io/fedora-lorax-templates.git branch = f39 +[submodule "external/lorax"] + path = external/lorax + url = https://github.com/weldr/lorax.git diff --git a/Containerfile b/Containerfile index f0b2174..fc097e5 100644 --- a/Containerfile +++ b/Containerfile @@ -1,6 +1,6 @@ -FROM fedora:39 +FROM fedora:40 -ARG VERSION=39 +ARG VERSION=40 ENV ARCH="x86_64" ENV IMAGE_NAME="base" diff --git a/Makefile b/Makefile index 5398b9e..c4cb913 100644 --- a/Makefile +++ b/Makefile @@ -84,6 +84,10 @@ build/deploy.iso: boot.iso container/$(IMAGE_NAME)-$(IMAGE_TAG) xorriso/input.t xorriso -dialog on < $(_BASE_DIR)/xorriso/input.txt implantisomd5 build/deploy.iso +lorax_repo: + git config advice.detachedHead false + cd external/lorax && git checkout tags/$(shell cd external/lorax && git tag -l lorax-$(VERSION).* --sort=creatordate | tail -n 1) + # Step 1: Generate Lorax Templates lorax_templates/post_%.tmpl: lorax_templates/scripts/post/% # Support interactive-defaults.ks @@ -163,11 +167,9 @@ repos/%.repo: /etc/yum.repos.d/%.repo %.repo: # Step 3: Build boot.iso using Lorax -boot.iso: $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES) +boot.iso: lorax_repo $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES) rm -Rf $(_BASE_DIR)/results || true mv /etc/rpm/macros.image-language-conf $(_TEMP_DIR)/macros.image-language-conf || true - cp /etc/os-release $(_TEMP_DIR)/os-release || true - sed -i 's/PLATFORM_ID=.*/PLATFORM_ID="$(_PLATFORM_ID)"/' /etc/os-release # Download the secure boot key if [ -n "$(SECURE_BOOT_KEY_URL)" ]; \ @@ -176,7 +178,7 @@ boot.iso: $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES) fi lorax -p $(IMAGE_NAME) -v $(VERSION) -r $(VERSION) -t $(VARIANT) \ - --isfinal --squashfs-only --buildarch=$(ARCH) --volid=$(_VOLID) \ + --isfinal --squashfs-only --buildarch=$(ARCH) --volid=$(_VOLID) --sharedir $(_BASE_DIR)/external/lorax/share/templates.d/99-generic \ $(_LORAX_ARGS) \ $(foreach file,$(_REPO_FILES),--repo $(_BASE_DIR)/$(file)) \ $(foreach file,$(_LORAX_TEMPLATES),--add-template $(_BASE_DIR)/$(file)) \ @@ -188,7 +190,6 @@ boot.iso: $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES) $(_BASE_DIR)/results/ mv $(_BASE_DIR)/results/images/boot.iso $(_BASE_DIR)/ mv -f $(_TEMP_DIR)/macros.image-language-conf /etc/rpm/macros.image-language-conf || true - mv -f $(_TEMP_DIR)/os-release /etc/os-release || true # Step 4: Download container image container/$(IMAGE_NAME)-$(IMAGE_TAG): @@ -223,7 +224,7 @@ clean: rm -f $(_BASE_DIR)/*.log || true install-deps: - dnf install -y lorax xorriso skopeo flatpak dbus-daemon ostree coreutils gettext + dnf install -y lorax xorriso skopeo flatpak dbus-daemon ostree coreutils gettext git test: test-iso test-vm @@ -253,4 +254,4 @@ test-vm: chmod +x $(foreach test,$(_TESTS),tests/vm/$(test)) for test in $(_TESTS); do ./tests/vm/$${test} deploy.iso; done -.PHONY: clean install-deps test test-iso test-vm container/$(IMAGE_NAME)-$(IMAGE_TAG) +.PHONY: clean install-deps test test-iso test-vm lorax_repo diff --git a/external/lorax b/external/lorax new file mode 160000 index 0000000..ffba307 --- /dev/null +++ b/external/lorax @@ -0,0 +1 @@ +Subproject commit ffba3078beab843c5d663f6443dca28d8e820948 From 4fac21d55fce0be8b20d4c90ce0acf641a8f7de6 Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:28:03 -0400 Subject: [PATCH 14/20] Allowing storing the Flatpak refs in files (#66) --- .github/workflows/build-and-test.yml | 13 ++++++-- Makefile | 27 +++++++++++++--- README.md | 32 +++++++++++-------- action.yml | 29 ++++++++++------- flatpak_refs/VLC | 2 ++ lorax_templates/flatpak_set_repo.tmpl | 4 +++ ...epo_updated.sh => flatpak_repo_updated.sh} | 0 .../{os-release.sh => install_os-release.sh} | 0 tests/vm/{flatpak.yml => flatpak_update.yml} | 0 9 files changed, 75 insertions(+), 32 deletions(-) create mode 100644 flatpak_refs/VLC rename tests/iso/{verify_flatpakrepo_updated.sh => flatpak_repo_updated.sh} (100%) rename tests/iso/{os-release.sh => install_os-release.sh} (100%) rename tests/vm/{flatpak.yml => flatpak_update.yml} (100%) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 640f37d..7419361 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -15,7 +15,7 @@ env: IMAGE_REPO: 'quay.io/fedora-ostree-desktops' IMAGE_TAG: '39' VARIANT: 'Server' - FLATPAK_REMOTE_REFS: app/org.videolan.VLC/x86_64/stable runtime/org.kde.Platform/x86_64/5.15-23.08 + FLATPAK_REMOTE_REFS_DIR: flatpak_refs SECURE_BOOT_KEY_URL: 'https://github.com/ublue-os/akmods/raw/main/certs/public_key.der' ENROLLMENT_PASSWORD: 'container-installer' @@ -175,7 +175,16 @@ jobs: - name: Run ISO checks run: | mv ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }} deploy.iso - make test-iso VERSION=${{ matrix.version }} FLATPAK_REMOTE_REFS="${{ env.FLATPAK_REMOTE_REFS}}" + make test-iso \ + ARCH=${{ env.ARCH}} \ + IMAGE_NAME=${{ env.IMAGE_NAME}} \ + IMAGE_REPO=${{ env.IMAGE_REPO}} \ + IMAGE_TAG=${{ env.IMAGE_TAG }} \ + VERSION=${{ matrix.version }} \ + VARIANT=${{ env.VARIANT }} \ + FLATPAK_REMOTE_REFS_DIR=${{ env.FLATPAK_REMOTE_REFS_DIR }} \ + SECURE_BOOT_KEY_URL=${{ env.SECURE_BOOT_KEY_URL }} \ + ENROLLMENT_PASSWORD=${{ env.ENROLLMENT_PASSWORD }} - name: Add Kickstart and Grub options to ISO run: | diff --git a/Makefile b/Makefile index c4cb913..c222730 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,8 @@ WEB_UI = false # Flatpak FLATPAK_REMOTE_NAME = flathub FLATPAK_REMOTE_URL = https://flathub.org/repo/flathub.flatpakrepo -FLATPAK_REMOTE_REFS = +FLATPAK_REMOTE_REFS = +FLATPAK_REMOTE_REFS_DIR = # Secure boot ENROLLMENT_PASSWORD = SECURE_BOOT_KEY_URL = @@ -60,6 +61,11 @@ else _PLATFORM_ID = platform:f$(VERSION) endif +ifneq ($(FLATPAK_REMOTE_REFS_DIR),) +COLLECTED_REFS = $(foreach file,$(shell ls $(FLATPAK_REMOTE_REFS_DIR)/*),$(shell cat $(file))) +FLATPAK_REMOTE_REFS += $(sort $(COLLECTED_REFS)) +endif + ifneq ($(FLATPAK_REMOTE_REFS),) _FLATPAK_REPO_GPG = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^GPGKey=' | cut -d= -f2) _FLATPAK_REPO_URL = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^URL=' | cut -d= -f2) @@ -166,6 +172,8 @@ repos/%.repo: /etc/yum.repos.d/%.repo # Don't do anything for custom repos %.repo: +flatpak_list: + # Step 3: Build boot.iso using Lorax boot.iso: lorax_repo $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES) rm -Rf $(_BASE_DIR)/results || true @@ -239,12 +247,23 @@ test-iso: sudo mount -o loop deploy.iso /mnt/iso sudo mount -t squashfs -o loop /mnt/iso/images/install.img /mnt/install - chmod +x $(foreach test,$(_TESTS),tests/iso/$(test)) + # install tests + chmod +x $(foreach test,$(filter install_%,$(_TESTS)),tests/iso/$(test)) for test in $(_TESTS); \ do \ $(foreach var,$(_VARS),$(var)=$($(var))) ./tests/iso/$${test}; \ done + # flapak tests + if [ -n "$(FLATPAK_REMOTE_REFS)" ]; \ + then \ + chmod +x $(foreach test,$(filter flatpak_%,$(_TESTS)),tests/iso/$(test)); \ + for test in $(_TESTS); \ + do \ + $(foreach var,$(_VARS),$(var)=$($(var))) ./tests/iso/$${test}; \ + done; \ + fi + # Cleanup sudo umount /mnt/install sudo umount /mnt/iso @@ -253,5 +272,5 @@ test-vm: $(eval _TESTS = $(filter-out README.md,$(shell ls tests/vm))) chmod +x $(foreach test,$(_TESTS),tests/vm/$(test)) for test in $(_TESTS); do ./tests/vm/$${test} deploy.iso; done - -.PHONY: clean install-deps test test-iso test-vm lorax_repo + +.PHONY: clean install-deps test test-iso test-vm lorax_repo flatpak_list diff --git a/README.md b/README.md index 54be4c1..f6c0280 100644 --- a/README.md +++ b/README.md @@ -38,20 +38,24 @@ See [Customizing](#customizing) for information about customizing the ISO that g The following variables can be used to customize the created ISO. ### Inputs -| Variable | Description | Default Value | -| -------------------- | ---------------------------------------------------------------------------- | ------------------------------ | -| additional_templates | Space delimited list of additional Lorax templates to include | \[empty\] | -| arch | Architecture for image to build | x86_64 | -| enrollment_password | Used for supporting secure boot (requires SECURE_BOOT_KEY_URL to be defined) | container-installer | -| extra_boot_params | Extra params used by grub to boot the anaconda installer | \[empty\] | -| image_name | Name of the source container image | base | -| image_repo | Repository containing the source container image | quay.io/fedora-ostree-desktops | -| image_tag | Tag of the source container image | *VERSION* | -| iso_name | Name of the ISO you wish to output when completed | build/deploy.iso | -| secure_boot_key_url | Secure boot key that is installed from URL location\*\* | \[empty\] | -| variant | Source container variant\* | Server | -| version | Fedora version of installer to build | 39 | -| web_ui | Enable Anaconda WebUI (experimental) | false | +| Variable | Description | Default Value | +| ---------------------- | ---------------------------------------------------------------------------- | -------------------------------------------- | +| additional_templates | Space delimited list of additional Lorax templates to include | \[empty\] | +| arch | Architecture for image to build | x86_64 | +| enrollment_password | Used for supporting secure boot (requires SECURE_BOOT_KEY_URL to be defined) | container-installer | +| extra_boot_params | Extra params used by grub to boot the anaconda installer | \[empty\] | +| flatpak_remote_name | Name of the Flatpak repo on the destination OS | flathub | +| flatpak_remote_refs | Space separated list of flatpak refs to install | \[empty\] | +| flatpak_remote_refs_dir | Directory that contains files that list the flatpak refs to install | \[empty\] | +| flatpak_remote_url | URL of the flatpakrepo file | https://flathub.org/repo/flathub.flatpakrepo | +| image_name | Name of the source container image | base | +| image_repo | Repository containing the source container image | quay.io/fedora-ostree-desktops | +| image_tag | Tag of the source container image | *VERSION* | +| iso_name | Name of the ISO you wish to output when completed | build/deploy.iso | +| secure_boot_key_url | Secure boot key that is installed from URL location\*\* | \[empty\] | +| variant | Source container variant\* | Server | +| version | Fedora version of installer to build | 39 | +| web_ui | Enable Anaconda WebUI (experimental) | false | \*Available options for VARIANT can be found by running `dnf provides system-release`. Variant will be the third item in the package name. Example: `fedora-release-kinoite-39-34.noarch` will be kinoite diff --git a/action.yml b/action.yml index a31dd46..f2eb50f 100644 --- a/action.yml +++ b/action.yml @@ -39,6 +39,10 @@ inputs: description: Space delimited list of refs to the flatpak packages to install required: false default: "" + flatpak_remote_refs_dir: + description: Directory that contains files that list the flatpak refs to install + required: false + default: "" flatpak_remote_url: description: The URL of the Flatpak remote flatpakrepo file required: false @@ -152,19 +156,20 @@ runs: fi docker run --privileged --volume ${{ github.workspace }}:/github/workspace/ ${cache} ghcr.io/jasonn3/build-container-installer:${tag} \ ADDITIONAL_TEMPLATES="${{ inputs.additional_templates }}" \ - ARCH=${{ inputs.arch }} \ - DNF_CACHE=/cache/dnf \ - ENROLLMENT_PASSWORD=${{ inputs.enrollment_password }} \ - FLATPAK_REMOTE_NAME=${{ inputs.flatpak_remote_name }} \ + ARCH="${{ inputs.arch }}" \ + DNF_CACHE="/cache/dnf" \ + ENROLLMENT_PASSWORD="${{ inputs.enrollment_password }}" \ + FLATPAK_REMOTE_NAME="${{ inputs.flatpak_remote_name }}" \ FLATPAK_REMOTE_REFS="${{ inputs.flatpak_remote_refs }}" \ - FLATPAK_REMOTE_URL=${{ inputs.flatpak_remote_url }} \ - IMAGE_NAME=${{ inputs.image_name }} \ - IMAGE_REPO=${{ inputs.image_repo }} \ - IMAGE_TAG=${{ inputs.image_tag || inputs.version }} \ - SECURE_BOOT_KEY_URL=${{ inputs.secure_boot_key_url }} \ - VARIANT=${{ inputs.variant }} \ - VERSION=${{ inputs.version }} \ - WEB_UI=${{ inputs.web_ui }} + FLATPAK_REMOTE_REFS_DIR="${{ inputs.flatpak_remote_refs_dir }}" \ + FLATPAK_REMOTE_URL="${{ inputs.flatpak_remote_url }}" \ + IMAGE_NAME="${{ inputs.image_name }}" \ + IMAGE_REPO="${{ inputs.image_repo }}" \ + IMAGE_TAG="${{ inputs.image_tag || inputs.version }}" \ + SECURE_BOOT_KEY_URL="${{ inputs.secure_boot_key_url }}" \ + VARIANT="${{ inputs.variant }}" \ + VERSION="${{ inputs.version }}" \ + WEB_UI="${{ inputs.web_ui }}" - name: Save dnf cache env: diff --git a/flatpak_refs/VLC b/flatpak_refs/VLC new file mode 100644 index 0000000..6e4a3fd --- /dev/null +++ b/flatpak_refs/VLC @@ -0,0 +1,2 @@ +app/org.videolan.VLC/x86_64/stable +runtime/org.kde.Platform/x86_64/5.15-23.08 \ No newline at end of file diff --git a/lorax_templates/flatpak_set_repo.tmpl b/lorax_templates/flatpak_set_repo.tmpl index 9cd006b..8feda96 100644 --- a/lorax_templates/flatpak_set_repo.tmpl +++ b/lorax_templates/flatpak_set_repo.tmpl @@ -4,4 +4,8 @@ append etc/anaconda/conf.d/anaconda.conf "flatpak_remote = ${flatpak_remote_name % else: replace "flatpak_manager\.add_remote\(\".*\", \".*\"\)" "flatpak_manager.add_remote(\"${flatpak_remote_name}\", \"${_flatpak_repo_url}\")" /usr/lib64/python*/site-packages/pyanaconda/modules/payloads/payload/rpm_ostree/flatpak_installation.py replace "flatpak_manager\.replace_installed_refs_remote\(\".*\"\)" "flatpak_manager.replace_installed_refs_remote(\"${flatpak_remote_name}\")" /usr/lib64/python*/site-packages/pyanaconda/modules/payloads/payload/rpm_ostree/flatpak_installation.py +% endif + +%if flatpak_remote_name != 'fedora' +systemctl disable flatpak-add-fedora-repos.service % endif \ No newline at end of file diff --git a/tests/iso/verify_flatpakrepo_updated.sh b/tests/iso/flatpak_repo_updated.sh similarity index 100% rename from tests/iso/verify_flatpakrepo_updated.sh rename to tests/iso/flatpak_repo_updated.sh diff --git a/tests/iso/os-release.sh b/tests/iso/install_os-release.sh similarity index 100% rename from tests/iso/os-release.sh rename to tests/iso/install_os-release.sh diff --git a/tests/vm/flatpak.yml b/tests/vm/flatpak_update.yml similarity index 100% rename from tests/vm/flatpak.yml rename to tests/vm/flatpak_update.yml From 6f1f78fa8916ce572b99414794f0cc442b996f1c Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Wed, 13 Mar 2024 09:25:32 -0400 Subject: [PATCH 15/20] Add stale workflow (#68) --- .github/workflows/stale.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..6ad1c0a --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,32 @@ +# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. +# +# You can adjust the behavior by modifying this file. +# For more information, see: +# https://github.com/actions/stale +name: Mark stale issues and pull requests + +on: + schedule: + - cron: '39 21 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v5 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Issue is stale and will be closed in 14 days if there is no further activity' + stale-pr-message: 'Pull request is stale and will be closed in 14 days if there is no further activity' + stale-issue-label: 'no-issue-activity' + stale-pr-label: 'no-pr-activity' + days-before-stale: 60 + days-before-close: 17 + remove-stale-when-updated: true + only-issue-labels: help wanted + From bbac3469b100817a55907ba1624456397a1296b3 Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:28:34 -0400 Subject: [PATCH 16/20] Only run some actions on 38 or older (#69) --- .github/workflows/build-and-test.yml | 5 + .gitignore | 1 + Makefile | 131 ++++++++---------- .../scripts/post/install_configure_upgrades | 4 +- 4 files changed, 64 insertions(+), 77 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7419361..7d67cdd 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -8,6 +8,11 @@ on: - 'v*' pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + # Test Values env: ARCH: 'x86_64' diff --git a/.gitignore b/.gitignore index ff22e19..dde3499 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /debugdata /build +/lorax_templates/post_* /pkglists /repos /results diff --git a/Makefile b/Makefile index c222730..e72c0bd 100644 --- a/Makefile +++ b/Makefile @@ -23,20 +23,68 @@ SECURE_BOOT_KEY_URL = # Cache DNF_CACHE = +# Functions +## Formatting = lowercase +# Get a list of templates for the feature +# $1 = feature +get_templates = $(shell ls lorax_templates/$(1)_*.tmpl) \ + $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/$(1)_*)),lorax_templates/post_$(file).tmpl) + +# Converts a post script to a template +# $1 = script to convert +# $2 = file on ISO to write +# $3 = whether to copy the '<%' lines to the template +convert_post_to_tmpl = header=0; \ + skip=0; \ + while read -r line; \ + do \ + if [[ $$line =~ ^\<\% ]]; \ + then \ + if [[ '$(3)' == 'true' ]]; \ + then \ + echo $$line >> lorax_templates/post_$(1).tmpl; \ + fi; \ + echo >> lorax_templates/post_$(1).tmpl; \ + else \ + if [[ $$header == 0 ]]; \ + then \ + if [[ $$line =~ ^\#\#\ (.*)$$ ]]; \ + then \ + echo "append $(2) \"%post --erroronfail $${BASH_REMATCH[1]}\"" >> lorax_templates/post_$(1).tmpl; \ + skip=1; \ + else \ + echo "append $(2) \"%post --erroronfail\"" >> lorax_templates/post_$(1).tmpl; \ + fi; \ + header=1; \ + fi; \ + if [[ $$skip == 0 ]]; \ + then \ + echo "append $(2) \"$${line//\"/\\\"}\"" >> lorax_templates/post_$(1).tmpl; \ + fi; \ + skip=0; \ + fi; \ + done < lorax_templates/scripts/post/$(1); \ + echo "append $(2) \"%end\"" >> lorax_templates/post_$(1).tmpl + # Generated/internal vars ## Formatting = _UPPERCASE _BASE_DIR = $(shell pwd) _IMAGE_REPO_ESCAPED = $(subst /,\/,$(IMAGE_REPO)) _IMAGE_REPO_DOUBLE_ESCAPED = $(subst \,\\\,$(_IMAGE_REPO_ESCAPED)) _LORAX_ARGS = -_LORAX_TEMPLATES = $(shell ls lorax_templates/install_*.tmpl) \ - $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/install_*)),lorax_templates/post_$(file).tmpl) +_LORAX_TEMPLATES = $(call get_templates,install) _REPO_FILES = $(subst /etc/yum.repos.d,repos,$(REPOS)) _TEMP_DIR = $(shell mktemp -d) -_TEMPLATE_VARS = ARCH IMAGE_NAME IMAGE_REPO _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED IMAGE_TAG REPOS VARIANT VERSION WEB_UI +_TEMPLATE_VARS = ARCH IMAGE_NAME IMAGE_REPO _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED IMAGE_TAG REPOS _RHEL VARIANT VERSION WEB_UI _VOLID = $(firstword $(subst -, ,$(IMAGE_NAME)))-$(ARCH)-$(IMAGE_TAG) ifeq ($(findstring redhat.repo,$(REPOS)),redhat.repo) +_RHEL = true +else +_RHEL = false +endif + +ifeq ($(_RHEL),true) _LORAX_ARGS += --nomacboot --noupgrade else ifeq ($(VARIANT),Server) _LORAX_ARGS += --macboot --noupgrade @@ -50,17 +98,10 @@ endif ifneq ($(DNF_CACHE),) _LORAX_ARGS += --cachedir $(DNF_CACHE) -_LORAX_TEMPLATES += $(shell ls lorax_templates/cache_*.tmpl) \ - $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/cache_*)),lorax_templates/post_$(file).tmpl) +_LORAX_TEMPLATES += $(call get_templates,cache) _TEMPLATE_VARS += DNF_CACHE endif -ifeq ($(findstring redhat.repo,$(REPOS)),redhat.repo) -_PLATFORM_ID = platform:el$(VERSION) -else -_PLATFORM_ID = platform:f$(VERSION) -endif - ifneq ($(FLATPAK_REMOTE_REFS_DIR),) COLLECTED_REFS = $(foreach file,$(shell ls $(FLATPAK_REMOTE_REFS_DIR)/*),$(shell cat $(file))) FLATPAK_REMOTE_REFS += $(sort $(COLLECTED_REFS)) @@ -70,16 +111,14 @@ ifneq ($(FLATPAK_REMOTE_REFS),) _FLATPAK_REPO_GPG = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^GPGKey=' | cut -d= -f2) _FLATPAK_REPO_URL = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^URL=' | cut -d= -f2) _LORAX_ARGS += -i flatpak-libs -_LORAX_TEMPLATES += $(shell ls lorax_templates/flatpak_*.tmpl) \ - $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/flatpak_*)),lorax_templates/post_$(file).tmpl) \ +_LORAX_TEMPLATES += $(call get_templates,flatpak) \ external/fedora-lorax-templates/ostree-based-installer/lorax-embed-flatpaks.tmpl _TEMPLATE_VARS += FLATPAK_REMOTE_NAME FLATPAK_REMOTE_REFS FLATPAK_REMOTE_URL _FLATPAK_REPO_GPG _FLATPAK_REPO_URL endif ifneq ($(SECURE_BOOT_KEY_URL),) -_LORAX_TEMPLATES += $(shell ls lorax_templates/secureboot_*.tmpl) \ - $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/secureboot_*)),lorax_templates/post_$(file).tmpl) +_LORAX_TEMPLATES += $(call get_templates,secureboot) _TEMPLATE_VARS += ENROLLMENT_PASSWORD endif @@ -97,68 +136,10 @@ lorax_repo: # Step 1: Generate Lorax Templates lorax_templates/post_%.tmpl: lorax_templates/scripts/post/% # Support interactive-defaults.ks - $(eval _ISO_FILE = usr/share/anaconda/interactive-defaults.ks) - - header=0; \ - skip=0; \ - while read -r line; \ - do \ - if [[ $$line =~ ^\<\% ]]; \ - then \ - echo $$line >> lorax_templates/post_$*.tmpl; \ - echo >> lorax_templates/post_$*.tmpl; \ - else \ - if [[ $$header == 0 ]]; \ - then \ - if [[ $$line =~ ^##\ (.*)$$ ]]; \ - then \ - echo "append $(_ISO_FILE) \"%post --erroronfail $${BASH_REMATCH[1]}\"" >> lorax_templates/post_$*.tmpl; \ - skip=1; \ - else \ - echo "append $(_ISO_FILE) \"%post --erroronfail\"" >> lorax_templates/post_$*.tmpl; \ - fi; \ - header=1; \ - fi; \ - if [[ $$skip == 0 ]]; \ - then \ - echo "append $(_ISO_FILE) \"$${line//\"/\\\"}\"" >> lorax_templates/post_$*.tmpl; \ - fi; \ - skip=0; \ - fi; \ - done < lorax_templates/scripts/post/$* - echo "append $(_ISO_FILE) \"%end\"" >> lorax_templates/post_$*.tmpl + ([ ${_RHEL} == false ] && [ $(VERSION) -le 38 ]) && ($(call convert_post_to_tmpl,$*,usr/share/anaconda/interactive-defaults.ks,true)) || true # Support new Anaconda method - $(eval _ISO_FILE = usr/share/anaconda/post-scripts/configure_upgrades.ks) - - header=0; \ - skip=0; \ - while read -r line; \ - do \ - if [[ $$line =~ ^\<\% ]]; \ - then \ - echo >> lorax_templates/post_$*.tmpl; \ - else \ - if [[ $$header == 0 ]]; \ - then \ - if [[ $$line =~ ^##\ (.*)$$ ]]; \ - then \ - echo "append $(_ISO_FILE) \"%post --erroronfail $${BASH_REMATCH[1]}\"" >> lorax_templates/post_$*.tmpl; \ - skip=1; \ - else \ - echo "append $(_ISO_FILE) \"%post --erroronfail\"" >> lorax_templates/post_$*.tmpl; \ - fi; \ - header=1; \ - fi; \ - if [[ $$skip == 0 ]]; \ - then \ - echo "append $(_ISO_FILE) \"$${line//\"/\\\"}\"" >> lorax_templates/post_$*.tmpl; \ - fi; \ - skip=0; \ - fi; \ - done < lorax_templates/scripts/post/$* - echo "append $(_ISO_FILE) \"%end\"" >> lorax_templates/post_$*.tmpl - + ([ ${_RHEL} == true ] || [ $(VERSION) -ge 39 ]) && ($(call convert_post_to_tmpl,$*,usr/share/anaconda/post-scripts/$*.sh,true)) || true repos: $(_REPO_FILES) diff --git a/lorax_templates/scripts/post/install_configure_upgrades b/lorax_templates/scripts/post/install_configure_upgrades index 761764d..f474d93 100644 --- a/lorax_templates/scripts/post/install_configure_upgrades +++ b/lorax_templates/scripts/post/install_configure_upgrades @@ -1,5 +1,5 @@ -<%page args="image_repo, _image_repo_double_escaped, image_name, image_tag"/> -if (which bootc &> /dev/null) +<%page args="image_repo, _image_repo_double_escaped, image_name, image_tag, _rhel, version"/> +if (which bootc &> /dev/null) && [ ${_rhel} == 'false' && ${version} -ge 39 ] then bootc switch --mutate-in-place --enforce-container-sigpolicy --transport registry ${image_repo}/${image_name}:${image_tag} else From 4ae7d1893ca91859b7169c265d5759c8b4d04171 Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:24:39 -0400 Subject: [PATCH 17/20] flatpak_set_repo fails to load (#72) --- .github/workflows/build-and-test.yml | 6 +++--- Makefile | 16 ++++++---------- action.yml | 16 ++++++++++++++-- flatpak_refs/Firefox | 5 +++++ lorax_templates/flatpak_set_repo.tmpl | 4 ++-- 5 files changed, 30 insertions(+), 17 deletions(-) create mode 100644 flatpak_refs/Firefox diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7d67cdd..5884ad6 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -20,7 +20,7 @@ env: IMAGE_REPO: 'quay.io/fedora-ostree-desktops' IMAGE_TAG: '39' VARIANT: 'Server' - FLATPAK_REMOTE_REFS_DIR: flatpak_refs + FLATPAK_REMOTE_REFS_DIR: /github/workspace/flatpak_refs SECURE_BOOT_KEY_URL: 'https://github.com/ublue-os/akmods/raw/main/certs/public_key.der' ENROLLMENT_PASSWORD: 'container-installer' @@ -103,7 +103,7 @@ jobs: type=ref,event=branch type=ref,event=pr - - name: Build ISO with new container + - name: Build ISO uses: ./ id: build with: @@ -113,7 +113,7 @@ jobs: image_tag: ${{ env.IMAGE_TAG }} version: ${{ matrix.version }} variant: ${{ env.VARIANT }} - flatpak_remote_refs: ${{ env.FLATPAK_REMOTE_REFS }} + flatpak_remote_refs_dir: ${{ env.FLATPAK_REMOTE_REFS_DIR }} secure_boot_key_url: ${{ env.SECURE_BOOT_KEY_URL }} enrollment_password: ${{ env.ENROLLMENT_PASSWORD }} iso_name: ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso diff --git a/Makefile b/Makefile index e72c0bd..1d6a94e 100644 --- a/Makefile +++ b/Makefile @@ -124,14 +124,15 @@ endif # Step 7: Build end ISO ## Default action -build/deploy.iso: boot.iso container/$(IMAGE_NAME)-$(IMAGE_TAG) xorriso/input.txt +build/deploy.iso: boot.iso container/$(IMAGE_NAME)-$(IMAGE_TAG) xorriso/input.txt mkdir $(_BASE_DIR)/build || true xorriso -dialog on < $(_BASE_DIR)/xorriso/input.txt implantisomd5 build/deploy.iso -lorax_repo: +external/lorax/branch-$(VERSION): git config advice.detachedHead false - cd external/lorax && git checkout tags/$(shell cd external/lorax && git tag -l lorax-$(VERSION).* --sort=creatordate | tail -n 1) + cd external/lorax && git reset --hard HEAD && git checkout tags/$(shell cd external/lorax && git tag -l lorax-$(VERSION).* --sort=creatordate | tail -n 1) + touch external/lorax/branch-$(VERSION) # Step 1: Generate Lorax Templates lorax_templates/post_%.tmpl: lorax_templates/scripts/post/% @@ -150,13 +151,8 @@ repos/%.repo: /etc/yum.repos.d/%.repo sed -i "s/\$$releasever/${VERSION}/g" $(_BASE_DIR)/repos/$*.repo sed -i "s/\$$basearch/${ARCH}/g" $(_BASE_DIR)/repos/$*.repo -# Don't do anything for custom repos -%.repo: - -flatpak_list: - # Step 3: Build boot.iso using Lorax -boot.iso: lorax_repo $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES) +boot.iso: external/lorax/branch-$(VERSION) $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES) rm -Rf $(_BASE_DIR)/results || true mv /etc/rpm/macros.image-language-conf $(_TEMP_DIR)/macros.image-language-conf || true @@ -254,4 +250,4 @@ test-vm: chmod +x $(foreach test,$(_TESTS),tests/vm/$(test)) for test in $(_TESTS); do ./tests/vm/$${test} deploy.iso; done -.PHONY: clean install-deps test test-iso test-vm lorax_repo flatpak_list +.PHONY: clean install-deps test test-iso test-vm diff --git a/action.yml b/action.yml index f2eb50f..84f9337 100644 --- a/action.yml +++ b/action.yml @@ -154,14 +154,26 @@ runs: then cache="${cache} -v /cache/dnf_new:/cache/dnf_new" fi + vars="" + if [[ -n "${{ inputs.flatpak_remote_refs }}" ]] && [[ -n "${{ inputs.flatpak_remote_refs_dir }}" ]] + then + echo "ERROR: flatpak_remote_refs is mutually exclusive to flatpak_remote_refs_dir" + exit 1 + else + if [[ -n "${{ inputs.flatpak_remote_refs }}" ]] + then + vars="${vars} FLATPAK_REMOTE_REFS=\"${{ inputs.flatpak_remote_refs }}\"" + else + vars="${vars} FLATPAK_REMOTE_REFS_DIR=\"${{ inputs.flatpak_remote_refs_dir }}\"" + fi + fi docker run --privileged --volume ${{ github.workspace }}:/github/workspace/ ${cache} ghcr.io/jasonn3/build-container-installer:${tag} \ ADDITIONAL_TEMPLATES="${{ inputs.additional_templates }}" \ ARCH="${{ inputs.arch }}" \ DNF_CACHE="/cache/dnf" \ ENROLLMENT_PASSWORD="${{ inputs.enrollment_password }}" \ FLATPAK_REMOTE_NAME="${{ inputs.flatpak_remote_name }}" \ - FLATPAK_REMOTE_REFS="${{ inputs.flatpak_remote_refs }}" \ - FLATPAK_REMOTE_REFS_DIR="${{ inputs.flatpak_remote_refs_dir }}" \ + ${vars} \ FLATPAK_REMOTE_URL="${{ inputs.flatpak_remote_url }}" \ IMAGE_NAME="${{ inputs.image_name }}" \ IMAGE_REPO="${{ inputs.image_repo }}" \ diff --git a/flatpak_refs/Firefox b/flatpak_refs/Firefox new file mode 100644 index 0000000..fdc2b93 --- /dev/null +++ b/flatpak_refs/Firefox @@ -0,0 +1,5 @@ +app/org.mozilla.firefox/x86_64/stable + +runtime/org.mozilla.firefox.Locale/x86_64/stable +runtime/org.freedesktop.Platform/x86_64/23.08 +runtime/org.freedesktop.Platform.Locale/x86_64/23.08 \ No newline at end of file diff --git a/lorax_templates/flatpak_set_repo.tmpl b/lorax_templates/flatpak_set_repo.tmpl index 8feda96..30992db 100644 --- a/lorax_templates/flatpak_set_repo.tmpl +++ b/lorax_templates/flatpak_set_repo.tmpl @@ -6,6 +6,6 @@ replace "flatpak_manager\.add_remote\(\".*\", \".*\"\)" "flatpak_manager.add_rem replace "flatpak_manager\.replace_installed_refs_remote\(\".*\"\)" "flatpak_manager.replace_installed_refs_remote(\"${flatpak_remote_name}\")" /usr/lib64/python*/site-packages/pyanaconda/modules/payloads/payload/rpm_ostree/flatpak_installation.py % endif -%if flatpak_remote_name != 'fedora' +%if flatpak_remote_name != 'fedora': systemctl disable flatpak-add-fedora-repos.service -% endif \ No newline at end of file +% endif From 10524ae6eada47237da9bed3f145c48a042236e4 Mon Sep 17 00:00:00 2001 From: Noel Miller <4983138+noelmiller@users.noreply.github.com> Date: Thu, 14 Mar 2024 09:17:25 -0500 Subject: [PATCH 18/20] Revert chmod command on build directory (#75) --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 672d924..e7c579f 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -36,4 +36,4 @@ mkdir /github/workspace/build || true # Copy resulting iso to github workspace and fix permissions cp build/deploy.iso /github/workspace/build -chmod -R ugo=rwx /github/workspace/build +chmod -R ugo=rwX /github/workspace/build From 3dc05c0efc418d311d020e3e08b4aa593c607a44 Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Thu, 14 Mar 2024 20:38:19 -0400 Subject: [PATCH 19/20] Fix post scripts and use multiple OS versions in tests (#70) --- .github/workflows/build-and-test.yml | 31 +++++++++-------------- Makefile | 38 +++++++++++++++++++--------- README.md | 36 +++++++++++++------------- tests/vm/flatpak_update.yml | 4 ++- 4 files changed, 59 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 5884ad6..a1aa320 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,4 +1,4 @@ -name: Create and publish an ISO +name: Tests on: push: @@ -26,6 +26,7 @@ env: jobs: build-container: + name: Build Container Image runs-on: ubuntu-latest permissions: contents: read @@ -67,6 +68,7 @@ jobs: password: ${{ github.token }} build-and-push-iso: + name: Build ISO runs-on: ubuntu-latest needs: - build-container @@ -110,13 +112,13 @@ jobs: arch: ${{ env.ARCH}} image_name: ${{ env.IMAGE_NAME}} image_repo: ${{ env.IMAGE_REPO}} - image_tag: ${{ env.IMAGE_TAG }} + image_tag: ${{ matrix.version }} version: ${{ matrix.version }} variant: ${{ env.VARIANT }} flatpak_remote_refs_dir: ${{ env.FLATPAK_REMOTE_REFS_DIR }} secure_boot_key_url: ${{ env.SECURE_BOOT_KEY_URL }} enrollment_password: ${{ env.ENROLLMENT_PASSWORD }} - iso_name: ${{ env.IMAGE_NAME }}-${{ env.IMAGE_TAG }}-${{ matrix.version }}.iso + iso_name: ${{ env.IMAGE_NAME }}-${{ matrix.version }}-${{ matrix.version }}.iso - name: Save output id: save_output @@ -138,6 +140,7 @@ jobs: overwrite: true test-qemu: + name: Test ISO runs-on: ubuntu-latest needs: - build-and-push-iso @@ -158,10 +161,11 @@ jobs: with: submodules: recursive - - name: Ensure qemu is installed + - name: Install test tools run: | sudo apt-get update - sudo apt-get install -y qemu qemu-utils xorriso unzip qemu-system-x86 netcat socat jq isomd5sum ansible make coreutils + sudo apt-get install -y make + sudo make install-test-deps PACKAGE_MANAGER=apt-get - name: Create disk run: | @@ -184,7 +188,7 @@ jobs: ARCH=${{ env.ARCH}} \ IMAGE_NAME=${{ env.IMAGE_NAME}} \ IMAGE_REPO=${{ env.IMAGE_REPO}} \ - IMAGE_TAG=${{ env.IMAGE_TAG }} \ + IMAGE_TAG=${{ matrix.version }} \ VERSION=${{ matrix.version }} \ VARIANT=${{ env.VARIANT }} \ FLATPAK_REMOTE_REFS_DIR=${{ env.FLATPAK_REMOTE_REFS_DIR }} \ @@ -250,7 +254,7 @@ jobs: timeout 1m bash -c "while ! (echo > /dev/tcp/127.0.0.1/4321); do sleep 0.1; done" (tail -f vm.stdin | nc localhost 4321 | tee vm.stdout) & - timeout 30m bash -c "while ! (grep 'login:' vm.stdout); do sleep 1; done" + timeout 30m bash -c "while ! (echo > /dev/tcp/${VM_IP}/${VM_PORT}); do sleep 1; done" if ! (echo > /dev/tcp/${VM_IP}/${VM_PORT}) then @@ -258,17 +262,6 @@ jobs: fi echo "VM ready for tests at IP ${VM_IP}:${VM_PORT}" - echo "Creating Ansible inventory" - cat << EOF > ansible_inventory - ungrouped: - hosts: - vm: - ansible_host: ${VM_IP} - ansible_port: ${VM_PORT} - ansible_user: ${VM_USER} - ansible_password: ${VM_PASS} - ansible_ssh_common_args: '-o StrictHostKeyChecking=no' - EOF - #make test-vm + make test-vm VM_IP=${VM_IP} VM_PORT=${VM_PORT} VM_USER=${VM_USER} VM_PASS=${VM_PASS} kill $QEMU_PID diff --git a/Makefile b/Makefile index 1d6a94e..5058868 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,8 @@ SECURE_BOOT_KEY_URL = # Cache DNF_CACHE = +PACKAGE_MANAGER = dnf + # Functions ## Formatting = lowercase # Get a list of templates for the feature @@ -75,7 +77,7 @@ _LORAX_ARGS = _LORAX_TEMPLATES = $(call get_templates,install) _REPO_FILES = $(subst /etc/yum.repos.d,repos,$(REPOS)) _TEMP_DIR = $(shell mktemp -d) -_TEMPLATE_VARS = ARCH IMAGE_NAME IMAGE_REPO _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED IMAGE_TAG REPOS _RHEL VARIANT VERSION WEB_UI +_TEMPLATE_VARS = ARCH _BASE_DIR IMAGE_NAME IMAGE_REPO _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED IMAGE_TAG REPOS _RHEL VARIANT VERSION WEB_UI _VOLID = $(firstword $(subst -, ,$(IMAGE_NAME)))-$(ARCH)-$(IMAGE_TAG) ifeq ($(findstring redhat.repo,$(REPOS)),redhat.repo) @@ -136,11 +138,7 @@ external/lorax/branch-$(VERSION): # Step 1: Generate Lorax Templates lorax_templates/post_%.tmpl: lorax_templates/scripts/post/% - # Support interactive-defaults.ks - ([ ${_RHEL} == false ] && [ $(VERSION) -le 38 ]) && ($(call convert_post_to_tmpl,$*,usr/share/anaconda/interactive-defaults.ks,true)) || true - - # Support new Anaconda method - ([ ${_RHEL} == true ] || [ $(VERSION) -ge 39 ]) && ($(call convert_post_to_tmpl,$*,usr/share/anaconda/post-scripts/$*.sh,true)) || true + $(call convert_post_to_tmpl,$*,usr/share/anaconda/post-scripts/$*.ks,true) repos: $(_REPO_FILES) @@ -209,7 +207,13 @@ clean: rm -f $(_BASE_DIR)/*.log || true install-deps: - dnf install -y lorax xorriso skopeo flatpak dbus-daemon ostree coreutils gettext git + if [ "$(PACKAGE_MANAGER)" =~ apt.* ]; then $(PACKAGE_MANAGER) update; fi + $(PACKAGE_MANAGER) install -y lorax xorriso skopeo flatpak dbus-daemon ostree coreutils gettext git + +install-test-deps: + if [ "$(PACKAGE_MANAGER)" =~ apt.* ]; then $(PACKAGE_MANAGER) update; fi + $(PACKAGE_MANAGER) install -y qemu qemu-utils xorriso unzip qemu-system-x86 netcat socat jq isomd5sum ansible make coreutils squashfs-tools + test: test-iso test-vm @@ -217,8 +221,6 @@ test-iso: $(eval _TESTS = $(filter-out README.md,$(shell ls tests/iso))) $(eval _VARS = VERSION FLATPAK_REMOTE_NAME _FLATPAK_REPO_URL) - sudo apt-get update - sudo apt-get install -y squashfs-tools sudo modprobe loop sudo mkdir /mnt/iso /mnt/install sudo mount -o loop deploy.iso /mnt/iso @@ -245,9 +247,21 @@ test-iso: sudo umount /mnt/install sudo umount /mnt/iso -test-vm: +ansible_inventory: + echo "ungrouped:" > ansible_inventory + echo " hosts:" >> ansible_inventory + echo " vm:" >> ansible_inventory + echo " ansible_host: ${VM_IP}" >> ansible_inventory + echo " ansible_port: ${VM_PORT}" >> ansible_inventory + echo " ansible_user: ${VM_USER}" >> ansible_inventory + echo " ansible_password: ${VM_PASS}" >> ansible_inventory + echo " ansible_become_pass: ${VM_PASS}" >> ansible_inventory + echo " ansible_ssh_common_args: '-o StrictHostKeyChecking=no'" >> ansible_inventory + +test-vm: ansible_inventory + ansible -i ansible_inventory -m ansible.builtin.wait_for_connection vm $(eval _TESTS = $(filter-out README.md,$(shell ls tests/vm))) chmod +x $(foreach test,$(_TESTS),tests/vm/$(test)) - for test in $(_TESTS); do ./tests/vm/$${test} deploy.iso; done + for test in $(_TESTS); do ./tests/vm/$${test}; done -.PHONY: clean install-deps test test-iso test-vm +.PHONY: clean install-deps install-test-deps test test-iso test-vm diff --git a/README.md b/README.md index f6c0280..44792b0 100644 --- a/README.md +++ b/README.md @@ -38,24 +38,24 @@ See [Customizing](#customizing) for information about customizing the ISO that g The following variables can be used to customize the created ISO. ### Inputs -| Variable | Description | Default Value | -| ---------------------- | ---------------------------------------------------------------------------- | -------------------------------------------- | -| additional_templates | Space delimited list of additional Lorax templates to include | \[empty\] | -| arch | Architecture for image to build | x86_64 | -| enrollment_password | Used for supporting secure boot (requires SECURE_BOOT_KEY_URL to be defined) | container-installer | -| extra_boot_params | Extra params used by grub to boot the anaconda installer | \[empty\] | -| flatpak_remote_name | Name of the Flatpak repo on the destination OS | flathub | -| flatpak_remote_refs | Space separated list of flatpak refs to install | \[empty\] | -| flatpak_remote_refs_dir | Directory that contains files that list the flatpak refs to install | \[empty\] | -| flatpak_remote_url | URL of the flatpakrepo file | https://flathub.org/repo/flathub.flatpakrepo | -| image_name | Name of the source container image | base | -| image_repo | Repository containing the source container image | quay.io/fedora-ostree-desktops | -| image_tag | Tag of the source container image | *VERSION* | -| iso_name | Name of the ISO you wish to output when completed | build/deploy.iso | -| secure_boot_key_url | Secure boot key that is installed from URL location\*\* | \[empty\] | -| variant | Source container variant\* | Server | -| version | Fedora version of installer to build | 39 | -| web_ui | Enable Anaconda WebUI (experimental) | false | +| Variable | Description | Default Value | Action | Container | Makefile | +| ----------------------- | ---------------------------------------------------------------------------- | -------------------------------------------- | ------------------ | ------------------ | ------------------ | +| additional_templates | Space delimited list of additional Lorax templates to include | \[empty\] | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| arch | Architecture for image to build | x86_64 | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| enrollment_password | Used for supporting secure boot (requires SECURE_BOOT_KEY_URL to be defined) | container-installer | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| extra_boot_params | Extra params used by grub to boot the anaconda installer | \[empty\] | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| flatpak_remote_name | Name of the Flatpak repo on the destination OS | flathub | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| flatpak_remote_refs | Space separated list of flatpak refs to install | \[empty\] | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| flatpak_remote_refs_dir | Directory that contains files that list the flatpak refs to install | \[empty\] | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| flatpak_remote_url | URL of the flatpakrepo file | https://flathub.org/repo/flathub.flatpakrepo | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| image_name | Name of the source container image | base | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| image_repo | Repository containing the source container image | quay.io/fedora-ostree-desktops | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| image_tag | Tag of the source container image | *VERSION* | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| iso_name | Name of the ISO you wish to output when completed | build/deploy.iso | :white_check_mark: | :x: | :x: | +| secure_boot_key_url | Secure boot key that is installed from URL location\*\* | \[empty\] | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| variant | Source container variant\* | Server | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| version | Fedora version of installer to build | 39 | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| web_ui | Enable Anaconda WebUI (experimental) | false | :white_check_mark: | :white_check_mark: | :white_check_mark: | \*Available options for VARIANT can be found by running `dnf provides system-release`. Variant will be the third item in the package name. Example: `fedora-release-kinoite-39-34.noarch` will be kinoite diff --git a/tests/vm/flatpak_update.yml b/tests/vm/flatpak_update.yml index 6f46263..7d004cd 100644 --- a/tests/vm/flatpak_update.yml +++ b/tests/vm/flatpak_update.yml @@ -5,6 +5,8 @@ gather_facts: no tasks: + # Verifies that the GPG key is functional - name: Test updating flatpak packages + become: true ansible.builtin.command: - cmd: /usr/bin/flatpak update -y + cmd: /usr/bin/flatpak update -y --noninteractive From 3d85c930108fea1d45529888da9909a28490057b Mon Sep 17 00:00:00 2001 From: Jason N <33561705+JasonN3@users.noreply.github.com> Date: Mon, 18 Mar 2024 14:32:01 -0400 Subject: [PATCH 20/20] Add tests (#76) --- .github/workflows/bot_run_tests.yml | 34 +++ .github/workflows/build-and-test.yml | 256 ++++++++++++++++-- .github/workflows/variables.yml | 27 ++ Makefile | 51 ++-- README.md | 2 + action.yml | 16 +- lorax_templates/flatpak_set_repo.tmpl | 3 - .../scripts/post/flatpak_configure | 7 +- tests/repo/vars.sh | 39 +++ tests/vm/flatpak_fedora_repo_disabled.yml | 16 ++ tests/vm/flatpak_installed.yml | 25 ++ tests/vm/install_image_source.yml | 25 ++ 12 files changed, 458 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/bot_run_tests.yml create mode 100644 .github/workflows/variables.yml create mode 100644 tests/repo/vars.sh create mode 100644 tests/vm/flatpak_fedora_repo_disabled.yml create mode 100644 tests/vm/flatpak_installed.yml create mode 100644 tests/vm/install_image_source.yml diff --git a/.github/workflows/bot_run_tests.yml b/.github/workflows/bot_run_tests.yml new file mode 100644 index 0000000..ba867ed --- /dev/null +++ b/.github/workflows/bot_run_tests.yml @@ -0,0 +1,34 @@ +name: Run tests on PR +on: issue_comment + +jobs: + pr_commented: + name: Check Permissions + if: > + github.event.issue.pull_request && + contains(github.event.comment.body, '/run tests') + runs-on: ubuntu-latest + steps: + - name: Check association + run: | + allowed=("OWNER" "COLLABORATOR") + value="\<${{ github.event.issue.author_association }}\>" + + if [[ ${allowed[@]} =~ $value ]] + then + exit 0 + else + exit 1 + fi + + run-tests: + name: Run Tests + permissions: + contents: read + packages: write + statuses: write + needs: + - pr_commented + uses: ./.github/workflows/build-and-test.yml + with: + pr: ${{ github.event.issue.number }} \ No newline at end of file diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a1aa320..7b90bfc 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -6,8 +6,16 @@ on: - 'main' tags: - 'v*' + pull_request: + workflow_call: + inputs: + pr: + required: true + type: string + + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -20,25 +28,62 @@ env: IMAGE_REPO: 'quay.io/fedora-ostree-desktops' IMAGE_TAG: '39' VARIANT: 'Server' - FLATPAK_REMOTE_REFS_DIR: /github/workspace/flatpak_refs + FLATPAK_REMOTE_REFS_DIR: flatpak_refs SECURE_BOOT_KEY_URL: 'https://github.com/ublue-os/akmods/raw/main/certs/public_key.der' ENROLLMENT_PASSWORD: 'container-installer' jobs: build-container: + if: > + github.event_name == 'push' || + github.event_name == 'issue_comment' || + ( github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'auto-test') ) name: Build Container Image + env: + JOB_NAME: Build Container Image runs-on: ubuntu-latest permissions: contents: read packages: write + statuses: write steps: - - name: Checkout repo + - name: Checkout uses: actions/checkout@v4 with: submodules: recursive fetch-depth: 0 fetch-tags: 'true' + - name: Switch branch + if: inputs.pr + env: + GITHUB_USER: ${{ github.actor }} + GITHUB_TOKEN: ${{ github.token }} + run: | + sudo apt-get update + sudo apt-get install -y hub + hub pr checkout ${{ inputs.pr }} + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Get Current Job Log URL + if: inputs.pr && always() + uses: Tiryoh/gha-jobid-action@v1 + id: jobs + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + job_name: "Run Tests / ${{ env.JOB_NAME }}" + + - name: Set status + if: inputs.pr && always() + uses: myrotvorets/set-commit-status-action@v2.0.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + context: ${{ env.JOB_NAME }} + sha: ${{ env.sha }} + targetUrl: ${{ steps.jobs.outputs.html_url }} + - name: Docker meta id: meta uses: docker/metadata-action@v5 @@ -66,15 +111,28 @@ jobs: tags: ${{ steps.build-image.outputs.tags }} username: ${{ github.actor }} password: ${{ github.token }} + + - name: Set status + if: inputs.pr && always() + uses: myrotvorets/set-commit-status-action@v2.0.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + context: ${{ env.JOB_NAME }} + sha: ${{ env.sha }} + targetUrl: ${{ steps.jobs.outputs.html_url }} build-and-push-iso: name: Build ISO + env: + JOB_NAME: Build ISO runs-on: ubuntu-latest needs: - build-container permissions: contents: read packages: write + statuses: write continue-on-error: false strategy: fail-fast: false @@ -88,8 +146,39 @@ jobs: iso_name-39: ${{ steps.save_output.outputs.iso_name-39 }} iso_name-40: ${{ steps.save_output.outputs.iso_name-40 }} steps: - - name: Checkout repo + - name: Checkout uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Switch branch + if: inputs.pr + env: + GITHUB_USER: ${{ github.actor }} + GITHUB_TOKEN: ${{ github.token }} + run: | + sudo apt-get update + sudo apt-get install -y hub + hub pr checkout ${{ inputs.pr }} + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Get Current Job Log URL + if: inputs.pr && always() + uses: Tiryoh/gha-jobid-action@v1 + id: jobs + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + job_name: "Run Tests / ${{ env.JOB_NAME }}" + + - name: Set status + if: inputs.pr && always() + uses: myrotvorets/set-commit-status-action@v2.0.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + context: ${{ env.JOB_NAME }} + sha: ${{ env.sha }} + targetUrl: ${{ steps.jobs.outputs.html_url }} - name: Lowercase Registry id: registry_case @@ -115,10 +204,10 @@ jobs: image_tag: ${{ matrix.version }} version: ${{ matrix.version }} variant: ${{ env.VARIANT }} - flatpak_remote_refs_dir: ${{ env.FLATPAK_REMOTE_REFS_DIR }} + flatpak_remote_refs_dir: /github/workspace/${{ env.FLATPAK_REMOTE_REFS_DIR }} secure_boot_key_url: ${{ env.SECURE_BOOT_KEY_URL }} enrollment_password: ${{ env.ENROLLMENT_PASSWORD }} - iso_name: ${{ env.IMAGE_NAME }}-${{ matrix.version }}-${{ matrix.version }}.iso + iso_name: ${{ env.IMAGE_NAME }}-${{ matrix.version }}-${{ github.event.pull_request.number || github.ref }}.iso - name: Save output id: save_output @@ -139,14 +228,26 @@ jobs: compression-level: 0 overwrite: true - test-qemu: + - name: Set status + if: inputs.pr && always() + uses: myrotvorets/set-commit-status-action@v2.0.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + context: ${{ env.JOB_NAME }} + sha: ${{ env.sha }} + targetUrl: ${{ steps.jobs.outputs.html_url }} + + test-iso: name: Test ISO + env: + JOB_NAME: Test ISO runs-on: ubuntu-latest needs: - build-and-push-iso permissions: contents: read - packages: write + statuses: write continue-on-error: false strategy: fail-fast: false @@ -156,21 +257,46 @@ jobs: - 39 - 40 steps: - - name: Checkout repo + - name: Checkout uses: actions/checkout@v4 with: submodules: recursive + - name: Switch branch + if: inputs.pr + env: + GITHUB_USER: ${{ github.actor }} + GITHUB_TOKEN: ${{ github.token }} + run: | + sudo apt-get update + sudo apt-get install -y hub + hub pr checkout ${{ inputs.pr }} + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Get Current Job Log URL + if: inputs.pr && always() + uses: Tiryoh/gha-jobid-action@v1 + id: jobs + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + job_name: "Run Tests / ${{ env.JOB_NAME }}" + + - name: Set status + if: inputs.pr && always() + uses: myrotvorets/set-commit-status-action@v2.0.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + context: ${{ env.JOB_NAME }} + sha: ${{ env.sha }} + targetUrl: ${{ steps.jobs.outputs.html_url }} + - name: Install test tools run: | sudo apt-get update sudo apt-get install -y make sudo make install-test-deps PACKAGE_MANAGER=apt-get - - name: Create disk - run: | - qemu-img create -f qcow2 disk.qcow2 50G - - name: Download generated ISO uses: actions/download-artifact@v4 with: @@ -195,8 +321,83 @@ jobs: SECURE_BOOT_KEY_URL=${{ env.SECURE_BOOT_KEY_URL }} \ ENROLLMENT_PASSWORD=${{ env.ENROLLMENT_PASSWORD }} + - name: Set status + if: inputs.pr && always() + uses: myrotvorets/set-commit-status-action@v2.0.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + context: ${{ env.JOB_NAME }} + sha: ${{ env.sha }} + targetUrl: ${{ steps.jobs.outputs.html_url }} + + test-deployment: + name: Test deployment + env: + JOB_NAME: Test deployment + runs-on: ubuntu-latest + needs: + - build-and-push-iso + permissions: + contents: read + statuses: write + continue-on-error: false + strategy: + fail-fast: false + matrix: + version: + - 38 + - 39 + - 40 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Switch branch + if: inputs.pr + env: + GITHUB_USER: ${{ github.actor }} + GITHUB_TOKEN: ${{ github.token }} + run: | + sudo apt-get update + sudo apt-get install -y hub + hub pr checkout ${{ inputs.pr }} + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Get Current Job Log URL + if: inputs.pr && always() + uses: Tiryoh/gha-jobid-action@v1 + id: jobs + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + job_name: "Run Tests / ${{ env.JOB_NAME }}" + + - name: Set status + if: inputs.pr && always() + uses: myrotvorets/set-commit-status-action@v2.0.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + context: ${{ env.JOB_NAME }} + sha: ${{ env.sha }} + targetUrl: ${{ steps.jobs.outputs.html_url }} + + - name: Install test tools + run: | + sudo apt-get update + sudo apt-get install -y make + sudo make install-test-deps PACKAGE_MANAGER=apt-get + + - name: Download generated ISO + uses: actions/download-artifact@v4 + with: + name: ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }} + - name: Add Kickstart and Grub options to ISO run: | + mv ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }} deploy.iso sudo mkdir /mnt/iso || true sudo mount -o loop deploy.iso /mnt/iso cp /mnt/iso/boot/grub2/grub.cfg grub.cfg @@ -225,9 +426,13 @@ jobs: -end EOF + - name: Create VM disk + run: | + qemu-img create -f qcow2 disk.qcow2 50G + - name: Install the test VM run: | - timeout 1h qemu-system-x86_64 -name "Anaconda" -boot d -m 4096 -cpu qemu64 -display none -cdrom test.iso -smp 2 -hda disk.qcow2 -serial telnet:localhost:4321,server,nowait & QEMU_PID=$! + timeout 1h qemu-system-x86_64 -name "Anaconda" -boot d -m 4096 -cpu qemu64 -display none -cdrom test.iso -smp 2 -hda disk.qcow2 -serial telnet:localhost:4321,server=on,wait=off & QEMU_PID=$! echo "PID: $QEMU_PID" timeout 1m bash -c "while ! (echo > /dev/tcp/127.0.0.1/4321); do sleep 0.1; done" (nc localhost 4321 | tee vm.stdout) & @@ -243,12 +448,12 @@ jobs: mkfifo vm.stdin qemu-system-x86_64 -name "Anaconda" \ -m 4096 -cpu qemu64 -display none -smp 2 \ - -chardev socket,path=/tmp/qga.sock,server,nowait,id=qga0 \ + -chardev socket,path=/tmp/qga.sock,server=on,wait=off,id=qga0 \ -device e1000,netdev=net0 \ -netdev user,id=net0,hostfwd=tcp::${VM_PORT}-:22 \ -device virtio-serial \ -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \ - -boot c -hda disk.qcow2 -serial telnet:localhost:4321,server,nowait & export QEMU_PID=$! + -boot c -hda disk.qcow2 -serial telnet:localhost:4321,server=on,wait=off & export QEMU_PID=$! echo "PID: $QEMU_PID" timeout 1m bash -c "while ! (echo > /dev/tcp/127.0.0.1/4321); do sleep 0.1; done" @@ -263,5 +468,24 @@ jobs: echo "VM ready for tests at IP ${VM_IP}:${VM_PORT}" - make test-vm VM_IP=${VM_IP} VM_PORT=${VM_PORT} VM_USER=${VM_USER} VM_PASS=${VM_PASS} + make test-vm VM_IP=${VM_IP} VM_PORT=${VM_PORT} VM_USER=${VM_USER} VM_PASS=${VM_PASS} \ + ARCH=${{ env.ARCH}} \ + IMAGE_NAME=${{ env.IMAGE_NAME}} \ + IMAGE_REPO=${{ env.IMAGE_REPO}} \ + IMAGE_TAG=${{ matrix.version }} \ + VERSION=${{ matrix.version }} \ + VARIANT=${{ env.VARIANT }} \ + FLATPAK_REMOTE_REFS_DIR=${{ env.FLATPAK_REMOTE_REFS_DIR }} \ + SECURE_BOOT_KEY_URL=${{ env.SECURE_BOOT_KEY_URL }} \ + ENROLLMENT_PASSWORD=${{ env.ENROLLMENT_PASSWORD }} kill $QEMU_PID + + - name: Set status + if: inputs.pr && always() + uses: myrotvorets/set-commit-status-action@v2.0.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + context: ${{ env.JOB_NAME }} + sha: ${{ env.sha }} + targetUrl: ${{ steps.jobs.outputs.html_url }} diff --git a/.github/workflows/variables.yml b/.github/workflows/variables.yml new file mode 100644 index 0000000..e697fa0 --- /dev/null +++ b/.github/workflows/variables.yml @@ -0,0 +1,27 @@ +name: Repo Tests + +on: + push: + branches: + - 'main' + tags: + - 'v*' + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + variables: + name: Check variables are listed + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Run test + run: | + /bin/bash tests/repo/vars.sh \ No newline at end of file diff --git a/Makefile b/Makefile index 5058868..acf2d0d 100644 --- a/Makefile +++ b/Makefile @@ -20,9 +20,12 @@ FLATPAK_REMOTE_REFS_DIR = # Secure boot ENROLLMENT_PASSWORD = SECURE_BOOT_KEY_URL = + +################### +# Hidden vars + # Cache DNF_CACHE = - PACKAGE_MANAGER = dnf # Functions @@ -32,6 +35,20 @@ PACKAGE_MANAGER = dnf get_templates = $(shell ls lorax_templates/$(1)_*.tmpl) \ $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/$(1)_*)),lorax_templates/post_$(file).tmpl) +# Get a list of tests for the feature +# $1 = test type +# $2 = feature +run_tests = tests="$(shell ls tests/$(1)/$(2)_*)"; \ + if [ -n "$$tests" ]; \ + then \ + chmod +x $$tests; \ + for test in $$tests; \ + do \ + $(foreach var,$(_VARS),$(var)=$($(var))) ./$${test}; \ + RC=$$?; if [ $$RC != 0 ]; then exit $$RC; fi; \ + done; \ + fi + # Converts a post script to a template # $1 = script to convert # $2 = file on ISO to write @@ -217,8 +234,10 @@ install-test-deps: test: test-iso test-vm +test-repo: + bash tests/repo/vars.sh + test-iso: - $(eval _TESTS = $(filter-out README.md,$(shell ls tests/iso))) $(eval _VARS = VERSION FLATPAK_REMOTE_NAME _FLATPAK_REPO_URL) sudo modprobe loop @@ -227,21 +246,10 @@ test-iso: sudo mount -t squashfs -o loop /mnt/iso/images/install.img /mnt/install # install tests - chmod +x $(foreach test,$(filter install_%,$(_TESTS)),tests/iso/$(test)) - for test in $(_TESTS); \ - do \ - $(foreach var,$(_VARS),$(var)=$($(var))) ./tests/iso/$${test}; \ - done - + $(call run_tests,iso,install) + # flapak tests - if [ -n "$(FLATPAK_REMOTE_REFS)" ]; \ - then \ - chmod +x $(foreach test,$(filter flatpak_%,$(_TESTS)),tests/iso/$(test)); \ - for test in $(_TESTS); \ - do \ - $(foreach var,$(_VARS),$(var)=$($(var))) ./tests/iso/$${test}; \ - done; \ - fi + if [ -n "$(FLATPAK_REMOTE_REFS)" ]; then $(call run_tests,iso,flatpak); fi # Cleanup sudo umount /mnt/install @@ -259,9 +267,14 @@ ansible_inventory: echo " ansible_ssh_common_args: '-o StrictHostKeyChecking=no'" >> ansible_inventory test-vm: ansible_inventory + $(eval _VARS = IMAGE_REPO IMAGE_NAME IMAGE_TAG) + ansible -i ansible_inventory -m ansible.builtin.wait_for_connection vm - $(eval _TESTS = $(filter-out README.md,$(shell ls tests/vm))) - chmod +x $(foreach test,$(_TESTS),tests/vm/$(test)) - for test in $(_TESTS); do ./tests/vm/$${test}; done + + # install tests + $(call run_tests,vm,install) + + # flapak tests + if [ -n "$(FLATPAK_REMOTE_REFS)" ]; then $(call run_tests,vm,flatpak); fi .PHONY: clean install-deps install-test-deps test test-iso test-vm diff --git a/README.md b/README.md index 44792b0..0e8611f 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ The following variables can be used to customize the created ISO. | image_repo | Repository containing the source container image | quay.io/fedora-ostree-desktops | :white_check_mark: | :white_check_mark: | :white_check_mark: | | image_tag | Tag of the source container image | *VERSION* | :white_check_mark: | :white_check_mark: | :white_check_mark: | | iso_name | Name of the ISO you wish to output when completed | build/deploy.iso | :white_check_mark: | :x: | :x: | +| repos | List of repo files for Lorax to use | /etc/yum.repos.d/*.repo | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| rootfs_size | The size (in GiB) for the squashfs runtime volume | 2 | :white_check_mark: | :white_check_mark: | :white_check_mark: | | secure_boot_key_url | Secure boot key that is installed from URL location\*\* | \[empty\] | :white_check_mark: | :white_check_mark: | :white_check_mark: | | variant | Source container variant\* | Server | :white_check_mark: | :white_check_mark: | :white_check_mark: | | version | Fedora version of installer to build | 39 | :white_check_mark: | :white_check_mark: | :white_check_mark: | diff --git a/action.yml b/action.yml index 84f9337..493dec4 100644 --- a/action.yml +++ b/action.yml @@ -62,6 +62,11 @@ inputs: description: "Name of the resulting ISO. Relative paths are relative to github.workspace" required: false default: build/deploy.iso + repos: + description: List of repo files for Lorax to use + required: false + rootfs_size: + description: The size (in GiB) for the squashfs runtime volume secure_boot_key_url: description: Secure boot key that is installed from URL location required: false @@ -131,16 +136,19 @@ runs: ACTION_REF: ${{ github.action_ref }} shell: bash run: | + image="ghcr.io/jasonn3/build-container-installer" # Check if running inside of the action repo if [[ -z "${ACTION_REPO}" || "${ACTION_REPO}" == "${{ github.repository }}" ]] then if [[ "${{ github.ref_name }}" =~ (.*)/merge ]] - then tag=pr-${BASH_REMATCH[1]} + then + tag="pr-${BASH_REMATCH[1]}" + image="docker.io/jasonn3/build-container-installer" else - tag=${{ github.ref_name }} + tag="${{ github.ref_name }}" fi else - tag=${ACTION_REF} + tag="${ACTION_REF}" fi if [[ "${{ inputs.enable_cache_dnf }}" == "true" ]] then @@ -167,7 +175,7 @@ runs: vars="${vars} FLATPAK_REMOTE_REFS_DIR=\"${{ inputs.flatpak_remote_refs_dir }}\"" fi fi - docker run --privileged --volume ${{ github.workspace }}:/github/workspace/ ${cache} ghcr.io/jasonn3/build-container-installer:${tag} \ + docker run --privileged --volume ${{ github.workspace }}:/github/workspace/ ${cache} ${image}:${tag} \ ADDITIONAL_TEMPLATES="${{ inputs.additional_templates }}" \ ARCH="${{ inputs.arch }}" \ DNF_CACHE="/cache/dnf" \ diff --git a/lorax_templates/flatpak_set_repo.tmpl b/lorax_templates/flatpak_set_repo.tmpl index 30992db..0d8db43 100644 --- a/lorax_templates/flatpak_set_repo.tmpl +++ b/lorax_templates/flatpak_set_repo.tmpl @@ -6,6 +6,3 @@ replace "flatpak_manager\.add_remote\(\".*\", \".*\"\)" "flatpak_manager.add_rem replace "flatpak_manager\.replace_installed_refs_remote\(\".*\"\)" "flatpak_manager.replace_installed_refs_remote(\"${flatpak_remote_name}\")" /usr/lib64/python*/site-packages/pyanaconda/modules/payloads/payload/rpm_ostree/flatpak_installation.py % endif -%if flatpak_remote_name != 'fedora': -systemctl disable flatpak-add-fedora-repos.service -% endif diff --git a/lorax_templates/scripts/post/flatpak_configure b/lorax_templates/scripts/post/flatpak_configure index 1c54a7f..5331c05 100644 --- a/lorax_templates/scripts/post/flatpak_configure +++ b/lorax_templates/scripts/post/flatpak_configure @@ -1,4 +1,4 @@ -<%page args="_flatpak_repo_gpg"/> +<%page args="_flatpak_repo_gpg, flatpak_remote_name"/> if [[ -d /ostree/deploy/default/var/lib/flatpak/repo ]] then @@ -9,3 +9,8 @@ then else echo "Could not find Flatpaks repo" fi + +if [[ "${flatpak_remote_name}" != 'fedora' ]] +then + systemctl disable flatpak-add-fedora-repos.service +fi diff --git a/tests/repo/vars.sh b/tests/repo/vars.sh new file mode 100644 index 0000000..2e4302e --- /dev/null +++ b/tests/repo/vars.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +vars=() + +while read -r line +do + if ! [[ $line =~ ^# ]] + then + vars+=$(echo $line | cut -d= -f1 | tr [:upper:] [:lower:]) + fi + if [[ $line =~ ^########## ]] + then + break + fi +done < Makefile + +result=0 + +for var in $vars +do + grep "^| ${var}" README.md > /dev/null + if [[ $? != 0 ]] + then + echo "$var not found in README.md" + result=1 + fi +done + +for var in $vars +do + grep "^ ${var}:" action.yml > /dev/null + if [[ $? != 0 ]] + then + echo "$var not found in action.yml" + result=1 + fi +done + +exit ${result} \ No newline at end of file diff --git a/tests/vm/flatpak_fedora_repo_disabled.yml b/tests/vm/flatpak_fedora_repo_disabled.yml new file mode 100644 index 0000000..f87ff80 --- /dev/null +++ b/tests/vm/flatpak_fedora_repo_disabled.yml @@ -0,0 +1,16 @@ +#!/usr/bin/env -S ansible-playbook -i ./ansible_inventory +--- +- name: Test for installed flatpaks + hosts: vm + gather_facts: no + + tasks: + - name: Collect facts about system services + service_facts: + register: services_state + + - name: Check that flatpak-add-fedora-repos is disabled + ansible.builtin.assert: + that: + - services_state['ansible_facts']['services']['flatpak-add-fedora-repos.service']['status'] == 'disabled' + fail_msg: 'flatpak-add-fedora-repos.service is not disabled' \ No newline at end of file diff --git a/tests/vm/flatpak_installed.yml b/tests/vm/flatpak_installed.yml new file mode 100644 index 0000000..da4f331 --- /dev/null +++ b/tests/vm/flatpak_installed.yml @@ -0,0 +1,25 @@ +#!/usr/bin/env -S ansible-playbook -i ./ansible_inventory +--- +- name: Test for installed flatpaks + hosts: vm + gather_facts: no + + tasks: + # Verifies that the flatpaks are installed + - name: Get list of installed Flatpaks + become: true + ansible.builtin.command: + cmd: /usr/bin/flatpak list + register: flatpaks + + - name: Check that VLC is installed + ansible.builtin.assert: + that: + - "'VLC' in flatpaks.stdout" + fail_msg: 'VLC is not installed' + + - name: Check that Firefox is installed + ansible.builtin.assert: + that: + - "'Firefox' in flatpaks.stdout" + fail_msg: 'Firefox is not installed' diff --git a/tests/vm/install_image_source.yml b/tests/vm/install_image_source.yml new file mode 100644 index 0000000..1b21331 --- /dev/null +++ b/tests/vm/install_image_source.yml @@ -0,0 +1,25 @@ +#!/usr/bin/env -S ansible-playbook -i ./ansible_inventory +--- +- name: Test Container Image source updates + hosts: vm + gather_facts: no + + tasks: + # Get list of origins + - name: Get origin + become: true + ansible.builtin.command: + cmd: /bin/bash -c "cat /ostree/deploy/default/deploy/*.origin" + register: origin + + - name: Get vars + ansible.builtin.set_fact: + image_repo: "{{ lookup('ansible.builtin.env', 'IMAGE_REPO') }}" + image_name: "{{ lookup('ansible.builtin.env', 'IMAGE_NAME') }}" + image_tag: "{{ lookup('ansible.builtin.env', 'IMAGE_TAG') }}" + + - name: Tests + ansible.builtin.assert: + that: + - (image_repo + '/' + image_name + ':' + image_tag) in origin.stdout + fail_msg: 'Origin not configured'