Merge branch 'tests' into flatpak-service

.github/workflows/build-and-test.yml

@@ -20,7 +20,7 @@ env:
   IMAGE_REPO: 'quay.io/fedora-ostree-desktops'
   IMAGE_TAG: '39'
   VARIANT: 'Server'
-  FLATPAK_REMOTE_REFS_DIR: /github/workspace/flatpak_refs
+  FLATPAK_REMOTE_REFS_DIR: flatpak_refs
   SECURE_BOOT_KEY_URL: 'https://github.com/ublue-os/akmods/raw/main/certs/public_key.der'
   ENROLLMENT_PASSWORD: 'container-installer'
 
@@ -115,7 +115,7 @@ jobs:
           image_tag: ${{ matrix.version }}
           version: ${{ matrix.version }}
           variant: ${{ env.VARIANT }}
-          flatpak_remote_refs_dir: ${{ env.FLATPAK_REMOTE_REFS_DIR }}
+          flatpak_remote_refs_dir: /github/workspace/${{ env.FLATPAK_REMOTE_REFS_DIR }}
           secure_boot_key_url: ${{ env.SECURE_BOOT_KEY_URL }}
           enrollment_password: ${{ env.ENROLLMENT_PASSWORD }}
           iso_name: ${{ env.IMAGE_NAME }}-${{ matrix.version }}-${{ matrix.version }}.iso
@@ -178,11 +178,13 @@ jobs:
     
       - name: Verify ISO
         run: |
+          set -e
           checkisomd5 ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }}
           sha256sum -c ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }}-CHECKSUM
 
       - name: Run ISO checks
         run: |
+          set -e
           mv ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }} deploy.iso
           make test-iso \
             ARCH=${{ env.ARCH}} \
@@ -197,6 +199,7 @@ jobs:
 
       - name: Add Kickstart and Grub options to ISO
         run: |
+          set -e
           sudo mkdir /mnt/iso || true
           sudo mount -o loop deploy.iso /mnt/iso
           cp /mnt/iso/boot/grub2/grub.cfg grub.cfg
@@ -227,7 +230,8 @@ jobs:
 
       - name: Install the test VM
         run: |
-          timeout 1h qemu-system-x86_64 -name "Anaconda" -boot d -m 4096 -cpu qemu64 -display none -cdrom test.iso -smp 2 -hda disk.qcow2 -serial telnet:localhost:4321,server,nowait & QEMU_PID=$!
+          set -e
+          timeout 1h qemu-system-x86_64 -name "Anaconda" -boot d -m 4096 -cpu qemu64 -display none -cdrom test.iso -smp 2 -hda disk.qcow2 -serial telnet:localhost:4321,server=on,wait=off & QEMU_PID=$!
           echo "PID: $QEMU_PID"
           timeout 1m bash -c "while ! (echo > /dev/tcp/127.0.0.1/4321); do sleep 0.1; done"
           (nc localhost 4321 | tee vm.stdout) &
@@ -240,15 +244,16 @@ jobs:
           VM_IP: "127.0.0.1"
           VM_PORT: "5555"
         run: |
+          set -e
           mkfifo vm.stdin
           qemu-system-x86_64 -name "Anaconda" \
             -m 4096 -cpu qemu64 -display none -smp 2 \
-            -chardev socket,path=/tmp/qga.sock,server,nowait,id=qga0 \
+            -chardev socket,path=/tmp/qga.sock,server=on,wait=off,id=qga0 \
             -device e1000,netdev=net0 \
             -netdev user,id=net0,hostfwd=tcp::${VM_PORT}-:22 \
             -device virtio-serial \
             -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \
-            -boot c -hda disk.qcow2 -serial telnet:localhost:4321,server,nowait & export QEMU_PID=$!
+            -boot c -hda disk.qcow2 -serial telnet:localhost:4321,server=on,wait=off & export QEMU_PID=$!
           echo "PID: $QEMU_PID"
 
           timeout 1m bash -c "while ! (echo > /dev/tcp/127.0.0.1/4321); do sleep 0.1; done"
@@ -263,5 +268,14 @@ jobs:
           
           echo "VM ready for tests at IP ${VM_IP}:${VM_PORT}"
           
-          make test-vm VM_IP=${VM_IP} VM_PORT=${VM_PORT} VM_USER=${VM_USER} VM_PASS=${VM_PASS}
+          make test-vm VM_IP=${VM_IP} VM_PORT=${VM_PORT} VM_USER=${VM_USER} VM_PASS=${VM_PASS} \
+            ARCH=${{ env.ARCH}} \
+            IMAGE_NAME=${{ env.IMAGE_NAME}} \
+            IMAGE_REPO=${{ env.IMAGE_REPO}} \
+            IMAGE_TAG=${{ matrix.version }} \
+            VERSION=${{ matrix.version }} \
+            VARIANT=${{ env.VARIANT }} \
+            FLATPAK_REMOTE_REFS_DIR=${{ env.FLATPAK_REMOTE_REFS_DIR }} \
+            SECURE_BOOT_KEY_URL=${{ env.SECURE_BOOT_KEY_URL }} \
+            ENROLLMENT_PASSWORD=${{ env.ENROLLMENT_PASSWORD }}
           kill $QEMU_PID

Makefile

@@ -32,6 +32,20 @@ PACKAGE_MANAGER = dnf
 get_templates = $(shell ls lorax_templates/$(1)_*.tmpl) \
                 $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/$(1)_*)),lorax_templates/post_$(file).tmpl)
 
+# Get a list of tests for the feature
+# $1 = test type
+# $2 = feature
+run_tests = tests="$(shell ls tests/$(1)/$(2)_*)"; \
+			if [ -n "$$tests" ]; \
+			then \
+			  chmod +x $$tests; \
+	  		  for test in $$tests; \
+	  		  do \
+				$(foreach var,$(_VARS),$(var)=$($(var))) ./$${test}; \
+				RC=$$?; if [ $$RC != 0 ]; then exit $$RC; fi; \
+	  		  done; \
+			fi
+
 # Converts a post script to a template
 # $1 = script to convert
 # $2 = file on ISO to write
@@ -218,7 +232,6 @@ install-test-deps:
 test: test-iso test-vm
 
 test-iso:
-	$(eval _TESTS = $(filter-out README.md,$(shell ls tests/iso)))
 	$(eval _VARS = VERSION FLATPAK_REMOTE_NAME _FLATPAK_REPO_URL)
 
 	sudo modprobe loop
@@ -227,21 +240,10 @@ test-iso:
 	sudo mount -t squashfs -o loop /mnt/iso/images/install.img /mnt/install
 
 	# install tests
-	chmod +x $(foreach test,$(filter install_%,$(_TESTS)),tests/iso/$(test))
-	for test in $(_TESTS); \
-	do \
-	  $(foreach var,$(_VARS),$(var)=$($(var))) ./tests/iso/$${test}; \
-	done
+	$(call run_tests,iso,install)
 	
 	# flapak tests
-	if [ -n "$(FLATPAK_REMOTE_REFS)" ]; \
-	then \
-		chmod +x $(foreach test,$(filter flatpak_%,$(_TESTS)),tests/iso/$(test)); \
-		for test in $(_TESTS); \
-		do \
-		$(foreach var,$(_VARS),$(var)=$($(var))) ./tests/iso/$${test}; \
-		done; \
-	fi
+	if [ -n "$(FLATPAK_REMOTE_REFS)" ]; then $(call run_tests,iso,flatpak); fi
 
 	# Cleanup
 	sudo umount /mnt/install
@@ -260,8 +262,11 @@ ansible_inventory:
 
 test-vm: ansible_inventory
 	ansible -i ansible_inventory -m ansible.builtin.wait_for_connection vm
-	$(eval _TESTS = $(filter-out README.md,$(shell ls tests/vm)))
-	chmod +x $(foreach test,$(_TESTS),tests/vm/$(test))
-	for test in $(_TESTS); do ./tests/vm/$${test}; done
+
+	# install tests
+	$(call run_tests,vm,install)
+
+	# flapak tests
+	if [ -n "$(FLATPAK_REMOTE_REFS)" ]; then $(call run_tests,vm,flatpak); fi
 
 .PHONY: clean install-deps install-test-deps test test-iso test-vm

tests/vm/flatpak_fedora_repo_disabled.yml

@@ -0,0 +1,16 @@
+#!/usr/bin/env -S ansible-playbook -i ./ansible_inventory
+---
+- name: Test for installed flatpaks
+  hosts: vm
+  gather_facts: no
+
+  tasks:
+  - name: Collect facts about system services
+    service_facts:
+    register: services_state
+
+  - name: Check that flatpak-add-fedora-repos is disabled
+    ansible.builtin.assert:
+      that:
+        - services_state['ansible_facts']['services']['flatpak-add-fedora-repos.service']['status'] == 'disabled'
+      fail_msg: 'flatpak-add-fedora-repos.service is not disabled'

tests/vm/flatpak_installed.yml

@@ -0,0 +1,25 @@
+#!/usr/bin/env -S ansible-playbook -i ./ansible_inventory
+---
+- name: Test for installed flatpaks
+  hosts: vm
+  gather_facts: no
+
+  tasks:
+  # Verifies that the flatpaks are installed
+  - name: Get list of installed Flatpaks
+    become: true
+    ansible.builtin.command:
+      cmd: /usr/bin/flatpak list
+    register: flatpaks
+
+  - name: Check that VLC is installed
+    ansible.builtin.assert:
+      that:
+        - "'VLC' in flatpaks.stdout"
+      fail_msg: 'VLC is not installed'
+
+  - name: Check that Firefox is installed
+    ansible.builtin.assert:
+      that:
+        - "'Firefox' in flatpaks.stdout"
+      fail_msg: 'Firefox is not installed'