From 9277acf58addc3d9c6354751cab3da5d66743976 Mon Sep 17 00:00:00 2001
From: "Jason N." <33561705+JasonN3@users.noreply.github.com>
Date: Thu, 14 Mar 2024 21:59:17 -0400
Subject: [PATCH 1/7] add test for flatpaks installed

---
 .github/workflows/build-and-test.yml | 11 +++++++-
 Makefile                             | 42 ++++++++++++++++++++--------
 tests/vm/flatpak_installed.yml       | 25 +++++++++++++++++
 3 files changed, 65 insertions(+), 13 deletions(-)
 create mode 100644 tests/vm/flatpak_installed.yml

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index a1aa320..b32a5f4 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -263,5 +263,14 @@ jobs:
           
           echo "VM ready for tests at IP ${VM_IP}:${VM_PORT}"
           
-          make test-vm VM_IP=${VM_IP} VM_PORT=${VM_PORT} VM_USER=${VM_USER} VM_PASS=${VM_PASS}
+          make test-vm VM_IP=${VM_IP} VM_PORT=${VM_PORT} VM_USER=${VM_USER} VM_PASS=${VM_PASS} \
+            ARCH=${{ env.ARCH}} \
+            IMAGE_NAME=${{ env.IMAGE_NAME}} \
+            IMAGE_REPO=${{ env.IMAGE_REPO}} \
+            IMAGE_TAG=${{ matrix.version }} \
+            VERSION=${{ matrix.version }} \
+            VARIANT=${{ env.VARIANT }} \
+            FLATPAK_REMOTE_REFS_DIR=${{ env.FLATPAK_REMOTE_REFS_DIR }} \
+            SECURE_BOOT_KEY_URL=${{ env.SECURE_BOOT_KEY_URL }} \
+            ENROLLMENT_PASSWORD=${{ env.ENROLLMENT_PASSWORD }}
           kill $QEMU_PID
diff --git a/Makefile b/Makefile
index 5058868..88089fb 100644
--- a/Makefile
+++ b/Makefile
@@ -32,6 +32,11 @@ PACKAGE_MANAGER = dnf
 get_templates = $(shell ls lorax_templates/$(1)_*.tmpl) \
                 $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/$(1)_*)),lorax_templates/post_$(file).tmpl)
 
+# Get a list of tests for the feature
+# $1 = test type
+# $2 = feature
+get_tests = $(shell ls tests/$(1)/$(2)_*)
+
 # Converts a post script to a template
 # $1 = script to convert
 # $2 = file on ISO to write
@@ -218,7 +223,6 @@ install-test-deps:
 test: test-iso test-vm
 
 test-iso:
-	$(eval _TESTS = $(filter-out README.md,$(shell ls tests/iso)))
 	$(eval _VARS = VERSION FLATPAK_REMOTE_NAME _FLATPAK_REPO_URL)
 
 	sudo modprobe loop
@@ -227,20 +231,20 @@ test-iso:
 	sudo mount -t squashfs -o loop /mnt/iso/images/install.img /mnt/install
 
 	# install tests
-	chmod +x $(foreach test,$(filter install_%,$(_TESTS)),tests/iso/$(test))
-	for test in $(_TESTS); \
+	chmod +x $(call get_tests,iso,install)
+	for test in $(call get_tests,iso,install); \
 	do \
-	  $(foreach var,$(_VARS),$(var)=$($(var))) ./tests/iso/$${test}; \
+	  $(foreach var,$(_VARS),$(var)=$($(var))) ./$${test}; \
 	done
 
 	# flapak tests
 	if [ -n "$(FLATPAK_REMOTE_REFS)" ]; \
 	then \
-		chmod +x $(foreach test,$(filter flatpak_%,$(_TESTS)),tests/iso/$(test)); \
-		for test in $(_TESTS); \
-		do \
-		$(foreach var,$(_VARS),$(var)=$($(var))) ./tests/iso/$${test}; \
-		done; \
+	  chmod +x $(call get_tests,iso,flatpak); \
+	  for test in $(call get_tests,iso,flatpak); \
+	  do \
+	    $(foreach var,$(_VARS),$(var)=$($(var))) ./$${test}; \
+	  done; \
 	fi
 
 	# Cleanup
@@ -260,8 +264,22 @@ ansible_inventory:
 
 test-vm: ansible_inventory
 	ansible -i ansible_inventory -m ansible.builtin.wait_for_connection vm
-	$(eval _TESTS = $(filter-out README.md,$(shell ls tests/vm)))
-	chmod +x $(foreach test,$(_TESTS),tests/vm/$(test))
-	for test in $(_TESTS); do ./tests/vm/$${test}; done
+
+	# install tests
+	chmod +x $(call get_tests,vm,install)
+	for test in $(call get_tests,vm,install); \
+	do \
+	  ./$${test}; \
+	done
+
+	# flapak tests
+	if [ -n "$(FLATPAK_REMOTE_REFS)" ]; \
+	then \
+	  chmod +x $(call get_tests,iso,flatpak); \
+	  for test in $(call get_tests,iso,flatpak); \
+	  do \
+	    $(foreach var,$(_VARS),$(var)=$($(var))) ./$${test}; \
+	  done; \
+	fi
 
 .PHONY: clean install-deps install-test-deps test test-iso test-vm
diff --git a/tests/vm/flatpak_installed.yml b/tests/vm/flatpak_installed.yml
new file mode 100644
index 0000000..d1630e4
--- /dev/null
+++ b/tests/vm/flatpak_installed.yml
@@ -0,0 +1,25 @@
+#!/usr/bin/env -S ansible-playbook -i ./ansible_inventory
+---
+- name: Test for installed flatpaks
+  hosts: vm
+  gather_facts: no
+
+  tasks:
+  # Verifies that the flatpaks are installed
+  - name: Get list of installed Flatpaks
+    become: true
+    ansible.builtin.command:
+      cmd: /usr/bin/flatpak list
+    register: flatpaks
+
+  - name: Check that VLC is installed
+    ansible.builtin.assert:
+      that:
+        - "'VLC' in flatpaks.stdout"
+      fail_msg: 'VLC is not installed'
+
+  - name: Check that VLC is installed
+    ansible.builtin.assert:
+      that:
+        - "'Firefox' in flatpaks.stdout"
+      fail_msg: 'Firefox is not installed'

From 5ff2a438ea1e8b2b0820928333a62ffa66fd019a Mon Sep 17 00:00:00 2001
From: "Jason N." <33561705+JasonN3@users.noreply.github.com>
Date: Fri, 15 Mar 2024 08:28:42 -0400
Subject: [PATCH 2/7] check if empty list

---
 Makefile | 42 ++++++++++++++----------------------------
 1 file changed, 14 insertions(+), 28 deletions(-)

diff --git a/Makefile b/Makefile
index 88089fb..079022b 100644
--- a/Makefile
+++ b/Makefile
@@ -35,7 +35,15 @@ get_templates = $(shell ls lorax_templates/$(1)_*.tmpl) \
 # Get a list of tests for the feature
 # $1 = test type
 # $2 = feature
-get_tests = $(shell ls tests/$(1)/$(2)_*)
+run_tests = tests=$(shell ls tests/$(1)/$(2)_*); \
+			if [ -n "$$tests" ]; \
+			then \
+			  chmod +x $$tests; \
+	  		  for test in $$tests; \
+	  		  do \
+				$(foreach var,$(_VARS),$(var)=$($(var))) ./$${test}; \
+	  		  done; \
+			fi
 
 # Converts a post script to a template
 # $1 = script to convert
@@ -231,21 +239,10 @@ test-iso:
 	sudo mount -t squashfs -o loop /mnt/iso/images/install.img /mnt/install
 
 	# install tests
-	chmod +x $(call get_tests,iso,install)
-	for test in $(call get_tests,iso,install); \
-	do \
-	  $(foreach var,$(_VARS),$(var)=$($(var))) ./$${test}; \
-	done
-
+	$(call run_tests,iso,install)
+	
 	# flapak tests
-	if [ -n "$(FLATPAK_REMOTE_REFS)" ]; \
-	then \
-	  chmod +x $(call get_tests,iso,flatpak); \
-	  for test in $(call get_tests,iso,flatpak); \
-	  do \
-	    $(foreach var,$(_VARS),$(var)=$($(var))) ./$${test}; \
-	  done; \
-	fi
+	[ -n "$(FLATPAK_REMOTE_REFS)" ] && ( $(call run_tests,iso,flatpak) )
 
 	# Cleanup
 	sudo umount /mnt/install
@@ -266,20 +263,9 @@ test-vm: ansible_inventory
 	ansible -i ansible_inventory -m ansible.builtin.wait_for_connection vm
 
 	# install tests
-	chmod +x $(call get_tests,vm,install)
-	for test in $(call get_tests,vm,install); \
-	do \
-	  ./$${test}; \
-	done
+	$(call run_tests,vm,install)
 
 	# flapak tests
-	if [ -n "$(FLATPAK_REMOTE_REFS)" ]; \
-	then \
-	  chmod +x $(call get_tests,iso,flatpak); \
-	  for test in $(call get_tests,iso,flatpak); \
-	  do \
-	    $(foreach var,$(_VARS),$(var)=$($(var))) ./$${test}; \
-	  done; \
-	fi
+	[ -n "$(FLATPAK_REMOTE_REFS)" ] && ( $(call run_tests,vm,flatpak) )
 
 .PHONY: clean install-deps install-test-deps test test-iso test-vm

From 8db6a3a25b17d00d1664755e18087323fa9163fc Mon Sep 17 00:00:00 2001
From: "Jason N." <33561705+JasonN3@users.noreply.github.com>
Date: Fri, 15 Mar 2024 09:12:16 -0400
Subject: [PATCH 3/7] use if

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 079022b..d797b7a 100644
--- a/Makefile
+++ b/Makefile
@@ -242,7 +242,7 @@ test-iso:
 	$(call run_tests,iso,install)
 	
 	# flapak tests
-	[ -n "$(FLATPAK_REMOTE_REFS)" ] && ( $(call run_tests,iso,flatpak) )
+	if [ -n "$(FLATPAK_REMOTE_REFS)" ]; then $(call run_tests,iso,flatpak); fi
 
 	# Cleanup
 	sudo umount /mnt/install
@@ -266,6 +266,6 @@ test-vm: ansible_inventory
 	$(call run_tests,vm,install)
 
 	# flapak tests
-	[ -n "$(FLATPAK_REMOTE_REFS)" ] && ( $(call run_tests,vm,flatpak) )
+	if [ -n "$(FLATPAK_REMOTE_REFS)" ]; then $(call run_tests,vm,flatpak); fi
 
 .PHONY: clean install-deps install-test-deps test test-iso test-vm

From 3fd3a2a2af3cfa2caf082c59f918cca8edc68f9c Mon Sep 17 00:00:00 2001
From: "Jason N." <33561705+JasonN3@users.noreply.github.com>
Date: Fri, 15 Mar 2024 12:17:56 -0400
Subject: [PATCH 4/7] fix refs dir

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index b32a5f4..1ad69ec 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -20,7 +20,7 @@ env:
   IMAGE_REPO: 'quay.io/fedora-ostree-desktops'
   IMAGE_TAG: '39'
   VARIANT: 'Server'
-  FLATPAK_REMOTE_REFS_DIR: /github/workspace/flatpak_refs
+  FLATPAK_REMOTE_REFS_DIR: flatpak_refs
   SECURE_BOOT_KEY_URL: 'https://github.com/ublue-os/akmods/raw/main/certs/public_key.der'
   ENROLLMENT_PASSWORD: 'container-installer'
 
@@ -115,7 +115,7 @@ jobs:
           image_tag: ${{ matrix.version }}
           version: ${{ matrix.version }}
           variant: ${{ env.VARIANT }}
-          flatpak_remote_refs_dir: ${{ env.FLATPAK_REMOTE_REFS_DIR }}
+          flatpak_remote_refs_dir: /github/workspace/${{ env.FLATPAK_REMOTE_REFS_DIR }}
           secure_boot_key_url: ${{ env.SECURE_BOOT_KEY_URL }}
           enrollment_password: ${{ env.ENROLLMENT_PASSWORD }}
           iso_name: ${{ env.IMAGE_NAME }}-${{ matrix.version }}-${{ matrix.version }}.iso

From 76f0b24d2a01c1f8ae4488bc5cf100ca8052fdf4 Mon Sep 17 00:00:00 2001
From: "Jason N." <33561705+JasonN3@users.noreply.github.com>
Date: Fri, 15 Mar 2024 15:21:03 -0400
Subject: [PATCH 5/7] add quotes

---
 .github/workflows/build-and-test.yml | 11 ++++++++---
 Makefile                             |  2 +-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 1ad69ec..cae216e 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -178,11 +178,13 @@ jobs:
     
       - name: Verify ISO
         run: |
+          set -e
           checkisomd5 ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }}
           sha256sum -c ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }}-CHECKSUM
 
       - name: Run ISO checks
         run: |
+          set -e
           mv ${{ needs['build-and-push-iso']['outputs'][format('iso_name-{0}', matrix.version)] }} deploy.iso
           make test-iso \
             ARCH=${{ env.ARCH}} \
@@ -197,6 +199,7 @@ jobs:
 
       - name: Add Kickstart and Grub options to ISO
         run: |
+          set -e
           sudo mkdir /mnt/iso || true
           sudo mount -o loop deploy.iso /mnt/iso
           cp /mnt/iso/boot/grub2/grub.cfg grub.cfg
@@ -227,7 +230,8 @@ jobs:
 
       - name: Install the test VM
         run: |
-          timeout 1h qemu-system-x86_64 -name "Anaconda" -boot d -m 4096 -cpu qemu64 -display none -cdrom test.iso -smp 2 -hda disk.qcow2 -serial telnet:localhost:4321,server,nowait & QEMU_PID=$!
+          set -e
+          timeout 1h qemu-system-x86_64 -name "Anaconda" -boot d -m 4096 -cpu qemu64 -display none -cdrom test.iso -smp 2 -hda disk.qcow2 -serial telnet:localhost:4321,server=on,wait=off & QEMU_PID=$!
           echo "PID: $QEMU_PID"
           timeout 1m bash -c "while ! (echo > /dev/tcp/127.0.0.1/4321); do sleep 0.1; done"
           (nc localhost 4321 | tee vm.stdout) &
@@ -240,15 +244,16 @@ jobs:
           VM_IP: "127.0.0.1"
           VM_PORT: "5555"
         run: |
+          set -e
           mkfifo vm.stdin
           qemu-system-x86_64 -name "Anaconda" \
             -m 4096 -cpu qemu64 -display none -smp 2 \
-            -chardev socket,path=/tmp/qga.sock,server,nowait,id=qga0 \
+            -chardev socket,path=/tmp/qga.sock,server=on,wait=off,id=qga0 \
             -device e1000,netdev=net0 \
             -netdev user,id=net0,hostfwd=tcp::${VM_PORT}-:22 \
             -device virtio-serial \
             -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \
-            -boot c -hda disk.qcow2 -serial telnet:localhost:4321,server,nowait & export QEMU_PID=$!
+            -boot c -hda disk.qcow2 -serial telnet:localhost:4321,server=on,wait=off & export QEMU_PID=$!
           echo "PID: $QEMU_PID"
 
           timeout 1m bash -c "while ! (echo > /dev/tcp/127.0.0.1/4321); do sleep 0.1; done"
diff --git a/Makefile b/Makefile
index d797b7a..7c18e8f 100644
--- a/Makefile
+++ b/Makefile
@@ -35,7 +35,7 @@ get_templates = $(shell ls lorax_templates/$(1)_*.tmpl) \
 # Get a list of tests for the feature
 # $1 = test type
 # $2 = feature
-run_tests = tests=$(shell ls tests/$(1)/$(2)_*); \
+run_tests = tests="$(shell ls tests/$(1)/$(2)_*)"; \
 			if [ -n "$$tests" ]; \
 			then \
 			  chmod +x $$tests; \

From 5cc2f5cc2d59fa6a29953d7dd484da052b6bb77d Mon Sep 17 00:00:00 2001
From: "Jason N." <33561705+JasonN3@users.noreply.github.com>
Date: Fri, 15 Mar 2024 15:57:20 -0400
Subject: [PATCH 6/7] check service

---
 tests/vm/flatpak_fedora_repo_disabled.yml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 tests/vm/flatpak_fedora_repo_disabled.yml

diff --git a/tests/vm/flatpak_fedora_repo_disabled.yml b/tests/vm/flatpak_fedora_repo_disabled.yml
new file mode 100644
index 0000000..2fc77a4
--- /dev/null
+++ b/tests/vm/flatpak_fedora_repo_disabled.yml
@@ -0,0 +1,15 @@
+#!/usr/bin/env -S ansible-playbook -i ./ansible_inventory
+---
+- name: Test for installed flatpaks
+  hosts: vm
+  gather_facts: no
+
+  tasks:
+  - name: Collect facts about system services
+    service_facts:
+    register: services_state
+
+  - name: Check that flatpak-add-fedora-repos is disabled
+    ansible.builtin.assert:
+      that:
+        - services_state['ansible_facts']['services']['flatpak-add-fedora-repos.service']['status'] == 'disabled'
\ No newline at end of file

From 9d326b1016a8ac7ad83c3f6d3550cfc7e18215dd Mon Sep 17 00:00:00 2001
From: "Jason N." <33561705+JasonN3@users.noreply.github.com>
Date: Fri, 15 Mar 2024 18:08:29 -0400
Subject: [PATCH 7/7] check return codes

---
 Makefile                                  | 1 +
 tests/vm/flatpak_fedora_repo_disabled.yml | 3 ++-
 tests/vm/flatpak_installed.yml            | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 7c18e8f..64001f2 100644
--- a/Makefile
+++ b/Makefile
@@ -42,6 +42,7 @@ run_tests = tests="$(shell ls tests/$(1)/$(2)_*)"; \
 	  		  for test in $$tests; \
 	  		  do \
 				$(foreach var,$(_VARS),$(var)=$($(var))) ./$${test}; \
+				RC=$$?; if [ $$RC != 0 ]; then exit $$RC; fi; \
 	  		  done; \
 			fi
 
diff --git a/tests/vm/flatpak_fedora_repo_disabled.yml b/tests/vm/flatpak_fedora_repo_disabled.yml
index 2fc77a4..f87ff80 100644
--- a/tests/vm/flatpak_fedora_repo_disabled.yml
+++ b/tests/vm/flatpak_fedora_repo_disabled.yml
@@ -12,4 +12,5 @@
   - name: Check that flatpak-add-fedora-repos is disabled
     ansible.builtin.assert:
       that:
-        - services_state['ansible_facts']['services']['flatpak-add-fedora-repos.service']['status'] == 'disabled'
\ No newline at end of file
+        - services_state['ansible_facts']['services']['flatpak-add-fedora-repos.service']['status'] == 'disabled'
+      fail_msg: 'flatpak-add-fedora-repos.service is not disabled'
\ No newline at end of file
diff --git a/tests/vm/flatpak_installed.yml b/tests/vm/flatpak_installed.yml
index d1630e4..da4f331 100644
--- a/tests/vm/flatpak_installed.yml
+++ b/tests/vm/flatpak_installed.yml
@@ -18,7 +18,7 @@
         - "'VLC' in flatpaks.stdout"
       fail_msg: 'VLC is not installed'
 
-  - name: Check that VLC is installed
+  - name: Check that Firefox is installed
     ansible.builtin.assert:
       that:
         - "'Firefox' in flatpaks.stdout"