Allow caching of dnf (#46)

.gitignore

@@ -3,7 +3,6 @@
 /pkglists
 /repos
 /results
-/lorax_templates/*.tmpl
 /xorriso/input.txt
 /xorriso/*.sh
 /original-pkgsizes.txt

Containerfile

@@ -15,10 +15,11 @@ RUN mkdir /build-container-installer
 COPY / /build-container-installer/
 
 WORKDIR /build-container-installer
+VOLUME /build-container-installer/build
+VOLUME /build-container-installer/repos
+VOLUME /cache
 
 RUN dnf install -y make && make install-deps
 
-VOLUME /build-container-installer/build
-
 ENTRYPOINT ["/bin/bash", "/build-container-installer/entrypoint.sh"]
 

Makefile

@@ -1,48 +1,77 @@
 # Configuration vars
 ## Formatting = UPPERCASE
+# General
+ADDITIONAL_TEMPLATES =
 ARCH = x86_64
-VERSION = 39
-IMAGE_REPO = quay.io/fedora-ostree-desktops
+EXTRA_BOOT_PARAMS =
 IMAGE_NAME = base
+IMAGE_REPO = quay.io/fedora-ostree-desktops
 IMAGE_TAG = $(VERSION)
+REPOS = $(subst :,\:,$(shell ls /etc/yum.repos.d/*.repo))
+ROOTFS_SIZE = 4
 VARIANT = Server
+VERSION = 39
 WEB_UI = false
-REPOS = /etc/yum.repos.d/fedora.repo /etc/yum.repos.d/fedora-updates.repo
-ADDITIONAL_TEMPLATES = 
+# Flatpak
 FLATPAK_REMOTE_NAME = flathub
 FLATPAK_REMOTE_URL = https://flathub.org/repo/flathub.flatpakrepo
 FLATPAK_REMOTE_REFS = 
+# Secure boot
 ENROLLMENT_PASSWORD =
 SECURE_BOOT_KEY_URL =
-ADDITIONAL_TEMPLATES =
-EXTRA_BOOT_PARAMS =
-ROOTFS_SIZE = 4
+# Cache
+DNF_CACHE = 
 
-# Generated vars
+# Generated/internal vars
 ## Formatting = _UPPERCASE
 _BASE_DIR = $(shell pwd)
 _IMAGE_REPO_ESCAPED = $(subst /,\/,$(IMAGE_REPO))
 _IMAGE_REPO_DOUBLE_ESCAPED = $(subst \,\\\,$(_IMAGE_REPO_ESCAPED))
 _VOLID = $(firstword $(subst -, ,$(IMAGE_NAME)))-$(ARCH)-$(IMAGE_TAG)
 _REPO_FILES = $(subst /etc/yum.repos.d,repos,$(REPOS))
-_LORAX_TEMPLATES = $(subst .in,,$(shell ls lorax_templates/*.tmpl.in)) $(foreach file,$(shell ls lorax_templates/scripts/post),lorax_templates/post_$(file).tmpl)
-_EXTERNAL_TEMPLATES = fedora-lorax-templates/ostree-based-installer/lorax-embed-flatpaks.tmpl
+_LORAX_TEMPLATES            = $(shell ls lorax_templates/install_*.tmpl)    $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/install_*)),lorax_templates/post_$(file).tmpl)
+_LORAX_TEMPLATES_FLATPAKS   = $(shell ls lorax_templates/flatpak_*.tmpl)    $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/flatpak_*)),lorax_templates/post_$(file).tmpl) external/fedora-lorax-templates/ostree-based-installer/lorax-embed-flatpaks.tmpl
+_LORAX_TEMPLATES_SECUREBOOT = $(shell ls lorax_templates/secureboot_*.tmpl) $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/secureboot_*)),lorax_templates/post_$(file).tmpl)
+_LORAX_TEMPLATES_CACHE      = $(shell ls lorax_templates/cache_*.tmpl)      $(foreach file,$(notdir $(shell ls lorax_templates/scripts/post/cache_*)),lorax_templates/post_$(file).tmpl)
+_LORAX_ARGS = 
 _FLATPAK_REPO_URL = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^URL=' | cut -d= -f2)
 _FLATPAK_REPO_GPG = $(shell curl -L $(FLATPAK_REMOTE_URL) | grep -i '^GPGKey=' | cut -d= -f2)
-_TEMPLATE_VARS = ARCH VERSION IMAGE_REPO IMAGE_NAME IMAGE_TAG VARIANT WEB_UI REPOS _IMAGE_REPO_ESCAPED _IMAGE_REPO_DOUBLE_ESCAPED FLATPAK_REMOTE_NAME FLATPAK_REMOTE_URL FLATPAK_REMOTE_REFS _FLATPAK_REPO_URL _FLATPAK_REPO_GPG ENROLLMENT_PASSWORD
+_TEMPLATE_VARS = ARCH IMAGE_NAME IMAGE_REPO _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED IMAGE_TAG REPOS VARIANT VERSION WEB_UI
 
-ifeq ($(VARIANT),Server)
-_LORAX_ARGS = --macboot --noupgrade
+
+ifeq ($(findstring redhat.repo,$(REPOS)),redhat.repo)
+_LORAX_ARGS += --nomacboot --noupgrade
+else ifeq ($(VARIANT),Server)
+_LORAX_ARGS += --macboot --noupgrade
 else
-_LORAX_ARGS = --nomacboot
+_LORAX_ARGS += --nomacboot
 endif
 
 ifeq ($(WEB_UI),true)
 _LORAX_ARGS += -i anaconda-webui
 endif
 
+ifneq ($(DNF_CACHE),)
+_LORAX_ARGS      += --cachedir $(DNF_CACHE)
+_LORAX_TEMPLATES += $(_LORAX_TEMPLATES_CACHE)
+_TEMPLATE_VARS   += DNF_CACHE
+endif
+
+ifeq ($(findstring redhat.repo,$(REPOS)),redhat.repo)
+_PLATFORM_ID = platform:el$(VERSION)
+else
+_PLATFORM_ID = platform:f$(VERSION)
+endif
+
 ifneq ($(FLATPAK_REMOTE_REFS),)
-_LORAX_ARGS += -i flatpak-libs
+_LORAX_ARGS      += -i flatpak-libs
+_LORAX_TEMPLATES += $(_LORAX_TEMPLATES_FLATPAKS)
+_TEMPLATE_VARS   += FLATPAK_REMOTE_NAME FLATPAK_REMOTE_REFS FLATPAK_REMOTE_URL _FLATPAK_REPO_GPG _FLATPAK_REPO_URL
+endif
+
+ifneq ($(SECURE_BOOT_KEY_URL),)
+_LORAX_TEMPLATES += $(_LORAX_TEMPLATES_SECUREBOOT)
+_TEMPLATE_VARS   += ENROLLMENT_PASSWORD
 endif
 
 # Step 7: Build end ISO
@@ -117,9 +146,8 @@ lorax_templates/post_%.tmpl: lorax_templates/scripts/post/%
 	done < lorax_templates/scripts/post/$*
 	echo "append $(_ISO_FILE) \"%end\"" >> lorax_templates/post_$*.tmpl
 
-lorax_templates/%.tmpl: lorax_templates/%.tmpl.in
-	$(eval _VARS = IMAGE_NAME IMAGE_TAG _IMAGE_REPO_DOUBLE_ESCAPED _IMAGE_REPO_ESCAPED)
-	$(foreach var,$(_VARS),$(var)=$($(var))) envsubst '$(foreach var,$(_VARS),$$$(var))' < $(_BASE_DIR)/lorax_templates/$*.tmpl.in > $(_BASE_DIR)/lorax_templates/$*.tmpl
+
+repos: $(_REPO_FILES)
 
 # Step 2: Replace vars in repo files
 repos/%.repo: /etc/yum.repos.d/%.repo
@@ -132,9 +160,11 @@ repos/%.repo: /etc/yum.repos.d/%.repo
 %.repo:
 
 # Step 3: Build boot.iso using Lorax
-boot.iso: $(_LORAX_TEMPLATES) $(_REPO_FILES)
+boot.iso: $(filter lorax_templates/%,$(_LORAX_TEMPLATES)) $(_REPO_FILES)
 	rm -Rf $(_BASE_DIR)/results || true
-	rm /etc/rpm/macros.image-language-conf || true
+	mv /etc/rpm/macros.image-language-conf /etc/rpm/macros.image-language-conf.orig || true
+	cp /etc/os-release /etc/os-release.orig || true
+	sed -i 's/PLATFORM_ID=.*/PLATFORM_ID="$(_PLATFORM_ID)"/' /etc/os-release
 
 	# Download the secure boot key
 	if [ -n "$(SECURE_BOOT_KEY_URL)" ]; \
@@ -154,6 +184,8 @@ boot.iso: $(_LORAX_TEMPLATES) $(_REPO_FILES)
 		$(foreach var,$(_TEMPLATE_VARS),--add-template-var "$(shell echo $(var) | tr '[:upper:]' '[:lower:]')=$($(var))") \
 		$(_BASE_DIR)/results/
 	mv $(_BASE_DIR)/results/images/boot.iso $(_BASE_DIR)/
+	mv -f /etc/rpm/macros.image-language-conf.orig /etc/rpm/macros.image-language-conf || true
+	mv -f /etc/os-release.orig /etc/os-release || true
 
 # Step 4: Download container image
 container/$(IMAGE_NAME)-$(IMAGE_TAG):
@@ -188,7 +220,7 @@ clean:
 	rm -f $(_BASE_DIR)/*.log || true
 
 install-deps:
-	dnf install -y lorax xorriso skopeo flatpak dbus-daemon ostree coreutils
+	dnf install -y lorax xorriso skopeo flatpak dbus-daemon ostree coreutils gettext
 
 test: test-iso test-vm
 
@@ -218,4 +250,4 @@ test-vm:
 	chmod +x $(foreach test,$(_TESTS),tests/vm/$(test))
 	for test in $(_TESTS); do ./tests/vm/$${test} deploy.iso; done
 	
-.PHONY: clean install-deps test test-iso test-vm
+.PHONY: clean install-deps test test-iso test-vm container/$(IMAGE_NAME)-$(IMAGE_TAG)

README.md

@@ -7,7 +7,7 @@ This action is used to enerate an ISO for installing an OSTree stored in a conta
 This action is designed to be called from a GitHub workflow using the following format
 ```yaml
 - name: Build ISO
-  uses: jasonn3/build-container-installer/v1.0.0
+  uses: jasonn3/build-container-installer@main
   id: build
   with:
     arch: ${{ env.ARCH}}

action.yml

@@ -13,6 +13,17 @@ inputs:
     description: Architecture for image to build
     required: true
     default: x86_64
+  dnf_cache_key:
+    description: Overrides the dnf cache key
+    required: false
+  enable_cache_dnf: 
+    description: Whether to enable caching for dnf
+    required: false
+    default: "true"
+  enable_cache_skopeo: 
+    description: Whether to enable caching for skopeo
+    required: false
+    default: "true"
   enrollment_password:
     description: Used for supporting secure boot (requires secure_boot_key_url to be defined)
     required: false
@@ -50,6 +61,9 @@ inputs:
   secure_boot_key_url:
     description: Secure boot key that is installed from URL location
     required: false
+  skopeo_cache_key:
+    description: Overrides the skopeo cache key
+    required: false
   variant:
     description: "Source container variant. Available options can be found by running `dnf provides system-release`. Variant will be the third item in the package name. Example: `fedora-release-kinoite-39-34.noarch` will be kinonite"
     required: true
@@ -74,15 +88,69 @@ outputs:
 runs:
   using: composite
   steps:
+    - name: Make cache directory
+      shell: bash
+      run: |
+        sudo mkdir /cache
+        sudo chmod 777 /cache
+
+    - name: Load dnf cache
+      id: load_dnf_cache
+      env:
+        dnf_cache_key: dnf-${{ inputs.version }}
+      if: inputs.enable_cache_dnf == 'true'
+      uses: actions/cache/restore@v4
+      with:
+        path: /cache/dnf
+        key: ${{ inputs.dnf_cache_key || env.dnf_cache_key }}
+
+    - name: Load skopeo cache
+      id: load_skopeo_cache
+      env:
+        skopeo_cache_key: skopeo-${{ inputs.image_name }}-${{ inputs.version || inputs.image_tag }}
+      if: inputs.enable_cache_skopeo == 'true'
+      uses: actions/cache/restore@v4
+      with:
+        path: /cache/skopeo
+        key: ${{ inputs.skopeo_cache_key || env.skopeo_cache_key }}
+
+    - name: Ensure cache directories exist
+      shell: bash
+      run: |
+        mkdir /cache/dnf || true
+        mkdir /cache/dnf_new || true
+        mkdir /cache/skopeo || true
+
     - name: Run docker image
       shell: bash
       run: |
         # Check if running inside of the action repo
-        if [[ -z "${{ github.action_ref }}" ]]; then if [[ "${{ github.ref_name }}" =~ (.*)/merge ]]; then tag=pr-${BASH_REMATCH[1]}; else tag=${{ github.ref_name }}; fi; fi
-        if [[ -z "${tag}" ]]; then tag=${{ github.action_ref }}; fi
-        docker run --privileged --volume ${{ github.workspace }}:/github/workspace ghcr.io/jasonn3/build-container-installer:${tag} \
+        if [[ -z "${{ github.action_repository }}" ]]
+        then
+          if [[ "${{ github.ref_name }}" =~ (.*)/merge ]]
+            then tag=pr-${BASH_REMATCH[1]}
+          else
+            tag=${{ github.ref_name }}
+          fi
+        else
+           tag=${{ github.action_ref }}
+        fi        
+        if [[ "${{ inputs.enable_cache_dnf }}" == "true" ]]
+        then
+          cache="${cache} -v /cache/dnf:/cache/dnf"
+        fi
+        if [[ "${{ inputs.enable_cache_skopeo }}" == "true" ]]
+        then
+          cache="${cache} -v /cache/skopeo:/cache/skopeo"
+        fi
+        if [[ "${{ steps.load_dnf_cache.outputs.cache-hit }}" != "true" ]]
+        then
+          cache="${cache} -v /cache/dnf_new:/cache/dnf_new"
+        fi
+        docker run --privileged --volume ${{ github.workspace }}:/github/workspace/ ${cache} ghcr.io/jasonn3/build-container-installer:${tag} \
           ADDITIONAL_TEMPLATES="${{ inputs.additional_templates }}" \
           ARCH=${{ inputs.arch }} \
+          DNF_CACHE=/cache/dnf \
           ENROLLMENT_PASSWORD=${{ inputs.enrollment_password }} \
           FLATPAK_REMOTE_NAME=${{ inputs.flatpak_remote_name }} \
           FLATPAK_REMOTE_REFS="${{ inputs.flatpak_remote_refs }}" \
@@ -95,6 +163,25 @@ runs:
           VERSION=${{ inputs.version }} \
           WEB_UI=${{ inputs.web_ui }}
 
+    - name: Save dnf cache
+      env:
+        dnf_cache_key: dnf-${{ inputs.version }}
+      if: inputs.enable_cache_dnf == 'true' && steps.load_dnf_cache.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: /cache/dnf_new
+        key: ${{ inputs.dnf_cache_key || env.dnf_cache_key }}
+
+    - name: Save skopeo cache
+      env:
+        skopeo_cache_key: skopeo-${{ inputs.image_name }}-${{ inputs.version || inputs.image_tag }}
+      if: inputs.enable_cache_skopeo == 'true' && steps.load_dnf_cache.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: /cache/skopeo
+        key: ${{ inputs.skopeo_cache_key || env.skopeo_cache_key }}
+          
+
     - name: Rename ISO file
       id: rename_iso
       shell: bash

entrypoint.sh

@@ -12,6 +12,16 @@ do
   export ${key}="${value}"
 done
 
+if [[ -d /cache/skopeo ]]
+then
+  ln -s /cache/skopeo /build-container-installer/container
+fi
+
+if [[ ! -d /cache/dnf ]]
+then
+  mkdir /cache/dnf
+fi
+
 # Pull container
 make container/${IMAGE_NAME}-${IMAGE_TAG} "$@"
 

lorax_templates/cache_copy_dnf.tmpl

@@ -0,0 +1,3 @@
+<%page args="dnf_cache"/>
+
+runcmd bash -c "if [[ -e ${dnf_cache}_new ]]; then cp -R ${dnf_cache}/* ${dnf_cache}_new/; fi"

lorax_templates/install_set_installer.tmpl

@@ -0,0 +1,4 @@
+<%page args="image_name, image_tag"/>
+
+append usr/share/anaconda/interactive-defaults.ks "ostreecontainer --url=/run/install/repo/${image_name}-${image_tag} --transport=oci --no-signature-verification"
+

lorax_templates/scripts/post/secureboot_enroll_key

@@ -1,4 +1,4 @@
-<%page args="enrollment_password "/>
+<%page args="enrollment_password"/>
 ## --nochroot
 
 set -oue pipefail

lorax_templates/set_installer.tmpl.in

@@ -1,2 +0,0 @@
-append usr/share/anaconda/interactive-defaults.ks "ostreecontainer --url=/run/install/repo/${IMAGE_NAME}-${IMAGE_TAG} --transport=oci --no-signature-verification"
-