From d7428d309686dba0e0d80c2caddeb55335f7c00e Mon Sep 17 00:00:00 2001 From: fithwum Date: Sat, 20 Dec 2025 08:52:51 -0800 Subject: [PATCH] Update .gitea/workflows/base-build.yml --- .gitea/workflows/base-build.yml | 366 ++++++++++++++++++++++---------- 1 file changed, 251 insertions(+), 115 deletions(-) diff --git a/.gitea/workflows/base-build.yml b/.gitea/workflows/base-build.yml index 8d76fa9..bf564a1 100644 --- a/.gitea/workflows/base-build.yml +++ b/.gitea/workflows/base-build.yml @@ -26,121 +26,272 @@ env: OUTPUT_DIR: ./output jobs: - build-and-push-rootfs-archives: + # build-and-push-rootfs-archives: + # runs-on: vm-docker-build2 + # outputs: + # archives_changed: ${{ steps.commit_archives.outputs.archives_changed }} + # steps: + # - name: Checkout source + # uses: actions/checkout@v3 + + # - name: Create output directory + # run: mkdir -p ./output + + # - name: Build all Debian rootfs versions into volumes and extract + # run: | + # versions=($VERSIONS) + # for version in "${versions[@]}"; do + # echo "[INFO] Building $version..." + # volume_name="build_output_$version" + # docker volume create "$volume_name" + + # docker build --build-arg VERSION=$version -t fithwum/debian-$version-base . + + # docker run --rm --privileged \ + # -v "$volume_name:/output" \ + # -e VERSION="$version" \ + # fithwum/debian-$version-base \ + # bash -c "/scripts/bootstrap-rootfs.sh \"$version\"" + + # # Extract the output file from the volume + # container_id=$(docker create -v "$volume_name:/output" debian) + # mkdir -p ./output/$version + # docker cp "$container_id:/output/$version/debian-$version.tar.bz2" ./output/$version/ + # docker rm "$container_id" + # done + + # - name: Validate that archives exist for each version + # run: | + # IFS=' ' read -r -a versions <<< "$VERSIONS" + # for version in "${versions[@]}"; do + # path="./output/$version/debian-$version.tar.bz2" + # if [[ ! -f "$path" ]]; then + # echo "[ERROR] Missing archive: $path" + # exit 1 + # else + # echo "[OK] Found: $path" + # fi + # done + + # - name: Clone upload repo + # run: | + # GIT_CREDENTIAL="${{ secrets.GIT_TOKEN || secrets.GIT_PASSWORD }}" + # git clone --depth=1 "https://${{ env.GIT_USERNAME }}:${GIT_CREDENTIAL}@gitea.fithwum.tech/fithwum/debian-base.git" upload-repo + + # - name: Clean old archives in upload-repo + # run: rm -rfv upload-repo/*/*.tar.bz2 + + # - name: Copy new archives to upload-repo + # run: | + # for filepath in ./output/*/debian-*.tar.bz2; do + # version_dir=$(basename "$(dirname "$filepath")") + # mkdir -p "upload-repo/$version_dir" + # cp "$filepath" "upload-repo/$version_dir/" + # done + + # - name: Calculate and store sha256sums in upload-repo + # run: | + # cd upload-repo + # rm -f sha256sums.txt + # for tarball in */debian-*.tar.bz2; do + # echo "[INFO] Processing: $tarball" + # checksum=$(sha256sum "$tarball" | awk '{print $1}') + # echo "$checksum $tarball" >> sha256sums.txt + # done + # echo "[INFO] SHA256 contents:" + # cat sha256sums.txt + + # - name: Commit and push files if changed + # id: commit_archives + # run: | + # cd upload-repo + # git config --global user.name "${{ env.GIT_USERNAME }}" + # git config --global user.email "${{ env.GIT_EMAIL }}" + + # if git status --porcelain | grep .; then + # git add **/*.tar.bz2 sha256sums.txt || true + # git commit -m "Update base images and checksum on $(date -u +'%Y-%m-%dT%H:%M:%SZ') [skip ci]" + # git push + # echo "archives_changed=true" >> $GITHUB_OUTPUT + # else + # echo "[INFO] No changes to commit." + # echo "archives_changed=false" >> $GITHUB_OUTPUT + # fi + + build-rootfs: + name: Build rootfs (${{ matrix.version }}) runs-on: vm-docker-build2 - outputs: - archives_changed: ${{ steps.commit_archives.outputs.archives_changed }} + + strategy: + fail-fast: false + matrix: + version: [bullseye, bookworm, trixie] + steps: - - name: Checkout source - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Create output directory - run: mkdir -p ./output - - - name: Build all Debian rootfs versions into volumes and extract + - name: Build rootfs tarball run: | - versions=($VERSIONS) - for version in "${versions[@]}"; do - echo "[INFO] Building $version..." - volume_name="build_output_$version" - docker volume create "$volume_name" + VERSION=${{ matrix.version }} - docker build --build-arg VERSION=$version -t fithwum/debian-$version-base . + docker build \ + --build-arg VERSION="$VERSION" \ + -t rootfs-$VERSION . - docker run --rm --privileged \ - -v "$volume_name:/output" \ - -e VERSION="$version" \ - fithwum/debian-$version-base \ - bash -c "/scripts/bootstrap-rootfs.sh \"$version\"" + docker run --rm rootfs-$VERSION \ + bash -c "/scripts/bootstrap-rootfs.sh $VERSION" - # Extract the output file from the volume - container_id=$(docker create -v "$volume_name:/output" debian) - mkdir -p ./output/$version - docker cp "$container_id:/output/$version/debian-$version.tar.bz2" ./output/$version/ - docker rm "$container_id" - done - - - name: Validate that archives exist for each version - run: | - IFS=' ' read -r -a versions <<< "$VERSIONS" - for version in "${versions[@]}"; do - path="./output/$version/debian-$version.tar.bz2" - if [[ ! -f "$path" ]]; then - echo "[ERROR] Missing archive: $path" - exit 1 - else - echo "[OK] Found: $path" - fi - done + test -f "debian-$VERSION.tar.bz2" - name: Clone upload repo run: | - GIT_CREDENTIAL="${{ secrets.GIT_TOKEN || secrets.GIT_PASSWORD }}" - git clone --depth=1 "https://${{ env.GIT_USERNAME }}:${GIT_CREDENTIAL}@gitea.fithwum.tech/fithwum/debian-base.git" upload-repo + git clone --depth=1 \ + https://${GIT_USERNAME}:${GIT_TOKEN}@gitea.fithwum.tech/fithwum/debian-base.git upload - - name: Clean old archives in upload-repo - run: rm -rfv upload-repo/*/*.tar.bz2 - - - name: Copy new archives to upload-repo + - name: Update archive + sha256 run: | - for filepath in ./output/*/debian-*.tar.bz2; do - version_dir=$(basename "$(dirname "$filepath")") - mkdir -p "upload-repo/$version_dir" - cp "$filepath" "upload-repo/$version_dir/" - done + VERSION=${{ matrix.version }} - - name: Calculate and store sha256sums in upload-repo - run: | - cd upload-repo - rm -f sha256sums.txt - for tarball in */debian-*.tar.bz2; do - echo "[INFO] Processing: $tarball" - checksum=$(sha256sum "$tarball" | awk '{print $1}') - echo "$checksum $tarball" >> sha256sums.txt - done - echo "[INFO] SHA256 contents:" - cat sha256sums.txt + mkdir -p upload/$VERSION + cp "debian-$VERSION.tar.bz2" "upload/$VERSION/" - - name: Commit and push files if changed - id: commit_archives + cd upload + sed -i "/debian-$VERSION.tar.bz2/d" sha256sums.txt 2>/dev/null || true + sha256sum "$VERSION/debian-$VERSION.tar.bz2" >> sha256sums.txt + + - name: Commit if changed run: | - cd upload-repo - git config --global user.name "${{ env.GIT_USERNAME }}" - git config --global user.email "${{ env.GIT_EMAIL }}" + VERSION=${{ matrix.version }} + + cd upload + git config user.name "$GIT_USERNAME" + git config user.email "$GIT_EMAIL" if git status --porcelain | grep .; then - git add **/*.tar.bz2 sha256sums.txt || true - git commit -m "Update base images and checksum on $(date -u +'%Y-%m-%dT%H:%M:%SZ') [skip ci]" + git add "$VERSION/debian-$VERSION.tar.bz2" sha256sums.txt + git commit -m "Update rootfs for $VERSION [skip ci]" git push - echo "archives_changed=true" >> $GITHUB_OUTPUT else - echo "[INFO] No changes to commit." - echo "archives_changed=false" >> $GITHUB_OUTPUT + echo "[INFO] No changes for $VERSION" fi + # build-and-push-docker-images: + # needs: build-and-push-rootfs-archives + # if: always() + # # if: needs.build-and-push-rootfs-archives.outputs.archives_changed == 'true' + # runs-on: vm-docker-build2 + # steps: + # - name: Checkout source + # uses: actions/checkout@v3 + + # - name: Wait for archives to appear in upload repo + # run: | + # echo "[INFO] Waiting for archives to appear in upload-repo..." + + # mkdir -p temp-check + # cd temp-check + + # # Retry loop for cloning the upload-repo + # for i in {1..10}; do + # echo "[INFO] Attempt $i: Cloning upload-repo..." + # if git clone --depth=1 "https://${{ secrets.GIT_USERNAME }}:${{ secrets.GIT_TOKEN }}@gitea.fithwum.tech/fithwum/debian-base.git"; then + # break + # fi + # echo "[WARN] Clone failed. Retrying in 10 seconds..." + # sleep 10 + # done + + # if [ ! -d "debian-base" ]; then + # echo "[ERROR] Failed to clone upload-repo after retries." + # exit 1 + # fi + + # cd debian-base + + # # Wait for all versions to show up + # missing_versions=() + # for version in $VERSIONS; do + # found=0 + # for i in {1..30}; do + # if [[ -f "$version/debian-$version.tar.bz2" ]]; then + # found=1 + # break + # else + # echo "[WAIT] $version not ready yet, sleeping 10s..." + # sleep 10 + # fi + # done + + # if [[ $found -eq 0 ]]; then + # missing_versions+=("$version") + # fi + # done + + # if [[ ${#missing_versions[@]} -gt 0 ]]; then + # echo "[ERROR] Missing archives for: ${missing_versions[*]}" + # exit 1 + # fi + + # - name: Generate Dockerfiles per version (if missing) + # run: | + # for version in $VERSIONS; do + # mkdir -p "$version" + # DOCKERFILE_PATH="$version/Dockerfile" + + # if [[ -f "$DOCKERFILE_PATH" ]]; then + # echo "[INFO] Skipping $DOCKERFILE_PATH (already exists)" + # continue + # fi + + # printf '%s\n' \ + # "FROM scratch" \ + # "LABEL maintainer=\"fithwum\"" \ + # "ADD debian-$version.tar.bz2 /" \ + # "CMD [\"/bin/bash\"]" > "$DOCKERFILE_PATH" + + # echo "[INFO] Created $DOCKERFILE_PATH" + # done + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Log in to Gitea Registry + # run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login gitea.fithwum.tech -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin + + # - name: Build and push Docker images + # run: | + # IMAGE_REGISTRY=gitea.fithwum.tech + # IMAGE_ORG=fithwum + # IMAGE_REPO=debian-base + + # for TAG in $VERSIONS; do + # FULL_IMAGE="${IMAGE_REGISTRY}/${IMAGE_ORG}/${IMAGE_REPO}:${TAG}" + # echo "[INFO] Building and pushing $FULL_IMAGE" + # docker buildx build --platform linux/amd64 --push -t "$FULL_IMAGE" "./$TAG" + # done + build-and-push-docker-images: needs: build-and-push-rootfs-archives if: always() - # if: needs.build-and-push-rootfs-archives.outputs.archives_changed == 'true' runs-on: vm-docker-build2 + strategy: + matrix: + version: [bullseye, bookworm, trixie] # replace with your $VERSIONS steps: - name: Checkout source uses: actions/checkout@v3 - name: Wait for archives to appear in upload repo run: | - echo "[INFO] Waiting for archives to appear in upload-repo..." + echo "[INFO] Waiting for archive for ${{ matrix.version }}..." + mkdir -p temp-check && cd temp-check - mkdir -p temp-check - cd temp-check - - # Retry loop for cloning the upload-repo for i in {1..10}; do echo "[INFO] Attempt $i: Cloning upload-repo..." if git clone --depth=1 "https://${{ secrets.GIT_USERNAME }}:${{ secrets.GIT_TOKEN }}@gitea.fithwum.tech/fithwum/debian-base.git"; then break fi - echo "[WARN] Clone failed. Retrying in 10 seconds..." + echo "[WARN] Clone failed. Retrying in 10s..." sleep 10 done @@ -151,49 +302,37 @@ jobs: cd debian-base - # Wait for all versions to show up - missing_versions=() - for version in $VERSIONS; do - found=0 - for i in {1..30}; do - if [[ -f "$version/debian-$version.tar.bz2" ]]; then - found=1 - break - else - echo "[WAIT] $version not ready yet, sleeping 10s..." - sleep 10 - fi - done - - if [[ $found -eq 0 ]]; then - missing_versions+=("$version") + # Wait for this version's archive + for i in {1..30}; do + if [[ -f "${{ matrix.version }}/debian-${{ matrix.version }}.tar.bz2" ]]; then + echo "[OK] Found archive for ${{ matrix.version }}" + break + else + echo "[WAIT] ${{ matrix.version }} not ready yet, sleeping 10s..." + sleep 10 fi done - if [[ ${#missing_versions[@]} -gt 0 ]]; then - echo "[ERROR] Missing archives for: ${missing_versions[*]}" + if [[ ! -f "${{ matrix.version }}/debian-${{ matrix.version }}.tar.bz2" ]]; then + echo "[ERROR] Missing archive for ${{ matrix.version }}" exit 1 fi - - name: Generate Dockerfiles per version (if missing) + - name: Generate Dockerfile for ${{ matrix.version }} run: | - for version in $VERSIONS; do - mkdir -p "$version" - DOCKERFILE_PATH="$version/Dockerfile" - - if [[ -f "$DOCKERFILE_PATH" ]]; then - echo "[INFO] Skipping $DOCKERFILE_PATH (already exists)" - continue - fi + mkdir -p "${{ matrix.version }}" + DOCKERFILE_PATH="${{ matrix.version }}/Dockerfile" + if [[ ! -f "$DOCKERFILE_PATH" ]]; then printf '%s\n' \ "FROM scratch" \ "LABEL maintainer=\"fithwum\"" \ - "ADD debian-$version.tar.bz2 /" \ + "ADD debian-${{ matrix.version }}.tar.bz2 /" \ "CMD [\"/bin/bash\"]" > "$DOCKERFILE_PATH" - echo "[INFO] Created $DOCKERFILE_PATH" - done + else + echo "[INFO] Dockerfile already exists for ${{ matrix.version }}" + fi - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -201,17 +340,14 @@ jobs: - name: Log in to Gitea Registry run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login gitea.fithwum.tech -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin - - name: Build and push Docker images + - name: Build and push Docker image for ${{ matrix.version }} run: | IMAGE_REGISTRY=gitea.fithwum.tech IMAGE_ORG=fithwum IMAGE_REPO=debian-base - - for TAG in $VERSIONS; do - FULL_IMAGE="${IMAGE_REGISTRY}/${IMAGE_ORG}/${IMAGE_REPO}:${TAG}" - echo "[INFO] Building and pushing $FULL_IMAGE" - docker buildx build --platform linux/amd64 --push -t "$FULL_IMAGE" "./$TAG" - done + FULL_IMAGE="${IMAGE_REGISTRY}/${IMAGE_ORG}/${IMAGE_REPO}:${{ matrix.version }}" + echo "[INFO] Building and pushing $FULL_IMAGE" + docker buildx build --platform linux/amd64 --push -t "$FULL_IMAGE" "./${{ matrix.version }}" generate-changelogs: needs: build-and-push-rootfs-archives