CI: manifest-diff workflow

.github/actions/cache-manifest/action.yml

@@ -0,0 +1,27 @@
+name: Cache manifest
+description: Annotates a rootfs manifest with a version label and caches it as the diff baseline, keyed by its own content hash
+
+inputs:
+  key_prefix:
+    description: 'Cache key prefix (e.g. ami-manifest-15-amd64, docker-manifest-17-production)'
+    required: true
+  version_label:
+    description: 'Version string to stamp into the manifest'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - id: hash
+      shell: bash
+      env:
+        VERSION_LABEL: ${{ inputs.version_label }}
+      run: |
+        printf '%s\n' "# version: $VERSION_LABEL" | cat - /tmp/manifest.txt > /tmp/manifest.new
+        mv /tmp/manifest.new /tmp/manifest.txt
+        echo "value=$(sha256sum /tmp/manifest.txt | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
+
+    - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: /tmp/manifest.txt
+        key: ${{ inputs.key_prefix }}-${{ steps.hash.outputs.value }}

.github/actions/diff-manifest/action.yml

@@ -0,0 +1,36 @@
+name: Diff manifest
+description: Restores the cached baseline manifest and diffs it against this build's manifest
+
+inputs:
+  key_prefix:
+    description: 'Cache key prefix matching the one used by cache-manifest (e.g. ami-manifest-15-amd64)'
+    required: true
+  publish_hint:
+    description: 'Shown when no baseline is cached yet, naming what publishes one'
+    required: true
+  artifact_name:
+    description: 'Name of the uploaded diff artifact'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - run: mv /tmp/manifest.txt /tmp/current-manifest.txt
+      shell: bash
+
+    - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+      with:
+        path: /tmp/manifest.txt
+        key: ${{ inputs.key_prefix }}-
+        restore-keys: ${{ inputs.key_prefix }}-
+
+    - shell: bash
+      env:
+        PUBLISH_HINT: ${{ inputs.publish_hint }}
+      run: nix run .#manifest-diff -- /tmp/manifest.txt /tmp/current-manifest.txt "$PUBLISH_HINT" /tmp/manifest-diff.txt
+
+    - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      with:
+        name: ${{ inputs.artifact_name }}
+        path: /tmp/manifest-diff.txt
+        retention-days: 7

.github/actions/post-manifest-comment/action.yml

@@ -0,0 +1,50 @@
+name: Post manifest comment
+description: Assembles per-leg manifest diffs into one collapsed comment and posts it, updating any existing one in place
+
+inputs:
+  marker:
+    description: 'Unique HTML comment marker identifying this comment (e.g. ami-manifest-diff)'
+    required: true
+  title:
+    description: 'Comment heading (e.g. AMI manifest diff)'
+    required: true
+  artifact_pattern:
+    description: 'Glob matching the uploaded per-leg diff artifacts (e.g. ami-manifest-diff-*)'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - uses: ./.github/actions/nix-install-ephemeral
+
+    - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+      with:
+        pattern: ${{ inputs.artifact_pattern }}
+        path: diffs
+
+    - id: render
+      shell: bash
+      env:
+        MARKER: ${{ inputs.marker }}
+        TITLE: ${{ inputs.title }}
+        ARTIFACT_PATTERN: ${{ inputs.artifact_pattern }}
+        RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        PREFIX="${ARTIFACT_PATTERN%-\*}"
+        nix run .#manifest-comment -- "$MARKER" "$TITLE" "$PREFIX" "$RUN_URL" > /tmp/comment.md
+        [ -s /tmp/comment.md ] && echo "has_comment=true" >> "$GITHUB_OUTPUT" || true
+
+    - uses: peter-evans/find-comment@v3
+      id: fc
+      if: steps.render.outputs.has_comment == 'true'
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        body-includes: "<!-- ${{ inputs.marker }} -->"
+
+    - uses: peter-evans/create-or-update-comment@v4
+      if: steps.render.outputs.has_comment == 'true'
+      with:
+        comment-id: ${{ steps.fc.outputs.comment-id }}
+        issue-number: ${{ github.event.pull_request.number }}
+        body-path: /tmp/comment.md
+        edit-mode: replace

.github/workflows/ami-release-nix.yml

@@ -105,6 +105,12 @@ jobs:
           postgres_version: ${{ matrix.postgres_version }}
           region: ${{ env.AWS_REGION }}
 
+      - name: Cache AMI manifest
+        uses: ./.github/actions/cache-manifest
+        with:
+          key_prefix: ami-manifest-${{ matrix.postgres_version }}-${{ matrix.target.arch }}
+          version_label: ${{ steps.build-ami.outputs.postgres_release_version }}
+
       - name: Setup post build env vars
         run: |
           POSTGRES_SUPABASE_VERSION=${{ steps.build-ami.outputs.postgres_release_version }}

.github/workflows/docker-image-test.yml

@@ -82,6 +82,19 @@ jobs:
             -t "supabase-postgres:${{ matrix.name }}-analyze" \
             .
 
+      - name: Snapshot Docker image manifest
+        run: |
+          docker run --rm -v "$PWD/testinfra/manifest-snapshot.sh:/manifest-snapshot.sh:ro" \
+            "pg-docker-test:${{ matrix.name }}" sh /manifest-snapshot.sh > /tmp/manifest.txt
+
+      - name: Diff Docker image manifest
+        if: github.event_name == 'pull_request'
+        uses: ./.github/actions/diff-manifest
+        with:
+          key_prefix: docker-manifest-${{ matrix.name }}
+          publish_hint: Publish one by running dockerhub-release-matrix.yml (push to develop).
+          artifact_name: docker-manifest-diff-${{ matrix.name }}
+
       - name: Run image size analysis
         if: ${{ matrix.target == '' }}
         run: |
@@ -115,3 +128,18 @@ jobs:
           docker ps -a --filter "name=pg-test-${{ matrix.name }}" -q | xargs -r docker rm -f || true
           docker rmi "pg-docker-test:${{ matrix.name }}" || true
           docker rmi "supabase-postgres:${{ matrix.name }}-analyze" || true
+
+  docker-manifest-comment:
+    if: always() && github.event_name == 'pull_request'
+    needs: docker-image-test
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+      - uses: ./.github/actions/post-manifest-comment
+        with:
+          marker: docker-manifest-diff
+          title: Docker image manifest diff
+          artifact_pattern: docker-manifest-diff-*

.github/workflows/dockerhub-release-matrix.yml

@@ -153,6 +153,20 @@ jobs:
           cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
           cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
           file: ${{ matrix.postgres.dockerfile }}
+
+      - name: Snapshot Docker image manifest
+        if: matrix.arch == 'arm64'
+        run: |
+          docker pull "${{ steps.image.outputs.pg_version }}_${{ matrix.arch }}"
+          docker run --rm -v "$PWD/testinfra/manifest-snapshot.sh:/manifest-snapshot.sh:ro" \
+            "${{ steps.image.outputs.pg_version }}_${{ matrix.arch }}" sh /manifest-snapshot.sh > /tmp/manifest.txt
+
+      - name: Cache Docker image manifest
+        if: matrix.arch == 'arm64'
+        uses: ./.github/actions/cache-manifest
+        with:
+          key_prefix: docker-manifest-${{ matrix.postgres.version }}
+          version_label: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }}
   merge_manifest:
     needs: [prepare, build, build_release_image]
     strategy:

.github/workflows/nix-build.yml

@@ -13,6 +13,7 @@ permissions:
   # required by testinfra-ami-build dependent workflows
   contents: write
   packages: write
+  pull-requests: write
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/testinfra-ami-build.yml

@@ -121,6 +121,21 @@ jobs:
           overwrite: true
           retention-days: 1
 
+      - name: Diff AMI manifest
+        if: github.event_name == 'pull_request'
+        uses: ./.github/actions/diff-manifest
+        with:
+          key_prefix: ami-manifest-${{ matrix.postgres_version }}-${{ matrix.target.arch }}
+          publish_hint: Publish one by running ami-release-nix.yml (push to develop).
+          artifact_name: ami-manifest-diff-${{ matrix.postgres_version }}-${{ matrix.target.arch }}
+
+      - name: TEMP seed baseline from PR build
+        if: github.event_name == 'pull_request'
+        uses: ./.github/actions/cache-manifest
+        with:
+          key_prefix: ami-manifest-${{ matrix.postgres_version }}-${{ matrix.target.arch }}
+          version_label: pr-${{ github.event.pull_request.head.sha }}
+
       - name: Run tests
         timeout-minutes: 10
         env:
@@ -166,3 +181,19 @@ jobs:
           cat ami-disk-usage/ami-disk-usage-*.json |
           sort -V |
           jq -rs '.[]|{version,arch,human,bytes}|"| \(join("|")) |"' >>"$GITHUB_STEP_SUMMARY"
+
+  manifest-diff-comment:
+    # always(): post whatever legs succeeded even if some matrix legs failed
+    if: always() && github.event_name == 'pull_request'
+    needs: build
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: supabase/postgres/.github/actions/shared-checkout@HEAD
+
+      - uses: ./.github/actions/post-manifest-comment
+        with:
+          marker: ami-manifest-diff
+          title: AMI manifest diff
+          artifact_pattern: ami-manifest-diff-*

nix/packages/default.nix

@@ -69,6 +69,8 @@
           http-mock-server = pkgs.callPackage ./http-mock-server.nix { };
           image-size-analyzer = pkgs.callPackage ./image-size-analyzer.nix { };
           local-infra-bootstrap = pkgs.callPackage ./local-infra-bootstrap.nix { };
+          manifest-diff = pkgs.callPackage ./manifest-diff.nix { };
+          manifest-comment = pkgs.callPackage ./manifest-comment.nix { };
           mecab-naist-jdic = pkgs.callPackage ./mecab-naist-jdic.nix { };
           migrate-tool = pkgs.callPackage ./migrate-tool.nix { psql_15 = self'.packages."psql_15/bin"; };
           overlayfs-on-package = pkgs.callPackage ./overlayfs-on-package.nix { };

nix/packages/manifest-comment.nix

@@ -0,0 +1,47 @@
+{
+  writeShellApplication,
+  coreutils,
+}:
+writeShellApplication {
+  name = "manifest-comment";
+  runtimeInputs = [
+    coreutils
+  ];
+  text = ''
+    if [ "$#" -lt 4 ]; then
+      echo "Usage: manifest-comment <marker> <title> <diffs_prefix> <run_url>" >&2
+      exit 1
+    fi
+    shopt -s nullglob
+    MARKER="$1"
+    TITLE="$2"
+    PREFIX="$3"
+    RUN_URL="$4"
+
+    fence=$'\x60\x60\x60'
+    dirs=("diffs/''${PREFIX}"-*)
+    if [ "''${#dirs[@]}" -eq 0 ]; then
+      exit 0
+    fi
+
+    echo "<!-- $MARKER -->"
+    echo "## $TITLE"
+    echo
+
+    for dir in "''${dirs[@]}"; do
+      leg="''${dir#diffs/"''${PREFIX}"-}"
+      file="$dir/manifest-diff.txt"
+      first="$(head -1 "$file")"
+      if [[ "$first" != "baseline: "* ]]; then
+        printf '<details>\n<summary>%s: %s</summary>\n</details>\n\n' "$leg" "$first"
+        continue
+      fi
+      body="$(tail -n +2 "$file")"
+      printf '<details>\n<summary>%s: changed (%s)</summary>\n\n%sdiff\n%s\n%s\n' "$leg" "$first" "$fence" "''${body:0:6000}" "$fence"
+      if [ "''${#body}" -gt 6000 ]; then
+        echo "truncated, [full output]($RUN_URL)"
+      fi
+      printf '</details>\n\n'
+    done
+  '';
+}

nix/packages/manifest-diff.nix

@@ -0,0 +1,48 @@
+{
+  writeShellApplication,
+  diffutils,
+  gnused,
+  coreutils,
+}:
+writeShellApplication {
+  name = "manifest-diff";
+  runtimeInputs = [
+    diffutils
+    gnused
+    coreutils
+  ];
+  text = ''
+    if [ "$#" -lt 4 ]; then
+      echo "Usage: manifest-diff <manifest_path> <current_manifest_path> <publish_hint> <diff_path>" >&2
+      exit 1
+    fi
+    MANIFEST_PATH="$1"
+    CURRENT_MANIFEST_PATH="$2"
+    PUBLISH_HINT="$3"
+    DIFF_PATH="$4"
+
+    if [ -s "$MANIFEST_PATH" ]; then
+      mv "$MANIFEST_PATH" /tmp/baseline-raw.txt
+      BASELINE_VERSION=$(sed -n 's/^# version: //p' /tmp/baseline-raw.txt | head -1)
+      tail -n +2 /tmp/baseline-raw.txt > /tmp/baseline.txt
+    else
+      touch /tmp/baseline.txt
+      BASELINE_VERSION=""
+    fi
+    mv "$CURRENT_MANIFEST_PATH" "$MANIFEST_PATH"
+    {
+      if [ -z "$BASELINE_VERSION" ]; then
+        echo "No baseline cached yet. $PUBLISH_HINT"
+      elif diff -q /tmp/baseline.txt "$MANIFEST_PATH" > /dev/null; then
+        echo "No changes vs baseline $BASELINE_VERSION."
+      else
+        echo "baseline: $BASELINE_VERSION"
+        diff \
+          --old-line-format='-%L' \
+          --new-line-format='+%L' \
+          --unchanged-group-format=$'... %dn unchanged\n' \
+          /tmp/baseline.txt "$MANIFEST_PATH" || true
+      fi
+    } | tee "$DIFF_PATH"
+  '';
+}

stage2-nix-psql.pkr.hcl

@@ -152,4 +152,22 @@ build {
     destination = "/tmp/ansible-stage2.log"
     direction   = "download"
   }
+
+  provisioner "file" {
+    source      = "testinfra/manifest-snapshot.sh"
+    destination = "/tmp/manifest-snapshot.sh"
+  }
+
+  provisioner "shell" {
+    inline = [
+      "sudo touch /etc/manifest-diff-test",
+      "sudo sh /tmp/manifest-snapshot.sh > /tmp/ami-manifest.txt"
+    ]
+  }
+
+  provisioner "file" {
+    source      = "/tmp/ami-manifest.txt"
+    destination = "/tmp/manifest.txt"
+    direction   = "download"
+  }
 }