ci: exercise the locked install path

.github/workflows/health-check-ansible.yaml

@@ -50,6 +50,9 @@ jobs:
         image:
           - ubuntu:22.04
           - ubuntu:24.04
+        locked:
+          - false
+          - true
     container:
       image: ${{ matrix.image }}
     steps:
@@ -80,7 +83,7 @@ jobs:
           bash ansible/scripts/install-ansible.sh
           export PATH="$HOME/.local/bin:$PATH"
           ansible-galaxy collection install -f -r ansible-galaxy-requirements.yaml
-          ansible-playbook autoware.dev_env.install_dev_env --skip-tags artifacts -v
+          ansible-playbook autoware.dev_env.install_dev_env --skip-tags artifacts -e use_locked_versions=${{ matrix.locked }} -v
 
       - name: 📊 Report installation disk footprint
         run: |
@@ -90,7 +93,7 @@ jobs:
           delta_gb=$(awk -v b="${delta}" 'BEGIN { printf "%.2f", b/1e9 }')
           echo "📦 Installation disk usage: ${delta_gb} GB"
           {
-            echo "## 📦 Installation disk usage (\`${{ matrix.image }}\`)"
+            echo "## 📦 Installation disk usage (\`${{ matrix.image }}\`, locked=${{ matrix.locked }})"
             echo ""
             echo "Root filesystem grew by **${delta_gb} GB** during the install."
           } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/validate-lockfiles.yaml

@@ -2,10 +2,24 @@ name: validate-lockfiles
 
 on:
   pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+    # Wide enough for whichever of the two jobs below could be affected. They no
+    # longer share one gate: the cheap lint runs on anything reaching this
+    # workflow, and the two cold container builds gate themselves on a
+    # changed-files job scoped to what the `base` target actually consumes.
+    # Markdown is excluded outright — 19 READMEs live under ansible/roles/**,
+    # and a docs-only PR must start neither job.
     paths:
-      - ansible/vars/locked-versions-*.yaml
+      - ansible/**
+      - ansible-galaxy-requirements.yaml
+      - docker/*.Dockerfile
       - docker/docker-bake.hcl
       - .github/workflows/validate-lockfiles.yaml
+      - "!**/*.md"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -28,3 +42,84 @@ jobs:
 
       - name: Validate lockfiles (format + digest/lockfile coverage)
         run: ./ansible/scripts/validate_lockfiles.sh
+
+  # What a locked `base` bake resolves against: the roles install_rmw runs, the
+  # lockfile it reads, the collection those roles are resolved from, and the two
+  # docker files the bake itself consumes. Deliberately narrower than the
+  # workflow trigger above — ansible/scripts/** and the other four playbooks
+  # reach the lint but no bake.
+  changed-files:
+    runs-on: ubuntu-latest
+    outputs:
+      should-run: ${{ steps.check.outputs.any_modified }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - id: check
+        uses: step-security/changed-files@v47
+        with:
+          files: |
+            ansible/roles/**
+            ansible/vars/**
+            ansible/playbooks/install_rmw.yaml
+            ansible-galaxy-requirements.yaml
+            docker/base.Dockerfile
+            docker/docker-bake.hcl
+            .github/workflows/validate-lockfiles.yaml
+          files_ignore: |
+            ansible/roles/artifacts/**
+            **/*.md
+
+  # The locked base bakes below are two cold, uncached container builds. Put
+  # them behind the same opt-in label the repo already uses for its other
+  # expensive lanes, wired to run-condition exactly as health-check-ansible.yaml,
+  # health-check-ansible-artifacts.yaml and health-check-pr.yaml wire theirs.
+  # Without run-condition this call site reports red — not skipped — on any PR
+  # that reaches the workflow without touching a bake input, and all four sites
+  # publish the same required `require-label / require-label` context.
+  require-label:
+    needs: changed-files
+    if: ${{ always() }}
+    uses: autowarefoundation/autoware-github-actions/.github/workflows/require-label.yaml@v1
+    with:
+      label: run:health-check
+      run-condition: ${{ needs.changed-files.outputs.should-run == 'true' }}
+
+  build-locked-base:
+    needs: require-label
+    if: needs.require-label.outputs.result == 'true'
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        rosdistro: [humble, jazzy]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build the locked base image
+        run: USE_LOCKFILE=true ROS_DISTRO=${{ matrix.rosdistro }} docker buildx bake -f docker/docker-bake.hcl base
+
+  # The matrix above emits one context per distro, none of which is a stable
+  # name a branch ruleset can require. Aggregate them into one, the same way
+  # health-check-ansible.yaml aggregates install-dev-env, so this lane can be
+  # promoted to a required check without the name changing again when a distro
+  # is added or dropped.
+  build-locked-base-required:
+    needs: build-locked-base
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Verify build-locked-base outcome
+        run: |
+          result="${{ needs.build-locked-base.result }}"
+          echo "build-locked-base: $result"
+          case "$result" in
+            success|skipped) exit 0 ;;
+            *) exit 1 ;;
+          esac

ansible/roles/version_lock/README.md

@@ -38,6 +38,43 @@ The file needs only a top-level `nvidia_pins` mapping, in the same format as a l
 
 Leaving `nvidia_lockfile_path` empty while overriding a version is not silently tolerated: the `cuda` role fails when no `cuda-*-<version>` entry covers this run, and the `tensorrt` role fails when the pinned `libnvinfer10` disagrees with `tensorrt_version`. Without those guards the CUDA half would install unfrozen (its version-suffixed names simply are not in the closure, and `verify.yaml` reports drift only for keys that are installed) and the TensorRT half would fail with an unattributable `no available installation candidate`.
 
+### Ubuntu archive snapshot
+
+An exact-version pin only binds a version APT can still reach. Ubuntu drops a
+superseded build from `<suite>-updates` and `<suite>-security` the moment it
+publishes the replacement, and a preference naming a version no configured
+source serves matches nothing at all — so APT quietly installs the current
+candidate and `tasks/verify.yaml` fails the play on the drift. This decays on
+its own: 17 days after the lockfiles were last regenerated, `wget` had moved on
+both distros with no change to this repository.
+
+In locked mode the role therefore also writes
+`/etc/apt/sources.list.d/ubuntu-snapshot.sources`, pointing at
+`snapshot.ubuntu.com` at the lockfile's `ubuntu_snapshot_date`
+(`tasks/ubuntu_snapshot_source.yaml`). That service republishes the archive as
+it stood at a given timestamp, so every `apt_pins` build stays downloadable
+after the live pockets move past it. It is the Ubuntu-archive counterpart of
+the dated `snapshots.ros.org` source the `ros2` role configures, it covers
+`amd64` and `arm64` from the same tree, and it is signed by the stock
+`ubuntu-archive-keyring`, so no extra key is installed.
+
+The source is added **alongside** the distribution's own sources, not in place
+of them:
+
+- The pins are the contract for Ubuntu-archive packages. At `Pin-Priority: 1001`
+  they select the locked build wherever it is served, so this source only has to
+  keep that build reachable — it does not have to win on its own.
+- `install_dev_env` runs on developer machines, and rewriting a user's
+  `/etc/apt/sources.list` is not this role's to do.
+- Packages outside `apt_pins` keep resolving exactly as they do without it, so
+  the change affects nothing but a pin that would otherwise be unreachable. A
+  newly introduced Ubuntu-archive package that nobody pinned is caught
+  separately, by `files/check_pin_completeness.py` in `tasks/verify.yaml`.
+
+`snapshot.ubuntu.com` is https-only, and the minimal Ubuntu images CI runs in
+ship no CA bundle, so the task installs `ca-certificates` from the sources
+configured at that point, before APT has to speak TLS.
+
 ### Snapshot reconcile
 
 In locked mode the role does not only pin future installs: right after the
@@ -65,11 +102,19 @@ freeze — release the hold if you want locked mode to govern it.
 On a machine already provisioned by `install_dev_env`, run:
 
 ```bash
-ROS_DISTRO=<distro> ROS_SNAPSHOT_DATE=<YYYY-MM-DD> ./ansible/scripts/generate_ansible_lockfile.sh
+ROS_DISTRO=<distro> ROS_SNAPSHOT_DATE=<YYYY-MM-DD> \
+  UBUNTU_SNAPSHOT_DATE=$(date -u +%Y%m%dT000000Z) \
+  ./ansible/scripts/generate_ansible_lockfile.sh
 ```
 
 `ROS_SNAPSHOT_DATE` must match the snapshot the machine was provisioned from.
 
+`UBUNTU_SNAPSHOT_DATE` is a `snapshot.ubuntu.com` timestamp of the form
+`20260805T000000Z`, and must be taken at the same time as the versions this run
+records. A timestamp from before the run would freeze the archive at builds that
+are not the ones pinned here, and every pin would then resolve to nothing.
+Today's date at midnight, as in the command above, is the normal choice.
+
 The script reads the package names from the existing lockfile for the current distro/architecture and fills in the versions currently installed on the machine. It rewrites `apt_pins` and `pip_pins`, and preserves `nvidia_pins` and `ros_overrides` verbatim.
 
 ### Recording the NVIDIA closure
@@ -231,11 +276,12 @@ vice-versa); regenerate both together so the digest matches `ros_snapshot_date`.
 
 ## Lockfile format
 
-A YAML mapping with five keys, one file per distro/arch
+A YAML mapping with six keys, one file per distro/arch
 (`ansible/vars/locked-versions-<rosdistro>-<arch>.yaml`):
 
 ```yaml
 ros_snapshot_date: "2026-04-13" # a real published date under snapshots.ros.org/<distro>/
+ubuntu_snapshot_date: 20260805T000000Z # a real published timestamp under snapshot.ubuntu.com/ubuntu/
 apt_pins: # Ubuntu-archive origin only, sorted by name; rendered as APT pins
   ccache: 4.9.1-1
   git-lfs: 3.4.1-1ubuntu0.4
@@ -251,6 +297,10 @@ ros_overrides: {} # exception pins for individual ROS packages; normally empty
   also carries a `Package: * / Pin: origin snapshots.ros.org / Pin-Priority: 1001` stanza so the
   snapshot outranks the rolling `packages.ros.org` repo even on a machine that already had it —
   otherwise both sit at priority 500 and apt would install the newer rolling build.
+- `ubuntu_snapshot_date` drives the dated `snapshot.ubuntu.com` source the role adds in locked
+  mode, which keeps every `apt_pins` build below downloadable after Ubuntu supersedes it in the
+  live pockets. Unlike `ros_snapshot_date` it does not by itself freeze a closure — the pins do
+  that — it only keeps the pinned builds reachable. See "Ubuntu archive snapshot" above.
 - `apt_pins` covers Ubuntu-archive packages. It is rendered into `/etc/apt/preferences.d/autoware-lock`
   with `Pin-Priority: 1001`.
 - `pip_pins` is the declared home for pip/pipx-managed packages. It is meant to be consumed

ansible/roles/version_lock/tasks/main.yaml

@@ -60,6 +60,21 @@
     state: absent
   when: not (use_locked_versions | default(false) | bool)
 
+# Keeps every pinned build downloadable after Ubuntu supersedes it in the live
+# pockets, which is what makes the pins above bind at all. Runs before the
+# python3-apt install below so that package, itself an apt_pin, already resolves
+# from the snapshot. See ubuntu_snapshot_source.yaml for the full rationale.
+- name: Configure the dated Ubuntu archive snapshot source
+  ansible.builtin.include_tasks: ubuntu_snapshot_source.yaml
+  when: use_locked_versions | default(false) | bool
+
+- name: Remove the dated Ubuntu archive snapshot source
+  become: true
+  ansible.builtin.file:
+    path: /etc/apt/sources.list.d/ubuntu-snapshot.sources
+    state: absent
+  when: not (use_locked_versions | default(false) | bool)
+
 # Both files/list_ros_snapshot_drift.py and files/check_pin_completeness.py
 # `import apt`, so python3-apt is a declared dependency of this role, not
 # incidental bootstrap. Installing it explicitly, before the baseline below,

ansible/roles/version_lock/tasks/ubuntu_snapshot_source.yaml

@@ -0,0 +1,58 @@
+# Configure the dated Ubuntu archive snapshot apt source (locked mode).
+#
+# An exact-version pin in /etc/apt/preferences.d/autoware-lock only binds a
+# version apt can actually reach. Ubuntu drops a superseded build from
+# <suite>-updates and <suite>-security as soon as it publishes the replacement,
+# so a pin taken at lockfile-generation time stops matching anything the moment
+# a security update lands. apt then silently installs the current candidate and
+# tasks/verify.yaml fails the play on the drift. This decayed on its own 17 days
+# after the lockfiles were last regenerated (wget, both distros), with no change
+# to the repository.
+#
+# snapshot.ubuntu.com republishes the archive as it stood at a given timestamp,
+# so pointing at the lockfile's own date makes every pinned build downloadable
+# again. It is the Ubuntu-archive counterpart to the dated snapshots.ros.org
+# source the ros2 role configures (roles/ros2/tasks/snapshot_source.yaml).
+#
+# Added ALONGSIDE the distribution's own sources rather than replacing them:
+#   - The pins are the contract for Ubuntu-archive packages, and at priority
+#     1001 they select the locked build wherever it is served. This source only
+#     has to keep that build reachable; it does not need to win by itself.
+#   - install_dev_env runs on developer machines, and rewriting a user's
+#     /etc/apt/sources.list is not this role's to do.
+#   - Packages outside apt_pins keep resolving exactly as they do today, so the
+#     change has no effect on anything but a pin that would otherwise be
+#     unreachable. A newly-introduced Ubuntu-archive package that nobody pinned
+#     is caught separately, by files/check_pin_completeness.py in verify.yaml.
+#
+# Callers do not import this file directly; tasks/main.yaml runs it in locked mode.
+- name: Assert lockfile provides an Ubuntu snapshot date
+  ansible.builtin.assert:
+    that:
+      - locked_packages.ubuntu_snapshot_date is defined
+      - locked_packages.ubuntu_snapshot_date | string | length > 0
+    fail_msg: >-
+      Locked mode requires 'ubuntu_snapshot_date' in {{ lockfile_path }}
+      (snapshot.ubuntu.com timestamp, e.g. 20260805T000000Z).
+    quiet: true
+
+# snapshot.ubuntu.com is https-only (plain http 301-redirects to it), and the
+# minimal Ubuntu images CI runs in ship no CA bundle. Install it from whatever
+# sources are configured now, before apt has to speak TLS. Not pinned: it is a
+# bootstrap prerequisite of this source, in the same spirit as python3-apt in
+# main.yaml, and it is installed before the pin-completeness baseline is taken.
+- name: Install ca-certificates so apt can reach the snapshot over https
+  become: true
+  ansible.builtin.apt:
+    name: ca-certificates
+    state: present
+    update_cache: true
+
+- name: Write dated Ubuntu snapshot apt source
+  become: true
+  ansible.builtin.template:
+    src: ubuntu-snapshot.sources.j2
+    dest: /etc/apt/sources.list.d/ubuntu-snapshot.sources
+    owner: root
+    group: root
+    mode: "0644"

ansible/roles/version_lock/templates/ubuntu-snapshot.sources.j2

@@ -0,0 +1,13 @@
+{# Rendered to /etc/apt/sources.list.d/ubuntu-snapshot.sources in locked mode. #}
+{# Added alongside the distribution's own sources, not in place of them: the #}
+{# exact-version pins in /etc/apt/preferences.d/autoware-lock (priority 1001) #}
+{# decide which candidate wins, and this source only has to make the pinned #}
+{# build downloadable again once the live pockets drop it. See #}
+{# tasks/ubuntu_snapshot_source.yaml for why. #}
+{# Signed by the stock Ubuntu archive key: snapshot.ubuntu.com republishes the #}
+{# original Release files, so no extra keyring is needed. #}
+Types: deb
+URIs: https://snapshot.ubuntu.com/ubuntu/{{ locked_packages.ubuntu_snapshot_date }}
+Suites: {{ ansible_distribution_release }} {{ ansible_distribution_release }}-updates {{ ansible_distribution_release }}-backports {{ ansible_distribution_release }}-security
+Components: main restricted universe multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg

ansible/scripts/generate_ansible_lockfile.sh

@@ -12,6 +12,20 @@ if [[ -z ${ROS_SNAPSHOT_DATE:-} ]]; then
     echo "Error: ROS_SNAPSHOT_DATE is not set (e.g. 2026-04-13)." >&2
     exit 1
 fi
+# The Ubuntu archive counterpart of ROS_SNAPSHOT_DATE. Take it at the same time
+# as the apt versions recorded below: snapshot.ubuntu.com is what keeps those
+# builds downloadable once the live pockets supersede them, so a timestamp from
+# before this run would freeze the archive at versions that are not the ones
+# pinned here.
+if [[ -z ${UBUNTU_SNAPSHOT_DATE:-} ]]; then
+    echo "Error: UBUNTU_SNAPSHOT_DATE is not set (snapshot.ubuntu.com timestamp, e.g. 20260805T000000Z)." >&2
+    echo "Hint: date -u +%Y%m%dT000000Z" >&2
+    exit 1
+fi
+if [[ ! $UBUNTU_SNAPSHOT_DATE =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then
+    echo "Error: UBUNTU_SNAPSHOT_DATE must look like 20260805T000000Z, got '${UBUNTU_SNAPSHOT_DATE}'." >&2
+    exit 1
+fi
 ARCH=$(dpkg --print-architecture)
 OUTPUT_FILE="${ANSIBLE_DIR}/vars/locked-versions-${ROS_DISTRO}-${ARCH}.yaml"
 
@@ -117,6 +131,12 @@ else:
 #
 # ros_snapshot_date freezes every ROS package snapshots.ros.org serves on that date.
 ros_snapshot_date: "${ROS_SNAPSHOT_DATE}"
+#
+# ubuntu_snapshot_date freezes what the Ubuntu archive can still serve: snapshot.ubuntu.com
+# republishes the pockets as they stood at that timestamp, so every apt_pins build below
+# stays downloadable after the live pockets supersede it. Take it at the same time as the
+# versions below, or a pin here will resolve to nothing.
+ubuntu_snapshot_date: ${UBUNTU_SNAPSHOT_DATE}
 apt_pins:
 HEADER
         for pkg in "${apt_packages[@]}"; do

ansible/scripts/validate_lockfiles.sh

@@ -24,6 +24,9 @@ if not isinstance(data, dict):
 date = data.get('ros_snapshot_date')
 if not isinstance(date, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}', date):
     sys.exit('Error: ros_snapshot_date must be a YYYY-MM-DD string')
+ubuntu_date = data.get('ubuntu_snapshot_date')
+if not isinstance(ubuntu_date, str) or not re.fullmatch(r'\d{8}T\d{6}Z', ubuntu_date):
+    sys.exit('Error: ubuntu_snapshot_date must be a snapshot.ubuntu.com timestamp, e.g. 20260805T000000Z')
 for key in ('apt_pins', 'pip_pins', 'nvidia_pins', 'ros_overrides'):
     section = data.get(key, {})
     if section is None:

ansible/vars/locked-versions-humble-amd64.yaml

@@ -6,6 +6,12 @@
 # ros_snapshot_date freezes every ROS package snapshots.ros.org serves on that date.
 # apt_pins are finalized by ansible/scripts/generate_ansible_lockfile.sh.
 ros_snapshot_date: "2026-05-14"
+#
+# ubuntu_snapshot_date freezes what the Ubuntu archive can still serve: snapshot.ubuntu.com
+# republishes the pockets as they stood at that timestamp, so every apt_pins build below
+# stays downloadable after the live pockets supersede it. Take it at the same time as the
+# versions below, or a pin here will resolve to nothing.
+ubuntu_snapshot_date: 20260805T000000Z
 apt_pins:
   build-essential: 12.9ubuntu3
   ccache: 4.5.1-1

ansible/vars/locked-versions-humble-arm64.yaml

@@ -5,6 +5,12 @@
 #
 # ros_snapshot_date freezes every ROS package snapshots.ros.org serves on that date.
 ros_snapshot_date: "2026-05-14"
+#
+# ubuntu_snapshot_date freezes what the Ubuntu archive can still serve: snapshot.ubuntu.com
+# republishes the pockets as they stood at that timestamp, so every apt_pins build below
+# stays downloadable after the live pockets supersede it. Take it at the same time as the
+# versions below, or a pin here will resolve to nothing.
+ubuntu_snapshot_date: 20260805T000000Z
 apt_pins:
   build-essential: 12.9ubuntu3
   ccache: 4.5.1-1