Make apt fail fast, so the retry has something to retry

.github/scripts/retry-with-apt-lock.sh

@@ -14,19 +14,56 @@
 #
 # Two things make the retry actually work, and both were learned the hard way:
 #
-#   1. The dpkg lock. apt runs as root, so killing the attempt leaves the
-#      apt-get child alive holding /var/lib/dpkg/lock-frontend, and the next
-#      attempt dies two seconds later with "Could not get lock". A retry that
-#      cannot succeed is worse than none: it buries the real reason under a
-#      second, different failure. So wait for the lock, then take it -- the
-#      holder is our own orphan and the runner is disposable.
+#   1. The apt locks, PLURAL. apt runs as root, so killing the attempt leaves the
+#      apt-get child alive holding a lock, and the next attempt dies two seconds
+#      later with "Could not get lock". A retry that cannot succeed is worse than
+#      none: it buries the real reason under a second, different failure. So wait
+#      for the locks, then take them -- the holder is our own orphan and the
+#      runner is disposable.
+#
+#      There are four, and which one matters depends on what apt was doing.
+#      Waiting on only /var/lib/dpkg/lock-frontend is how the first version of
+#      this shipped, and it made the retry useless for exactly the case it was
+#      written for: `apt-get update` takes /var/lib/apt/lists/lock and nothing
+#      else, so the wait saw a free lock, retried immediately, and produced
+#
+#        E: Could not get lock /var/lib/apt/lists/lock. It is held by process 2420 (apt-get)
+#
+#      twice in a row in under two seconds. Three attempts, one real one.
 #
 #   2. `set -e`. GitHub runs `run:` blocks as `bash -e`, so a bare failing
 #      command aborts the step then and there. A retry loop written as
 #      `timeout ... ; rc=$?` never reaches its second attempt: the step exits 124
 #      with no output at all, which is exactly how the first version of this
 #      shipped and why it is a script now rather than eight inline copies.
 #
+#   3. apt's own timeouts, which is the part that makes the retry meaningful
+#      rather than merely survivable. Reading the logs of the four stalls above:
+#
+#        04:47:02  Get:5 https://archive.ubuntu.com/ubuntu noble-security InRelease [126 kB]
+#        05:16:29  ##[error]The operation was canceled.
+#
+#      Twenty-nine minutes of silence, mid-fetch of a 126 kB index file. Two of
+#      the four hung on that same file. What happened before it is the other half
+#      of the story: azure.archive.ubuntu.com was `Ign:`d four times over thirty
+#      seconds, so apt had already failed over through /etc/apt/apt-mirrors.txt to
+#      the public archive, which is not provisioned for this fleet.
+#
+#      apt did not consider any of that an error. `Acquire::http::Timeout`
+#      defaults to 120s AND is an idle timeout, so a connection that is open and
+#      trickling never trips it -- apt will wait out the heat death of the
+#      universe for a socket that is technically still alive. Cutting the timeout
+#      to 20s with internal retries turns a 29-minute hang into a ~1 minute
+#      failure, which matters for a reason beyond speed: the wall-clock kill below
+#      is what orphans the dpkg lock in the first place, so an apt that fails on
+#      its own is an apt we never have to kill.
+#
+#      Deliberately NOT pinning a mirror. The evidence does not support it: in one
+#      stall Azure was dead and the public archive hung, in another Azure was
+#      serving fine and the transfer stalled at a 13.6 MB package. Neither is
+#      reliably better, and the mirrorlist failover is already the right mechanism
+#      -- it just needs to be allowed to give up.
+#
 # Usage:
 #   bash .github/scripts/retry-with-apt-lock.sh apt-get update
 #   bash .github/scripts/retry-with-apt-lock.sh apt-get install -y foo bar
@@ -35,14 +72,21 @@
 # Environment:
 #   RETRY_ATTEMPTS         attempts before giving up   (default 3)
 #   RETRY_ATTEMPT_TIMEOUT  seconds per attempt         (default 480)
+#   APT_ACQUIRE_TIMEOUT    seconds apt waits on a stalled transfer (default 20)
+#   APT_ACQUIRE_RETRIES    apt's own internal retries  (default 3)
 #
 # Deliberately no `set -e`: this script reads exit codes itself, and -e would
 # abort it on the very first failing attempt -- the bug described above.
 set -uo pipefail
 
 ATTEMPTS="${RETRY_ATTEMPTS:-3}"
 ATTEMPT_TIMEOUT="${RETRY_ATTEMPT_TIMEOUT:-480}"
-DPKG_LOCK="/var/lib/dpkg/lock-frontend"
+# Every lock apt takes. dpkg's two cover install/configure, lists covers `update`,
+# and archives covers the download cache; an orphan can be holding any of them.
+APT_LOCKS="/var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock"
+APT_TIMEOUT="${APT_ACQUIRE_TIMEOUT:-20}"
+APT_RETRIES="${APT_ACQUIRE_RETRIES:-3}"
+APT_CONF="/etc/apt/apt.conf.d/99-unsloth-ci-fail-fast"
 
 if [ "$#" -eq 0 ]; then
   echo "::error::retry-with-apt-lock.sh needs a command to run" >&2
@@ -54,21 +98,59 @@ fi
 # is explicable.
 have_fuser() { command -v fuser > /dev/null 2>&1; }
 
-release_dpkg_lock() {
+# Make apt give up on a dead transfer instead of waiting on it forever. Written
+# before the first attempt rather than baked into the runner image so it applies
+# to `playwright install --with-deps` too, which shells out to apt without ever
+# naming it. Best effort throughout: a runner where this cannot be written still
+# runs the command, just without the fast failure.
+configure_apt_fail_fast() {
+  [ -d /etc/apt/apt.conf.d ] || return 0
+  conf="Acquire::Retries \"${APT_RETRIES}\";
+Acquire::http::Timeout \"${APT_TIMEOUT}\";
+Acquire::https::Timeout \"${APT_TIMEOUT}\";
+Acquire::ftp::Timeout \"${APT_TIMEOUT}\";"
+  if ! printf '%s\n' "$conf" | sudo tee "$APT_CONF" > /dev/null 2>&1; then
+    echo "::warning::could not write ${APT_CONF}; apt keeps its 120s idle timeout"
+    return 0
+  fi
+  echo "apt configured to fail fast: ${APT_TIMEOUT}s transfer timeout, ${APT_RETRIES} internal retries"
+}
+
+# Held locks, as a single string. Absent files are not locks, and `fuser` on one
+# is an error rather than an answer, so they are skipped rather than waited on.
+held_apt_locks() {
+  held=""
+  for lock in $APT_LOCKS; do
+    [ -e "$lock" ] || continue
+    if sudo fuser "$lock" > /dev/null 2>&1; then
+      held="$held $lock"
+    fi
+  done
+  printf '%s' "$held"
+}
+
+release_apt_locks() {
   if ! have_fuser; then
-    echo "::warning::fuser unavailable; cannot wait on ${DPKG_LOCK}, retrying blind"
+    echo "::warning::fuser unavailable; cannot wait on the apt locks, retrying blind"
     sleep 15
     return 0
   fi
   for _ in $(seq 1 24); do
-    sudo fuser "$DPKG_LOCK" > /dev/null 2>&1 || return 0
+    holding="$(held_apt_locks)"
+    [ -n "$holding" ] || return 0
     sleep 5
   done
-  echo "::warning::${DPKG_LOCK} still held after 120s; terminating the holder"
-  sudo fuser -k "$DPKG_LOCK" > /dev/null 2>&1 || true
+  holding="$(held_apt_locks)"
+  [ -n "$holding" ] || return 0
+  echo "::warning::still held after 120s, terminating the holders:${holding}"
+  for lock in $holding; do
+    sudo fuser -k "$lock" > /dev/null 2>&1 || true
+  done
   sleep 5
 }
 
+configure_apt_fail_fast
+
 for attempt in $(seq 1 "$ATTEMPTS"); do
   rc=0
   # `|| rc=$?` keeps this exempt from any -e a caller may have set, and records
@@ -91,7 +173,7 @@ for attempt in $(seq 1 "$ATTEMPTS"); do
   echo "::warning::attempt ${attempt}/${ATTEMPTS} of '$*' ${reason}"
 
   [ "$attempt" -ge "$ATTEMPTS" ] && break
-  release_dpkg_lock
+  release_apt_locks
 done
 
 echo "::error::'$*' failed ${ATTEMPTS} times; the attempt warnings above say which way each one went"

.github/workflows/consolidated-tests-ci.yml

@@ -2414,7 +2414,7 @@ jobs:
           # `install_llama_cpp` requirement-check is a no-op.
           RETRY_ATTEMPTS=2 RETRY_ATTEMPT_TIMEOUT=300 \
             bash .github/scripts/retry-with-apt-lock.sh sudo sh -c \
-            'apt-get update -qq && apt-get install -y -qq build-essential cmake git curl libgomp1 libssl-dev libcurl4-openssl-dev'
+            'apt-get install -y -qq build-essential cmake git curl libgomp1 libssl-dev libcurl4-openssl-dev || { apt-get update -qq && apt-get install -y -qq build-essential cmake git curl libgomp1 libssl-dev libcurl4-openssl-dev; }'
           python <<'PY'
           import os, shutil, subprocess, sys, pathlib
           # Apply the same CPU spoof the pytest shims use BEFORE any

.github/workflows/desktop-app-clean-machine-ci.yml

@@ -399,9 +399,9 @@ jobs:
           # Deliberately no build-essential/cmake/git: a user installing a .deb has none.
           # Xvfb and WebKit are runtime requirements and apt pulls the .deb's declared
           # deps, so a wrong dependency list fails here.
-          RETRY_ATTEMPTS=3 RETRY_ATTEMPT_TIMEOUT=150 \
+          RETRY_ATTEMPTS=2 RETRY_ATTEMPT_TIMEOUT=360 \
             bash .github/scripts/retry-with-apt-lock.sh sudo sh -c \
-            'apt-get update -qq && apt-get install -y -qq --no-install-recommends xvfb'
+            'apt-get install -y -qq --no-install-recommends xvfb || { apt-get update -qq && apt-get install -y -qq --no-install-recommends xvfb; }'
           if [ "${{ matrix.kind }}" = "deb" ]; then
             sudo apt-get install -y ./dl/*.deb || {
               echo "::error::the .deb does not declare its runtime dependencies correctly"

.github/workflows/interrupted-install-ci.yml

@@ -123,13 +123,17 @@ jobs:
         # with no reason and every later step skipped. update and install go as one
         # unit, since retrying the install after a stalled update re-reads the same
         # broken package list.
-        timeout-minutes: 13
+        # Two long attempts, not three short ones. 150s killed apt mid-`update`
+        # against a mirror that was degraded rather than dead, and every attempt
+        # then hit the same wall -- three kills and no result. The bound exists to
+        # stop an infinite hang, not to race a slow mirror.
+        timeout-minutes: 15
         env:
-          RETRY_ATTEMPTS: '3'
-          RETRY_ATTEMPT_TIMEOUT: '150'
+          RETRY_ATTEMPTS: '2'
+          RETRY_ATTEMPT_TIMEOUT: '360'
         run: |
           bash .github/scripts/retry-with-apt-lock.sh sudo sh -c \
-            'apt-get update -qq && apt-get install -y -qq --no-install-recommends cmake git build-essential libcurl4-openssl-dev'
+            'apt-get install -y -qq --no-install-recommends cmake git build-essential libcurl4-openssl-dev || { apt-get update -qq && apt-get install -y -qq --no-install-recommends cmake git build-essential libcurl4-openssl-dev; }'
 
       - name: Install, interrupted at "${{ matrix.label }}"
         env:

.github/workflows/lint-ci.yml

@@ -135,13 +135,17 @@ jobs:
       # update and install are one unit: retrying the install alone after a stalled
       # update just re-reads the same broken package list.
       - name: Linux deps for shellcheck
-        timeout-minutes: 13
+        # Two long attempts, not three short ones. 150s killed apt mid-`update`
+        # against a mirror that was degraded rather than dead, and every attempt
+        # then hit the same wall -- three kills and no result. The bound exists to
+        # stop an infinite hang, not to race a slow mirror.
+        timeout-minutes: 15
         env:
-          RETRY_ATTEMPTS: '3'
-          RETRY_ATTEMPT_TIMEOUT: '150'
+          RETRY_ATTEMPTS: '2'
+          RETRY_ATTEMPT_TIMEOUT: '360'
         run: |
           bash .github/scripts/retry-with-apt-lock.sh sudo sh -c \
-            'apt-get update -qq && apt-get install -y --no-install-recommends shellcheck'
+            'apt-get install -y --no-install-recommends shellcheck || { apt-get update -qq && apt-get install -y --no-install-recommends shellcheck; }'
 
       - name: Python AST/syntax check (every committed .py must compile)
         # python -m compileall uses the same parser the interpreter

.github/workflows/local-agent-guides-ci.yml

@@ -157,13 +157,17 @@ jobs:
         # with no reason and every later step skipped. update and install go as one
         # unit, since retrying the install after a stalled update re-reads the same
         # broken package list.
-        timeout-minutes: 13
+        # Two long attempts, not three short ones. 150s killed apt mid-`update`
+        # against a mirror that was degraded rather than dead, and every attempt
+        # then hit the same wall -- three kills and no result. The bound exists to
+        # stop an infinite hang, not to race a slow mirror.
+        timeout-minutes: 15
         env:
-          RETRY_ATTEMPTS: '3'
-          RETRY_ATTEMPT_TIMEOUT: '150'
+          RETRY_ATTEMPTS: '2'
+          RETRY_ATTEMPT_TIMEOUT: '360'
         run: |
           bash .github/scripts/retry-with-apt-lock.sh sudo sh -c \
-            'apt-get update && apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq'
+            'apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq || { apt-get update && apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq; }'
 
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
         with:
@@ -388,13 +392,17 @@ jobs:
         # with no reason and every later step skipped. update and install go as one
         # unit, since retrying the install after a stalled update re-reads the same
         # broken package list.
-        timeout-minutes: 13
+        # Two long attempts, not three short ones. 150s killed apt mid-`update`
+        # against a mirror that was degraded rather than dead, and every attempt
+        # then hit the same wall -- three kills and no result. The bound exists to
+        # stop an infinite hang, not to race a slow mirror.
+        timeout-minutes: 15
         env:
-          RETRY_ATTEMPTS: '3'
-          RETRY_ATTEMPT_TIMEOUT: '150'
+          RETRY_ATTEMPTS: '2'
+          RETRY_ATTEMPT_TIMEOUT: '360'
         run: |
           bash .github/scripts/retry-with-apt-lock.sh sudo sh -c \
-            'apt-get update && apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq'
+            'apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq || { apt-get update && apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq; }'
 
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
         with:
@@ -577,13 +585,17 @@ jobs:
         # with no reason and every later step skipped. update and install go as one
         # unit, since retrying the install after a stalled update re-reads the same
         # broken package list.
-        timeout-minutes: 13
+        # Two long attempts, not three short ones. 150s killed apt mid-`update`
+        # against a mirror that was degraded rather than dead, and every attempt
+        # then hit the same wall -- three kills and no result. The bound exists to
+        # stop an infinite hang, not to race a slow mirror.
+        timeout-minutes: 15
         env:
-          RETRY_ATTEMPTS: '3'
-          RETRY_ATTEMPT_TIMEOUT: '150'
+          RETRY_ATTEMPTS: '2'
+          RETRY_ATTEMPT_TIMEOUT: '360'
         run: |
           bash .github/scripts/retry-with-apt-lock.sh sudo sh -c \
-            'apt-get update && apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq'
+            'apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq || { apt-get update && apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq; }'
 
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
         with:
@@ -743,13 +755,17 @@ jobs:
         # with no reason and every later step skipped. update and install go as one
         # unit, since retrying the install after a stalled update re-reads the same
         # broken package list.
-        timeout-minutes: 13
+        # Two long attempts, not three short ones. 150s killed apt mid-`update`
+        # against a mirror that was degraded rather than dead, and every attempt
+        # then hit the same wall -- three kills and no result. The bound exists to
+        # stop an infinite hang, not to race a slow mirror.
+        timeout-minutes: 15
         env:
-          RETRY_ATTEMPTS: '3'
-          RETRY_ATTEMPT_TIMEOUT: '150'
+          RETRY_ATTEMPTS: '2'
+          RETRY_ATTEMPT_TIMEOUT: '360'
         run: |
           bash .github/scripts/retry-with-apt-lock.sh sudo sh -c \
-            'apt-get update && apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq'
+            'apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq || { apt-get update && apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq; }'
 
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
         with:

.github/workflows/release-desktop.yml

@@ -559,13 +559,17 @@ jobs:
         # with no reason and every later step skipped. update and install go as one
         # unit, since retrying the install after a stalled update re-reads the same
         # broken package list.
-        timeout-minutes: 13
+        # Two long attempts, not three short ones. 150s killed apt mid-`update`
+        # against a mirror that was degraded rather than dead, and every attempt
+        # then hit the same wall -- three kills and no result. The bound exists to
+        # stop an infinite hang, not to race a slow mirror.
+        timeout-minutes: 15
         env:
-          RETRY_ATTEMPTS: '3'
-          RETRY_ATTEMPT_TIMEOUT: '150'
+          RETRY_ATTEMPTS: '2'
+          RETRY_ATTEMPT_TIMEOUT: '360'
         run: |
           bash .github/scripts/retry-with-apt-lock.sh sudo sh -c \
-            'apt-get update && apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev libxdo-dev libssl-dev patchelf'
+            'apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev libxdo-dev libssl-dev patchelf || { apt-get update && apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev libxdo-dev libssl-dev patchelf; }'
 
       # ── Node.js ──
       - name: Setup Node.js

.github/workflows/studio-api-smoke.yml

@@ -76,13 +76,17 @@ jobs:
         # with no reason and every later step skipped. update and install go as one
         # unit, since retrying the install after a stalled update re-reads the same
         # broken package list.
-        timeout-minutes: 13
+        # Two long attempts, not three short ones. 150s killed apt mid-`update`
+        # against a mirror that was degraded rather than dead, and every attempt
+        # then hit the same wall -- three kills and no result. The bound exists to
+        # stop an infinite hang, not to race a slow mirror.
+        timeout-minutes: 15
         env:
-          RETRY_ATTEMPTS: '3'
-          RETRY_ATTEMPT_TIMEOUT: '150'
+          RETRY_ATTEMPTS: '2'
+          RETRY_ATTEMPT_TIMEOUT: '360'
         run: |
           bash .github/scripts/retry-with-apt-lock.sh sudo sh -c \
-            'apt-get update && apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq'
+            'apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq || { apt-get update && apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev jq; }'
 
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
         with: