Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 98 additions & 4 deletions .github/workflows/free-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,32 @@ jobs:
with:
path: ~/.cache/ms-playwright
key: linux-playwright-chromium-${{ steps.playwright.outputs.version }}
# --with-deps fetches about 38 MB of system packages from archive.ubuntu.com
# on every run. On 2026-11-19 that mirror served some runners at 85 to
# 280 kB/s: 17 of 456 first attempts hit the 5-minute limit, and each
# retry then died in seconds on the dpkg lock that the killed attempt's
# apt-get still held (run 44485949701). apt now waits for that lock, and
# the packages one run downloads are cached under a key made from the
# runner's installed package list and the Playwright version, so a later
# run on the same image installs them with dpkg and never touches the mirror.
- name: Make apt wait for its lock and keep the packages it downloads
run: printf 'DPkg::Lock::Timeout "240";\nAPT::Keep-Downloaded-Packages "true";\nBinary::apt::APT::Keep-Downloaded-Packages "true";\n' | sudo tee /etc/apt/apt.conf.d/90gstack-ci > /dev/null
- name: Fingerprint the runner's installed system packages
id: apt_base
env:
PLAYWRIGHT_VERSION: ${{ steps.playwright.outputs.version }}
run: echo "cache_key=linux-apt-debs-$(dpkg-query -W -f='${binary:Package}=${Version}\n' | sha256sum | cut -c1-16)-playwright-$PLAYWRIGHT_VERSION" >> "$GITHUB_OUTPUT"
- name: Restore the system packages an earlier run downloaded
id: apt_cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: ~/.cache/gstack-apt-debs
key: ${{ steps.apt_base.outputs.cache_key }}
- name: Install the cached system packages without the network
id: apt_offline
if: steps.apt_cache.outputs.cache-hit == 'true'
continue-on-error: true
run: sudo dpkg -i "$HOME"/.cache/gstack-apt-debs/*.deb
# The cache restores the browser; --with-deps still runs apt for system
# libraries. On 2026-10-21 apt stalled on an archive.ubuntu.com fetch and
# held free-suite (12) for the whole 20-minute job timeout (run
Expand All @@ -482,11 +508,17 @@ jobs:
id: playwright_install
timeout-minutes: 5
continue-on-error: true
run: npx playwright install --with-deps chromium
env:
SYSTEM_PACKAGES: ${{ steps.apt_offline.outcome }}
run: |
if [ "$SYSTEM_PACKAGES" = success ]; then npx playwright install chromium; else npx playwright install --with-deps chromium; fi
- name: Retry the Playwright Chromium install once
if: steps.playwright_install.outcome == 'failure'
timeout-minutes: 5
run: npx playwright install --with-deps chromium
env:
SYSTEM_PACKAGES: ${{ steps.apt_offline.outcome }}
run: |
if [ "$SYSTEM_PACKAGES" = success ]; then npx playwright install chromium; else npx playwright install --with-deps chromium; fi
- name: Configure the bundled Chromium sandbox helper
run: |
set -euo pipefail
Expand Down Expand Up @@ -516,6 +548,21 @@ jobs:
sudo apparmor_parser -r "$profile"
- name: Install Xvfb + X11 utilities + gate tools
run: sudo apt-get install -y --no-install-recommends xvfb x11-utils poppler-utils fonts-noto-color-emoji zsh
- name: Keep the system packages this run downloaded
id: apt_keep
if: steps.apt_cache.outputs.cache-hit != 'true'
run: |
mkdir -p ~/.cache/gstack-apt-debs
if compgen -G '/var/cache/apt/archives/*.deb' > /dev/null; then
cp /var/cache/apt/archives/*.deb ~/.cache/gstack-apt-debs/
echo "save=true" >> "$GITHUB_OUTPUT"
fi
- name: Save the system packages for the next run on this image
if: steps.apt_keep.outputs.save == 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: ~/.cache/gstack-apt-debs
key: ${{ steps.apt_cache.outputs.cache-primary-key }}
- name: Configure git identity (tests init temp repos)
run: |
git config --global user.email "free-tests-ci@gstack.test"
Expand Down Expand Up @@ -595,6 +642,32 @@ jobs:
path: ~/.cache/ms-playwright
key: linux-playwright-chromium-${{ steps.playwright.outputs.version }}

# --with-deps fetches about 38 MB of system packages from archive.ubuntu.com
# on every run. On 2026-11-19 that mirror served some runners at 85 to
# 280 kB/s: 17 of 456 first attempts hit the 5-minute limit, and each
# retry then died in seconds on the dpkg lock that the killed attempt's
# apt-get still held (run 44485949701). apt now waits for that lock, and
# the packages one run downloads are cached under a key made from the
# runner's installed package list and the Playwright version, so a later
# run on the same image installs them with dpkg and never touches the mirror.
- name: Make apt wait for its lock and keep the packages it downloads
run: printf 'DPkg::Lock::Timeout "240";\nAPT::Keep-Downloaded-Packages "true";\nBinary::apt::APT::Keep-Downloaded-Packages "true";\n' | sudo tee /etc/apt/apt.conf.d/90gstack-ci > /dev/null
- name: Fingerprint the runner's installed system packages
id: apt_base
env:
PLAYWRIGHT_VERSION: ${{ steps.playwright.outputs.version }}
run: echo "cache_key=linux-apt-debs-$(dpkg-query -W -f='${binary:Package}=${Version}\n' | sha256sum | cut -c1-16)-playwright-$PLAYWRIGHT_VERSION" >> "$GITHUB_OUTPUT"
- name: Restore the system packages an earlier run downloaded
id: apt_cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: ~/.cache/gstack-apt-debs
key: ${{ steps.apt_base.outputs.cache_key }}
- name: Install the cached system packages without the network
id: apt_offline
if: steps.apt_cache.outputs.cache-hit == 'true'
continue-on-error: true
run: sudo dpkg -i "$HOME"/.cache/gstack-apt-debs/*.deb
# The cache restores the browser; --with-deps still runs apt for system
# libraries. On 2026-10-21 apt stalled on an archive.ubuntu.com fetch and
# held free-suite (12) for the whole 20-minute job timeout (run
Expand All @@ -605,12 +678,18 @@ jobs:
id: playwright_install
timeout-minutes: 5
continue-on-error: true
run: npx playwright install --with-deps chromium
env:
SYSTEM_PACKAGES: ${{ steps.apt_offline.outcome }}
run: |
if [ "$SYSTEM_PACKAGES" = success ]; then npx playwright install chromium; else npx playwright install --with-deps chromium; fi

- name: Retry the Playwright Chromium install once
if: steps.playwright_install.outcome == 'failure'
timeout-minutes: 5
run: npx playwright install --with-deps chromium
env:
SYSTEM_PACKAGES: ${{ steps.apt_offline.outcome }}
run: |
if [ "$SYSTEM_PACKAGES" = success ]; then npx playwright install chromium; else npx playwright install --with-deps chromium; fi

- name: Configure the bundled Chromium sandbox helper
run: |
Expand Down Expand Up @@ -655,6 +734,21 @@ jobs:
# skip instead of running.
- name: Install Xvfb + X11 utilities + gate tools
run: sudo apt-get install -y --no-install-recommends xvfb x11-utils poppler-utils fonts-noto-color-emoji zsh
- name: Keep the system packages this run downloaded
id: apt_keep
if: steps.apt_cache.outputs.cache-hit != 'true'
run: |
mkdir -p ~/.cache/gstack-apt-debs
if compgen -G '/var/cache/apt/archives/*.deb' > /dev/null; then
cp /var/cache/apt/archives/*.deb ~/.cache/gstack-apt-debs/
echo "save=true" >> "$GITHUB_OUTPUT"
fi
- name: Save the system packages for the next run on this image
if: steps.apt_keep.outputs.save == 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: ~/.cache/gstack-apt-debs
key: ${{ steps.apt_cache.outputs.cache-primary-key }}

- name: Configure git identity (tests init temp repos)
run: |
Expand Down
1 change: 1 addition & 0 deletions changelog.d/flaky-fix-2026-11-19.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **Free-suite CI no longer fails before any test runs when Ubuntu's package mirror is slow.** Playwright's system packages and the gate tools are cached per runner image and Playwright version, so a run on a known image installs them with dpkg instead of downloading about 38 MB from archive.ubuntu.com, and apt waits for a timed-out attempt's lock instead of failing the retry in seconds. On 2026-11-19, 27 free-suite and quick-tier jobs failed this way. Two flaky free tests are fixed too: `first-use` now lists several user drives in sorted order on every filesystem, and the gstack-lease lock test waits for the owner line instead of reading a file that was created but not yet written. Two eval checkers stop refusing correct reads. The read-only command grammar accepts `\"` inside double quotes, as bash and zsh do, and the coverage audit follows a `cd` into a subfolder followed by `cd ..` in the next call.
2 changes: 1 addition & 1 deletion lib/first-use.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ export function recordRoot(env: NodeJS.ProcessEnv = process.env): Result<{ root:
if (env.GSTACK_CAPY !== '1') return { root: path.join(resolveStateRoot(env), 'first-use') };
const drive = path.join(env.HOME || os.homedir(), '.capy', 'drive');
let found: string[] = [];
try { found = fs.readdirSync(drive).filter(d => d.startsWith('user-') && fs.statSync(path.join(drive, d)).isDirectory()); } catch { /* no drive mounted */ }
try { found = fs.readdirSync(drive).filter(d => d.startsWith('user-') && fs.statSync(path.join(drive, d)).isDirectory()).sort(); } catch { /* no drive mounted */ }
if (found.length !== 1) return fail('FIRST_USE_NO_PRIVATE_DRIVE', found.length ? `${found.length} user-level drive folders (${found.join(', ')})` : `no user-level drive folder under ${drive}`);
return { root: path.join(drive, found[0]!, 'first-use') };
}
Expand Down
61 changes: 61 additions & 0 deletions test/coverage-audit-evidence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import captured_wc_anchor from './fixtures/coverage-audit-wc-anchor.json';
import captured_separator from './fixtures/coverage-audit-separator-collision.json';
import captured_two_file_grep from './fixtures/coverage-audit-two-file-grep.json';
import captured_plan_eng_wc_bytes from './fixtures/coverage-audit-plan-eng-wc-bytes.json';
import captured_relative_cd from './fixtures/coverage-audit-relative-cd.json';

const clone = <T>(v:T):T => structuredClone(v);
const diagram = '```text\nsrc/billing.ts\n├── processPayment: happy path [TESTED]\n└── refundPayment [UNTESTED]\n```';
Expand Down Expand Up @@ -1399,3 +1400,63 @@ describe('coverage audit native evidence: zsh abandons the line at an echo ====
expect(reads(command, 'src/billing.ts was read' + error)).toEqual(neither);
});
});

describe('coverage audit native evidence: cd .. back to the cwd after an earlier cd into review/ (census 44351657692 slice 15)', () => {
const capture = captured_relative_cd, [failed, listed, readBoth] = capture.calls;
const files = { cwd: capture.cwd, source: { path: capture.cwd + '/src/billing.ts', content: capture.files.source },
tests: { path: capture.cwd + '/test/billing.test.ts', content: capture.files.tests } };
const both = { sourceRead: true, testsRead: true }, neither = { sourceRead: false, testsRead: false };
type Call = { toolUseId: string; command: string; isError: boolean; output: string };
const event = (type: 'assistant' | 'user', content: unknown[], parent: string | null = null) =>
({ type, session_id: capture.sessionId, parent_tool_use_id: parent, message: { role: type, content } });
const use = (call: Call) => ({ type: 'tool_use', id: call.toolUseId, name: 'Bash', input: { command: call.command } });
const result = (call: Call) => ({ type: 'tool_result', tool_use_id: call.toolUseId, is_error: call.isError, content: call.output });
const transcript = (calls: Call[]) => [{ type: 'system', subtype: 'init', session_id: capture.sessionId, cwd: capture.cwd },
...calls.flatMap(call => [event('assistant', [use(call)]), event('user', [result(call)])])];
const reads = (calls: Call[]) => coverageAuditReadEvidence(transcript(calls), files);
const call = (id: string, command: string, output = '', isError = false): Call => ({ toolUseId: id, command, isError, output });
const nonce = capture.files.source.match(/coverage-read-evidence: (\S+)/)![1]!;

test('the recorded calls credit both reads and the recorded run passes', () => {
expect(capture.provenance.recordedPassed).toBe(false);
expect(capture.files.source).toBe(fixture.files.source + '\n// coverage-read-evidence: ' + nonce + '\n');
expect([failed!.isError, listed!.isError, readBoth!.isError]).toEqual([true, false, false]);
expect(listed!.command).toBe(`cd ${capture.cwd}/review; ls -R; wc -c SKILL.md sections/* specialists/* 2>&1`);
expect(readBoth!.command).toBe('cd .. && cat -n src/billing.ts && echo ---- && cat -n test/billing.test.ts');
expect(reads(capture.calls)).toEqual(both);
expect(coverageAuditVerdict({ exitReason: 'success', browseErrors: [], output: capture.finalOutput, transcript: transcript(capture.calls) } as any, files))
.toEqual({ ...both, diagram: true, passed: true, failures: [] });
const deeper = call('toolu_deeper', `cd ${capture.cwd}/review/sections; ls`, 'adversarial.md\n');
expect(reads([failed!, listed!, deeper, { ...readBoth!, command: readBoth!.command.replace('cd ..', 'cd ../..') }])).toEqual(both);
});

test('cd .. gets no credit unless the earlier calls prove the shell sat one level below the cwd', () => {
expect(reads([readBoth!])).toEqual(neither);
expect(reads([failed!, readBoth!])).toEqual(neither);
expect(reads([{ ...listed!, isError: true }, readBoth!])).toEqual(neither);
expect(reads([{ ...listed!, output: `(eval):cd:1: no such file or directory: ${capture.cwd}/review\n` + listed!.output }, readBoth!])).toEqual(neither);
for (const command of [`cd /foreign/review; ls -R`, `cd ${capture.cwd}/../review; ls -R`, `cd review; ls -R`, `cd ${capture.cwd}; ls -R`,
`cd ${capture.cwd}/review; ls -R; cd sections`, `cd ${capture.cwd}/review; pushd sections`, `cd ${capture.cwd}/review; source x.sh`,
`cd ${capture.cwd}/review; ls & cd sections`, `cd ${capture.cwd}/review; ls $(cd sections)`, `cd ${capture.cwd}/review; ls; python3 x.py`,
`cd ${capture.cwd}/review || ls`, `cd "${capture.cwd}/review"/sections/..; ls`]) {
expect(reads([{ ...listed!, command }, readBoth!]), command).toEqual(neither);
}
const moved = call('toolu_moved', `cd ${capture.cwd}/review/sections; ls`, 'adversarial.md\n');
expect(reads([failed!, listed!, moved, readBoth!])).toEqual(neither);
expect(reads([failed!, listed!, call('toolu_unknown', 'cd sections && ls', 'adversarial.md\n'), readBoth!])).toEqual(neither);
});

test('a concurrent or subagent Bash call voids the proved shell; the reads keep their own grammar', () => {
const parallel = [{ type: 'system', subtype: 'init', session_id: capture.sessionId, cwd: capture.cwd },
event('assistant', [use(listed!)]), event('assistant', [use(readBoth!)]), event('user', [result(listed!)]), event('user', [result(readBoth!)])];
expect(coverageAuditReadEvidence(parallel, files)).toEqual(neither);
const child = [...transcript([listed!]).slice(0, 3), event('assistant', [{ type: 'tool_use', id: 'toolu_child', name: 'Bash', input: { command: 'cd /tmp' } }], 'toolu_agent'),
...transcript([readBoth!]).slice(1)];
expect(coverageAuditReadEvidence(child, files)).toEqual(neither);
for (const command of ['cd .. ; cat -n src/billing.ts; echo ----; cat -n test/billing.test.ts', 'cd ../src && cat -n billing.ts && echo ---- && cat -n ../test/billing.test.ts',
'cd ~/.. && cat -n src/billing.ts && echo ---- && cat -n test/billing.test.ts', 'cd .. && cat -n src/billing.ts && echo ---- && cat -n test/billing.test.ts; rm -f x']) {
expect(reads([listed!, { ...readBoth!, command }]), command).toEqual(neither);
}
expect(reads([listed!, { ...readBoth!, output: readBoth!.output.replace(nonce, nonce.slice(1) + '0') }])).toMatchObject({ sourceRead: false });
});
});
26 changes: 26 additions & 0 deletions test/docsync-command-grammar.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,32 @@ test.each(JOINED_READ_LINES)('captured joined read line from %s is a declared re
} finally { owned.clean(); }
});

// Gate 44321344888 slice 7 @ 3ecd7168, ship-docsync-failure (claude-opus-5-5): after prepare for docs-a2 the parent checked
// the new prompt and candidate with one grep whose double-quoted pattern ends in an escaped quote, then a tail. The shared
// grammar rejected every escape inside double quotes, so this read failed the run as an undeclared command.
test('captured grep with an escaped double quote inside its pattern is a declared read; other escapes and joins stay refused', () => {
const owned = fixtureDocs('current');
try {
const actor = path.join(import.meta.dir, 'helpers/docsync-fault-actor.ts');
const prompt = path.join(owned.home, 'prompt-docs-a2.md'), candidate = path.join(owned.home, 'candidate-docs-a2.json');
fs.writeFileSync(prompt, 'RETRY_NOTE: x\nAudit id docs-a2\n');
fs.writeFileSync(candidate, '{"handbook/reference/commands/widget.md.tmpl": "b64635c"}\n');
const captured = 'grep -n "RETRY_NOTE\\|Audit id\\|widget.md.tmpl\\"" {home}/prompt-docs-a2.md {home}/candidate-docs-a2.json; tail -n 3 {home}/prompt-docs-a2.md';
const command = captured.replaceAll('{home}', owned.home);
const run = spawnSync('bash', ['-c', command], { cwd: owned.repo, encoding: 'utf8', timeout: 10000 });
expect(run.status).toBe(0);
expect(run.stdout).toContain('widget.md.tmpl"');
expect(docsCommandAllowed(command, owned, [actor])).toBe(true);
expect(docsToolFailures(nativeResult(command, run.stdout), owned, [actor])).toEqual([]);
for (const denied of [
command.replace('widget.md.tmpl\\"', 'widget.md.tmpl\\$HOME'), command.replace('widget.md.tmpl\\"', 'widget.md.tmpl\\\\'),
command.replace('widget.md.tmpl\\"', 'widget.md.tmpl\\`id\\`'), `${command} > ${path.join(owned.home, 'out.md')}`, `${command}; rm -f ${prompt}`,
command.replace(candidate, path.join(owned.home, 'actor-state.json')), command.replace(candidate, '/etc/passwd'),
'grep -n "widget.md.tmpl\\" README.md', 'grep -n "x\\"" README.md; git add app.ts',
]) expect(docsCommandAllowed(denied, owned, [actor]), denied).toBe(false);
} finally { owned.clean(); }
});

test('operand-free tag listings are reads, and every tag write stays refused', () => {
for (const listing of ['git tag', 'git tag -l', 'git tag --list', 'git status; git tag; git log --oneline -3'])
expect(docsCommandAllowed(listing, fixture), listing).toBe(true);
Expand Down
Loading
Loading