import assert from 'node:assert/strict'; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { test } from 'node:test'; import { INPUT_CHECKSUMS_FILENAME, captureTaskInputs } from './input-checksums'; import { SECRET_PLACEHOLDER, readEnvSecrets, scrubTaskSecrets } from './secret-scrub'; const ENV_KEY = 'plain24charLocalEnvValue'; const WORKER_KEY = `sk-plwkr01-${'x'.repeat(48)}`; const OLDER_KEY = 'ABC1DEF2GH3IJ4KLMNOP5Q6R'; const DIRECT_KEY = `sk-ant-api03-${'z'.repeat(95)}`; // Shapes of the fixture values real tasks carry in their sessions. const DUMMY_DIRECT_KEY = 'sk-ant-test-1234567890'; const DUMMY_NAMED = 'ANTHROPIC_API_KEY=test-key-for-local-dev'; function scratch(): string { return mkdtempSync(join(tmpdir(), 'secret-scrub-')); } function put(root: string, rel: string, text: string): void { mkdirSync(dirname(join(root, rel)), { recursive: true }); writeFileSync(join(root, rel), text); } function jsonl(...texts: string[]): string { return ( texts.map((t) => JSON.stringify({ type: 'user', message: { content: t } })).join('\n') + '\n' ); } test('readEnvSecrets keeps credential values and skips settings and short values', () => { const root = scratch(); try { put( root, '.env', [ `export ANTHROPIC_API_KEY="${ENV_KEY}"`, 'ANTHROPIC_BASE_URL=https://proxy.example.com/v1/some/long/path', 'SHORT_TOKEN=abc', '# OTHER_KEY=commented-out-value-123456', ].join('\r\n') ); put(root, 'explore/.env', `OPENAI_API_KEY='${WORKER_KEY}'\n`); assert.deepEqual(readEnvSecrets(root).sort(), [ENV_KEY, WORKER_KEY].sort()); } finally { rmSync(root, { recursive: true, force: true }); } }); test('scrubs real keys out of transcripts, keeps dummies, and keeps each line valid JSON', () => { const task = scratch(); try { const session = jsonl( `ANTHROPIC_API_KEY=${ENV_KEY}\nHOME=/root`, `declare -x ANTHROPIC_API_KEY="${ENV_KEY}"`, `'X-Api-Key'=>'${WORKER_KEY}'`, `key ${DIRECT_KEY} in a tool result`, `fixtures: ${DUMMY_DIRECT_KEY} and ${DUMMY_NAMED}` ); put(task, 'session-full.jsonl', session); put(task, 'environment/session.jsonl', session); put(task, 'environment/session/subagents/agent-1.jsonl', session); put(task, 'reference-runs/run-1/agent/trajectory.json', JSON.stringify({ out: WORKER_KEY })); const scan = scrubTaskSecrets(task, [ENV_KEY]); assert.deepEqual(scan.scrubbed.sort(), [ 'environment/session.jsonl', 'environment/session/subagents/agent-1.jsonl', 'reference-runs/run-1/agent/trajectory.json', 'session-full.jsonl', ]); assert.deepEqual(scan.flagged, []); const full = readFileSync(join(task, 'session-full.jsonl'), 'utf8'); for (const secret of [ENV_KEY, WORKER_KEY, DIRECT_KEY]) { assert.ok(!full.includes(secret), 'a key survived the scrub'); } assert.ok(full.includes(`ANTHROPIC_API_KEY=${SECRET_PLACEHOLDER}`)); assert.ok(full.includes(DUMMY_DIRECT_KEY) && full.includes(DUMMY_NAMED), 'a fixture was lost'); for (const line of full.trimEnd().split('\n')) JSON.parse(line); } finally { rmSync(task, { recursive: true, force: true }); } }); test('a transcript with only fixture values is left untouched', () => { const task = scratch(); try { const session = jsonl(`${DUMMY_NAMED}\n${DUMMY_DIRECT_KEY}`); put(task, 'environment/session.jsonl', session); const scan = scrubTaskSecrets(task, []); assert.deepEqual(scan.scrubbed, []); assert.equal(readFileSync(join(task, 'environment/session.jsonl'), 'utf8'), session); } finally { rmSync(task, { recursive: true, force: true }); } }); test('an older-shape key is scrubbed from transcripts by the name before it, with no .env', () => { const task = scratch(); try { put( task, 'session-full.jsonl', jsonl( `ANTHROPIC_API_KEY=${OLDER_KEY}\nHOME=/root`, `'X-Api-Key'=>'${OLDER_KEY}'`, `order ${OLDER_KEY} shipped` ) ); put(task, 'environment/workspace.patch', `+ANTHROPIC_API_KEY=${OLDER_KEY}\n`); const scan = scrubTaskSecrets(task, []); assert.deepEqual(scan.scrubbed, ['session-full.jsonl']); assert.deepEqual(scan.flagged, []); const full = readFileSync(join(task, 'session-full.jsonl'), 'utf8'); assert.ok(full.includes(`ANTHROPIC_API_KEY=${SECRET_PLACEHOLDER}`)); assert.ok(full.includes(`'X-Api-Key'=>'${SECRET_PLACEHOLDER}'`)); assert.ok(full.includes(`order ${OLDER_KEY} shipped`), 'a string with no key name was redacted'); } finally { rmSync(task, { recursive: true, force: true }); } }); test('agent output loses real keys but keeps fixtures, even realistic ones', () => { const task = scratch(); try { const fixture = `ANTHROPIC_API_KEY=${DIRECT_KEY}\n`; put(task, 'reference-runs/run-1/agent-output/test/fixture.env', fixture); put(task, 'reference-runs/run-1/agent-output/notes.txt', `copied ${WORKER_KEY}\n`); const scan = scrubTaskSecrets(task, []); assert.deepEqual(scan.scrubbed, ['reference-runs/run-1/agent-output/notes.txt']); const kept = readFileSync( join(task, 'reference-runs/run-1/agent-output/test/fixture.env'), 'utf8' ); assert.equal(kept, fixture); } finally { rmSync(task, { recursive: true, force: true }); } }); test('flags a real key in an authored file without editing it, and ignores fixtures', () => { const task = scratch(); try { const patch = `+ANTHROPIC_API_KEY=${ENV_KEY}\n`; put(task, 'environment/workspace.patch', patch); put( task, 'environment/workspace/test/setup.ts', `process.env.ANTHROPIC_API_KEY = '${DIRECT_KEY}';\n` ); put(task, 'environment/corpus/dump.txt', ENV_KEY); put(task, 'environment/workspace/node_modules/pkg/index.js', ENV_KEY); const scan = scrubTaskSecrets(task, [ENV_KEY]); assert.deepEqual(scan.flagged, ['environment/workspace.patch']); assert.deepEqual(scan.scrubbed, []); assert.equal(readFileSync(join(task, 'environment/workspace.patch'), 'utf8'), patch); } finally { rmSync(task, { recursive: true, force: true }); } }); test('detector reports lose real keys instead of being flagged', () => { const task = scratch(); try { put(task, 'detectors/detector-credential-leakage.md', `found ${WORKER_KEY} in the session\n`); const scan = scrubTaskSecrets(task, []); assert.deepEqual(scan.scrubbed, ['detectors/detector-credential-leakage.md']); assert.deepEqual(scan.flagged, []); } finally { rmSync(task, { recursive: true, force: true }); } }); test('a file that is not UTF-8 keeps every byte but the key', () => { const task = scratch(); try { const head = Buffer.from([0xff, 0xfe, 0x80]); const rel = 'reference-runs/run-1/agent-output/blob.bin'; put(task, rel, ''); writeFileSync(join(task, rel), Buffer.concat([head, Buffer.from(WORKER_KEY), head])); scrubTaskSecrets(task, []); const want = Buffer.concat([head, Buffer.from(SECRET_PLACEHOLDER), head]); assert.ok(readFileSync(join(task, rel)).equals(want)); } finally { rmSync(task, { recursive: true, force: true }); } }); test('redacting the resumed session restamps the runs recorded against it', () => { const task = scratch(); const stamp = (run: string) => join(task, 'reference-runs', run, INPUT_CHECKSUMS_FILENAME); try { put(task, 'environment/session.jsonl', jsonl(`ANTHROPIC_API_KEY=${ENV_KEY}`)); put(task, `reference-runs/run-1/${INPUT_CHECKSUMS_FILENAME}`, JSON.stringify(captureTaskInputs(task))); const other = captureTaskInputs(task); const edited = { ...other, inputs: { ...other.inputs, sessionJsonl: 'an-earlier-session' } }; put(task, `reference-runs/run-2/${INPUT_CHECKSUMS_FILENAME}`, JSON.stringify(edited)); scrubTaskSecrets(task, [ENV_KEY]); const now = captureTaskInputs(task).inputs.sessionJsonl; assert.equal(JSON.parse(readFileSync(stamp('run-1'), 'utf8')).inputs.sessionJsonl, now); assert.equal( JSON.parse(readFileSync(stamp('run-2'), 'utf8')).inputs.sessionJsonl, 'an-earlier-session', 'a run recorded against a different session was restamped' ); } finally { rmSync(task, { recursive: true, force: true }); } });