228 lines
8.2 KiB
TypeScript
228 lines
8.2 KiB
TypeScript
import assert from 'node:assert/strict';
|
|
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { dirname, join } from 'node:path';
|
|
import { test } from 'node:test';
|
|
|
|
import { INPUT_CHECKSUMS_FILENAME, captureTaskInputs } from './input-checksums';
|
|
import { SECRET_PLACEHOLDER, readEnvSecrets, scrubTaskSecrets } from './secret-scrub';
|
|
|
|
const ENV_KEY = 'plain24charLocalEnvValue';
|
|
const WORKER_KEY = `sk-plwkr01-${'x'.repeat(48)}`;
|
|
const OLDER_KEY = 'ABC1DEF2GH3IJ4KLMNOP5Q6R';
|
|
const DIRECT_KEY = `sk-ant-api03-${'z'.repeat(95)}`;
|
|
// Shapes of the fixture values real tasks carry in their sessions.
|
|
const DUMMY_DIRECT_KEY = 'sk-ant-test-1234567890';
|
|
const DUMMY_NAMED = 'ANTHROPIC_API_KEY=test-key-for-local-dev';
|
|
|
|
function scratch(): string {
|
|
return mkdtempSync(join(tmpdir(), 'secret-scrub-'));
|
|
}
|
|
|
|
function put(root: string, rel: string, text: string): void {
|
|
mkdirSync(dirname(join(root, rel)), { recursive: true });
|
|
writeFileSync(join(root, rel), text);
|
|
}
|
|
|
|
function jsonl(...texts: string[]): string {
|
|
return (
|
|
texts.map((t) => JSON.stringify({ type: 'user', message: { content: t } })).join('\n') + '\n'
|
|
);
|
|
}
|
|
|
|
test('readEnvSecrets keeps credential values and skips settings and short values', () => {
|
|
const root = scratch();
|
|
try {
|
|
put(
|
|
root,
|
|
'.env',
|
|
[
|
|
`export ANTHROPIC_API_KEY="${ENV_KEY}"`,
|
|
'ANTHROPIC_BASE_URL=https://proxy.example.com/v1/some/long/path',
|
|
'SHORT_TOKEN=abc',
|
|
'# OTHER_KEY=commented-out-value-123456',
|
|
].join('\r\n')
|
|
);
|
|
put(root, 'explore/.env', `OPENAI_API_KEY='${WORKER_KEY}'\n`);
|
|
assert.deepEqual(readEnvSecrets(root).sort(), [ENV_KEY, WORKER_KEY].sort());
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('scrubs real keys out of transcripts, keeps dummies, and keeps each line valid JSON', () => {
|
|
const task = scratch();
|
|
try {
|
|
const session = jsonl(
|
|
`ANTHROPIC_API_KEY=${ENV_KEY}\nHOME=/root`,
|
|
`declare -x ANTHROPIC_API_KEY="${ENV_KEY}"`,
|
|
`'X-Api-Key'=>'${WORKER_KEY}'`,
|
|
`key ${DIRECT_KEY} in a tool result`,
|
|
`fixtures: ${DUMMY_DIRECT_KEY} and ${DUMMY_NAMED}`
|
|
);
|
|
put(task, 'session-full.jsonl', session);
|
|
put(task, 'environment/session.jsonl', session);
|
|
put(task, 'environment/session/subagents/agent-1.jsonl', session);
|
|
put(task, 'reference-runs/run-1/agent/trajectory.json', JSON.stringify({ out: WORKER_KEY }));
|
|
|
|
const scan = scrubTaskSecrets(task, [ENV_KEY]);
|
|
|
|
assert.deepEqual(scan.scrubbed.sort(), [
|
|
'environment/session.jsonl',
|
|
'environment/session/subagents/agent-1.jsonl',
|
|
'reference-runs/run-1/agent/trajectory.json',
|
|
'session-full.jsonl',
|
|
]);
|
|
assert.deepEqual(scan.flagged, []);
|
|
const full = readFileSync(join(task, 'session-full.jsonl'), 'utf8');
|
|
for (const secret of [ENV_KEY, WORKER_KEY, DIRECT_KEY]) {
|
|
assert.ok(!full.includes(secret), 'a key survived the scrub');
|
|
}
|
|
assert.ok(full.includes(`ANTHROPIC_API_KEY=${SECRET_PLACEHOLDER}`));
|
|
assert.ok(full.includes(DUMMY_DIRECT_KEY) && full.includes(DUMMY_NAMED), 'a fixture was lost');
|
|
for (const line of full.trimEnd().split('\n')) JSON.parse(line);
|
|
} finally {
|
|
rmSync(task, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('a transcript with only fixture values is left untouched', () => {
|
|
const task = scratch();
|
|
try {
|
|
const session = jsonl(`${DUMMY_NAMED}\n${DUMMY_DIRECT_KEY}`);
|
|
put(task, 'environment/session.jsonl', session);
|
|
|
|
const scan = scrubTaskSecrets(task, []);
|
|
|
|
assert.deepEqual(scan.scrubbed, []);
|
|
assert.equal(readFileSync(join(task, 'environment/session.jsonl'), 'utf8'), session);
|
|
} finally {
|
|
rmSync(task, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('an older-shape key is scrubbed from transcripts by the name before it, with no .env', () => {
|
|
const task = scratch();
|
|
try {
|
|
put(
|
|
task,
|
|
'session-full.jsonl',
|
|
jsonl(
|
|
`ANTHROPIC_API_KEY=${OLDER_KEY}\nHOME=/root`,
|
|
`'X-Api-Key'=>'${OLDER_KEY}'`,
|
|
`order ${OLDER_KEY} shipped`
|
|
)
|
|
);
|
|
put(task, 'environment/workspace.patch', `+ANTHROPIC_API_KEY=${OLDER_KEY}\n`);
|
|
|
|
const scan = scrubTaskSecrets(task, []);
|
|
|
|
assert.deepEqual(scan.scrubbed, ['session-full.jsonl']);
|
|
assert.deepEqual(scan.flagged, []);
|
|
const full = readFileSync(join(task, 'session-full.jsonl'), 'utf8');
|
|
assert.ok(full.includes(`ANTHROPIC_API_KEY=${SECRET_PLACEHOLDER}`));
|
|
assert.ok(full.includes(`'X-Api-Key'=>'${SECRET_PLACEHOLDER}'`));
|
|
assert.ok(full.includes(`order ${OLDER_KEY} shipped`), 'a string with no key name was redacted');
|
|
} finally {
|
|
rmSync(task, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('agent output loses real keys but keeps fixtures, even realistic ones', () => {
|
|
const task = scratch();
|
|
try {
|
|
const fixture = `ANTHROPIC_API_KEY=${DIRECT_KEY}\n`;
|
|
put(task, 'reference-runs/run-1/agent-output/test/fixture.env', fixture);
|
|
put(task, 'reference-runs/run-1/agent-output/notes.txt', `copied ${WORKER_KEY}\n`);
|
|
|
|
const scan = scrubTaskSecrets(task, []);
|
|
|
|
assert.deepEqual(scan.scrubbed, ['reference-runs/run-1/agent-output/notes.txt']);
|
|
const kept = readFileSync(
|
|
join(task, 'reference-runs/run-1/agent-output/test/fixture.env'),
|
|
'utf8'
|
|
);
|
|
assert.equal(kept, fixture);
|
|
} finally {
|
|
rmSync(task, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('flags a real key in an authored file without editing it, and ignores fixtures', () => {
|
|
const task = scratch();
|
|
try {
|
|
const patch = `+ANTHROPIC_API_KEY=${ENV_KEY}\n`;
|
|
put(task, 'environment/workspace.patch', patch);
|
|
put(
|
|
task,
|
|
'environment/workspace/test/setup.ts',
|
|
`process.env.ANTHROPIC_API_KEY = '${DIRECT_KEY}';\n`
|
|
);
|
|
put(task, 'environment/corpus/dump.txt', ENV_KEY);
|
|
put(task, 'environment/workspace/node_modules/pkg/index.js', ENV_KEY);
|
|
|
|
const scan = scrubTaskSecrets(task, [ENV_KEY]);
|
|
|
|
assert.deepEqual(scan.flagged, ['environment/workspace.patch']);
|
|
assert.deepEqual(scan.scrubbed, []);
|
|
assert.equal(readFileSync(join(task, 'environment/workspace.patch'), 'utf8'), patch);
|
|
} finally {
|
|
rmSync(task, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('detector reports lose real keys instead of being flagged', () => {
|
|
const task = scratch();
|
|
try {
|
|
put(task, 'detectors/detector-credential-leakage.md', `found ${WORKER_KEY} in the session\n`);
|
|
|
|
const scan = scrubTaskSecrets(task, []);
|
|
|
|
assert.deepEqual(scan.scrubbed, ['detectors/detector-credential-leakage.md']);
|
|
assert.deepEqual(scan.flagged, []);
|
|
} finally {
|
|
rmSync(task, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('a file that is not UTF-8 keeps every byte but the key', () => {
|
|
const task = scratch();
|
|
try {
|
|
const head = Buffer.from([0xff, 0xfe, 0x80]);
|
|
const rel = 'reference-runs/run-1/agent-output/blob.bin';
|
|
put(task, rel, '');
|
|
writeFileSync(join(task, rel), Buffer.concat([head, Buffer.from(WORKER_KEY), head]));
|
|
|
|
scrubTaskSecrets(task, []);
|
|
|
|
const want = Buffer.concat([head, Buffer.from(SECRET_PLACEHOLDER), head]);
|
|
assert.ok(readFileSync(join(task, rel)).equals(want));
|
|
} finally {
|
|
rmSync(task, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('redacting the resumed session restamps the runs recorded against it', () => {
|
|
const task = scratch();
|
|
const stamp = (run: string) => join(task, 'reference-runs', run, INPUT_CHECKSUMS_FILENAME);
|
|
try {
|
|
put(task, 'environment/session.jsonl', jsonl(`ANTHROPIC_API_KEY=${ENV_KEY}`));
|
|
put(task, `reference-runs/run-1/${INPUT_CHECKSUMS_FILENAME}`, JSON.stringify(captureTaskInputs(task)));
|
|
const other = captureTaskInputs(task);
|
|
const edited = { ...other, inputs: { ...other.inputs, sessionJsonl: 'an-earlier-session' } };
|
|
put(task, `reference-runs/run-2/${INPUT_CHECKSUMS_FILENAME}`, JSON.stringify(edited));
|
|
|
|
scrubTaskSecrets(task, [ENV_KEY]);
|
|
|
|
const now = captureTaskInputs(task).inputs.sessionJsonl;
|
|
assert.equal(JSON.parse(readFileSync(stamp('run-1'), 'utf8')).inputs.sessionJsonl, now);
|
|
assert.equal(
|
|
JSON.parse(readFileSync(stamp('run-2'), 'utf8')).inputs.sessionJsonl,
|
|
'an-earlier-session',
|
|
'a run recorded against a different session was restamped'
|
|
);
|
|
} finally {
|
|
rmSync(task, { recursive: true, force: true });
|
|
}
|
|
});
|