toolkit 1.0.1

worker-toolkit-potion-polyglot-v1.0.1/scripts/lib/secret-scrub.ts

@@ -0,0 +1,128 @@
+/**
+ * Keys and tokens a task must never ship: the scrub that swaps them out of a task's
+ * transcripts and run output, and the check that finds them anywhere else in the task.
+ */
+
+import { createHash } from 'crypto';
+import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from 'fs';
+import { join, relative } from 'path';
+
+import { INPUT_CHECKSUMS_FILENAME } from './input-checksums';
+
+export const SECRET_PLACEHOLDER = '[redacted]';
+
+const SECRET_NAME_RE = /KEY|TOKEN|SECRET|PASSWORD/i;
+// Short values are left alone so a scrub can't eat ordinary words.
+const TOKEN_VALUE_RE = /^[A-Za-z0-9._~+/=-]{16,}$/;
+const WORKER_KEY_RE = /sk-plwkr01-[A-Za-z0-9_-]{16,}/g;
+// Full-length only: short `sk-ant-test-…` strings are fixtures that tasks are built around.
+const DIRECT_KEY_RE = /sk-ant-[a-z]+\d{2}-[A-Za-z0-9_-]{80,}/g;
+// Older worker keys are 24 uppercase letters and digits, so they're only recognizable after a key name.
+const OLDER_KEY_RE =
+  /(?<=(?:ANTHROPIC_API_KEY|OPENAI_API_KEY|[Xx]-[Aa][Pp][Ii]-[Kk][Ee][Yy])[^A-Za-z0-9]{1,6})[A-Z0-9]{24}(?![A-Za-z0-9_-])/g;
+const MAX_SCAN_BYTES = 20 * 1024 * 1024;
+// Files the worker didn't write: run output and detector reports.
+const GENERATED_RE = /^(?:(?:reference-runs|rubric-regrades)\/[^/]+\/agent-output\/|detectors\/)/;
+const TRANSCRIPT_RE = /^(?:session-full\.jsonl$|environment\/session|reference-runs\/|rubric-regrades\/)/;
+const RESUMED_SESSION = 'environment/session.jsonl';
+
+/** Credential values from the toolkit's `.env` and `explore/.env`. */
+export function readEnvSecrets(root: string): string[] {
+  const values = new Set<string>();
+  for (const file of [join(root, '.env'), join(root, 'explore', '.env')]) {
+    if (!existsSync(file)) continue;
+    for (const line of readFileSync(file, 'utf8').split(/\r?\n/)) {
+      const m = line.trim().match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
+      if (!m || !SECRET_NAME_RE.test(m[1])) continue;
+      const value = m[2].trim().replace(/^(['"])(.*)\1$/, '$2');
+      if (TOKEN_VALUE_RE.test(value)) values.add(value);
+    }
+  }
+  return [...values];
+}
+
+function scrub(text: string, patterns: readonly RegExp[]): { text: string; count: number } {
+  let count = 0;
+  let out = text;
+  for (const re of patterns) {
+    out = out.replace(re, () => {
+      count++;
+      return SECRET_PLACEHOLDER;
+    });
+  }
+  return { text: out, count };
+}
+
+const sha256 = (bytes: Buffer): string => createHash('sha256').update(bytes).digest('hex');
+
+/** Restamp runs recorded against the unredacted resumed session, so a redaction alone never stales them. */
+function restampRuns(taskDir: string, before: string, after: string): void {
+  const runsDir = join(taskDir, 'reference-runs');
+  if (!existsSync(runsDir)) return;
+  for (const run of readdirSync(runsDir, { withFileTypes: true })) {
+    const file = join(runsDir, run.name, INPUT_CHECKSUMS_FILENAME);
+    if (!run.isDirectory() || !existsSync(file)) continue;
+    try {
+      const record = JSON.parse(readFileSync(file, 'utf8'));
+      if (record?.inputs?.sessionJsonl !== before) continue;
+      record.inputs.sessionJsonl = after;
+      writeFileSync(file, JSON.stringify(record, null, 2) + '\n');
+    } catch {
+      // A malformed record already reads as unverifiable; leave it.
+    }
+  }
+}
+
+export interface TaskSecretScan {
+  /** Transcripts, run output and detector reports whose keys were replaced in place. */
+  scrubbed: string[];
+  /** Files the worker wrote that hold a key, left for them to fix. */
+  flagged: string[];
+}
+
+/** Scrub keys out of a task's transcripts and generated files in place, and list any other file that holds one. */
+export function scrubTaskSecrets(taskDir: string, envSecrets: readonly string[]): TaskSecretScan {
+  const exact = envSecrets.map((v) => new RegExp(v.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&'), 'g'));
+  // Code and fixtures can hold realistic fake keys of these shapes, so only transcripts are scrubbed for them.
+  const strict = [...exact, WORKER_KEY_RE];
+  const broad = [...strict, DIRECT_KEY_RE, OLDER_KEY_RE];
+  const scan: TaskSecretScan = { scrubbed: [], flagged: [] };
+  let session = null as { before: string; after: string } | null;
+
+  const visit = (abs: string, rel: string): void => {
+    const generated = GENERATED_RE.test(rel);
+    const transcript = !generated && TRANSCRIPT_RE.test(rel);
+    if (!transcript && statSync(abs).size > MAX_SCAN_BYTES) return;
+    const bytes = readFileSync(abs);
+    // latin1 maps each byte to one character, so every byte but the key's is written back as it was.
+    const text = bytes.toString('latin1');
+    if (generated || transcript) {
+      const result = scrub(text, transcript ? broad : strict);
+      if (result.count === 0) return;
+      const scrubbed = Buffer.from(result.text, 'latin1');
+      writeFileSync(abs, scrubbed);
+      scan.scrubbed.push(rel);
+      if (rel === RESUMED_SESSION) session = { before: sha256(bytes), after: sha256(scrubbed) };
+    } else if (strict.some((re) => new RegExp(re.source).test(text))) {
+      scan.flagged.push(rel);
+    }
+  };
+
+  const walk = (dir: string): void => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const abs = join(dir, entry.name);
+      const rel = relative(taskDir, abs).split('\\').join('/');
+      if (rel === 'environment/corpus' || entry.isSymbolicLink()) continue;
+      if (entry.name === 'node_modules' || entry.name === '.git') continue;
+      try {
+        if (entry.isDirectory()) walk(abs);
+        else if (entry.isFile()) visit(abs, rel);
+      } catch {
+        // Skipped: ingest runs this scrub again on the unpacked submission.
+      }
+    }
+  };
+  if (existsSync(taskDir)) walk(taskDir);
+  if (session) restampRuns(taskDir, session.before, session.after);
+  return scan;
+}