Project-2 baseline

worker-toolkit-potion-polyglot/scripts/submit-task.ts

@@ -23,6 +23,7 @@ import {
   diffTaskInputs,
   readTaskInputChecksums,
 } from './lib/input-checksums';
+import { checkWorkspacePatchSize } from './lib/patch-size';
 import {
   bannerize,
   checkTaskInfraIntegrity,
@@ -94,6 +95,12 @@ export interface TaskStalenessReport {
   generatedAt: string;
   referenceRuns: ReferenceRunStaleness[];
   detectors: DetectorReportStaleness[];
+  /**
+   * Detector reports that name reference runs no longer present under
+   * reference-runs/ — a regrade adopted with --replace, or a re-copied trial,
+   * re-minted the run folder after the report was written. Absent when none.
+   */
+  detectorRunReferences?: Array<{ report: string; missingRuns: string[] }>;
 }
 
 export interface ValidationResult {
@@ -522,13 +529,13 @@ export function validateTask(
   // (fetch-submission and the review pipeline both pass the repo root as cwd),
   // where .claude/skills additionally holds pipeline-only detectors that
   // workers cannot run and must never be warned about.
-  // raccoon-worker-toolkit/static/.claude/skills is therefore preferred;
-  // in a packed toolkit only .claude/skills exists, and it IS the shipped set.
+  // toolkit/static/.claude/skills (the toolkit submodule's shipped set) is therefore
+  // preferred; in a packed toolkit only .claude/skills exists, and it IS the shipped set.
   // Missing reports stay a warning, not an error: the package is reviewable
   // without them, but a reviewer has to regenerate the set by hand, which stalls
   // the queue. A zero-byte report counts as missing — a stub carries no verdict.
   const detectorSkillsDir = [
-    join(cwd, 'raccoon-worker-toolkit', 'static', '.claude', 'skills'),
+    join(cwd, 'toolkit', 'static', '.claude', 'skills'),
     join(cwd, '.claude', 'skills'),
   ].find((p) => existsSync(p));
   const expectedDetectors = detectorSkillsDir
@@ -795,6 +802,36 @@ export function validateTask(
     : [];
   const refCount = runDirs.length;
 
+  // Detector reports that name runs no longer in the package. The run-reading
+  // detectors key their findings on reference-run folder names
+  // (reward-<r>-<id>), and a holistic regrade adopted with --replace re-mints
+  // that folder under the new reward and trial id — so a report written before
+  // the regrade describes grades and runs that are gone. Input stamps cannot
+  // see this (the run set is not a stamped input), so compare the names each
+  // report cites against the directories that exist.
+  if (existsSync(detectorsDir) && runDirs.length > 0) {
+    const runToken = /\breward-\d+(?:\.\d+)?-[A-Za-z0-9][A-Za-z0-9_-]*/g;
+    const present = new Set(runDirs);
+    const orphaned: Array<{ report: string; missingRuns: string[] }> = [];
+    for (const report of readdirSync(detectorsDir).filter((f) => f.endsWith('.md'))) {
+      const cited = new Set(readFileSync(join(detectorsDir, report), 'utf8').match(runToken) ?? []);
+      const missing = [...cited].filter((id) => !present.has(id)).sort();
+      if (missing.length > 0) orphaned.push({ report, missingRuns: missing });
+    }
+    if (orphaned.length > 0) {
+      staleness.detectorRunReferences = orphaned;
+      const detail = orphaned.map((o) => `${o.report} → ${o.missingRuns.join(', ')}`).join('; ');
+      logger.warn(
+        { orphaned },
+        `These detector reports name reference runs that are no longer in reference-runs/ — a regrade adopted with --replace, or a re-copied trial, re-minted the run folder after the report was written: ${detail}. Re-run those detectors against the current runs and re-package.`
+      );
+      warnings.push(
+        `${orphaned.length} detector report(s) name reference runs no longer in reference-runs/ — ${detail} — ` +
+          're-run them against the current runs'
+      );
+    }
+  }
+
   const scores: number[] = [];
   const correctnessScores: number[] = [];
   const RECOMMENDED_REF_COUNT = 4;
@@ -962,6 +999,12 @@ export function validateTask(
     );
   }
 
+  const patchSize = checkWorkspacePatchSize(join(taskDir, 'environment', 'workspace.patch'));
+  for (const e of patchSize.errors) logger.error(e);
+  for (const w of patchSize.warnings) logger.warn(w);
+  errors.push(...patchSize.errors);
+  warnings.push(...patchSize.warnings);
+
   // Workspace
   const workspaceDir = join(taskDir, 'environment', 'workspace');
   let workspaceFiles = 0;