chore: init commit
in worker.../repo/GITFOLDER.zip is the .git folder.
This commit is contained in:
@@ -0,0 +1,405 @@
|
||||
/**
|
||||
* copy-reference-run.ts - Copy Harbor job trials into a task's reference-runs directory.
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/copy-reference-run.ts <trial-path> [trial-path...]
|
||||
*
|
||||
* Examples:
|
||||
* # Copy a single trial
|
||||
* npx tsx scripts/copy-reference-run.ts harbor-jobs/2026-04-02__11-43-55/my-task-slug__3Df3Bjr
|
||||
*
|
||||
* # Copy all trials from a job
|
||||
* npx tsx scripts/copy-reference-run.ts harbor-jobs/2026-04-02__11-43-55/my-task-slug__*
|
||||
*
|
||||
* What gets copied:
|
||||
* - verifier/agent-output/ (answer.md, etc.)
|
||||
* - verifier/reward.txt + reward-correctness.txt (behavioural + correctness scores)
|
||||
* - verifier/reward.json (the split grader's machine-readable dual-score record)
|
||||
* - verifier/signals-status.txt (whether every deterministic check actually ran,
|
||||
* i.e. whether the correctness score is signal-backed)
|
||||
* - verifier/grade.md + every grade-<N>.md grader sample
|
||||
* - verifier/grader-result(-<N>).json, grader-stderr(-<N>).log, grader-samples.txt
|
||||
* - agent/claude-code.txt or agent/codex.txt (the harness's own log), agent/trajectory.json
|
||||
* - session.jsonl — the resumable session log, hoisted to the top of the run dir so
|
||||
* `view-harbor-session.ts <run-dir>/session.jsonl` can load it without further
|
||||
* indirection. Its location is per harness: Claude Code writes
|
||||
* `agent/sessions/projects/-workspace/<id>.jsonl` with the id printed in
|
||||
* `agent/claude-code.txt`; codex writes `agent/sessions/<Y>/<M>/<D>/rollout-*.jsonl`.
|
||||
* - config.json, result.json, trial.log
|
||||
* - input-checksums.json — sha256 checksums of the task inputs the run was
|
||||
* generated against (prompt, session snapshot, workspace patch, gitref),
|
||||
* so submit-task.ts can warn when the run goes stale. Copied from the
|
||||
* trial dir when harbor-run stamped one at launch time (capturedBy:
|
||||
* 'run' — immune to edits made between the run and this copy);
|
||||
* otherwise captured here at copy time as a fallback (capturedBy:
|
||||
* 'copy').
|
||||
*
|
||||
* What is NOT copied:
|
||||
* - agent/sessions/, agent/setup/ (workspace data — the resumable JSONL
|
||||
* is hoisted out as `session.jsonl` above; everything else here is
|
||||
* untyped workspace state)
|
||||
* - artifacts/
|
||||
*/
|
||||
|
||||
import './lib/check-devcontainer';
|
||||
|
||||
import {
|
||||
chmodSync,
|
||||
copyFileSync,
|
||||
existsSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
readlinkSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from 'fs';
|
||||
import { basename, join } from 'path';
|
||||
|
||||
import {
|
||||
captureTaskInputs,
|
||||
INPUT_CHECKSUMS_FILENAME,
|
||||
readTaskInputChecksums,
|
||||
} from './lib/input-checksums';
|
||||
import { didRepair, manualRepairHint, normalizeTreePermissions } from './lib/tree-permissions';
|
||||
import { readSessionId } from './session-id';
|
||||
|
||||
/**
|
||||
* Copy helpers that stand in for `cpSync`, which this script must not call.
|
||||
*
|
||||
* `cpSync`'s work happens in a C++ helper (Node 22+) that a macOS docker bind mount does
|
||||
* not satisfy: `cpSyncCopyDir` fails EACCES for every directory copy into one, and
|
||||
* `cpSyncOverrideFile` fails EACCES when a single-file copy would overwrite an existing
|
||||
* destination. Workers run this from the toolkit, whose harbor-jobs and harbor-tasks both
|
||||
* live on that mount — so `copy-reference-run` died partway through agent-output/, leaving
|
||||
* a half-copied reference run behind. mkdir/copyFile/chmod on those same paths all work.
|
||||
*/
|
||||
function copyPath(src: string, dest: string) {
|
||||
const st = lstatSync(src);
|
||||
if (st.isSymbolicLink()) {
|
||||
rmSync(dest, { force: true });
|
||||
symlinkSync(readlinkSync(src), dest);
|
||||
return;
|
||||
}
|
||||
if (st.isDirectory()) {
|
||||
copyTree(src, dest);
|
||||
return;
|
||||
}
|
||||
// Unlink first: copyFileSync onto an existing file keeps that file's mode.
|
||||
rmSync(dest, { force: true });
|
||||
copyFileSync(src, dest);
|
||||
chmodSync(dest, statSync(src).mode & 0o777);
|
||||
}
|
||||
|
||||
function copyTree(src: string, dest: string) {
|
||||
mkdirSync(dest, { recursive: true });
|
||||
for (const entry of readdirSync(src, { withFileTypes: true })) {
|
||||
copyPath(join(src, entry.name), join(dest, entry.name));
|
||||
}
|
||||
}
|
||||
|
||||
const HARBOR_TASKS_DIR = 'harbor-tasks';
|
||||
|
||||
function findTaskDir(trialPrefix: string): string | null {
|
||||
if (!existsSync(HARBOR_TASKS_DIR)) return null;
|
||||
const entries = readdirSync(HARBOR_TASKS_DIR, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory() && entry.name.startsWith(trialPrefix)) {
|
||||
return join(HARBOR_TASKS_DIR, entry.name);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Newest `rollout-*.jsonl` anywhere under a codex `sessions/` tree, or null. */
|
||||
function newestRollout(sessionsDir: string): string | null {
|
||||
if (!existsSync(sessionsDir)) return null;
|
||||
const found: Array<{ path: string; mtime: number }> = [];
|
||||
const walk = (dir: string) => {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(full);
|
||||
else if (entry.name.startsWith('rollout-') && entry.name.endsWith('.jsonl')) {
|
||||
found.push({ path: full, mtime: statSync(full).mtimeMs });
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(sessionsDir);
|
||||
if (found.length === 0) return null;
|
||||
found.sort((a, b) => b.mtime - a.mtime);
|
||||
return found[0].path;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the task dir from the trial's result.json `task_name` — the FULL,
|
||||
* unambiguous slug. Harbor TRUNCATES long task names in the trial DIRNAME, so two
|
||||
* distinct tasks sharing a truncated prefix (e.g. `foo--hash` and `foo--hash-2`,
|
||||
* both truncating to `foo--ha`) collide: a dirname-prefix scan returns whichever
|
||||
* sorts first and misroutes the other's trials (observed in the wild as base +
|
||||
* `-2` reference-runs sharing trial IDs). result.json is written per-trial with
|
||||
* the real task_name, so it disambiguates exactly. Returns null when result.json
|
||||
* is absent/unparseable or names a task dir that doesn't exist (caller then falls
|
||||
* back to the prefix scan).
|
||||
*/
|
||||
function findTaskDirByResultJson(trialPath: string): string | null {
|
||||
const resultPath = join(trialPath, 'result.json');
|
||||
if (!existsSync(resultPath)) return null;
|
||||
let taskName: unknown;
|
||||
try {
|
||||
taskName = (JSON.parse(readFileSync(resultPath, 'utf-8')) as { task_name?: unknown }).task_name;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (typeof taskName !== 'string' || taskName.length === 0) return null;
|
||||
// Hub-published task_names are org-prefixed (`<org>/<slug>`); the dir is bare.
|
||||
// Inlined (not the shared bareSlug helper) because this script ships in the
|
||||
// worker toolkit and must not import outside its shipped file set.
|
||||
const dir = join(HARBOR_TASKS_DIR, taskName.replace(/^[^/]+\//, ''));
|
||||
return existsSync(dir) ? dir : null;
|
||||
}
|
||||
|
||||
function copyTrial(trialPath: string, destName?: string) {
|
||||
trialPath = trialPath.replace(/\/$/, '');
|
||||
|
||||
if (!existsSync(trialPath)) {
|
||||
console.error(`Error: ${trialPath} does not exist`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const rewardPath = join(trialPath, 'verifier', 'reward.txt');
|
||||
if (!existsSync(rewardPath)) {
|
||||
console.error(`Error: No reward.txt found in ${trialPath}/verifier/`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const reward = readFileSync(rewardPath, 'utf-8').trim();
|
||||
const trialDir = basename(trialPath);
|
||||
|
||||
// Trial dir format: <task-slug-truncated>__<trialId>
|
||||
const separatorIndex = trialDir.lastIndexOf('__');
|
||||
if (separatorIndex === -1) {
|
||||
console.error(
|
||||
`Error: Trial directory '${trialDir}' does not match expected format <slug>__<trialId>`
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const trialPrefix = trialDir.substring(0, separatorIndex);
|
||||
const trialId = trialDir.substring(separatorIndex + 2);
|
||||
|
||||
// Prefer the exact task_name from result.json (handles truncated-prefix
|
||||
// collisions like `foo--hash` vs `foo--hash-2`); fall back to the dirname
|
||||
// prefix scan only when result.json can't resolve it.
|
||||
const taskDir = findTaskDirByResultJson(trialPath) ?? findTaskDir(trialPrefix);
|
||||
if (!taskDir) {
|
||||
console.error(
|
||||
`Error: Could not find task directory matching prefix '${trialPrefix}' in ${HARBOR_TASKS_DIR}/`
|
||||
);
|
||||
console.error('Available tasks:');
|
||||
readdirSync(HARBOR_TASKS_DIR).forEach((d) => console.error(` ${d}`));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// destName (--dest-name) makes a RECORDED rollout id authoritative: the run
|
||||
// is copied to exactly that name instead of the minted reward-<r>-<id> —
|
||||
// used by the SxS pipeline so task.toml preference_rollouts, this dir, and
|
||||
// the publish-manifest run_id stay byte-identical by construction.
|
||||
const dest = join(taskDir, 'reference-runs', destName ?? `reward-${reward}-${trialId}`);
|
||||
|
||||
if (existsSync(dest)) {
|
||||
console.warn(`Warning: ${dest} already exists, overwriting`);
|
||||
rmSync(dest, { recursive: true });
|
||||
}
|
||||
|
||||
mkdirSync(dest, { recursive: true });
|
||||
|
||||
// Copy verifier outputs. The grader writes more than one behavioural grade: the
|
||||
// behavioural reward (reward.txt) AND the separate correctness reward
|
||||
// (reward-correctness.txt, plus the machine-readable dual-score reward.json
|
||||
// from the split grader), signals-status.txt, its aggregate reasoning (grade.md)
|
||||
// PLUS one grade-<N>.md per grader sample, the structured grade(-<N>).json
|
||||
// (the source of truth grade.md/reward.txt are rendered from), the
|
||||
// machine-readable grader-result(-<N>).json, the grader-stderr(-<N>).log,
|
||||
// render-stderr(-<N>).log, and grader-samples.txt.
|
||||
// Copy the whole set — glob so it stays agnostic to the sample count. (Earlier
|
||||
// this took only reward.txt/grade.md/grader-stderr.log and silently dropped the
|
||||
// correctness score and every per-sample record.)
|
||||
//
|
||||
// signals-status.txt is what qualifies the correctness score: it records whether
|
||||
// every deterministic check actually produced a verdict ("ok") or one or more was
|
||||
// killed before finishing ("degraded" — the correctness score is then NOT
|
||||
// signal-backed). Without it a copied run is indistinguishable from a run whose
|
||||
// checks all passed, so it must travel with reward-correctness.txt.
|
||||
const verifierDir = join(trialPath, 'verifier');
|
||||
if (existsSync(verifierDir)) {
|
||||
for (const entry of readdirSync(verifierDir, { withFileTypes: true })) {
|
||||
if (!entry.isFile()) continue;
|
||||
const f = entry.name;
|
||||
if (
|
||||
f === 'reward.txt' ||
|
||||
f === 'reward-correctness.txt' ||
|
||||
f === 'reward.json' ||
|
||||
f === 'signals-status.txt' ||
|
||||
f === 'grader-samples.txt' ||
|
||||
/^grade(-\d+)?\.md$/.test(f) ||
|
||||
/^grade(-\d+)?\.json$/.test(f) ||
|
||||
/^grader-result(-\d+)?\.json$/.test(f) ||
|
||||
/^grader-stderr(-\d+)?\.log$/.test(f) ||
|
||||
/^render-stderr(-\d+)?\.log$/.test(f)
|
||||
) {
|
||||
copyPath(join(verifierDir, f), join(dest, f));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const agentOutputDir = join(verifierDir, 'agent-output');
|
||||
if (existsSync(agentOutputDir)) {
|
||||
copyTree(agentOutputDir, join(dest, 'agent-output'));
|
||||
}
|
||||
|
||||
// Copy agent session log and trajectory (not workspace)
|
||||
const agentDir = join(trialPath, 'agent');
|
||||
if (existsSync(agentDir)) {
|
||||
mkdirSync(join(dest, 'agent'), { recursive: true });
|
||||
// The agent's own log is named per harness (claude-code.txt / codex.txt); copying only
|
||||
// the Claude one left codex reference runs with nothing but the trajectory.
|
||||
for (const file of ['claude-code.txt', 'codex.txt', 'trajectory.json']) {
|
||||
const src = join(agentDir, file);
|
||||
if (existsSync(src)) copyPath(src, join(dest, 'agent', file));
|
||||
}
|
||||
// The grader (and downstream worldbench export / replay) reads
|
||||
// agent/trajectory.json. If it's missing, scream so we don't silently
|
||||
// ship a reference run that's only half-useful.
|
||||
if (!existsSync(join(agentDir, 'trajectory.json'))) {
|
||||
console.warn(
|
||||
`WARNING: ${trialPath}/agent/trajectory.json is missing. ` +
|
||||
`Harbor's adapter for this harness failed to write it (typically because ` +
|
||||
`the converter choked on the session log). Downstream consumers ` +
|
||||
`(grader replay, worldbench export) need this file — investigate ` +
|
||||
`before relying on this reference run.`
|
||||
);
|
||||
}
|
||||
// Hoist the resumable session JSONL to <run-dir>/session.jsonl. The id
|
||||
// gets embedded in claude-code.txt's first non-empty line; we use it to
|
||||
// locate the sibling JSONL Claude Code wrote in the same trial.
|
||||
// Layout is per harness, so each needs a case here — the same reason
|
||||
// harness-session.mjs has one finder per CLI. A harness with no case gets no hoisted
|
||||
// session, which is what codex got before this: nothing at all.
|
||||
const claudeCodeTxt = join(agentDir, 'claude-code.txt');
|
||||
if (existsSync(claudeCodeTxt)) {
|
||||
// Claude Code: the id is in claude-code.txt, the JSONL is its sibling.
|
||||
const sessionId = readSessionId(claudeCodeTxt);
|
||||
if (sessionId) {
|
||||
const jsonl = join(agentDir, 'sessions', 'projects', '-workspace', `${sessionId}.jsonl`);
|
||||
if (existsSync(jsonl)) copyPath(jsonl, join(dest, 'session.jsonl'));
|
||||
}
|
||||
} else {
|
||||
// codex: sessions/<Y>/<M>/<D>/rollout-<ts>-<id>.jsonl, newest wins.
|
||||
const rollout = newestRollout(join(agentDir, 'sessions'));
|
||||
if (rollout) copyPath(rollout, join(dest, 'session.jsonl'));
|
||||
}
|
||||
}
|
||||
|
||||
// Copy top-level metadata
|
||||
for (const file of ['config.json', 'result.json', 'trial.log']) {
|
||||
const src = join(trialPath, file);
|
||||
if (existsSync(src)) copyPath(src, join(dest, file));
|
||||
}
|
||||
|
||||
// Record the checksums of the task inputs this run was generated against
|
||||
// (prompt, session snapshot, workspace patch, gitref, grader guidance).
|
||||
// submit-task.ts re-captures at packaging time and warns when any of them
|
||||
// changed — the run then describes an older revision of the task than the
|
||||
// one being shipped. Preferred source: the launch-time stamp harbor-run
|
||||
// wrote into the trial dir (scripts/stamp-trial-inputs.ts, capturedBy:
|
||||
// 'run') — it records the inputs the agent actually ran against, so an
|
||||
// input edited BETWEEN harbor-run and this copy is still caught. Fallback
|
||||
// (trials from an older harbor-run, or a failed stamp): capture here at
|
||||
// copy time, marked capturedBy: 'copy' so staleness.json is honest about
|
||||
// the weaker evidence.
|
||||
const trialStampPath = join(trialPath, INPUT_CHECKSUMS_FILENAME);
|
||||
const trialStamp = readTaskInputChecksums(trialStampPath);
|
||||
// A stamp that names a DIFFERENT task got mis-routed (e.g. concurrent
|
||||
// harbor-runs sharing a cwd) — its hashes describe some other task's
|
||||
// inputs, so treat it as absent rather than importing false evidence.
|
||||
const stampSlug = trialStamp?.taskSlug;
|
||||
const stampMisrouted = typeof stampSlug === 'string' && stampSlug !== basename(taskDir);
|
||||
if (trialStamp && !stampMisrouted) {
|
||||
copyPath(trialStampPath, join(dest, INPUT_CHECKSUMS_FILENAME));
|
||||
} else {
|
||||
if (stampMisrouted) {
|
||||
console.warn(
|
||||
`Warning: ${trialStampPath} was captured for task '${stampSlug}', not ` +
|
||||
`'${basename(taskDir)}' — ignoring it and capturing at copy time instead`
|
||||
);
|
||||
}
|
||||
writeFileSync(
|
||||
join(dest, INPUT_CHECKSUMS_FILENAME),
|
||||
JSON.stringify(captureTaskInputs(taskDir, 'copy'), null, 2) + '\n'
|
||||
);
|
||||
}
|
||||
|
||||
// Files captured from a run can land unreadable to you, which makes packaging
|
||||
// fail later. Fix that now. Guarded so it can never fail a copy that worked.
|
||||
let perms = null;
|
||||
try {
|
||||
perms = normalizeTreePermissions(dest);
|
||||
} catch (err) {
|
||||
console.warn(`Warning: could not normalize permissions on ${dest}: ${String(err)}`);
|
||||
console.warn(` The run copied fine. If packaging later fails on permissions:`);
|
||||
console.warn(` ${manualRepairHint(dest)}`);
|
||||
}
|
||||
if (perms && perms.failures.length > 0) {
|
||||
console.warn(
|
||||
`Warning: could not normalize permissions on ${perms.failures.length} path(s) under ${dest}.`
|
||||
);
|
||||
console.warn(
|
||||
` If packaging later fails with 'Cannot stat: Permission denied', run:\n` +
|
||||
` ${manualRepairHint(dest)}`
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`Copied to ${dest}`);
|
||||
console.log(` reward: ${reward}`);
|
||||
console.log(` task: ${taskDir}`);
|
||||
console.log(` trial: ${trialId}`);
|
||||
if (perms && didRepair(perms)) {
|
||||
console.log(
|
||||
` perms: normalized ${perms.ownerFixed.length} owner / ${perms.modeFixed.length} mode`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Main
|
||||
const rawArgs = process.argv.slice(2);
|
||||
let destName: string | undefined;
|
||||
const args: string[] = [];
|
||||
for (let i = 0; i < rawArgs.length; i++) {
|
||||
if (rawArgs[i] === '--dest-name') {
|
||||
destName = rawArgs[++i];
|
||||
if (!destName) {
|
||||
console.error('Error: --dest-name requires a value');
|
||||
process.exit(1);
|
||||
}
|
||||
} else {
|
||||
args.push(rawArgs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
if (args.length === 0) {
|
||||
console.error(
|
||||
'Usage: npx tsx scripts/copy-reference-run.ts [--dest-name <run-id>] <trial-path> [trial-path...]'
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
if (destName && args.length !== 1) {
|
||||
console.error('Error: --dest-name applies to exactly one trial path');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
for (const trialPath of args) {
|
||||
copyTrial(trialPath, destName);
|
||||
}
|
||||
Reference in New Issue
Block a user