after moving all to cipher

This commit is contained in:
2026-08-19 10:19:57 +00:00
parent 4df62d2609
commit 9abade1a81
100 changed files with 1286 additions and 4335 deletions

View File

@@ -302,6 +302,22 @@ Narrow Correctness (see the system prompt's attribution notes)."
fi
# The prompt section injected into the grader prompt(s). Empty when no signals.
# The grader runs in the agent's container, with Bash and Read — so on a task that opted into
# a browser it can drive the app and look at a screenshot itself, rather than judging rendered
# behaviour from the code. Probed, not assumed: most images have no `pw`, and a prompt that
# promised one would send the grader after a missing binary.
#
# Capability only. When to use it is task-specific and belongs in grader guidance; steering it
# from the shared prompt would tilt grades on every task at once.
BROWSER_SECTION=""
if command -v pw >/dev/null 2>&1; then
BROWSER_SECTION='## Browser
Chromium is available in this environment via Playwright. `pw <script.js>` runs Node with
`require("playwright")` resolvable (CommonJS — `import` will not find it). You can load the
app and `Read` a screenshot you take.'
fi
SIGNALS_SECTION=""
if [ -n "$DETERMINISTIC_SIGNALS" ]; then
SIGNALS_SECTION="## Deterministic Signals
@@ -479,6 +495,8 @@ $RUBRIC_CRITERIA
$SIGNALS_SECTION
$BROWSER_SECTION
## RUBRIC GRADING (output protocol)
This grading run scores the agent's response against the task-specific rubric
@@ -549,6 +567,8 @@ $GRADER_GUIDANCE
$SIGNALS_SECTION
$BROWSER_SECTION
## Final instruction
$AGENTIC_FINAL"
@@ -585,6 +605,13 @@ GRADER_SAMPLES="${GRADER_SAMPLES:-3}"
mkdir -p /tmp/outputs /logs/verifier
[ -e /tmp/files ] || ln -sfn /workspace /tmp/files
# The prompt goes to claude on stdin, not as a command-line argument. A single
# argument is capped at 128 KiB, and the prompt carries the whole deterministic-
# signals block, so a task whose checks are verbose can exceed it — and the exec
# then fails before claude starts, leaving an empty grader-result-N.json and no
# reward. Reading it from a file has no size limit.
GRADER_PROMPT_PATH=/tmp/grader-prompt.txt
N_VALID=0
SUM=0
# Recovery ladder for a sample whose grade.json doesn't validate. The grader
@@ -635,11 +662,13 @@ Do not change any judgment. Do not shorten any rationale." \
>"/logs/verifier/grader-result-$I.json" \
2>"/logs/verifier/grader-stderr-$I.log"
else
printf '%s' "$GRADER_PROMPT" > "$GRADER_PROMPT_PATH"
cd /tmp/files && claude \
--model "$GRADER_MODEL" \
--allowedTools Read Glob Grep Bash Write \
--output-format json \
-p "$GRADER_PROMPT" \
-p \
<"$GRADER_PROMPT_PATH" \
>"/logs/verifier/grader-result-$I.json" \
2>"/logs/verifier/grader-stderr-$I.log"
fi