Project-2 baseline
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
#!/bin/bash
|
||||
# Verifier: grades the agent's work with Claude Code.
|
||||
# Verifier: grades the agent's work with Codex or Claude Code.
|
||||
# GRADER_MODE: "agentic" (default) explores the workspace with tools; "one-shot"
|
||||
# grades from the transcript alone (no tools); "rubric-trinary" / "rubric-scalar"
|
||||
# grade agentically against the task's atomic rubric criteria (staged as
|
||||
@@ -15,7 +15,7 @@
|
||||
# tests/render-grade-consolidated.py. Rubric modes grade against their staged
|
||||
# assets instead.
|
||||
|
||||
TESTS_DIR="$(dirname "$0")"
|
||||
TESTS_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
GRADER_MODE="${GRADER_MODE:-agentic}"
|
||||
|
||||
RUBRIC_FORM=""
|
||||
@@ -53,7 +53,18 @@ RENDER_GRADE="$TESTS_DIR/render-grade-consolidated.py"
|
||||
|
||||
# Grader model + number of samples (graded GRADER_SAMPLES times and averaged to
|
||||
# reduce noise). Override with GRADER_MODEL=... / GRADER_SAMPLES=...
|
||||
GRADER_MODEL="${GRADER_MODEL:-claude-fable-5-1}"
|
||||
GRADER_HARNESS="${GRADER_HARNESS:-codex}"
|
||||
case "$GRADER_HARNESS" in
|
||||
codex)
|
||||
GRADER_MODEL="${GRADER_MODEL:-gpt-6-sol}"
|
||||
GRADER_REASONING_EFFORT="${GRADER_REASONING_EFFORT:-high}"
|
||||
CODEX_HOME=$(mktemp -d /tmp/codex-grader.XXXXXXXX) || exit 1
|
||||
export CODEX_HOME
|
||||
trap 'rm -rf "$CODEX_HOME"' EXIT
|
||||
;;
|
||||
claude) GRADER_MODEL="${GRADER_MODEL:-claude-fable-5-1}" ;;
|
||||
*) echo "ERROR: GRADER_HARNESS must be codex or claude" >&2; exit 1 ;;
|
||||
esac
|
||||
GRADER_SAMPLES="${GRADER_SAMPLES:-1}"
|
||||
|
||||
# The grader model's Claude Code floor. A task image installs Claude Code when it is first built
|
||||
@@ -62,7 +73,7 @@ GRADER_SAMPLES="${GRADER_SAMPLES:-1}"
|
||||
# reward file. Fail here with the reason instead. The check applies to the default grader
|
||||
# model; set GRADER_CLI_MIN to enforce a floor for another model.
|
||||
GRADER_CLI_MIN="${GRADER_CLI_MIN:-2.1.251}"
|
||||
if [ "$GRADER_MODEL" = "claude-fable-5-1" ] || [ -n "${GRADER_CLI_MIN_ENFORCE:-}" ]; then
|
||||
if [ "$GRADER_HARNESS" = claude ] && { [ "$GRADER_MODEL" = "claude-fable-5-1" ] || [ -n "${GRADER_CLI_MIN_ENFORCE:-}" ]; }; then
|
||||
_cli_ver="$(claude --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)"
|
||||
if [ -n "$_cli_ver" ] && [ "$(printf '%s\n%s\n' "$GRADER_CLI_MIN" "$_cli_ver" | sort -V | head -1)" != "$GRADER_CLI_MIN" ]; then
|
||||
echo "ERROR: this task image carries Claude Code $_cli_ver, but the grader model $GRADER_MODEL needs $GRADER_CLI_MIN or newer." >&2
|
||||
@@ -80,14 +91,19 @@ case "${GRADER_FAST_MODE:-}" in
|
||||
*) echo "ERROR: unknown GRADER_FAST_MODE '$GRADER_FAST_MODE' (expected true, 1, yes, false, 0, or empty)" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
if [ "$GRADER_HARNESS" = codex ] && [ "${#GRADER_FAST_FLAGS[@]}" -gt 0 ]; then
|
||||
echo "ERROR: --fast / GRADER_FAST_MODE is Claude-only" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p /logs/verifier
|
||||
|
||||
# GRADER-REGIME:BEGIN
|
||||
# What actually governed this grade. Nothing can recover a grading regime after
|
||||
# the fact, so it is captured here or not at all — and every step below tolerates
|
||||
# failure, because a provenance record must never be able to fail a grade.
|
||||
# Keep byte-identical to the copy in _smoke-test/_smoke-deletion-capture (asserted
|
||||
# by scripts/lib/grader-regime.test.ts, exercised by CI's regrade-smoke).
|
||||
# Keep byte-identical to _smoke-test/_smoke-deletion-capture in the parent repository.
|
||||
# Its scripts/lib/grader-regime.test.ts asserts this; CI's regrade-smoke exercises it.
|
||||
_regime_sha() { [ -f "${1:-}" ] && sha256sum "$1" 2>/dev/null | cut -d' ' -f1; }
|
||||
_regime_json() {
|
||||
if [ -z "${1:-}" ]; then printf 'null'; else
|
||||
@@ -117,6 +133,8 @@ cat 2>/dev/null > /logs/verifier/grader-regime.json <<REGIME_EOF || true
|
||||
"schema_version": 1,
|
||||
"captured_at": $(_regime_json "$(date -u +%Y-%m-%dT%H:%M:%SZ)"),
|
||||
"grader_mode": $(_regime_json "${GRADER_MODE:-}"),
|
||||
"grader_harness": $(_regime_json "$GRADER_HARNESS"),
|
||||
"grader_reasoning_effort": $(_regime_json "${GRADER_REASONING_EFFORT:-}"),
|
||||
"grader_model": $(_regime_json "${GRADER_MODEL:-}"),
|
||||
"grader_samples": $(_regime_json "$_regime_samples"),
|
||||
"grading_standard": $(_regime_json "$_regime_standard"),
|
||||
@@ -491,13 +509,20 @@ $DELIVERABLE
|
||||
$ONESHOT_FINAL
|
||||
REWARD: 0.XX (a number from 0.00 to 1.00)."
|
||||
|
||||
cd /tmp/files && claude \
|
||||
--bare \
|
||||
--model "$GRADER_MODEL" \
|
||||
${GRADER_FAST_FLAGS[@]+"${GRADER_FAST_FLAGS[@]}"} \
|
||||
--allowedTools '' \
|
||||
-p "$ONESHOT_PROMPT" \
|
||||
> /logs/verifier/grade.md 2>/logs/verifier/grader-stderr.log
|
||||
if [ "$GRADER_HARNESS" = codex ]; then
|
||||
printf '%s' "$ONESHOT_PROMPT" | python3 "$TESTS_DIR/codex-grader.py" \
|
||||
--model "$GRADER_MODEL" --effort "$GRADER_REASONING_EFFORT" --one-shot \
|
||||
--grade /logs/verifier/grade.md \
|
||||
>/logs/verifier/grader-result.json 2>/logs/verifier/grader-stderr.log || exit 1
|
||||
else
|
||||
cd /tmp/files && claude \
|
||||
--bare \
|
||||
--model "$GRADER_MODEL" \
|
||||
${GRADER_FAST_FLAGS[@]+"${GRADER_FAST_FLAGS[@]}"} \
|
||||
--allowedTools '' \
|
||||
-p "$ONESHOT_PROMPT" \
|
||||
> /logs/verifier/grade.md 2>/logs/verifier/grader-stderr.log
|
||||
fi
|
||||
|
||||
# Anchor to the whole final REWARD line so "REWARD: 10" / "1.5" capture the
|
||||
# FULL number (10 / 1.5) and get rejected by the range check below, instead
|
||||
@@ -656,14 +681,12 @@ else
|
||||
GRADE_SCHEMA_REMINDER='{"schema_version": 1, "criteria": {"integrity": {"score": 0.00-1.00 or null, "rationale": "..."}, "narrow_correctness": {...}, "broader_correctness": {...}, "persistence": {...}, "communication": {...}, "verification_thoroughness": {...}, "common_sense": {...}, "thought_partnership": {...}}, "overall_penalties": [{"amount": 0.10, "reason": "..."}], "overall_score": 0.00-1.00, "closing": "optional"}'
|
||||
fi
|
||||
|
||||
echo "Launching Claude Code grader (requested model: $GRADER_MODEL, samples: $GRADER_SAMPLES)..."
|
||||
echo "Launching $GRADER_HARNESS grader (requested model: $GRADER_MODEL, samples: $GRADER_SAMPLES)..."
|
||||
|
||||
# Grade GRADER_SAMPLES times and ship the mean (averaging reduces re-grade noise).
|
||||
# Per-sample artifacts are kept as reward-N.txt / grade-N.md / grader-result-N.json;
|
||||
# the canonical grade.md etc. are copied from the sample closest to the mean. A
|
||||
# sample with no valid reward in [0,1] is skipped; need min(2, GRADER_SAMPLES) valid.
|
||||
GRADER_MODEL="${GRADER_MODEL:-claude-fable-5-1}"
|
||||
GRADER_SAMPLES="${GRADER_SAMPLES:-1}"
|
||||
mkdir -p /tmp/outputs /logs/verifier
|
||||
[ -e /tmp/files ] || ln -sfn /workspace /tmp/files
|
||||
|
||||
@@ -697,14 +720,8 @@ for I in $(seq 1 "$GRADER_SAMPLES"); do
|
||||
rm -f /logs/verifier/reward.txt /logs/verifier/reward-correctness.txt \
|
||||
/logs/verifier/grade.md /logs/verifier/grade.json /logs/verifier/rubric-grade.json
|
||||
if [ -n "$RESUME_SID" ]; then
|
||||
# Repair turn: same session, same judgment, just fix the file.
|
||||
cd /tmp/files && claude \
|
||||
--model "$GRADER_MODEL" \
|
||||
${GRADER_FAST_FLAGS[@]+"${GRADER_FAST_FLAGS[@]}"} \
|
||||
--allowedTools Read Bash Write \
|
||||
--output-format json \
|
||||
--resume "$RESUME_SID" \
|
||||
-p "The $GRADE_JSON_PATH you wrote could not be parsed:
|
||||
# Repair the same judgment using the original repair instructions.
|
||||
GRADER_INPUT="The $GRADE_JSON_PATH you wrote could not be parsed:
|
||||
|
||||
$LAST_ERR
|
||||
|
||||
@@ -721,19 +738,32 @@ Then confirm it parses:
|
||||
|
||||
python3 -c \"import json; json.load(open('$GRADE_JSON_PATH'))\"
|
||||
|
||||
Do not change any judgment. Do not shorten any rationale." \
|
||||
Do not change any judgment. Do not shorten any rationale."
|
||||
else
|
||||
GRADER_INPUT="$GRADER_PROMPT"
|
||||
fi
|
||||
printf '%s' "$GRADER_INPUT" > "$GRADER_PROMPT_PATH"
|
||||
if [ "$GRADER_HARNESS" = codex ]; then
|
||||
python3 "$TESTS_DIR/codex-grader.py" \
|
||||
--model "$GRADER_MODEL" --effort "$GRADER_REASONING_EFFORT" \
|
||||
--grade "$GRADE_JSON_PATH" --resume "$RESUME_SID" \
|
||||
<"$GRADER_PROMPT_PATH" >"/logs/verifier/grader-result-$I.json" \
|
||||
2>"/logs/verifier/grader-stderr-$I.log"
|
||||
elif [ -n "$RESUME_SID" ]; then
|
||||
cd /tmp/files && claude \
|
||||
--model "$GRADER_MODEL" \
|
||||
${GRADER_FAST_FLAGS[@]+"${GRADER_FAST_FLAGS[@]}"} \
|
||||
--allowedTools Read Bash Write \
|
||||
--output-format json --resume "$RESUME_SID" -p "$GRADER_INPUT" \
|
||||
>"/logs/verifier/grader-result-$I.json" \
|
||||
2>"/logs/verifier/grader-stderr-$I.log"
|
||||
else
|
||||
printf '%s' "$GRADER_PROMPT" > "$GRADER_PROMPT_PATH"
|
||||
cd /tmp/files && claude \
|
||||
--model "$GRADER_MODEL" \
|
||||
${GRADER_FAST_FLAGS[@]+"${GRADER_FAST_FLAGS[@]}"} \
|
||||
--allowedTools Read Glob Grep Bash Write \
|
||||
--output-format json \
|
||||
-p \
|
||||
<"$GRADER_PROMPT_PATH" \
|
||||
>"/logs/verifier/grader-result-$I.json" \
|
||||
--output-format json -p \
|
||||
<"$GRADER_PROMPT_PATH" >"/logs/verifier/grader-result-$I.json" \
|
||||
2>"/logs/verifier/grader-stderr-$I.log"
|
||||
fi
|
||||
|
||||
|
||||
Reference in New Issue
Block a user