Project-2 baseline

worker-toolkit-potion-polyglot/scripts/harbor-regrade

@@ -194,6 +194,9 @@ if [ ${#REF_DIRS[@]} -gt 1 ]; then
     # each one is a sandbox that bills until something else reaps it.
     trap '[ ${#BATCH_PIDS[@]} -gt 0 ] && kill -TERM "${BATCH_PIDS[@]}" 2>/dev/null; exit 143' TERM
     trap '[ ${#BATCH_PIDS[@]} -gt 0 ] && kill -TERM "${BATCH_PIDS[@]}" 2>/dev/null; exit 130' INT
+    # One stamp for the whole fan-out: a second round is a NEW job to harbor, and it
+    # refuses (or silently resumes) a job dir whose name the previous round took.
+    ROUND_STAMP="$(date +%s)-$$"
     IDX=0
     TOTAL=${#REF_DIRS[@]}
     while [ "$IDX" -lt "$TOTAL" ]; do
@@ -205,7 +208,7 @@ if [ ${#REF_DIRS[@]} -gt 1 ]; then
             RUN_ID=$(basename "$REF")
             # --job-name, not a nested output dir: children keep harbor's own
             # harbor-jobs/<job>/<trial> shape. Indexed so duplicate args cannot collide.
-            JOB_NAME="regrade-$((IDX + 1))-$RUN_ID"
+            JOB_NAME="regrade-$ROUND_STAMP-$((IDX + 1))-$RUN_ID"
             LOG="$OUT_BASE/$JOB_NAME.log"
             echo "  starting $RUN_ID (log: $LOG)"
             "$SCRIPT_DIR/harbor-regrade" "$TASK_DIR_ABS" "$REF" \
@@ -246,6 +249,16 @@ if [ ${#REF_DIRS[@]} -gt 1 ]; then
                 echo "The rest are still in their job dirs; their logs above say why." >&2
             fi
             ;;
+        *)
+            # A replaced holistic grade re-mints the run folder under its new reward and
+            # trial id, so every detector report keyed on run names is now out of date.
+            if [ -n "$REPLACE" ]; then
+                echo "" >&2
+                echo "Each replaced run's folder was re-minted under its new reward and trial id, so" >&2
+                echo "detector reports written before now may name runs that are gone. Run" >&2
+                echo "submit-task.ts: it names each report to re-run." >&2
+            fi
+            ;;
     esac
     exit 0
 fi
@@ -561,6 +574,16 @@ if [ ${#TRIALS[@]} -eq 0 ]; then
     echo "Note: no trial directory under $JOB_DIR, so there is no grade." >&2
     exit "$HARBOR_EXIT"
 fi
+# harbor exits 0 when a trial dies (e.g. a failed image build), so the reward is the signal.
+UNGRADED=0
+for _t in "${TRIALS[@]}"; do
+    if [ ! -f "$_t/verifier/reward.txt" ]; then
+        UNGRADED=$((UNGRADED + 1))
+        echo "Error: $(rel "$_t") produced no grade (no verifier/reward.txt)." >&2
+        [ -f "$_t/exception.txt" ] && echo "       See $(rel "$_t")/exception.txt" >&2
+    fi
+done
+[ "$UNGRADED" -gt 0 ] && exit 1
 if [ ${#TRIALS[@]} -gt 1 ]; then
     echo "Note: ${#TRIALS[@]} trials under $JOB_DIR (-k grades one run repeatedly)." >&2
     echo "      Pick the one to keep and file it:  $FILE_CMD <trial-dir>" >&2