Project-2 baseline

worker-toolkit-potion-polyglot/task-shared/render-rubric-grade.py

@@ -18,9 +18,11 @@ the grader agent score each atomic rubric criterion independently and write
      certain_dealbreaker=5 (Critical), possible_dealbreaker=2 (Major),
      unlikely_dealbreaker=1 (Minor); dodged_bullet criteria are weighted by
      their severity like every other category. Criteria whose manifest
-     category is extra_credit carry weight 1 and are included only when their
-     value is > 0 (fulfilled extra credit joins the weighted mean; unfulfilled
-     extra credit is excluded rather than penalized). A non-extra-credit
+     category is extra_credit carry weight 1 scaled by how far they were
+     fulfilled, and enter the mean at full value: a pass joins at weight 1, a
+     partial at weight 0.5, a scalar score s at weight s, and unfulfilled extra
+     credit is left out. Extra credit therefore only ever raises the reward. A
+     non-extra-credit
      criterion with a null/missing severity falls back to
      unlikely_dealbreaker (weight 1) with a warning on stderr,
   4. rewrites rubric-grade.json in normalized form (generator stamp).
@@ -49,7 +51,7 @@ import os
 import sys
 from typing import Any, Dict, List
 
-RENDER_RUBRIC_GRADE_VERSION = "render-rubric-grade/2.0.0"
+RENDER_RUBRIC_GRADE_VERSION = "render-rubric-grade/2.1.0"
 SCHEMA_VERSION = 1
 
 FORMS = ("trinary", "scalar")
@@ -244,30 +246,41 @@ def aggregate(grade: Dict[str, Any], expected: List[Dict[str, Any]]) -> Dict[str
     """Severity-weighted mean over criteria in cents.
 
     reward_cents = round_half_up(sum(weight_i * cents_i) / sum(weight_i))
-    over included criteria. extra_credit (weight 1) is included only when its
-    value is > 0; every other criterion is always included at its severity
-    weight.
+    over included criteria. Every criterion other than extra_credit is always
+    included at its severity weight. An extra_credit criterion with value > 0
+    is included at full value (100 cents) with its weight scaled by its value,
+    so a partial counts as a pass at half weight and extra credit can only
+    raise the reward; one with value 0 is left out.
+
+    Sums are kept in hundredths of a weight unit so the arithmetic stays exact.
     """
-    weighted_cents = 0
-    total_weight = 0
+    weighted = 0  # sum of weight * cents, in hundredths of a weight unit
+    total = 0  # sum of weight, in hundredths of a weight unit
     n_included = 0
     excluded_extra_credit = 0
+    partial_extra_credit = 0
     for criterion in expected:
         entry = grade["by_id"][criterion["id"]]
-        if criterion["category"] == "extra_credit" and entry["_cents"] == 0:
-            excluded_extra_credit += 1
+        cents = entry["_cents"]
+        if criterion["category"] == "extra_credit":
+            if cents == 0:
+                excluded_extra_credit += 1
+                continue
+            if cents < 100:
+                partial_extra_credit += 1
+            n_included += 1
+            weighted += criterion["weight"] * cents * 100
+            total += criterion["weight"] * cents
             continue
         n_included += 1
-        weighted_cents += criterion["weight"] * entry["_cents"]
-        total_weight += criterion["weight"]
-    if total_weight:
-        reward_cents = _round_half_up(weighted_cents, total_weight)
-    else:
-        reward_cents = 0
+        weighted += criterion["weight"] * cents * 100
+        total += criterion["weight"] * 100
+    reward_cents = _round_half_up(weighted, total) if total else 0
     return {
         "n_included": n_included,
         "n_excluded_extra_credit": excluded_extra_credit,
-        "total_weight": total_weight,
+        "n_partial_extra_credit": partial_extra_credit,
+        "total_weight": total / 100.0,
         "reward_cents": reward_cents,
     }
 
@@ -292,6 +305,13 @@ def render_markdown(
             excluded,
             "on" if excluded == 1 else "a",
         )
+    partial = agg["n_partial_extra_credit"]
+    if partial:
+        detail += "; %d partly fulfilled extra-credit criteri%s counted at %s" % (
+            partial,
+            "on" if partial == 1 else "a",
+            "half weight" if form == "trinary" else "a weight equal to the score",
+        )
     sections = ["Rubric score (%s): %s (%s)" % (form, _fmt(agg["reward_cents"]), detail)]
 
     for criterion in expected:
@@ -380,8 +400,16 @@ def main() -> int:
         f.write(normalized_json(grade, expected, args.form))
 
     print(
-        "render-rubric-grade: ok reward=%s form=%s criteria=%d excluded_extra_credit=%d total_weight=%d"
-        % (reward, args.form, agg["n_included"], agg["n_excluded_extra_credit"], agg["total_weight"])
+        "render-rubric-grade: ok reward=%s form=%s criteria=%d excluded_extra_credit=%d "
+        "partial_extra_credit=%d total_weight=%g"
+        % (
+            reward,
+            args.form,
+            agg["n_included"],
+            agg["n_excluded_extra_credit"],
+            agg["n_partial_extra_credit"],
+            agg["total_weight"],
+        )
     )
     return 0