lots of change - all to start my 3rd redo
This commit is contained in:
@@ -0,0 +1,328 @@
|
||||
#!/bin/bash
|
||||
# Read the harness registry and derive per-harness credentials from it.
|
||||
#
|
||||
# Source it — the whole point is exporting into the caller's environment, which a subshell
|
||||
# would lose:
|
||||
#
|
||||
# HARNESS_SCRIPTS_DIR=/workspace/scripts . /workspace/scripts/lib/harness-credentials.sh
|
||||
# harness_setup_credentials
|
||||
#
|
||||
# Three callers: `harbor-run`, which needs only this; `refresh-harness-auth`, which
|
||||
# re-derives and rewrites the auth files before an interactive launch; and
|
||||
# `setup-harnesses.sh`, which sources it and adds installs, config writing and launchers
|
||||
# on top.
|
||||
#
|
||||
# No -e here — this file is SOURCED, and shell options belong to the caller's shell (both
|
||||
# post-creates run with -e). An unguarded failure below therefore aborts container
|
||||
# creation, which is why every failure site is individually guarded rather than relying on
|
||||
# this line.
|
||||
set -uo pipefail
|
||||
|
||||
_HARNESS_REGISTRY_DIR="${HARNESS_SCRIPTS_DIR:-/workspace/scripts}"
|
||||
|
||||
# The registry is read with tomllib (stdlib from 3.11), and `python3` is not always new
|
||||
# enough — macOS ships 3.9, and a container may symlink an older managed interpreter. Pick
|
||||
# the first one that can actually import it rather than assuming.
|
||||
_raccoon_python() {
|
||||
local p
|
||||
for p in "${RACCOON_PYTHON:-}" python3 python3.13 python3.12 python3.11; do
|
||||
[ -n "$p" ] || continue
|
||||
command -v "$p" >/dev/null 2>&1 || continue
|
||||
if "$p" -c "import tomllib" >/dev/null 2>&1; then
|
||||
printf '%s' "$p"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
_harness_query() {
|
||||
local py
|
||||
py=$(_raccoon_python) || return 1
|
||||
"$py" "$_HARNESS_REGISTRY_DIR/resolve_harness.py" "$@"
|
||||
}
|
||||
|
||||
# Drop every whitespace character from a value read out of .env. A Windows-saved .env leaves a
|
||||
# \r on each value, which reaches the proxy as a 401; no key or base URL legitimately contains
|
||||
# whitespace anywhere, so deleting rather than trimming needs no cases.
|
||||
_harness_trim() {
|
||||
local out
|
||||
# Fall back to the raw value: a trim that cannot run must never turn a working key into an
|
||||
# empty one, which is what an unavailable `tr` would otherwise do to every caller.
|
||||
out="$(printf '%s' "$1" | tr -d '[:space:]' 2>/dev/null)" || out="$1"
|
||||
printf '%s' "${out:-$1}"
|
||||
}
|
||||
|
||||
# The proxy root: the worker's ANTHROPIC_BASE_URL minus its provider path.
|
||||
_harness_proxy_root() {
|
||||
local base_url
|
||||
base_url="$(_harness_trim "${ANTHROPIC_BASE_URL:-}")"
|
||||
[ -n "$base_url" ] || return 1
|
||||
base_url="${base_url%"${base_url##*[!/]}"}"
|
||||
# ".../llm_proxy/projects/<id>/anthropic" -> ".../llm_proxy/projects/<id>", so each
|
||||
# harness's proxy_path composes onto the project route. Requires a path to strip: a base
|
||||
# URL that is a bare host with no path — a provider's own API root rather than the
|
||||
# proxy — would yield "https:/", handed to codex as a base URL and failing obscurely.
|
||||
case "${base_url#*://}" in
|
||||
*/*) printf '%s' "${base_url%/*}" ;;
|
||||
*) return 2 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
harness_setup_credentials() {
|
||||
# `|| rc=$?` and not a bare assignment: this is sourced into a `set -e` shell (see the
|
||||
# note at the top), and a bare failing assignment would exit the caller's post-create
|
||||
# outright — silently, since the failure paths below are what do the explaining.
|
||||
local root rc=0
|
||||
root="$(_harness_proxy_root)" || rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
if [ "$rc" -eq 2 ]; then
|
||||
echo "harness-setup: ANTHROPIC_BASE_URL (${ANTHROPIC_BASE_URL:-}) has no provider" >&2
|
||||
echo "harness-setup: path, so it is not the proxy URL other harnesses derive their" >&2
|
||||
echo "harness-setup: credentials from. claude will work; codex will not be" >&2
|
||||
echo "harness-setup: authenticated. Use the base URL you were given." >&2
|
||||
else
|
||||
echo "harness-setup: ANTHROPIC_BASE_URL unset — skipping credential derivation" >&2
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
ANTHROPIC_BASE_URL="$(_harness_trim "${ANTHROPIC_BASE_URL:-}")"
|
||||
export ANTHROPIC_BASE_URL
|
||||
local key
|
||||
key="$(_harness_trim "${ANTHROPIC_API_KEY:-}")"
|
||||
if [ -z "$key" ]; then
|
||||
echo "harness-setup: ANTHROPIC_API_KEY unset — skipping credential derivation" >&2
|
||||
return 0
|
||||
fi
|
||||
# harbor-run sources .env itself and passes ANTHROPIC_* through to the trial sandbox, so
|
||||
# cleaning only the derived per-harness copies would leave a claude trial carrying the CR.
|
||||
export ANTHROPIC_API_KEY="$key"
|
||||
|
||||
local id key_env base_url_env proxy_path
|
||||
while IFS=$'\t' read -r id key_env base_url_env proxy_path; do
|
||||
[ -n "$key_env" ] || continue
|
||||
# ${!name} is an indirect expansion. Only set when empty: an explicit key wins.
|
||||
if [ -z "${!key_env:-}" ]; then
|
||||
export "$key_env=$key"
|
||||
fi
|
||||
if [ -n "$base_url_env" ] && [ -n "$proxy_path" ] && [ -z "${!base_url_env:-}" ]; then
|
||||
export "$base_url_env=$root/$proxy_path"
|
||||
fi
|
||||
echo "harness-setup: $id credentials ready ($key_env, ${base_url_env:-no base url})" >&2
|
||||
done < <(_harness_query --authoring-credentials 2>/dev/null || true)
|
||||
}
|
||||
|
||||
# Write the auth file for harnesses that read credentials from disk rather than $ENV.
|
||||
harness_write_auth() {
|
||||
local id auth_path key_env target key py
|
||||
py=$(_raccoon_python) || {
|
||||
echo "harness-setup: no python3.11+ with tomllib — skipping auth files" >&2
|
||||
return 0
|
||||
}
|
||||
while IFS=$'\t' read -r id auth_path key_env; do
|
||||
[ -n "$auth_path" ] && [ -n "$key_env" ] || continue
|
||||
# Last mile: an explicit OPENAI_API_KEY bypasses the derivation above, so trim here
|
||||
# too — this is the value that reaches the file the harness authenticates with.
|
||||
key="$(_harness_trim "${!key_env:-}")"
|
||||
if [ -z "$key" ]; then
|
||||
echo "harness-setup: $key_env unset — skipping $id auth file" >&2
|
||||
continue
|
||||
fi
|
||||
target=$(eval "printf '%s' \"$auth_path\"") || {
|
||||
echo "harness-setup: WARNING $id auth_path could not be expanded — skipping" >&2
|
||||
continue
|
||||
}
|
||||
mkdir -p "$(dirname "$target")" || {
|
||||
echo "harness-setup: WARNING $id auth dir not creatable — skipping $target" >&2
|
||||
continue
|
||||
}
|
||||
# json.dumps, not printf: a key containing a quote or backslash would otherwise
|
||||
# produce a file the CLI cannot parse, and the failure would surface as an auth
|
||||
# error rather than a malformed file.
|
||||
# 0600 tmp + rename, never a redirect onto the target: a redirect truncates the live
|
||||
# file first, so a write dying mid-flight leaves codex an EMPTY auth.json.
|
||||
if ! RACCOON_AUTH_K="$key_env" RACCOON_AUTH_V="$key" RACCOON_AUTH_TARGET="$target" \
|
||||
"$py" -c 'import json, os
|
||||
target = os.environ["RACCOON_AUTH_TARGET"]
|
||||
tmp = target + ".raccoon-tmp." + str(os.getpid())
|
||||
try:
|
||||
with os.fdopen(os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w") as fh:
|
||||
json.dump({os.environ["RACCOON_AUTH_K"]: os.environ["RACCOON_AUTH_V"]}, fh)
|
||||
fh.write("\n")
|
||||
os.replace(tmp, target)
|
||||
except OSError:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
raise SystemExit(1)
|
||||
'; then
|
||||
echo "harness-setup: WARNING $id auth file NOT written — $target unwritable." >&2
|
||||
echo "harness-setup: the key already on disk (if any) is left untouched." >&2
|
||||
continue
|
||||
fi
|
||||
echo "harness-setup: $id auth -> $target" >&2
|
||||
done < <(_harness_query --auth-files 2>/dev/null || true)
|
||||
}
|
||||
|
||||
# Re-set just the root keys of a harness's config file (codex's `openai_base_url`),
|
||||
# leaving every other line — the explore surface's [hooks] table included — untouched.
|
||||
harness_refresh_config_keys() {
|
||||
local id config_path blob target py
|
||||
py=$(_raccoon_python) || return 0
|
||||
# The surface only decides what a CREATE writes. An update takes the root keys off the
|
||||
# front of the same blob, so a surface's tables survive byte-for-byte either way.
|
||||
while IFS=$'\t' read -r id config_path blob; do
|
||||
[ -n "$config_path" ] && [ -n "$blob" ] || continue
|
||||
target=$(eval "printf '%s' \"$config_path\"") || continue
|
||||
mkdir -p "$(dirname "$target")" || continue
|
||||
if printf '%s' "$blob" | base64 -d |
|
||||
RACCOON_CONFIG_TARGET="$target" "$py" -c '
|
||||
import os, re, sys, tomllib
|
||||
|
||||
HEADER = "# Generated from harness-registry.toml — edits here are overwritten."
|
||||
|
||||
target = os.environ["RACCOON_CONFIG_TARGET"]
|
||||
text = sys.stdin.read()
|
||||
# Empty counts as unresolved: writing an empty base URL would break a container whose
|
||||
# config is currently right, which is the one thing this must never do.
|
||||
if [m for m in re.finditer(r"\$\{(\w+)\}", text) if not os.environ.get(m.group(1))]:
|
||||
raise SystemExit(1)
|
||||
text = os.path.expandvars(text)
|
||||
|
||||
# Root keys, plus keys inside a [model_providers.*] table: codex reserves its built-in
|
||||
# provider ids, so the proxy URL it must follow lives in a provider table, not at the
|
||||
# root. Every other table, [hooks] on the explore surface included, is left alone.
|
||||
REFRESHABLE_TABLE = re.compile(r"\[model_providers\.[^]]+\]$")
|
||||
wanted = []
|
||||
section = None
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("["):
|
||||
section = stripped if REFRESHABLE_TABLE.match(stripped) else False
|
||||
continue
|
||||
if section is False:
|
||||
continue
|
||||
m = re.match(r"\s*\"?([A-Za-z0-9_.-]+)\"?\s*=", line)
|
||||
if m:
|
||||
wanted.append((section, m.group(1), line.rstrip()))
|
||||
if not wanted:
|
||||
raise SystemExit(0)
|
||||
|
||||
|
||||
def section_path(header):
|
||||
"""[model_providers.llm-proxy] -> ("model_providers", "llm-proxy")."""
|
||||
return tuple(header.strip("[]").split("."))
|
||||
|
||||
|
||||
def lookup(doc, header, key):
|
||||
"""The value a parsed config holds for a wanted key, or KeyError."""
|
||||
node = doc
|
||||
if header:
|
||||
for part in section_path(header):
|
||||
node = node[part]
|
||||
return node[key]
|
||||
|
||||
mode = None
|
||||
if os.path.exists(target):
|
||||
try:
|
||||
with open(target, encoding="utf-8") as fh:
|
||||
lines = fh.read().splitlines()
|
||||
mode = os.stat(target).st_mode & 0o777
|
||||
except OSError:
|
||||
raise SystemExit(1)
|
||||
def span(header):
|
||||
"""The line range a section owns, or None when the file has no such section.
|
||||
|
||||
Root is everything above the first table header: a key appended below one
|
||||
would be reparented into it, so searches and inserts stay inside the span.
|
||||
"""
|
||||
heads = [i for i, l in enumerate(lines) if l.lstrip().startswith("[")]
|
||||
if header is None:
|
||||
return 0, (heads[0] if heads else len(lines))
|
||||
at = next((i for i in heads if lines[i].strip() == header), None)
|
||||
if at is None:
|
||||
return None
|
||||
after = next((i for i in heads if i > at), len(lines))
|
||||
return at + 1, after
|
||||
|
||||
# Grouped, root first, so a section this file lacks can be written whole.
|
||||
grouped = {}
|
||||
for header, key, line in wanted:
|
||||
grouped.setdefault(header, []).append((key, line))
|
||||
ordered = sorted(grouped, key=lambda h: (h is not None, h or ""))
|
||||
|
||||
changed = False
|
||||
for header in ordered:
|
||||
if span(header) is None:
|
||||
# A config written before this section existed. Write the whole table
|
||||
# rather than leave a root key naming a provider that is not there.
|
||||
if lines and lines[-1].strip():
|
||||
lines.append("")
|
||||
lines.append(header)
|
||||
lines.extend(line for _, line in grouped[header])
|
||||
changed = True
|
||||
continue
|
||||
for key, line in grouped[header]:
|
||||
# Re-read the span: an insert for an earlier key moved it.
|
||||
start, end = span(header)
|
||||
# The quoted spelling is the same key: replace rather than duplicate.
|
||||
pat = re.compile(r"\s*\"?" + re.escape(key) + r"\"?\s*=")
|
||||
at = next((i for i in range(start, end) if pat.match(lines[i])), None)
|
||||
if at is None:
|
||||
if end < len(lines) and lines[end].strip():
|
||||
lines.insert(end, "")
|
||||
lines.insert(end, line)
|
||||
changed = True
|
||||
elif lines[at] != line:
|
||||
lines[at] = line
|
||||
changed = True
|
||||
if not changed:
|
||||
raise SystemExit(0)
|
||||
out = "\n".join(lines).rstrip("\n") + "\n"
|
||||
else:
|
||||
# No file means container-create could not write one, so write what it would have:
|
||||
# on the explore surface that is the capture hooks too, not just the root keys.
|
||||
out = HEADER + "\n" + text
|
||||
|
||||
try:
|
||||
doc = tomllib.loads(out)
|
||||
except tomllib.TOMLDecodeError:
|
||||
raise SystemExit(1)
|
||||
# Parsing is not enough: a line edit can land inside a multi-line value, which still
|
||||
# parses while leaving the key unset. Require every key to have landed on the value the
|
||||
# blob asks for, in its own section — skipping sections this file does not carry.
|
||||
blob_doc = tomllib.loads(text)
|
||||
for header, key, _ in wanted:
|
||||
try:
|
||||
expected = lookup(blob_doc, header, key)
|
||||
except (KeyError, TypeError):
|
||||
raise SystemExit(1)
|
||||
try:
|
||||
got = lookup(doc, header, key)
|
||||
except (KeyError, TypeError):
|
||||
if header is None:
|
||||
raise SystemExit(1)
|
||||
continue
|
||||
if got != expected:
|
||||
raise SystemExit(1)
|
||||
|
||||
# Pid-suffixed: two launches at once must not write the same scratch path.
|
||||
tmp = target + ".raccoon-tmp." + str(os.getpid())
|
||||
try:
|
||||
with open(tmp, "w", encoding="utf-8") as fh:
|
||||
fh.write(out)
|
||||
if mode is not None:
|
||||
os.chmod(tmp, mode)
|
||||
os.replace(tmp, target)
|
||||
except OSError:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
raise SystemExit(1)
|
||||
'; then
|
||||
echo "harness-setup: $id config keys refreshed -> $target" >&2
|
||||
fi
|
||||
done < <(_harness_query --container-configs --surface "${RACCOON_SURFACE:-authoring}" 2>/dev/null || true)
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
"""harness_registry.py — Python loader for ``scripts/harness-registry.toml``.
|
||||
|
||||
The ONE loader for the registry: TS callers shell into ``resolve_harness.py`` rather than
|
||||
parse the TOML themselves, which is why the toolkit ships no TOML parser for TS (its
|
||||
package.json has no zod/smol-toml).
|
||||
|
||||
This module supersedes ``benchmark_models_lib``'s ``HARNESS_BY_IMPORT_PATH`` and
|
||||
``LEGACY_BARE_MODEL_AGENTS``; those should read from here rather than keep private
|
||||
copies.
|
||||
|
||||
Harbor-free and dependency-free (stdlib ``tomllib``) so it can be imported from a
|
||||
sandbox agent, a plain unit test, or the devcontainer python alike.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tomllib
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
REGISTRY_PATH = Path(__file__).resolve().parent.parent / "harness-registry.toml"
|
||||
|
||||
MODEL_ID_SHAPES = frozenset({"bare", "provider/model", "provider:model"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Harness:
|
||||
"""One harness, as declared in harness-registry.toml."""
|
||||
|
||||
id: str
|
||||
label: str
|
||||
agent_import_path: str
|
||||
model_id_shape: str
|
||||
writes_atif: bool
|
||||
capture: bool
|
||||
seed_native: bool
|
||||
seed_atif: bool
|
||||
agent_import_path_single_turn: str | None = None
|
||||
# Browser-opt-in variants (`[metadata] browser = true`). A harness that has no variant
|
||||
# keeps its normal class: codex, for instance, gains the browser and its disclosure but
|
||||
# has no `Read` equivalent to switch toolsets for.
|
||||
agent_import_path_browser: str | None = None
|
||||
agent_import_path_single_turn_browser: str | None = None
|
||||
import_path_aliases: tuple[str, ...] = ()
|
||||
legacy_bare_model_rows: bool = False
|
||||
default_model: str | None = None
|
||||
effort_kwarg: str = ""
|
||||
effort_default: str | None = None
|
||||
# Agent kwarg that opts a trial into the harness's fast/priority serving mode
|
||||
# (claude-code: fast mode). Empty means the harness has none and --fast refuses.
|
||||
fast_kwarg: str = ""
|
||||
key_env: str | None = None
|
||||
base_url_env: str | None = None
|
||||
proxy_path: str | None = None
|
||||
flaky_hangs: bool = False
|
||||
enabled: bool = True
|
||||
# Worker-container fields; see the registry header.
|
||||
authoring: bool = False
|
||||
cli: str | None = None
|
||||
install: str | None = None
|
||||
skills_dir: str | None = None
|
||||
auth_path: str | None = None
|
||||
auth_key_env: str | None = None
|
||||
explore_launch: str | None = None
|
||||
config_path: str | None = None
|
||||
# Config the harness needs wherever it runs, trial sandbox included.
|
||||
agent_config: str | None = None
|
||||
# Config for both worker containers (explore and authoring).
|
||||
container_config: str | None = None
|
||||
# Config for the EXPLORE container only — the capture hooks, whose commands ship in
|
||||
# explore/plugins/. Writing them in authoring would register hooks against files that
|
||||
# are not there, firing on every prompt.
|
||||
explore_config: str | None = None
|
||||
# Fields added for a later phase, kept verbatim so this loader doesn't have to
|
||||
# be edited in lockstep with the schema.
|
||||
extra: dict = field(default_factory=dict, compare=False)
|
||||
|
||||
def agent_import_path_for(self, *, multi_turn: bool, browser: bool = False) -> str:
|
||||
"""Agent class to launch. Multi-turn tasks need the resuming class; a
|
||||
single-turn task given it would try to resume a session that isn't there.
|
||||
|
||||
``browser`` selects the opt-in variant, which for claude also carries the ``Read``
|
||||
built-in — a different toolset is a different agent, so it is a different class with
|
||||
its own name rather than a flag on the canonical one. Harnesses without a variant fall
|
||||
through to their normal class."""
|
||||
if browser:
|
||||
variant = (
|
||||
self.agent_import_path_browser
|
||||
if multi_turn
|
||||
else (self.agent_import_path_single_turn_browser or self.agent_import_path_browser)
|
||||
)
|
||||
if variant:
|
||||
return variant
|
||||
if multi_turn:
|
||||
return self.agent_import_path
|
||||
return self.agent_import_path_single_turn or self.agent_import_path
|
||||
|
||||
def row_label(self, model: str) -> str:
|
||||
"""Row identity for one trial: bare model for legacy harnesses (so
|
||||
published manifests keep their labels), else ``<harness>:<model>``."""
|
||||
return model if self.legacy_bare_model_rows else f"{self.id}:{model}"
|
||||
|
||||
def agent_config_overrides(self) -> dict[str, str]:
|
||||
"""``agent_config`` as flat ``dotted.key -> value`` pairs in CLI-override form.
|
||||
|
||||
Values are rendered bare — ``disabled``, not ``"disabled"``. Every consumer
|
||||
interpolates these into a shell command, which would strip the quotes anyway;
|
||||
emitting them would only make the result depend on how many shell layers the
|
||||
string crosses. Bare is what the CLIs document (``-c model="o3"`` reaches the
|
||||
binary as ``model=o3``).
|
||||
|
||||
These settings ride the command line as ``-c dotted.key=value`` everywhere the
|
||||
harness runs, never a config file. A trial sandbox rules the file out: the
|
||||
harness's own runner appends root keys to it, and TOML has no way back to the
|
||||
root scope once a table has opened, so a table we appended would swallow them.
|
||||
Overrides compose in any order and beat the file, so the same rendering serves
|
||||
the explore launcher too — one declaration, one mechanism.
|
||||
"""
|
||||
if not self.agent_config:
|
||||
return {}
|
||||
try:
|
||||
parsed = tomllib.loads(self.agent_config)
|
||||
except tomllib.TOMLDecodeError as exc:
|
||||
raise HarnessRegistryError(
|
||||
f"{self.id}: agent_config is not valid TOML ({exc})"
|
||||
) from exc
|
||||
|
||||
flat: dict[str, str] = {}
|
||||
|
||||
def walk(node: dict, prefix: str) -> None:
|
||||
for key, value in node.items():
|
||||
path = f"{prefix}{key}"
|
||||
if isinstance(value, dict):
|
||||
walk(value, f"{path}.")
|
||||
elif isinstance(value, bool):
|
||||
flat[path] = "true" if value else "false"
|
||||
elif isinstance(value, (int, float)):
|
||||
flat[path] = str(value)
|
||||
elif isinstance(value, str):
|
||||
if value != value.strip() or any(c in value for c in " \"'\\"):
|
||||
raise HarnessRegistryError(
|
||||
f"{self.id}: agent_config key {path!r} has a value needing "
|
||||
"shell quoting, which the -c override form cannot carry"
|
||||
)
|
||||
flat[path] = value
|
||||
else:
|
||||
raise HarnessRegistryError(
|
||||
f"{self.id}: agent_config key {path!r} has type "
|
||||
f"{type(value).__name__}, which has no -c override form"
|
||||
)
|
||||
|
||||
walk(parsed, "")
|
||||
return flat
|
||||
|
||||
def container_config_text(self, *, surface: str) -> str | None:
|
||||
"""Config file body for a worker container. `surface` is "explore" or
|
||||
"authoring"; explore additionally gets `explore_config`. Root keys come from
|
||||
`container_config` first, so appending a table section stays valid TOML."""
|
||||
parts = [self.container_config]
|
||||
if surface == "explore":
|
||||
parts.append(self.explore_config)
|
||||
kept = [part.strip("\n") for part in parts if part and part.strip()]
|
||||
return "\n\n".join(kept) + "\n" if kept else None
|
||||
|
||||
def agent_config_flags(self) -> str:
|
||||
"""``agent_config`` as a ``-c key=value`` command-line string."""
|
||||
return " ".join(
|
||||
f"-c {key}={value}"
|
||||
for key, value in sorted(self.agent_config_overrides().items())
|
||||
)
|
||||
|
||||
def explore_launch_command(self) -> str | None:
|
||||
"""``explore_launch`` with the registry's own values substituted in.
|
||||
|
||||
The worker's Explore session and the trial must run the same agent, so the
|
||||
model, effort and reductions are declared once here and rendered into both.
|
||||
A literal in the launch string would be a second declaration, and the two
|
||||
would drift the first time one of them was updated alone.
|
||||
|
||||
Only these three placeholders are substituted; ``$@`` and
|
||||
``$RACCOON_TOOLSET_NOTE`` stay for the launcher's own shell to expand.
|
||||
"""
|
||||
if not self.explore_launch:
|
||||
return None
|
||||
return (
|
||||
self.explore_launch.replace("$RACCOON_AGENT_FLAGS", self.agent_config_flags())
|
||||
.replace("$RACCOON_MODEL", self.default_model or "")
|
||||
.replace("$RACCOON_EFFORT", self.effort_default or "")
|
||||
)
|
||||
|
||||
def known_import_paths(self) -> tuple[str, ...]:
|
||||
paths = [self.agent_import_path, *self.import_path_aliases]
|
||||
if self.agent_import_path_single_turn:
|
||||
paths.append(self.agent_import_path_single_turn)
|
||||
return tuple(paths)
|
||||
|
||||
|
||||
_KNOWN_FIELDS = frozenset(
|
||||
{
|
||||
"id",
|
||||
"label",
|
||||
"agent_import_path",
|
||||
"agent_import_path_single_turn",
|
||||
"agent_import_path_browser",
|
||||
"agent_import_path_single_turn_browser",
|
||||
"import_path_aliases",
|
||||
"legacy_bare_model_rows",
|
||||
"default_model",
|
||||
"model_id_shape",
|
||||
"effort_kwarg",
|
||||
"effort_default",
|
||||
"fast_kwarg",
|
||||
"key_env",
|
||||
"base_url_env",
|
||||
"proxy_path",
|
||||
"writes_atif",
|
||||
"capture",
|
||||
"seed_native",
|
||||
"seed_atif",
|
||||
"flaky_hangs",
|
||||
"enabled",
|
||||
"authoring",
|
||||
"cli",
|
||||
"install",
|
||||
"skills_dir",
|
||||
"auth_path",
|
||||
"auth_key_env",
|
||||
"explore_launch",
|
||||
"config_path",
|
||||
"agent_config",
|
||||
"container_config",
|
||||
"explore_config",
|
||||
}
|
||||
)
|
||||
|
||||
_REQUIRED_FIELDS = (
|
||||
"id",
|
||||
"label",
|
||||
"agent_import_path",
|
||||
"model_id_shape",
|
||||
"writes_atif",
|
||||
"capture",
|
||||
"seed_native",
|
||||
"seed_atif",
|
||||
)
|
||||
|
||||
|
||||
class HarnessRegistryError(ValueError):
|
||||
"""Malformed registry. Raised rather than tolerated: a broken registry is a
|
||||
broken deployment, and silently defaulting would pick the wrong agent."""
|
||||
|
||||
|
||||
def _references_agent_flags(launch: str) -> bool:
|
||||
return "$RACCOON_AGENT_FLAGS" in launch or "${RACCOON_AGENT_FLAGS}" in launch
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HarnessRegistry:
|
||||
version: int
|
||||
harnesses: tuple[Harness, ...]
|
||||
|
||||
def all(self) -> tuple[Harness, ...]:
|
||||
return self.harnesses
|
||||
|
||||
def enabled(self) -> tuple[Harness, ...]:
|
||||
return tuple(h for h in self.harnesses if h.enabled)
|
||||
|
||||
def authoring(self) -> tuple[Harness, ...]:
|
||||
"""Harnesses a worker can author with — what the worker containers install.
|
||||
Narrower than enabled(): a harness can be runnable in a trial without having
|
||||
an authoring story (no CLI to converse with, or no capture)."""
|
||||
return tuple(h for h in self.harnesses if h.enabled and h.authoring)
|
||||
|
||||
def find(self, harness_id: str) -> Harness | None:
|
||||
return next((h for h in self.harnesses if h.id == harness_id), None)
|
||||
|
||||
def require(self, harness_id: str) -> Harness:
|
||||
harness = self.find(harness_id)
|
||||
if harness is not None:
|
||||
return harness
|
||||
available = ", ".join(sorted(h.id for h in self.enabled()))
|
||||
raise HarnessRegistryError(
|
||||
f'Unknown harness "{harness_id}". Available: {available}'
|
||||
)
|
||||
|
||||
def by_import_path(self, agent: str) -> Harness | None:
|
||||
"""Resolve an agent identity — a ``name()`` or import path from
|
||||
``result.json`` ``config.agent``, or a manifest row — to its harness."""
|
||||
needle = (agent or "").strip()
|
||||
if not needle:
|
||||
return None
|
||||
for harness in self.harnesses:
|
||||
if needle == harness.id or needle in harness.known_import_paths():
|
||||
return harness
|
||||
return None
|
||||
|
||||
|
||||
def _build(entry: dict, index: int) -> Harness:
|
||||
for name in _REQUIRED_FIELDS:
|
||||
if name not in entry:
|
||||
raise HarnessRegistryError(
|
||||
f"harness[{index}]: missing required field '{name}'"
|
||||
)
|
||||
shape = entry["model_id_shape"]
|
||||
if shape not in MODEL_ID_SHAPES:
|
||||
raise HarnessRegistryError(
|
||||
f"harness[{index}] ({entry['id']}): model_id_shape {shape!r} not one of "
|
||||
f"{sorted(MODEL_ID_SHAPES)}"
|
||||
)
|
||||
# These three reach `eval` in setup-harnesses.sh, which is how they support the
|
||||
# `${CODEX_HOME:-$HOME/.codex}` default-value syntax that python's expandvars cannot
|
||||
# express. Under eval a backtick or $( would EXECUTE, so refuse them here — the registry
|
||||
# is ours, but "ours" is not an argument that survives a careless future edit.
|
||||
for shell_field in ("config_path", "auth_path", "skills_dir"):
|
||||
value = entry.get(shell_field)
|
||||
if not isinstance(value, str):
|
||||
continue
|
||||
# A backtick or $( executes outright. A double quote closes the string these are
|
||||
# interpolated into, and a semicolon then starts a new command inside it — same
|
||||
# outcome, one step removed.
|
||||
bad = [t for t in ("`", "$(", '"', ";") if t in value]
|
||||
if bad:
|
||||
raise HarnessRegistryError(
|
||||
f"harness[{index}] ({entry['id']}): {shell_field} contains "
|
||||
f"{', '.join(repr(t) for t in bad)} ({value!r}). This value is shell-"
|
||||
f"expanded, so that would execute; use plain $VAR or ${{VAR:-default}} only."
|
||||
)
|
||||
|
||||
launch = entry.get("explore_launch")
|
||||
if entry.get("agent_config") and launch and not _references_agent_flags(launch):
|
||||
raise HarnessRegistryError(
|
||||
f"harness[{index}] ({entry['id']}): declares agent_config but its "
|
||||
"explore_launch does not pass $RACCOON_AGENT_FLAGS. The worker's session "
|
||||
"would then run with a different toolset than the trial it is authoring "
|
||||
"for, which is the drift agent_config exists to prevent."
|
||||
)
|
||||
return Harness(
|
||||
id=entry["id"],
|
||||
label=entry["label"],
|
||||
agent_import_path=entry["agent_import_path"],
|
||||
agent_import_path_single_turn=entry.get("agent_import_path_single_turn"),
|
||||
agent_import_path_browser=entry.get("agent_import_path_browser"),
|
||||
agent_import_path_single_turn_browser=entry.get("agent_import_path_single_turn_browser"),
|
||||
import_path_aliases=tuple(entry.get("import_path_aliases", ())),
|
||||
legacy_bare_model_rows=bool(entry.get("legacy_bare_model_rows", False)),
|
||||
default_model=entry.get("default_model"),
|
||||
model_id_shape=shape,
|
||||
effort_kwarg=entry.get("effort_kwarg", ""),
|
||||
effort_default=entry.get("effort_default"),
|
||||
fast_kwarg=entry.get("fast_kwarg", ""),
|
||||
key_env=entry.get("key_env"),
|
||||
base_url_env=entry.get("base_url_env"),
|
||||
proxy_path=entry.get("proxy_path"),
|
||||
writes_atif=bool(entry["writes_atif"]),
|
||||
capture=bool(entry["capture"]),
|
||||
seed_native=bool(entry["seed_native"]),
|
||||
seed_atif=bool(entry["seed_atif"]),
|
||||
flaky_hangs=bool(entry.get("flaky_hangs", False)),
|
||||
enabled=bool(entry.get("enabled", True)),
|
||||
authoring=bool(entry.get("authoring", False)),
|
||||
cli=entry.get("cli"),
|
||||
install=entry.get("install"),
|
||||
skills_dir=entry.get("skills_dir"),
|
||||
auth_path=entry.get("auth_path"),
|
||||
auth_key_env=entry.get("auth_key_env"),
|
||||
explore_launch=entry.get("explore_launch"),
|
||||
config_path=entry.get("config_path"),
|
||||
agent_config=entry.get("agent_config"),
|
||||
container_config=entry.get("container_config"),
|
||||
explore_config=entry.get("explore_config"),
|
||||
extra={k: v for k, v in entry.items() if k not in _KNOWN_FIELDS},
|
||||
)
|
||||
|
||||
|
||||
_cache: dict[Path, HarnessRegistry] = {}
|
||||
|
||||
|
||||
def load_harness_registry(path: Path | str = REGISTRY_PATH) -> HarnessRegistry:
|
||||
"""Parse and validate the registry. Raises HarnessRegistryError on a malformed
|
||||
file, a duplicate id, or an import path claimed by two harnesses (which would
|
||||
make ``by_import_path`` depend on declaration order)."""
|
||||
resolved = Path(path).resolve()
|
||||
if resolved in _cache:
|
||||
return _cache[resolved]
|
||||
|
||||
with open(resolved, "rb") as handle:
|
||||
doc = tomllib.load(handle)
|
||||
|
||||
if "version" not in doc:
|
||||
raise HarnessRegistryError("harness-registry: missing 'version'")
|
||||
entries = doc.get("harness") or []
|
||||
if not entries:
|
||||
raise HarnessRegistryError("harness-registry: no [[harness]] entries")
|
||||
|
||||
harnesses = tuple(_build(entry, i) for i, entry in enumerate(entries))
|
||||
|
||||
seen_ids: set[str] = set()
|
||||
for harness in harnesses:
|
||||
if harness.id in seen_ids:
|
||||
raise HarnessRegistryError(
|
||||
f"harness-registry: duplicate harness id: {harness.id}"
|
||||
)
|
||||
seen_ids.add(harness.id)
|
||||
|
||||
owners: dict[str, str] = {}
|
||||
for harness in harnesses:
|
||||
for import_path in harness.known_import_paths():
|
||||
owner = owners.get(import_path)
|
||||
if owner is not None and owner != harness.id:
|
||||
raise HarnessRegistryError(
|
||||
f'harness-registry: import path "{import_path}" claimed by both '
|
||||
f'"{owner}" and "{harness.id}"'
|
||||
)
|
||||
owners[import_path] = harness.id
|
||||
|
||||
registry = HarnessRegistry(version=int(doc["version"]), harnesses=harnesses)
|
||||
_cache[resolved] = registry
|
||||
return registry
|
||||
Reference in New Issue
Block a user