84aac0ebfd
Useful as prompts grow longer and we benchmark alternate models — bare PASS/FAIL hides regressions like a prompt that doubles latency without changing correctness. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
78 lines
2.3 KiB
Python
78 lines
2.3 KiB
Python
"""Run all eval cases and report which pass/fail.
|
|
|
|
Each case has a `name`, an `input` string, and an `expected` dict. The current
|
|
v0 only exercises the `db_kind` classifier. As more classifiers come online,
|
|
each case will grow expected sections for them too.
|
|
|
|
Run from the repo root:
|
|
|
|
uv run python eval/run_eval.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
from tolkien.llm import classify
|
|
from tolkien.schemas import DbKindResult
|
|
|
|
CASES_DIR = Path(__file__).parent / "cases"
|
|
|
|
|
|
def _diff_fields(expected: dict, actual: dict) -> list[str]:
|
|
"""Return human-readable lines describing field-level mismatches."""
|
|
diffs: list[str] = []
|
|
for key, want in expected.items():
|
|
got = actual.get(key)
|
|
if isinstance(want, list) and isinstance(got, list):
|
|
want_set, got_set = set(want), set(got)
|
|
missing = want_set - got_set
|
|
extra = got_set - want_set
|
|
if missing or extra:
|
|
diffs.append(f" {key}: missing={sorted(missing)} extra={sorted(extra)}")
|
|
elif want != got:
|
|
diffs.append(f" {key}: want={want!r} got={got!r}")
|
|
return diffs
|
|
|
|
|
|
def main() -> int:
|
|
cases = sorted(CASES_DIR.glob("*.yml"))
|
|
if not cases:
|
|
print(f"No cases found in {CASES_DIR}", file=sys.stderr)
|
|
return 2
|
|
|
|
failures = 0
|
|
total_start = time.perf_counter()
|
|
for case_path in cases:
|
|
case = yaml.safe_load(case_path.read_text())
|
|
name = case["name"]
|
|
case_start = time.perf_counter()
|
|
result = classify("db_kind", case["input"], DbKindResult)
|
|
elapsed = time.perf_counter() - case_start
|
|
actual = result.model_dump()
|
|
diffs = _diff_fields(case["expected"], actual)
|
|
status = "PASS" if not diffs else "FAIL"
|
|
print(f"{status} {name} ({elapsed:.2f}s)")
|
|
if diffs:
|
|
failures += 1
|
|
for line in diffs:
|
|
print(line)
|
|
total_elapsed = time.perf_counter() - total_start
|
|
|
|
if failures:
|
|
print(
|
|
f"\n{failures}/{len(cases)} cases failed. Total {total_elapsed:.2f}s.",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
print(f"\nAll {len(cases)} cases passed. Total {total_elapsed:.2f}s.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|