feat(eval): print per-case + total wall-clock time

Useful as prompts grow longer and we benchmark alternate models — bare
PASS/FAIL hides regressions like a prompt that doubles latency without
changing correctness.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
João Pedro Battistella Nadas
2026-05-29 15:14:20 +02:00
parent 058362b87f
commit 84aac0ebfd

View File

@@ -12,6 +12,7 @@ Run from the repo root:
from __future__ import annotations
import sys
import time
from pathlib import Path
import yaml
@@ -45,24 +46,30 @@ def main() -> int:
return 2
failures = 0
total_start = time.perf_counter()
for case_path in cases:
case = yaml.safe_load(case_path.read_text())
name = case["name"]
case_start = time.perf_counter()
result = classify("db_kind", case["input"], DbKindResult)
elapsed = time.perf_counter() - case_start
actual = result.model_dump()
diffs = _diff_fields(case["expected"], actual)
status = "PASS" if not diffs else "FAIL"
print(f"{status} {name} ({elapsed:.2f}s)")
if diffs:
failures += 1
print(f"FAIL {name}")
for line in diffs:
print(line)
else:
print(f"PASS {name}")
total_elapsed = time.perf_counter() - total_start
if failures:
print(f"\n{failures}/{len(cases)} cases failed.", file=sys.stderr)
print(
f"\n{failures}/{len(cases)} cases failed. Total {total_elapsed:.2f}s.",
file=sys.stderr,
)
return 1
print(f"\nAll {len(cases)} cases passed.")
print(f"\nAll {len(cases)} cases passed. Total {total_elapsed:.2f}s.")
return 0