feat(eval): print per-case + total wall-clock time
Useful as prompts grow longer and we benchmark alternate models — bare PASS/FAIL hides regressions like a prompt that doubles latency without changing correctness. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -12,6 +12,7 @@ Run from the repo root:
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
@@ -45,24 +46,30 @@ def main() -> int:
|
|||||||
return 2
|
return 2
|
||||||
|
|
||||||
failures = 0
|
failures = 0
|
||||||
|
total_start = time.perf_counter()
|
||||||
for case_path in cases:
|
for case_path in cases:
|
||||||
case = yaml.safe_load(case_path.read_text())
|
case = yaml.safe_load(case_path.read_text())
|
||||||
name = case["name"]
|
name = case["name"]
|
||||||
|
case_start = time.perf_counter()
|
||||||
result = classify("db_kind", case["input"], DbKindResult)
|
result = classify("db_kind", case["input"], DbKindResult)
|
||||||
|
elapsed = time.perf_counter() - case_start
|
||||||
actual = result.model_dump()
|
actual = result.model_dump()
|
||||||
diffs = _diff_fields(case["expected"], actual)
|
diffs = _diff_fields(case["expected"], actual)
|
||||||
|
status = "PASS" if not diffs else "FAIL"
|
||||||
|
print(f"{status} {name} ({elapsed:.2f}s)")
|
||||||
if diffs:
|
if diffs:
|
||||||
failures += 1
|
failures += 1
|
||||||
print(f"FAIL {name}")
|
|
||||||
for line in diffs:
|
for line in diffs:
|
||||||
print(line)
|
print(line)
|
||||||
else:
|
total_elapsed = time.perf_counter() - total_start
|
||||||
print(f"PASS {name}")
|
|
||||||
|
|
||||||
if failures:
|
if failures:
|
||||||
print(f"\n{failures}/{len(cases)} cases failed.", file=sys.stderr)
|
print(
|
||||||
|
f"\n{failures}/{len(cases)} cases failed. Total {total_elapsed:.2f}s.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
return 1
|
return 1
|
||||||
print(f"\nAll {len(cases)} cases passed.")
|
print(f"\nAll {len(cases)} cases passed. Total {total_elapsed:.2f}s.")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user