"""Run all eval cases and report which pass/fail.

Each case has a `name`, an `input` string, and an `expected` dict. The current
v0 only exercises the `db_kind` classifier. As more classifiers come online,
each case will grow expected sections for them too.

Run from the repo root:

    uv run python eval/run_eval.py
"""

from __future__ import annotations

import sys
import time
from pathlib import Path

import yaml

from tolkien.llm import classify
from tolkien.schemas import DbKindResult

CASES_DIR = Path(__file__).parent / "cases"


def _diff_fields(expected: dict, actual: dict) -> list[str]:
    """Return human-readable lines describing field-level mismatches."""
    diffs: list[str] = []
    for key, want in expected.items():
        got = actual.get(key)
        if isinstance(want, list) and isinstance(got, list):
            want_set, got_set = set(want), set(got)
            missing = want_set - got_set
            extra = got_set - want_set
            if missing or extra:
                diffs.append(f"  {key}: missing={sorted(missing)} extra={sorted(extra)}")
        elif want != got:
            diffs.append(f"  {key}: want={want!r} got={got!r}")
    return diffs


def main() -> int:
    cases = sorted(CASES_DIR.glob("*.yml"))
    if not cases:
        print(f"No cases found in {CASES_DIR}", file=sys.stderr)
        return 2

    failures = 0
    total_start = time.perf_counter()
    for case_path in cases:
        case = yaml.safe_load(case_path.read_text())
        name = case["name"]
        case_start = time.perf_counter()
        result = classify("db_kind", case["input"], DbKindResult)
        elapsed = time.perf_counter() - case_start
        actual = result.model_dump()
        diffs = _diff_fields(case["expected"], actual)
        status = "PASS" if not diffs else "FAIL"
        print(f"{status} {name} ({elapsed:.2f}s)")
        if diffs:
            failures += 1
            for line in diffs:
                print(line)
    total_elapsed = time.perf_counter() - total_start

    if failures:
        print(
            f"\n{failures}/{len(cases)} cases failed. Total {total_elapsed:.2f}s.",
            file=sys.stderr,
        )
        return 1
    print(f"\nAll {len(cases)} cases passed. Total {total_elapsed:.2f}s.")
    return 0


if __name__ == "__main__":
    sys.exit(main())