feat: scaffold tolkien v0 — local-LLM classifier wrapper + CLI + eval
Initial skeleton for the deterministic compound-AI pipeline that turns `tolkien deploy <name>` into a valinor scaffold. v0 wires one classifier (`db_kind`) end-to-end against the Ollama-on-radagast backend so we can iterate prompts with confidence before adding the rest. - pyproject.toml: uv project, Python 3.13, httpx/pydantic/typer/pyyaml. - src/tolkien/llm.py: single-turn /api/chat wrapper. JSON-mode + temp 0 + 30m keep-alive. Pydantic schema validation on parse. - src/tolkien/cli.py: `tolkien classify db-kind --doc <file>` runs the classifier and prints JSON. `tolkien deploy <name>` is stubbed. - src/tolkien/prompts/db_kind.md: tight system prompt with 1-shot example and explicit "return ONLY JSON" guard. - src/tolkien/schemas.py: DbKindResult pydantic model. - eval/cases/mealie.yml + run_eval.py: regression harness. Currently one case (mealie); failures print field-level diffs. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
70
eval/run_eval.py
Normal file
70
eval/run_eval.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""Run all eval cases and report which pass/fail.
|
||||
|
||||
Each case has a `name`, an `input` string, and an `expected` dict. The current
|
||||
v0 only exercises the `db_kind` classifier. As more classifiers come online,
|
||||
each case will grow expected sections for them too.
|
||||
|
||||
Run from the repo root:
|
||||
|
||||
uv run python eval/run_eval.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from tolkien.llm import classify
|
||||
from tolkien.schemas import DbKindResult
|
||||
|
||||
CASES_DIR = Path(__file__).parent / "cases"
|
||||
|
||||
|
||||
def _diff_fields(expected: dict, actual: dict) -> list[str]:
|
||||
"""Return human-readable lines describing field-level mismatches."""
|
||||
diffs: list[str] = []
|
||||
for key, want in expected.items():
|
||||
got = actual.get(key)
|
||||
if isinstance(want, list) and isinstance(got, list):
|
||||
want_set, got_set = set(want), set(got)
|
||||
missing = want_set - got_set
|
||||
extra = got_set - want_set
|
||||
if missing or extra:
|
||||
diffs.append(f" {key}: missing={sorted(missing)} extra={sorted(extra)}")
|
||||
elif want != got:
|
||||
diffs.append(f" {key}: want={want!r} got={got!r}")
|
||||
return diffs
|
||||
|
||||
|
||||
def main() -> int:
|
||||
cases = sorted(CASES_DIR.glob("*.yml"))
|
||||
if not cases:
|
||||
print(f"No cases found in {CASES_DIR}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
failures = 0
|
||||
for case_path in cases:
|
||||
case = yaml.safe_load(case_path.read_text())
|
||||
name = case["name"]
|
||||
result = classify("db_kind", case["input"], DbKindResult)
|
||||
actual = result.model_dump()
|
||||
diffs = _diff_fields(case["expected"], actual)
|
||||
if diffs:
|
||||
failures += 1
|
||||
print(f"FAIL {name}")
|
||||
for line in diffs:
|
||||
print(line)
|
||||
else:
|
||||
print(f"PASS {name}")
|
||||
|
||||
if failures:
|
||||
print(f"\n{failures}/{len(cases)} cases failed.", file=sys.stderr)
|
||||
return 1
|
||||
print(f"\nAll {len(cases)} cases passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user