From 058362b87fe9ee400577c8b70d582497eaec661d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro=20Battistella=20Nadas?= Date: Fri, 29 May 2026 15:09:11 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20scaffold=20tolkien=20v0=20=E2=80=94=20l?= =?UTF-8?q?ocal-LLM=20classifier=20wrapper=20+=20CLI=20+=20eval?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Initial skeleton for the deterministic compound-AI pipeline that turns `tolkien deploy ` into a valinor scaffold. v0 wires one classifier (`db_kind`) end-to-end against the Ollama-on-radagast backend so we can iterate prompts with confidence before adding the rest. - pyproject.toml: uv project, Python 3.13, httpx/pydantic/typer/pyyaml. - src/tolkien/llm.py: single-turn /api/chat wrapper. JSON-mode + temp 0 + 30m keep-alive. Pydantic schema validation on parse. - src/tolkien/cli.py: `tolkien classify db-kind --doc ` runs the classifier and prints JSON. `tolkien deploy ` is stubbed. - src/tolkien/prompts/db_kind.md: tight system prompt with 1-shot example and explicit "return ONLY JSON" guard. - src/tolkien/schemas.py: DbKindResult pydantic model. - eval/cases/mealie.yml + run_eval.py: regression harness. Currently one case (mealie); failures print field-level diffs. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 21 ++++++++++ .python-version | 1 + README.md | 49 ++++++++++++++++++++++++ eval/cases/mealie.yml | 36 +++++++++++++++++ eval/run_eval.py | 70 ++++++++++++++++++++++++++++++++++ pyproject.toml | 29 ++++++++++++++ src/tolkien/__init__.py | 1 + src/tolkien/__main__.py | 4 ++ src/tolkien/cli.py | 46 ++++++++++++++++++++++ src/tolkien/llm.py | 70 ++++++++++++++++++++++++++++++++++ src/tolkien/prompts/db_kind.md | 19 +++++++++ src/tolkien/schemas.py | 19 +++++++++ 12 files changed, 365 insertions(+) create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 README.md create mode 100644 eval/cases/mealie.yml create mode 100644 eval/run_eval.py create mode 100644 pyproject.toml create mode 100644 src/tolkien/__init__.py create mode 100644 src/tolkien/__main__.py create mode 100644 src/tolkien/cli.py create mode 100644 src/tolkien/llm.py create mode 100644 src/tolkien/prompts/db_kind.md create mode 100644 src/tolkien/schemas.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e3a3e11 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +.pytest_cache/ +.ruff_cache/ + +# uv / build +.venv/ +dist/ +build/ +*.egg-info/ + +# Tolkien-specific +out/ # scaffold output directory (v0) +.cache/ + +# Editor / OS +.vscode/ +.idea/ +.DS_Store diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/README.md b/README.md new file mode 100644 index 0000000..87e374b --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# tolkien + +Local-LLM orchestrator that turns `tolkien deploy mealie` into a complete valinor +GitOps scaffold (config.yaml, values.yaml, CNPG cluster, Vault secrets, Terraform +stubs) ready for the operator to review and PR. + +## Architecture + +Compound-AI pipeline, not a ReAct loop. Each step in the deploy workflow is a +single LLM call that produces structured JSON — the surrounding logic is plain +Python. This shape works reliably on a CPU-served 7B model where a free-form +agent loop would drift. + +``` +deploy + ├── search.find_helm_chart() + ├── classify.has_chart(results) — LLM, JSON + ├── fetch.app_docs(url) + ├── classify.db_kind(docs) — LLM, JSON + ├── classify.needs_object_storage(docs)— LLM, JSON + ├── classify.secrets_vs_config(env) — LLM, JSON + ├── classify.storage_class(docs) — LLM, JSON + └── render.scaffold(facts) — Jinja2 → ./out// +``` + +## Backend + +Ollama running on radagast (`http://radagast.jpnadas.xyz:11434`), serving +Qwen2.5-Coder-7B-Instruct Q4_K_M. See valinor/`ansible/playbooks/radagast-setup.yml` +for the host setup. + +## v0 scope + +- Single classifier (`db_kind`) wired end-to-end against the live model +- CLI: `tolkien classify db-kind --doc ` prints JSON +- Eval harness with one case (mealie) +- Output to stdout / `./out/`; no git push, no FastAPI, no Redis + +v1 layers on FastAPI + Redis + a Gitea bot to push branches; v2 wires a Gitea +webhook for issue-comment triggers. + +## Develop + +```bash +uv sync +uv run tolkien --help +uv run pytest +uv run ruff check . +``` diff --git a/eval/cases/mealie.yml b/eval/cases/mealie.yml new file mode 100644 index 0000000..3661845 --- /dev/null +++ b/eval/cases/mealie.yml @@ -0,0 +1,36 @@ +# Eval case for the `db_kind` classifier. +# The pipeline's correctness on real apps is gated on this kind of fixture — +# re-run `eval/run_eval.py` after every prompt change. + +name: mealie +input: | + App: mealie (recipe manager). Excerpt from official docker-compose docs: + + environment: + DB_ENGINE: postgres + POSTGRES_USER: mealie + POSTGRES_PASSWORD: change-me + POSTGRES_SERVER: postgres + POSTGRES_PORT: 5432 + POSTGRES_DB: mealie + ALLOW_SIGNUP: "true" + BASE_URL: https://mealie.example.com + OPENAI_API_KEY: sk-... # optional, enables recipe parsing via OpenAI + TZ: Europe/Amsterdam + +expected: + needs_postgres: true + db_env_vars: + - DB_ENGINE + - POSTGRES_USER + - POSTGRES_PASSWORD + - POSTGRES_SERVER + - POSTGRES_PORT + - POSTGRES_DB + secrets: + - POSTGRES_PASSWORD + - OPENAI_API_KEY + config: + - ALLOW_SIGNUP + - BASE_URL + - TZ diff --git a/eval/run_eval.py b/eval/run_eval.py new file mode 100644 index 0000000..14d48d5 --- /dev/null +++ b/eval/run_eval.py @@ -0,0 +1,70 @@ +"""Run all eval cases and report which pass/fail. + +Each case has a `name`, an `input` string, and an `expected` dict. The current +v0 only exercises the `db_kind` classifier. As more classifiers come online, +each case will grow expected sections for them too. + +Run from the repo root: + + uv run python eval/run_eval.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import yaml + +from tolkien.llm import classify +from tolkien.schemas import DbKindResult + +CASES_DIR = Path(__file__).parent / "cases" + + +def _diff_fields(expected: dict, actual: dict) -> list[str]: + """Return human-readable lines describing field-level mismatches.""" + diffs: list[str] = [] + for key, want in expected.items(): + got = actual.get(key) + if isinstance(want, list) and isinstance(got, list): + want_set, got_set = set(want), set(got) + missing = want_set - got_set + extra = got_set - want_set + if missing or extra: + diffs.append(f" {key}: missing={sorted(missing)} extra={sorted(extra)}") + elif want != got: + diffs.append(f" {key}: want={want!r} got={got!r}") + return diffs + + +def main() -> int: + cases = sorted(CASES_DIR.glob("*.yml")) + if not cases: + print(f"No cases found in {CASES_DIR}", file=sys.stderr) + return 2 + + failures = 0 + for case_path in cases: + case = yaml.safe_load(case_path.read_text()) + name = case["name"] + result = classify("db_kind", case["input"], DbKindResult) + actual = result.model_dump() + diffs = _diff_fields(case["expected"], actual) + if diffs: + failures += 1 + print(f"FAIL {name}") + for line in diffs: + print(line) + else: + print(f"PASS {name}") + + if failures: + print(f"\n{failures}/{len(cases)} cases failed.", file=sys.stderr) + return 1 + print(f"\nAll {len(cases)} cases passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ebd89f4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "tolkien" +version = "0.1.0" +description = "Local-LLM orchestrator that scaffolds valinor app deployments from a one-line spec." +readme = "README.md" +requires-python = ">=3.13" +dependencies = [ + "httpx>=0.27", + "pydantic>=2.8", + "pyyaml>=6.0", + "typer>=0.12", +] + +[project.scripts] +tolkien = "tolkien.cli:app" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/tolkien"] + +[tool.ruff] +line-length = 100 +target-version = "py313" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM"] diff --git a/src/tolkien/__init__.py b/src/tolkien/__init__.py new file mode 100644 index 0000000..3dc1f76 --- /dev/null +++ b/src/tolkien/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/src/tolkien/__main__.py b/src/tolkien/__main__.py new file mode 100644 index 0000000..0a6f6c5 --- /dev/null +++ b/src/tolkien/__main__.py @@ -0,0 +1,4 @@ +from tolkien.cli import app + +if __name__ == "__main__": + app() diff --git a/src/tolkien/cli.py b/src/tolkien/cli.py new file mode 100644 index 0000000..1a9fce2 --- /dev/null +++ b/src/tolkien/cli.py @@ -0,0 +1,46 @@ +"""Tolkien CLI — v0 surface area. + +v0 exposes the building blocks (classify) directly so we can iterate prompts +against the live model. The full `deploy` pipeline lands once the individual +classifiers are reliable on the eval cases. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated + +import typer + +from tolkien.llm import LLMError, classify +from tolkien.schemas import DbKindResult + +app = typer.Typer(no_args_is_help=True, add_completion=False) +classify_app = typer.Typer(no_args_is_help=True) +app.add_typer(classify_app, name="classify", help="Run a single classifier against the LLM.") + + +@classify_app.command("db-kind") +def cli_classify_db_kind( + doc: Annotated[Path, typer.Option("--doc", help="Path to the documentation snippet.")], +) -> None: + """Classify the DB requirements + env vars of an app from its docs.""" + user_input = doc.read_text() + try: + result = classify("db_kind", user_input, DbKindResult) + except LLMError as e: + typer.echo(f"LLM error: {e}", err=True) + raise typer.Exit(1) from e + typer.echo(json.dumps(result.model_dump(), indent=2)) + + +@app.command() +def deploy(name: Annotated[str, typer.Argument(help="App name, e.g. 'mealie'.")]) -> None: + """Generate a valinor scaffold for the given app. (Stubbed — coming next.)""" + typer.echo(f"deploy {name}: pipeline not yet implemented.", err=True) + raise typer.Exit(2) + + +if __name__ == "__main__": + app() diff --git a/src/tolkien/llm.py b/src/tolkien/llm.py new file mode 100644 index 0000000..149156d --- /dev/null +++ b/src/tolkien/llm.py @@ -0,0 +1,70 @@ +"""Thin Ollama /api/chat wrapper with JSON-mode + pydantic schema validation. + +Each call is a single turn: system prompt + user message in, structured JSON out. +No streaming, no tool use, no multi-turn — those are footguns at 7B-class. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import httpx +from pydantic import BaseModel, ValidationError + +OLLAMA_HOST = "http://radagast.jpnadas.xyz:11434" +MODEL = "qwen2.5-coder:7b-instruct" +KEEP_ALIVE = "30m" +TIMEOUT_SECONDS = 120.0 + +PROMPTS_DIR = Path(__file__).parent / "prompts" + + +class LLMError(Exception): + """LLM produced output that couldn't be parsed or validated.""" + + +def _read_prompt(name: str) -> str: + return (PROMPTS_DIR / f"{name}.md").read_text() + + +def classify[T: BaseModel]( + prompt_name: str, + user_input: str, + schema: type[T], + *, + host: str = OLLAMA_HOST, + model: str = MODEL, +) -> T: + """Run a single-turn classify call and validate the output against `schema`. + + Raises LLMError if the model output isn't valid JSON or doesn't match the schema. + """ + system_prompt = _read_prompt(prompt_name) + + response = httpx.post( + f"{host}/api/chat", + json={ + "model": model, + "stream": False, + "format": "json", + "keep_alive": KEEP_ALIVE, + "options": {"temperature": 0}, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_input}, + ], + }, + timeout=TIMEOUT_SECONDS, + ) + response.raise_for_status() + raw = response.json()["message"]["content"] + + try: + return schema.model_validate_json(raw) + except ValidationError as e: + raise LLMError( + f"Schema validation failed for prompt '{prompt_name}'.\n" + f"Raw output:\n{raw}\n\n" + f"Errors:\n{json.dumps(e.errors(), indent=2)}" + ) from e diff --git a/src/tolkien/prompts/db_kind.md b/src/tolkien/prompts/db_kind.md new file mode 100644 index 0000000..e7bc7d0 --- /dev/null +++ b/src/tolkien/prompts/db_kind.md @@ -0,0 +1,19 @@ +You analyze an app's deployment documentation and emit JSON describing what it needs. + +Schema (return EXACTLY these keys): +- needs_postgres: boolean. True only if PostgreSQL is required (not optional, not SQLite, not MariaDB). +- db_env_vars: array of env-var names that configure the database connection. +- secrets: array of env-var names that hold sensitive values (passwords, tokens, keys). Strict — only include vars that are clearly secret. +- config: array of env-var names that are plain configuration (URLs, booleans, ints). + +Example input: +environment: + DB_TYPE: sqlite + WORKERS: 4 + SECRET_KEY: change-me + BASE_URL: https://app.example.com + +Example output: +{"needs_postgres": false, "db_env_vars": ["DB_TYPE"], "secrets": ["SECRET_KEY"], "config": ["WORKERS", "BASE_URL"]} + +Return ONLY the JSON object. No prose, no markdown fences. diff --git a/src/tolkien/schemas.py b/src/tolkien/schemas.py new file mode 100644 index 0000000..4109c41 --- /dev/null +++ b/src/tolkien/schemas.py @@ -0,0 +1,19 @@ +from pydantic import BaseModel, Field + + +class DbKindResult(BaseModel): + needs_postgres: bool = Field( + description="True only if PostgreSQL is required (not optional, not SQLite, not MariaDB)." + ) + db_env_vars: list[str] = Field( + default_factory=list, + description="Env var names that configure the database connection.", + ) + secrets: list[str] = Field( + default_factory=list, + description="Env var names that hold sensitive values (passwords, tokens, keys).", + ) + config: list[str] = Field( + default_factory=list, + description="Env var names that are plain non-sensitive configuration.", + )