downshift 0.1.0__py3-none-any.whl

downshift/evalgen.py

@@ -0,0 +1,220 @@
+"""Generate eval sets with any configured model (the no-Bob path).
+
+`generate_eval_set` asks a model for test cases for one call site, then keeps
+only the cases that pass the same checks as `downshift check-evals`. Cases are
+renumbered, so the model never has to get ids right. The CLI command
+`evalgen` is a thin wrapper that writes one `<slug>.jsonl` per call site.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from downshift.evals import (
+    EvalCase,
+    EvalSet,
+    allowed_values,
+    expression_placeholders,
+    graded_fields,
+    site_placeholders,
+    validate_eval_set,
+)
+from downshift.llm import LLMClient
+from downshift.schema import CallSite
+
+_FENCE = re.compile(r"^```(?:json)?\s*|\s*```$")
+
+SYSTEM_PROMPT = (
+    "You write test cases for one LLM feature in a software product. "
+    "Every expected answer must be certainly correct. Reply with JSON only."
+)
+
+
+class EvalGenSkip(Exception):
+    """Raised when a call site cannot get generated evals; the message says why."""
+
+
+@dataclass
+class GenerationResult:
+    site_id: str
+    grading: str
+    cases: list[EvalCase] = field(default_factory=list)
+    dropped: list[str] = field(default_factory=list)
+    attempts: int = 0
+
+
+def grading_for(site: CallSite) -> str:
+    """The audit's grading, or a safe default for an unaudited call site."""
+    if site.grading:
+        return site.grading
+    return "json_fields" if site.output_format == "json" else "judge"
+
+
+def generate_eval_set(
+    client: LLMClient,
+    model: str,
+    site: CallSite,
+    *,
+    count: int = 20,
+    shared: dict[str, str] | None = None,
+    max_attempts: int = 3,
+    temperature: float = 0.7,
+) -> GenerationResult:
+    """Ask `model` for cases until `count` valid ones are kept or attempts run out."""
+    _check_site(site)
+    shared = {k: v for k, v in (shared or {}).items() if k in site_placeholders(site)}
+    grading = grading_for(site)
+    result = GenerationResult(site_id=site.id, grading=grading)
+    seen_inputs: set[str] = set()
+
+    while len(result.cases) < count and result.attempts < max_attempts:
+        result.attempts += 1
+        wanted = count - len(result.cases)
+        messages = build_messages(site, grading, wanted, shared)
+        completion = client.complete(model, messages, temperature=temperature, json_mode=True)
+        try:
+            raw_cases = parse_cases(completion.text)
+        except ValueError as exc:
+            result.dropped.append(f"attempt {result.attempts}: {exc}")
+            continue
+        for raw in raw_cases:
+            if len(result.cases) >= count:
+                break
+            case, problem = _to_case(raw, site, grading, shared, len(result.cases) + 1)
+            if case is None:
+                result.dropped.append(problem)
+                continue
+            key = json.dumps(case.inputs, sort_keys=True, ensure_ascii=False)
+            if key in seen_inputs:
+                result.dropped.append(f"duplicate inputs: {case.notes}")
+                continue
+            seen_inputs.add(key)
+            result.cases.append(case)
+    return result
+
+
+def build_messages(
+    site: CallSite, grading: str, count: int, shared: dict[str, str]
+) -> list[dict[str, str]]:
+    inputs = [name for name in site_placeholders(site) if name not in shared]
+    prompt_lines = [f"[{m.role}]\n{m.content}" for m in site.messages or []]
+    parts = [
+        f"Feature: {site.purpose or site.function}",
+        f"Output contract: {site.output_contract or 'not documented'}",
+        "Prompt template sent to the model ({name} is filled per case):",
+        "\n\n".join(prompt_lines),
+    ]
+    if shared:
+        parts.append("Fixed inputs, the same for every case (do not repeat them in cases):")
+        parts.extend(f"--- {k} ---\n{v}" for k, v in shared.items())
+    parts.append(f"Write {count} diverse, realistic test cases.")
+    parts.append(
+        f"Each case has keys: inputs (an object with exactly these keys: "
+        f"{', '.join(inputs)}), expected, notes (what the case tests)."
+    )
+    parts.append(_expected_rule(site, grading))
+    parts.append(
+        "Include edge cases: very short text, other languages, typos, "
+        "missing details. Skip any case whose answer could be argued."
+    )
+    parts.append('Reply with: {"cases": [ ... ]}')
+    return [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user", "content": "\n\n".join(parts)},
+    ]
+
+
+def parse_cases(text: str) -> list[Any]:
+    """Pull the case list out of a model reply. Raises ValueError if there is none."""
+    cleaned = _FENCE.sub("", text.strip())
+    try:
+        data = json.loads(cleaned)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"reply is not valid JSON ({exc.msg})") from exc
+    if isinstance(data, dict):
+        data = data.get("cases")
+    if not isinstance(data, list):
+        raise ValueError('reply has no "cases" list')
+    return data
+
+
+def write_eval_set(
+    path: Path, cases: list[EvalCase], shared_files: dict[str, Path] | None = None
+) -> None:
+    """Write cases as JSONL, with a `shared` first line pointing at files if given."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    lines = []
+    if shared_files:
+        header = {
+            key: {"file": Path(os.path.relpath(target, path.parent)).as_posix()}
+            for key, target in shared_files.items()
+        }
+        lines.append(json.dumps({"shared": header}, ensure_ascii=False))
+    for case in cases:
+        record = {
+            "id": case.id,
+            "inputs": case.inputs,
+            "expected": case.expected,
+            "grading": case.grading,
+            "notes": case.notes,
+        }
+        lines.append(json.dumps(record, ensure_ascii=False))
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+# --- helpers ------------------------------------------------------------------
+
+
+def _check_site(site: CallSite) -> None:
+    if not site.prompt_resolved:
+        raise EvalGenSkip("prompt is not resolved; run the Bob auditor first")
+    expressions = expression_placeholders(site)
+    if expressions:
+        names = ", ".join("{" + e + "}" for e in expressions)
+        raise EvalGenSkip(f"prompt has expression placeholders {names}; run the Bob auditor first")
+
+
+def _expected_rule(site: CallSite, grading: str) -> str:
+    if grading == "exact":
+        labels = allowed_values(site)
+        if labels:
+            return f"expected: exactly one of {', '.join(labels)} (a string)."
+        return "expected: the exact short answer as a string."
+    if grading == "json_fields":
+        fields = graded_fields(site)
+        if fields:
+            return f"expected: an object with exactly these keys: {', '.join(fields)}."
+        return "expected: the JSON object the feature should return."
+    return (
+        "expected: a rubric string listing what a correct reply must and must not do, "
+        "point by point."
+    )
+
+
+def _to_case(
+    raw: Any, site: CallSite, grading: str, shared: dict[str, str], number: int
+) -> tuple[EvalCase | None, str]:
+    if not isinstance(raw, dict):
+        return None, f"not an object: {str(raw)[:60]}"
+    notes = raw.get("notes")
+    notes = notes if isinstance(notes, str) and notes.strip() else "generated"
+    inputs = raw.get("inputs")
+    if not isinstance(inputs, dict):
+        return None, f"inputs is not an object: {notes}"
+    case = EvalCase(
+        id=f"{site.function}-{number:02d}",
+        inputs=inputs,
+        expected=raw.get("expected"),
+        grading=grading,
+        notes=notes,
+    )
+    check = validate_eval_set(EvalSet(path=Path("-"), cases=[case], shared=shared), site)
+    if check.errors:
+        reason = check.errors[0].split(": ", 1)[-1]
+        return None, f"{reason}: {notes}"
+    return case, ""

downshift/evals.py

@@ -0,0 +1,395 @@
+"""Eval sets: one JSONL file of test cases per call site.
+
+File layout (``<slug>.jsonl``, slug from `slug_for`)::
+
+    {"shared": {"policy": {"file": "../data/refund_policy.md"}}}   <- optional first line
+    {"id": "c01", "inputs": {...}, "expected": ..., "grading": "exact", "notes": "..."}
+    ...
+
+`shared` inputs are merged under every case's own inputs, so long fixed
+values (a policy document) are written once. A shared value is a JSON value
+or ``{"file": "relative/path"}``, resolved against the eval file's folder.
+
+`validate_eval_set` checks a file against its call site: the input keys must
+match the prompt placeholders, and `expected` must fit the grading type.
+The CLI command `check-evals` is a thin wrapper around `check_eval_dir`.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from downshift.schema import GRADINGS, CallSite
+
+EVAL_SUFFIX = ".jsonl"
+CASE_FIELDS = frozenset({"id", "inputs", "expected", "grading", "notes"})
+MIN_CASES = 20
+
+_BRACED = re.compile(r"\{([^{}\n]+)\}")
+_GRADED_FIELDS = re.compile(r"Graded fields?:\s*([^\n]+)", re.IGNORECASE)
+_VALUE_SET = re.compile(r"from the set \{([^{}]+)\}", re.IGNORECASE)
+
+
+class EvalError(ValueError):
+    """Raised when an eval file cannot be parsed at all."""
+
+
+# --- call site helpers --------------------------------------------------------
+
+
+def slug_for(site_id: str) -> str:
+    """`supportdesk/triage.py::classify_category` -> `supportdesk.triage__classify_category`."""
+    path, _, function = site_id.partition("::")
+    module = path[:-3] if path.endswith(".py") else path
+    module = module.replace("\\", "/").strip("/").replace("/", ".")
+    return f"{module}__{function}" if function else module
+
+
+def placeholders(template: str) -> list[str]:
+    """Placeholders in a prompt as the scanner renders it, in order, without duplicates.
+
+    `{name}` and `{ticket['body']}` both count. Braces around text that is not
+    a Python expression (literal JSON such as `{"a": 1}`) are ignored.
+    """
+    found: list[str] = []
+    for match in _BRACED.finditer(template):
+        inner = match.group(1).strip()
+        if not inner or not _is_expression(inner):
+            continue
+        if inner not in found:
+            found.append(inner)
+    return found
+
+
+def site_placeholders(site: CallSite) -> list[str]:
+    found: list[str] = []
+    for message in site.messages or []:
+        for name in placeholders(message.content):
+            if name not in found:
+                found.append(name)
+    return found
+
+
+def expression_placeholders(site: CallSite) -> list[str]:
+    """Placeholders that are not plain names; eval inputs cannot fill them."""
+    return [name for name in site_placeholders(site) if not name.isidentifier()]
+
+
+def graded_fields(site: CallSite) -> list[str] | None:
+    """Field names after 'Graded fields:' in the output contract, or None if absent."""
+    if not site.output_contract:
+        return None
+    match = _GRADED_FIELDS.search(site.output_contract)
+    if not match:
+        return None
+    names: list[str] = []
+    for part in match.group(1).split(","):
+        token = part.strip().rstrip(".;").strip("`'\"")
+        if not token.isidentifier():
+            break
+        names.append(token)
+    return names or None
+
+
+def allowed_values(site: CallSite) -> list[str] | None:
+    """Labels from 'from the set {a, b, c}' in the output contract, or None if absent."""
+    if not site.output_contract:
+        return None
+    match = _VALUE_SET.search(site.output_contract)
+    if not match:
+        return None
+    values = [v.strip().strip("`'\"") for v in match.group(1).split(",")]
+    return [v for v in values if v] or None
+
+
+def render_messages(site: CallSite, inputs: dict[str, Any]) -> list[dict[str, str]]:
+    """Fill a call site's prompt with one case's inputs (for Phase 5 runs)."""
+    rendered = []
+    for message in site.messages or []:
+        content = message.content
+        for name in placeholders(content):
+            if name in inputs:
+                content = content.replace("{" + name + "}", _as_text(inputs[name]))
+        rendered.append({"role": message.role, "content": content})
+    return rendered
+
+
+# --- loading ------------------------------------------------------------------
+
+
+@dataclass
+class EvalCase:
+    id: str
+    inputs: dict[str, Any]
+    expected: Any
+    grading: str
+    notes: str | None = None
+    line: int = 0
+
+
+@dataclass
+class EvalSet:
+    path: Path
+    cases: list[EvalCase]
+    shared: dict[str, Any] = field(default_factory=dict)
+    problems: list[str] = field(default_factory=list)
+
+    @property
+    def slug(self) -> str:
+        return self.path.name[: -len(EVAL_SUFFIX)]
+
+    def inputs_for(self, case: EvalCase) -> dict[str, Any]:
+        return {**self.shared, **case.inputs}
+
+
+def load_eval_set(path: Path) -> EvalSet:
+    """Parse a JSONL eval file. Bad lines become `problems`; unreadable files raise."""
+    try:
+        text = path.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        raise EvalError(f"eval file not found: {path}") from None
+    except (OSError, UnicodeDecodeError) as exc:
+        raise EvalError(f"{path}: could not read ({exc})") from exc
+
+    eval_set = EvalSet(path=path, cases=[])
+    first_record = True
+    for number, raw in enumerate(text.splitlines(), start=1):
+        if not raw.strip():
+            continue
+        where = f"line {number}"
+        try:
+            record = json.loads(raw)
+        except json.JSONDecodeError as exc:
+            eval_set.problems.append(f"{where}: invalid JSON ({exc.msg})")
+            first_record = False
+            continue
+        if not isinstance(record, dict):
+            eval_set.problems.append(f"{where}: expected a JSON object")
+            first_record = False
+            continue
+
+        if "shared" in record:
+            if not first_record:
+                eval_set.problems.append(f"{where}: 'shared' is only allowed on the first line")
+            else:
+                eval_set.shared = _load_shared(record, path, where, eval_set.problems)
+            first_record = False
+            continue
+        first_record = False
+
+        case = _case_from_record(record, number, eval_set.problems)
+        if case is not None:
+            eval_set.cases.append(case)
+    return eval_set
+
+
+def _load_shared(record: dict[str, Any], path: Path, where: str, problems: list[str]) -> dict:
+    extra = sorted(set(record) - {"shared"})
+    if extra:
+        problems.append(f"{where}: unknown field(s) next to 'shared': {', '.join(extra)}")
+    shared = record["shared"]
+    if not isinstance(shared, dict):
+        problems.append(f"{where}: 'shared' must be an object")
+        return {}
+    values: dict[str, Any] = {}
+    for key, value in shared.items():
+        if isinstance(value, dict) and set(value) == {"file"}:
+            target = path.parent / str(value["file"])
+            try:
+                values[key] = target.read_text(encoding="utf-8")
+            except OSError:
+                problems.append(f"{where}: shared input {key!r}: file not found: {value['file']}")
+        else:
+            values[key] = value
+    return values
+
+
+def _case_from_record(record: dict[str, Any], number: int, problems: list[str]) -> EvalCase | None:
+    where = f"line {number}"
+    unknown = sorted(set(record) - CASE_FIELDS)
+    if unknown:
+        problems.append(f"{where}: unknown field(s) {', '.join(unknown)}")
+    missing = [name for name in ("id", "inputs", "expected", "grading") if name not in record]
+    if missing:
+        problems.append(f"{where}: missing {', '.join(missing)}")
+        return None
+
+    case_id, inputs, grading = record["id"], record["inputs"], record["grading"]
+    notes = record.get("notes")
+    ok = True
+    if not isinstance(case_id, str) or not case_id.strip():
+        problems.append(f"{where}: id must be a non-empty string")
+        ok = False
+    if not isinstance(inputs, dict):
+        problems.append(f"{where}: inputs must be an object")
+        ok = False
+    if grading not in GRADINGS:
+        problems.append(f"{where}: grading must be one of {', '.join(sorted(GRADINGS))}")
+        ok = False
+    if notes is not None and not isinstance(notes, str):
+        problems.append(f"{where}: notes must be a string")
+        ok = False
+    if not ok:
+        return None
+    return EvalCase(
+        id=case_id,
+        inputs=inputs,
+        expected=record["expected"],
+        grading=grading,
+        notes=notes,
+        line=number,
+    )
+
+
+# --- validation ---------------------------------------------------------------
+
+
+@dataclass
+class EvalReport:
+    path: Path
+    site_id: str | None = None
+    cases: int = 0
+    grading: str | None = None
+    errors: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def ok(self) -> bool:
+        return not self.errors
+
+
+def validate_eval_set(eval_set: EvalSet, site: CallSite) -> EvalReport:
+    report = EvalReport(
+        path=eval_set.path, site_id=site.id, cases=len(eval_set.cases), grading=site.grading
+    )
+    report.errors.extend(eval_set.problems)
+
+    names = site_placeholders(site)
+    expressions = [name for name in names if not name.isidentifier()]
+    if site.messages is None or not site.prompt_resolved:
+        report.errors.append("call site prompt is not resolved; re-run the Bob auditor")
+    if expressions:
+        report.errors.append(
+            "prompt has expression placeholders "
+            + ", ".join("{" + e + "}" for e in expressions)
+            + "; rename them to plain names in the audit"
+        )
+    wanted = set(names) - set(expressions)
+
+    fields = graded_fields(site)
+    labels = allowed_values(site)
+    if site.grading == "json_fields" and fields is None:
+        report.warnings.append("output_contract has no 'Graded fields:' list")
+
+    seen: set[str] = set()
+    for case in eval_set.cases:
+        where = f"line {case.line} ({case.id})"
+        if case.id in seen:
+            report.errors.append(f"{where}: duplicate id")
+        seen.add(case.id)
+
+        if site.grading is not None and case.grading != site.grading:
+            report.errors.append(
+                f"{where}: grading {case.grading} does not match call site grading {site.grading}"
+            )
+
+        given = set(eval_set.inputs_for(case))
+        if not expressions:
+            missing = sorted(wanted - given)
+            extra = sorted(given - wanted)
+            if missing:
+                report.errors.append(f"{where}: inputs missing {', '.join(missing)}")
+            if extra:
+                report.errors.append(f"{where}: inputs not in the prompt: {', '.join(extra)}")
+
+        report.errors.extend(f"{where}: {p}" for p in _check_expected(case, fields, labels))
+
+    if len(eval_set.cases) < MIN_CASES:
+        report.warnings.append(f"only {len(eval_set.cases)} cases (aim for {MIN_CASES} or more)")
+    return report
+
+
+def _check_expected(case: EvalCase, fields: list[str] | None, labels: list[str] | None) -> list:
+    expected = case.expected
+    if case.grading == "exact":
+        if not isinstance(expected, str) or not expected.strip():
+            return ["exact grading needs expected to be a non-empty string"]
+        if labels is not None and expected not in labels:
+            return [f"expected {expected!r} is not one of {', '.join(labels)}"]
+        return []
+    if case.grading == "json_fields":
+        if not isinstance(expected, dict) or not expected:
+            return ["json_fields grading needs expected to be a non-empty object"]
+        if fields is None:
+            return []
+        problems = []
+        missing = [name for name in fields if name not in expected]
+        extra = sorted(set(expected) - set(fields))
+        if missing:
+            problems.append(f"expected missing graded field(s) {', '.join(missing)}")
+        if extra:
+            problems.append(f"expected has fields that are not graded: {', '.join(extra)}")
+        return problems
+    if not isinstance(expected, str) or not expected.strip():
+        return ["judge grading needs expected to be a reference answer or rubric string"]
+    return []
+
+
+def check_eval_dir(directory: Path, sites: list[CallSite]) -> list[EvalReport]:
+    """Validate every `*.jsonl` in a folder against the call sites it is named after.
+
+    Returns one report per eval file, plus one per call site that has no file.
+    """
+    by_slug = {slug_for(site.id): site for site in sites}
+    reports: list[EvalReport] = []
+    covered: set[str] = set()
+
+    for path in sorted(directory.glob(f"*{EVAL_SUFFIX}")):
+        slug = path.name[: -len(EVAL_SUFFIX)]
+        site = by_slug.get(slug)
+        if site is None:
+            reports.append(
+                EvalReport(path=path, errors=[f"no call site with slug {slug!r} in the audit"])
+            )
+            continue
+        covered.add(slug)
+        try:
+            eval_set = load_eval_set(path)
+        except EvalError as exc:
+            reports.append(EvalReport(path=path, site_id=site.id, errors=[str(exc)]))
+            continue
+        reports.append(validate_eval_set(eval_set, site))
+
+    for slug, site in by_slug.items():
+        if slug not in covered:
+            reports.append(
+                EvalReport(
+                    path=directory / f"{slug}{EVAL_SUFFIX}",
+                    site_id=site.id,
+                    grading=site.grading,
+                    warnings=["no eval file"],
+                )
+            )
+    return reports
+
+
+# --- helpers ------------------------------------------------------------------
+
+
+def _is_expression(text: str) -> bool:
+    if text[0] in "\"'":
+        return False
+    try:
+        node = ast.parse(text, mode="eval").body
+    except SyntaxError:
+        return False
+    return not isinstance(node, ast.Constant)
+
+
+def _as_text(value: Any) -> str:
+    return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)