evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/core/importer.py

@@ -0,0 +1,257 @@
+"""One importer code path for `sync`: loads and
+validates YAML, lints against the other datasets in the workspace, checks that
+every fixture's bytes were uploaded, then hands the cases and overlays to the
+store, which diffs by content hash.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+import yaml
+from pydantic import BaseModel, Field, ValidationError
+
+from evalkit.contracts.cases import (
+    REGISTRY_FILENAME,
+    Case,
+    FixtureEntry,
+    Overlay,
+    content_hash,
+    is_case_yaml,
+    load_cases_from_yaml,
+)
+from evalkit.contracts.enums import DatasetKind, SyncSource
+from evalkit.core.access import WorkspaceRef
+from evalkit.core.fixtures import load_registries
+from evalkit.core.lint import Finding, fixture_findings, has_errors
+from evalkit.core.lint import lint as lint_cases
+from evalkit.core.overlays import OverlayBlock, incoming_overlay, overlay_clashes
+from evalkit.store.base import (
+    CarriedReview,
+    CaseFile,
+    IncomingCase,
+    IncomingOverlay,
+    SyncResult,
+)
+
+
+class CaseCatalog(Protocol):
+    async def workspace_cases(
+        self, workspace: WorkspaceRef, *, exclude_dataset: str | None
+    ) -> dict[str, str]: ...
+
+    async def overlays_on(
+        self, workspace: WorkspaceRef, case_ids: list[str], *, exclude_dataset: str
+    ) -> dict[str, list[OverlayBlock]]: ...
+
+    async def known_blobs(self, shas: list[str]) -> set[str]: ...
+
+    async def has_dataset(self, workspace: WorkspaceRef, name: str) -> bool: ...
+
+    async def sync(
+        self,
+        *,
+        workspace: WorkspaceRef,
+        dataset: str,
+        kind: DatasetKind,
+        incoming: list[IncomingCase],
+        overlays: list[IncomingOverlay],
+        full: bool,
+        actor_id: int,
+        source: SyncSource,
+        detail: dict,
+    ) -> SyncResult: ...
+
+
+class ReviewReport(BaseModel):
+    now_unreviewed: list[str] = Field(default_factory=list)
+    carried: list[CarriedReview] = Field(default_factory=list)
+
+
+class ImportReport(BaseModel):
+    created: int = 0
+    patched: int = 0
+    deleted: int = 0
+    overlays_applied: int = 0
+    findings: list[Finding] = Field(default_factory=list)
+    aborted: bool = False
+    review: ReviewReport = Field(default_factory=ReviewReport)
+
+
+def load_files(
+    files: list[Path],
+) -> tuple[list[tuple[Case, Path]], list[tuple[Overlay, str]], list[Finding]]:
+    cases: list[tuple[Case, Path]] = []
+    overlays: list[tuple[Overlay, str]] = []
+    findings: list[Finding] = []
+    for path in files:
+        if not is_case_yaml(path):
+            continue
+        try:
+            file_cases, file_overlays = load_cases_from_yaml(path)
+        except (ValidationError, ValueError, yaml.YAMLError) as exc:
+            findings.append(
+                Finding(
+                    severity="error",
+                    code="E-SCHEMA",
+                    case_id=path.name,
+                    message=str(exc),
+                )
+            )
+            continue
+        cases.extend((case, path) for case in file_cases)
+        overlays.extend((ov, path.name) for ov in file_overlays)
+    return cases, overlays, findings
+
+
+def case_files(case: Case, entries: dict[str, FixtureEntry]) -> list[CaseFile]:
+    return [
+        CaseFile(key=key, sha256=entries[key].sha256, context=entries[key].context)
+        for key in case.fixtures
+        if key in entries
+    ]
+
+
+async def missing_blobs(
+    catalog: CaseCatalog, cases: list[Case], entries: dict[str, FixtureEntry]
+) -> list[Finding]:
+    wanted = [(case.id, file) for case in cases for file in case_files(case, entries)]
+    known = await catalog.known_blobs(sorted({file.sha256 for _, file in wanted}))
+    return [
+        Finding(
+            severity="error",
+            code="E-FIXTURE-MISSING",
+            case_id=case_id,
+            message=(
+                f"fixture {file.key!r} (sha256 {file.sha256}) was never uploaded; the evalkit "
+                "CLI uploads it from the `fixtures:` directory in evalkit.yaml when you sync"
+            ),
+        )
+        for case_id, file in wanted
+        if file.sha256 not in known
+    ]
+
+
+def _path_tags(source_path: Path | None, root: Path | None) -> list[str]:
+    if source_path is None or root is None:
+        return []
+    return list(source_path.resolve().relative_to(root.resolve()).parent.parts)
+
+
+def incoming_case(
+    case: Case, source: str, files: list[CaseFile], extra_tags: list[str]
+) -> IncomingCase:
+    return IncomingCase(
+        case_id=case.id,
+        content_hash=content_hash(case, [f.model_dump() for f in files]),
+        input=dict(case.input),
+        expected=case.expected,
+        evaluators=case.evaluators.as_metadata(),
+        tags=list(case.tags),
+        path_tags=[tag for tag in dict.fromkeys(extra_tags) if tag not in case.tags],
+        source=source,
+        files=files,
+    )
+
+
+async def import_cases(
+    catalog: CaseCatalog,
+    *,
+    workspace: WorkspaceRef,
+    dataset: str,
+    kind: DatasetKind,
+    sourced_cases: list[tuple[Case, Path | None]],
+    overlays: list[tuple[Overlay, str]],
+    findings: list[Finding],
+    full: bool,
+    known_custom_evaluators: set[str],
+    actor_id: int,
+    source: SyncSource,
+    detail: dict,
+    entries: dict[str, FixtureEntry] | None = None,
+    root: Path | None = None,
+) -> ImportReport:
+    cases = [case for case, _ in sourced_cases]
+    entries = entries or {}
+    findings = list(findings)
+    findings.extend(
+        lint_cases(
+            cases,
+            [overlay for overlay, _ in overlays],
+            known_custom_evaluators=known_custom_evaluators,
+            other_cases=await catalog.workspace_cases(workspace, exclude_dataset=dataset),
+        )
+    )
+    existing = await catalog.overlays_on(
+        workspace, [overlay.extends for overlay, _ in overlays], exclude_dataset=dataset
+    )
+    findings.extend(overlay_clashes([overlay for overlay, _ in overlays], existing))
+    findings.extend(fixture_findings(cases, entries, warn_unused=False))
+    if has_errors(findings):
+        return ImportReport(findings=findings, aborted=True)
+    findings.extend(await missing_blobs(catalog, cases, entries))
+    if has_errors(findings):
+        return ImportReport(findings=findings, aborted=True)
+    if not sourced_cases and not overlays and not await catalog.has_dataset(workspace, dataset):
+        return ImportReport(findings=findings)
+    result = await catalog.sync(
+        workspace=workspace,
+        dataset=dataset,
+        kind=kind,
+        incoming=[
+            incoming_case(
+                case,
+                source=dataset if source_path is None else source_path.name,
+                files=case_files(case, entries),
+                extra_tags=_path_tags(source_path, root),
+            )
+            for case, source_path in sourced_cases
+        ],
+        overlays=[incoming_overlay(overlay, file) for overlay, file in overlays],
+        full=full,
+        actor_id=actor_id,
+        source=source,
+        detail=detail,
+    )
+    return ImportReport(
+        created=len(result.created),
+        patched=len(result.patched),
+        deleted=len(result.removed),
+        overlays_applied=result.overlays,
+        findings=findings,
+        review=ReviewReport(now_unreviewed=result.now_unreviewed, carried=result.carried),
+    )
+
+
+async def import_dataset(
+    catalog: CaseCatalog,
+    *,
+    workspace: WorkspaceRef,
+    dataset: str,
+    files: list[Path],
+    full: bool,
+    known_custom_evaluators: set[str],
+    actor_id: int,
+    detail: dict,
+    root: Path | None = None,
+) -> ImportReport:
+    sourced_cases, overlays, findings = load_files(files)
+    entries, registry_findings = load_registries([p for p in files if p.name == REGISTRY_FILENAME])
+    sourced: list[tuple[Case, Path | None]] = list(sourced_cases)
+    return await import_cases(
+        catalog,
+        workspace=workspace,
+        dataset=dataset,
+        kind=DatasetKind.SUITE,
+        sourced_cases=sourced,
+        overlays=overlays,
+        findings=findings + registry_findings,
+        full=full,
+        known_custom_evaluators=known_custom_evaluators,
+        actor_id=actor_id,
+        source=SyncSource.GIT,
+        detail=detail,
+        entries=entries,
+        root=root,
+    )

evalkit/core/judge.py

@@ -0,0 +1,78 @@
+import httpx
+from pydantic import BaseModel, ValidationError
+
+from evalkit.contracts.settings import JudgeSettings
+
+
+class JudgeUsage(BaseModel):
+    input_tokens: int = 0
+    output_tokens: int = 0
+
+
+def judge_headers(session_id: str | None, evaluator_name: str) -> dict[str, str]:
+    headers = {"x-litellm-tags": f"eval-judge,evaluator:{evaluator_name}"}
+    if session_id:
+        headers["x-litellm-session-id"] = session_id
+    return headers
+
+
+async def call_judge[T: BaseModel](
+    cfg: JudgeSettings,
+    *,
+    api_key: str | None,
+    system_prompt: str,
+    user_prompt: str,
+    output_type: type[T],
+    session_id: str | None = None,
+    evaluator_name: str,
+    model: str | None = None,
+) -> tuple[T, JudgeUsage]:
+    usage = JudgeUsage()
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_prompt},
+    ]
+    async with httpx.AsyncClient(
+        base_url=cfg.base_url,
+        headers={"Authorization": f"Bearer {api_key}"} if api_key else {},
+        timeout=300,
+    ) as client:
+        for attempt in range(2):
+            response = await client.post(
+                "/chat/completions",
+                json={
+                    "model": model or cfg.model,
+                    "messages": messages,
+                    "response_format": {
+                        "type": "json_schema",
+                        "json_schema": {
+                            "name": output_type.__name__,
+                            "schema": output_type.model_json_schema(),
+                        },
+                    },
+                    "max_tokens": cfg.max_tokens,
+                    "reasoning_effort": cfg.reasoning_effort,
+                },
+                headers=judge_headers(session_id, evaluator_name),
+            )
+            response.raise_for_status()
+            body = response.json()
+            call_usage = body.get("usage") or {}
+            usage.input_tokens += call_usage.get("prompt_tokens") or 0
+            usage.output_tokens += call_usage.get("completion_tokens") or 0
+            content = body["choices"][0]["message"]["content"] or ""
+            try:
+                return output_type.model_validate_json(content), usage
+            except ValidationError as exc:
+                if attempt == 1:
+                    raise
+                messages = messages + [
+                    {"role": "assistant", "content": content},
+                    {
+                        "role": "user",
+                        "content": "Your previous response was not valid JSON for the "
+                        f"required schema: {exc.errors()[0]['msg']}. "
+                        "Respond again with only the corrected JSON object.",
+                    },
+                ]
+    raise AssertionError("unreachable")

evalkit/core/lint.py

@@ -0,0 +1,265 @@
+"""Deterministic quality lint over validated eval cases — no LLM.
+
+Severity is calibrated to confidence and blast radius: errors block sync and
+fire only where the case is genuinely broken; warnings are heuristics that
+never block. evalkit knows no client tools, so the jargon check looks for
+tool-ish vocabulary patterns (mcp__ prefixes, double-underscore tokens,
+snake_case call names) rather than a known-tool list.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Literal
+
+from evalkit.contracts.cases import Case, FixtureEntry, Overlay
+from evalkit.contracts.harness import input_errors
+
+_DATE_ANCHOR = re.compile(
+    r"\d{4}-\d{2}-\d{2}"
+    r"|\d{1,2}[-/ ](?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*[-/ ]?\d{2,4}"
+    r"|\b(?:january|february|march|april|may|june|july|august|september|october"
+    r"|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)[a-z]*\s+\d{4}"
+    r"|\b20\d{2}\b"
+    r"|\bFY\s?\d{2,4}\b"
+    r"|\bQ[1-4]\b"
+    r"|\bas of\b"
+    r"|\bas on\b"
+    r"|\bbetween\b.{1,40}\band\b",
+    re.IGNORECASE,
+)
+
+_NUMERIC_VALUE = re.compile(
+    r"\d+\.\d+|\d+\s*%|[₹$€£]\s*\d|\b(?:million|billion|crore|crores|lakh|lakhs|bps)\b",
+    re.IGNORECASE,
+)
+
+_DOUBLE_UNDERSCORE_TOKEN = re.compile(r"\b\w+__\w+\b")
+_SNAKE_CALL = re.compile(r"\b[a-z][a-z0-9]+(?:_[a-z0-9]+)+\s*\(")
+
+
+@dataclass
+class Finding:
+    severity: Literal["error", "warning"]
+    code: str
+    case_id: str
+    message: str
+
+
+def _blackbox_texts(case: Case) -> list[tuple[str, str]]:
+    texts: list[tuple[str, str]] = [("input.question", str(case.input.get("question", "")))]
+    rubrics = case.evaluators.output.answer_rubrics
+    if rubrics is not None:
+        for r in rubrics.rubrics:
+            texts.append(("answer_rubrics.rubric", r.rubric))
+    tr = case.evaluators.trajectory.trajectory_rubrics
+    if tr is not None:
+        for rubric in tr.rubrics:
+            texts.append(("trajectory_rubrics.rubric", rubric))
+    pj = case.evaluators.output.presentation_judge
+    if pj is not None:
+        for pr in pj.rubrics:
+            texts.append(("presentation_judge.rubric", pr.rubric or pr.forbid or ""))
+    return texts
+
+
+def _jargon_hits(text: str) -> set[str]:
+    hits: set[str] = set()
+    if "mcp__" in text.lower():
+        hits.add("mcp__ prefix")
+    hits.update(_DOUBLE_UNDERSCORE_TOKEN.findall(text))
+    hits.update(m.rstrip("( \t") for m in _SNAKE_CALL.findall(text))
+    return hits
+
+
+def _e_jargon(case: Case) -> list[Finding]:
+    findings = []
+    for surface, text in _blackbox_texts(case):
+        hits = _jargon_hits(text)
+        if hits:
+            findings.append(
+                Finding(
+                    severity="error",
+                    code="E-JARGON",
+                    case_id=case.id,
+                    message=(
+                        f"{surface} leaks agent internals ({', '.join(sorted(hits))}) "
+                        "— black-box surfaces must stay free of tool/skill names; "
+                        "move tool checks into tool_trajectory_match / required_tools_called "
+                        "or a custom evaluator"
+                    ),
+                )
+            )
+    return findings
+
+
+def _e_reserved_tags(case_id: str, tags: list[str]) -> list[Finding]:
+    return [
+        Finding(
+            severity="error",
+            code="E-RESERVED-TAG",
+            case_id=case_id,
+            message=(
+                f"tag '{t}' is reserved for run-time case selection "
+                "(the working split) — pick another tag name"
+            ),
+        )
+        for t in tags
+        if t.startswith("working:")
+    ]
+
+
+def _w_custom_unknown(
+    case_id: str, custom: dict, known_custom_evaluators: set[str]
+) -> list[Finding]:
+    return [
+        Finding(
+            severity="warning",
+            code="W-CUSTOM-UNKNOWN",
+            case_id=case_id,
+            message=(
+                f"custom evaluator '{name}' is not a remote evaluator here: create it with "
+                "evalkit evaluators create, or pass a Python evaluator with this name to "
+                "run_experiment"
+            ),
+        )
+        for name in sorted(set(custom) - known_custom_evaluators)
+    ]
+
+
+def _w_undated(case: Case) -> list[Finding]:
+    rubrics = case.evaluators.output.answer_rubrics
+    if rubrics is None:
+        return []
+    rubric_text = " ".join(r.rubric for r in rubrics.rubrics)
+    if not _NUMERIC_VALUE.search(rubric_text):
+        return []
+    if _DATE_ANCHOR.search(str(case.input.get("question", "")) + " " + rubric_text):
+        return []
+    return [
+        Finding(
+            severity="warning",
+            code="W-UNDATED",
+            case_id=case.id,
+            message=(
+                "rubrics pin numeric values but neither the input nor the rubrics "
+                "carry a date anchor — pinned ground truth will drift"
+            ),
+        )
+    ]
+
+
+def _w_thin_rubrics(case: Case) -> list[Finding]:
+    rubrics = case.evaluators.output.answer_rubrics
+    if rubrics is None or len(rubrics.rubrics) >= 2:
+        return []
+    return [
+        Finding(
+            severity="warning",
+            code="W-THIN-RUBRICS",
+            case_id=case.id,
+            message="answer_rubrics carries a single rubric — thin coverage for a whole case",
+        )
+    ]
+
+
+def fixture_findings(
+    cases: list[Case], entries: dict[str, FixtureEntry], *, warn_unused: bool = True
+) -> list[Finding]:
+    findings: list[Finding] = []
+    used: set[str] = set()
+    for case in cases:
+        for key in case.fixtures:
+            used.add(key)
+            if key not in entries:
+                findings.append(
+                    Finding(
+                        severity="error",
+                        code="E-FIXTURE-UNKNOWN-KEY",
+                        case_id=case.id,
+                        message=f"fixture key {key!r} is not declared in fixtures.yaml",
+                    )
+                )
+    for key in entries:
+        if warn_unused and key not in used:
+            findings.append(
+                Finding(
+                    severity="warning",
+                    code="W-FIXTURE-UNUSED",
+                    case_id=key,
+                    message=f"fixture {key!r} is declared but no case references it",
+                )
+            )
+    return findings
+
+
+def lint(
+    cases: list[Case],
+    overlays: list[Overlay],
+    *,
+    known_custom_evaluators: set[str],
+    other_cases: Mapping[str, str] | None = None,
+    input_schema: dict | None = None,
+) -> list[Finding]:
+    findings: list[Finding] = []
+
+    seen: set[str] = set()
+    for case in cases:
+        if case.id in seen:
+            findings.append(
+                Finding(
+                    severity="error",
+                    code="E-DUP-ID",
+                    case_id=case.id,
+                    message="duplicate case id in this import",
+                )
+            )
+        seen.add(case.id)
+    seen_targets: set[str] = set()
+    for overlay in overlays:
+        if overlay.extends in seen_targets:
+            findings.append(
+                Finding(
+                    severity="error",
+                    code="E-DUP-ID",
+                    case_id=overlay.extends,
+                    message="multiple overlays extend the same case in this import",
+                )
+            )
+        seen_targets.add(overlay.extends)
+
+    for case in cases:
+        findings.extend(_e_jargon(case))
+        findings.extend(_e_reserved_tags(case.id, case.tags))
+        findings.extend(_w_custom_unknown(case.id, case.evaluators.custom, known_custom_evaluators))
+        findings.extend(_w_undated(case))
+        findings.extend(_w_thin_rubrics(case))
+
+    if input_schema is not None:
+        for case in cases:
+            for error in input_errors(input_schema, case.input):
+                findings.append(
+                    Finding(severity="error", code="E-INPUT-SCHEMA", case_id=case.id, message=error)
+                )
+
+    for overlay in overlays:
+        findings.extend(_e_reserved_tags(overlay.extends, overlay.tags))
+        findings.extend(
+            _w_custom_unknown(overlay.extends, overlay.evaluators.custom, known_custom_evaluators)
+        )
+        if other_cases is not None and overlay.extends not in other_cases:
+            findings.append(
+                Finding(
+                    severity="error",
+                    code="E-OVERLAY-TARGET",
+                    case_id=overlay.extends,
+                    message="overlay extends a case id no other dataset in this workspace holds",
+                )
+            )
+    return findings
+
+
+def has_errors(findings: list[Finding]) -> bool:
+    return any(f.severity == "error" for f in findings)

evalkit/core/overlays.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from pydantic import BaseModel, Field
+
+from evalkit.contracts.cases import Overlay
+from evalkit.core.lint import Finding
+from evalkit.store.base import IncomingOverlay
+
+
+class OverlayBlock(BaseModel):
+    source: str
+    agents: list[str]
+    tags: list[str] = Field(default_factory=list)
+    evaluators: dict = Field(default_factory=dict)
+    source_file: str | None = None
+
+
+def overlay_evaluators(overlay: Overlay) -> dict:
+    families = {
+        "custom": overlay.evaluators.custom,
+        "trajectory": overlay.evaluators.trajectory.as_metadata(),
+    }
+    return {family: block for family, block in families.items() if block}
+
+
+def incoming_overlay(overlay: Overlay, source_file: str | None) -> IncomingOverlay:
+    return IncomingOverlay(
+        target_case_id=overlay.extends,
+        agents=list(overlay.agents),
+        tags=list(overlay.tags),
+        evaluators=overlay_evaluators(overlay),
+        source_file=source_file,
+    )
+
+
+def apply_overlay_blocks(evaluators: dict, blocks: list[OverlayBlock], agent: str | None) -> dict:
+    out = {family: dict(block) for family, block in evaluators.items()}
+    if not agent:
+        return out
+    for block in sorted(blocks, key=lambda b: b.source):
+        if agent.partition("@")[0] not in block.agents:
+            continue
+        for family, named in block.evaluators.items():
+            out[family] = {**out.get(family, {}), **named}
+    return out
+
+
+def _names(evaluators: dict) -> set[tuple[str, str]]:
+    return {(family, name) for family, block in evaluators.items() for name in block}
+
+
+def overlay_clashes(
+    overlays: list[Overlay], existing: Mapping[str, list[OverlayBlock]]
+) -> list[Finding]:
+    findings: list[Finding] = []
+    for overlay in overlays:
+        ours = _names(overlay_evaluators(overlay))
+        for block in sorted(existing.get(overlay.extends, []), key=lambda b: b.source):
+            agents = sorted(set(overlay.agents) & set(block.agents))
+            names = sorted(ours & _names(block.evaluators))
+            if agents and names:
+                findings.append(
+                    Finding(
+                        severity="error",
+                        code="E-OVERLAY-CLASH",
+                        case_id=overlay.extends,
+                        message=(
+                            f"{block.source} already sets "
+                            f"{', '.join(f'{family}.{name}' for family, name in names)} "
+                            f"for agents {', '.join(agents)}"
+                        ),
+                    )
+                )
+    return findings

evalkit/core/paths.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+
+class UnsafePathError(ValueError):
+    pass
+
+
+def is_safe_path(path: str) -> bool:
+    return all(segment not in ("", ".", "..") for segment in path.lstrip("/").split("/"))
+
+
+def require_safe_path(path: str) -> str:
+    if not is_safe_path(path):
+        raise UnsafePathError(f"unsafe path {path!r}")
+    return path.lstrip("/")
+
+
+def is_safe_run_id(run_id: str) -> bool:
+    return "/" not in run_id and is_safe_path(run_id)
+
+
+def require_safe_run_id(run_id: str) -> str:
+    if not is_safe_run_id(run_id):
+        raise UnsafePathError(f"unsafe run id {run_id!r}")
+    return run_id