evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/client/sync.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from evalkit.client.api import SyncApi
+from evalkit.contracts.cases import REGISTRY_FILENAME, is_case_yaml, load_cases_from_yaml
+from evalkit.core.fixtures import load_registries, sha256_hex
+from evalkit.core.workspace import Reference, parse_reference
+from evalkit.store.base import GitRevision, Selection
+
+
+@dataclass
+class ResolvedPath:
+    ref: Reference
+    dataset_dir: Path | None
+    selection: Selection
+    subpath: str | None = None
+
+
+def _case_ids(file: Path) -> list[str]:
+    cases, _ = load_cases_from_yaml(file)
+    return [case.id for case in cases]
+
+
+def _selection_for(subpath: str, dataset_dir: Path | None) -> Selection:
+    if not subpath:
+        return Selection(all=True)
+    on_disk = dataset_dir / subpath if dataset_dir is not None else None
+    if on_disk is not None and on_disk.is_dir():
+        return Selection(tag=on_disk.name)
+    if on_disk is not None and on_disk.is_file():
+        return Selection(case_ids=_case_ids(on_disk))
+    if Path(subpath).suffix in (".yaml", ".yml"):
+        raise ValueError(f"{subpath} is not on disk; selecting a file needs its case ids")
+    return Selection(tag=Path(subpath).name)
+
+
+def _on_disk(candidate: Path, cases_root: Path | None) -> tuple[Path, str]:
+    resolved = candidate.resolve()
+    if cases_root is not None and resolved.is_relative_to(cases_root.resolve()):
+        relative = resolved.relative_to(cases_root.resolve())
+        if not relative.parts:
+            raise ValueError(f"{candidate} is the cases root; pass a dataset directory under it")
+        return cases_root.resolve() / relative.parts[0], Path(*relative.parts[1:]).as_posix()
+    if resolved.is_dir():
+        return resolved, ""
+    return resolved.parent, resolved.name
+
+
+def resolve_path(arg: str, cases_root: Path | None, my_workspace: str) -> ResolvedPath:
+    candidate = Path(arg)
+    if candidate.exists():
+        dataset_dir, subpath = _on_disk(candidate, cases_root)
+        ref = Reference(dataset_dir.name, my_workspace)
+        if subpath == ".":
+            subpath = ""
+        return ResolvedPath(ref, dataset_dir, _selection_for(subpath, dataset_dir), subpath or None)
+    ref_text, _, subpath = arg.partition("/")
+    ref = parse_reference(ref_text, my_workspace)
+    own = ref.workspace == my_workspace
+    if own and cases_root is None:
+        raise ValueError(
+            f"{ref.name} is in your workspace, so it syncs from disk: set `cases:` in "
+            "evalkit.yaml, or pass the dataset's directory"
+        )
+    on_disk = cases_root / ref.name if cases_root is not None else None
+    if on_disk is not None and not on_disk.is_dir():
+        on_disk = None
+    if own and on_disk is None:
+        raise ValueError(f"no dataset directory {ref.name!r} under {cases_root}")
+    return ResolvedPath(
+        ref, on_disk if own else None, _selection_for(subpath, on_disk), subpath or None
+    )
+
+
+def resolve_dataset(dataset: str, workspace: str | None, my_workspace: str) -> Reference:
+    ref = parse_reference(dataset, my_workspace)
+    if "@" in dataset and workspace:
+        raise ValueError("dataset already names a workspace; drop the workspace argument")
+    if workspace:
+        return Reference(ref.name, workspace)
+    return ref
+
+
+def _git(args: list[str], cwd: Path) -> str | None:
+    if shutil.which("git") is None:
+        return None
+    result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True)
+    return result.stdout.strip() if result.returncode == 0 else None
+
+
+def git_revision(path: Path) -> GitRevision:
+    cwd = path if path.is_dir() else path.parent
+    sha = _git(["rev-parse", "HEAD"], cwd)
+    if sha is None:
+        return GitRevision()
+    status = _git(["status", "--porcelain", "--", str(path)], cwd)
+    return GitRevision(
+        sha=sha, branch=_git(["rev-parse", "--abbrev-ref", "HEAD"], cwd), dirty=bool(status)
+    )
+
+
+def _nearest_registry(start: Path) -> Path | None:
+    for directory in (start, *start.parents):
+        candidate = directory / REGISTRY_FILENAME
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def sync_files(dataset_dir: Path, subpath: str | None = None) -> list[tuple[str, Path]]:
+    start = dataset_dir / subpath if subpath else dataset_dir
+    if not start.exists():
+        raise ValueError(
+            f"{subpath} is not a folder or file under {dataset_dir}; sync needs one on disk"
+        )
+    if start.is_dir():
+        files = sorted(
+            f
+            for pattern in ("*.yaml", "*.yml")
+            for f in start.rglob(pattern)
+            if f.is_file()
+            and not any(part.startswith(".") for part in f.relative_to(dataset_dir).parts)
+            and is_case_yaml(f.relative_to(dataset_dir))
+        )
+    else:
+        files = [start]
+    out = [(f.relative_to(dataset_dir).as_posix(), f) for f in files]
+    registry = _nearest_registry(dataset_dir)
+    if registry is not None:
+        out.append((REGISTRY_FILENAME, registry))
+    return out
+
+
+def upload_fixtures(
+    client: SyncApi, files: list[tuple[str, Path]], fixtures_dir: Path
+) -> list[str]:
+    registry = [path for name, path in files if name == REGISTRY_FILENAME]
+    entries, _ = load_registries(registry)
+    referenced: set[str] = set()
+    for name, path in files:
+        if name == REGISTRY_FILENAME:
+            continue
+        cases, _ = load_cases_from_yaml(path)
+        referenced.update(key for case in cases for key in case.fixtures)
+    uploaded: list[str] = []
+    for key in sorted(referenced):
+        entry = entries.get(key)
+        if entry is None:
+            continue
+        url = f"/api/fixtures/sha256/{entry.sha256}"
+        if client.head(url).status_code == 200:
+            continue
+        local = fixtures_dir / key
+        if not local.is_file():
+            raise ValueError(
+                f"fixture {key!r} is registered but {local} does not exist and the server has "
+                f"no bytes with sha256 {entry.sha256}"
+            )
+        data = local.read_bytes()
+        if sha256_hex(data) != entry.sha256:
+            raise ValueError(
+                f"fixture {key!r} on disk has sha256 {sha256_hex(data)} but {REGISTRY_FILENAME} "
+                f"says {entry.sha256}; update the registry entry"
+            )
+        client.put(url, content=data, retry=True)
+        uploaded.append(key)
+    return uploaded
+
+
+def sync_dataset(
+    client: SyncApi,
+    dataset_dir: Path,
+    *,
+    name: str,
+    workspace: str,
+    revision: GitRevision,
+    subpath: str | None = None,
+    fixtures_dir: Path | None = None,
+    actor: str | None = None,
+) -> dict:
+    params: dict[str, object] = {
+        "dataset": name,
+        "workspace": workspace,
+        "prune": subpath is None,
+    }
+    for key, value in revision.model_dump().items():
+        if value is not None:
+            params[key] = value
+    if actor is not None:
+        params["actor"] = actor
+    files = sync_files(dataset_dir, subpath)
+    if fixtures_dir is not None:
+        upload_fixtures(client, files, fixtures_dir)
+    return client.post_files("/api/sync", files, params=params).json()

evalkit/config.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, model_validator
+
+from evalkit.credentials import CredentialsConfig, load_dotenv_beside
+
+
+class MissingEnvError(ValueError):
+    def __init__(self, source: Path, key: str, name: str):
+        self.source = source
+        self.key = key
+        self.name = name
+        super().__init__(f"{source}: {key} needs env {name} (or env:{name}:<default>)")
+
+
+class ConfigModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", hide_input_in_errors=True)
+
+
+class ServerConfig(ConfigModel):
+    base_url: str
+    accent_color: str | None = None
+
+
+class DbConfig(ConfigModel):
+    url: str
+    migrate_on_start: bool = False
+
+
+class BlobConfig(ConfigModel):
+    url: str
+
+
+class CostMapConfig(ConfigModel):
+    source: str = "bundled"
+
+
+class AuthConfig(ConfigModel):
+    token_ttl_hours: int = 720
+
+
+class McpOAuthConfig(ConfigModel):
+    issuer: str
+    jwks_uri: str
+    audience: str | None = None
+    algorithm: str = "RS256"
+    scope: str = "evalkit:read"
+
+
+class McpConfig(ConfigModel):
+    oauth: McpOAuthConfig | None = None
+
+
+class OtelConfig(ConfigModel):
+    settle_seconds: int = 10
+    max_batch_bytes: int = 16 * 1024 * 1024
+    max_session_bytes: int = 384 * 1024 * 1024
+    import_page_traces: int = 25
+
+
+class SentryConfig(ConfigModel):
+    dsn: str
+    environment: str | None = None
+    traces_sample_rate: float = 0.0
+    max_request_body_size: Literal["never", "small", "medium", "always"] = "never"
+
+
+MB = 1024 * 1024
+
+
+class LimitsConfig(ConfigModel):
+    fixture_upload_bytes: int = 500 * MB
+    experiment_blob_bytes: int = 500 * MB
+    case_upload_bytes: int = 50 * MB
+    request_bytes: int = 10 * MB
+
+
+class RunnerConfig(ConfigModel):
+    cost_map: CostMapConfig = CostMapConfig()
+
+
+# TODO: remove in 0.5.0, two minor versions after 0.3.0 moved credentials and
+# judge_default to the database; _shell and _moved_to_db_commands only serve that check.
+def _shell(value: object) -> str:
+    text = str(value)
+    return f"${text.removeprefix('env:')}" if text.startswith("env:") else text
+
+
+def _moved_to_db_commands(data: dict) -> list[str]:
+    credentials = data.get("credentials") or {}
+    judge = data.get("judge_default") or {}
+    commands = [
+        f"evalkit llm-providers create <name> --base-url {_shell(url)} "
+        f"--api-key-env {str(ref).removeprefix('env:')}"
+        for url, ref in (credentials.get("judges") or {}).items()
+    ]
+    if judge:
+        commands.append(
+            f"evalkit judge-default set --provider <name> "
+            f"--model {_shell(judge.get('model', '<model>'))}"
+        )
+    commands += [
+        f"evalkit agents edit {ref} --api-key-env {str(env).removeprefix('env:')}"
+        for ref, env in (credentials.get("agents") or {}).items()
+    ]
+    commands += [
+        f"evalkit evaluators edit {ref} --api-key-env {str(env).removeprefix('env:')}"
+        for ref, env in (credentials.get("evaluators") or {}).items()
+    ]
+    return commands
+
+
+class Config(RunnerConfig):
+    server: ServerConfig
+    db: DbConfig
+    blob: BlobConfig
+    auth: AuthConfig = AuthConfig()
+    mcp: McpConfig = McpConfig()
+    otel: OtelConfig = OtelConfig()
+    limits: LimitsConfig = LimitsConfig()
+    sentry: SentryConfig | None = None
+    encryption_key: str | None = None
+    allow_private_endpoints: bool = False
+
+    # TODO: remove in 0.5.0, two minor versions after 0.3.0 moved credentials and
+    # judge_default to the database.
+    @model_validator(mode="before")
+    @classmethod
+    def _keys_moved_to_db(cls, data: object) -> object:
+        if isinstance(data, dict) and ("credentials" in data or "judge_default" in data):
+            commands = "\n".join(f"  {c}" for c in _moved_to_db_commands(data))
+            raise ValueError(
+                "credentials and judge_default moved to the database. Remove them from the "
+                "config, start the server, then run (as an admin, with the keys in your env):\n"
+                f"{commands}"
+            )
+        return data
+
+
+LOCAL_CONFIG_FILENAME = "evalkit.yaml"
+
+
+class LocalConfig(RunnerConfig):
+    path: Path
+    credentials: CredentialsConfig = CredentialsConfig()
+    cases: Path | None = None
+    fixtures: Path | None = None
+
+    @classmethod
+    def find(cls, start: Path) -> LocalConfig | None:
+        start = start.resolve()
+        for directory in (start, *start.parents):
+            candidate = directory / LOCAL_CONFIG_FILENAME
+            if candidate.is_file():
+                return load_local_config(candidate)
+        return None
+
+    def environ(self) -> dict[str, str]:
+        return load_dotenv_beside(self.path)
+
+    def resolve_env(self, *, scoring: bool = True) -> LocalConfig:
+        data = self.model_dump()
+        if scoring:
+            environ = self.environ()
+            data["cost_map"] = resolve_env_refs(data["cost_map"], self.path, "cost_map", environ)
+        return LocalConfig.model_validate(data)
+
+
+UNRESOLVED_KEYS = frozenset({"credentials"})
+
+
+def resolve_env_refs(obj: object, source: Path, key: str, environ: Mapping[str, str]) -> object:
+    if isinstance(obj, dict):
+        return {
+            k: v
+            if k in UNRESOLVED_KEYS and not key
+            else resolve_env_refs(v, source, _join(key, k), environ)
+            for k, v in obj.items()
+        }
+    if isinstance(obj, list):
+        return [
+            resolve_env_refs(v, source, f"{key}[{_item_label(v, i)}]", environ)
+            for i, v in enumerate(obj)
+        ]
+    if isinstance(obj, str) and obj.startswith("env:"):
+        name, has_default, default = obj[4:].partition(":")
+        if has_default:
+            return environ.get(name, default)
+        if name not in environ:
+            raise MissingEnvError(source, key, name)
+        return environ[name]
+    return obj
+
+
+def _join(key: str, child: object) -> str:
+    return f"{key}.{child}" if key else str(child)
+
+
+def _item_label(item: object, index: int) -> object:
+    if isinstance(item, dict) and isinstance(item.get("name"), str):
+        return item["name"]
+    return index
+
+
+def load_config(path: Path) -> Config:
+    raw = yaml.safe_load(path.read_text())
+    data = resolve_env_refs(raw, path, "", os.environ)
+    return Config.model_validate(data)
+
+
+def load_local_config(path: Path) -> LocalConfig:
+    data = yaml.safe_load(path.read_text()) or {}
+    assert isinstance(data, dict)
+    base = path.resolve().parent
+    cfg = LocalConfig.model_validate({**data, "path": path.resolve()})
+    return cfg.model_copy(
+        update={
+            "cases": (base / cfg.cases).resolve() if cfg.cases is not None else None,
+            "fixtures": (base / cfg.fixtures).resolve() if cfg.fixtures is not None else None,
+        }
+    )

evalkit/contracts/atif.py

@@ -0,0 +1,115 @@
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+ACCEPTED_SCHEMA_VERSIONS = frozenset({"ATIF-v1.8"})
+
+
+class TrajectoryValidationError(Exception):
+    pass
+
+
+class AtifModel(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+
+class AtifAgent(AtifModel):
+    name: str
+    version: str | None = None
+    model_name: str | None = None
+    tool_definitions: list[dict[str, Any]] | None = None
+    extra: dict[str, Any] | None = None
+
+
+class ContentPart(AtifModel):
+    type: str
+    text: str | None = None
+    media_type: str | None = None
+    path: str | None = None
+
+
+class ToolCall(AtifModel):
+    tool_call_id: str
+    function_name: str
+    arguments: dict[str, Any]
+    extra: dict[str, Any] | None = None
+
+
+class SubagentTrajectoryRef(AtifModel):
+    trajectory_id: str | None = None
+    trajectory_path: str | None = None
+    session_id: str | None = None
+    extra: dict[str, Any] | None = None
+
+
+class ObservationResult(AtifModel):
+    source_call_id: str | None = None
+    content: str | list[ContentPart] | None = None
+    subagent_trajectory_ref: list[SubagentTrajectoryRef] | None = None
+    extra: dict[str, Any] | None = None
+
+
+class Observation(AtifModel):
+    results: list[ObservationResult]
+
+
+class StepMetrics(AtifModel):
+    prompt_tokens: int | None = None
+    completion_tokens: int | None = None
+    cached_tokens: int | None = None
+    cost_usd: float | None = None
+    extra: dict[str, Any] | None = None
+
+
+class FinalMetrics(AtifModel):
+    total_prompt_tokens: int | None = None
+    total_completion_tokens: int | None = None
+    total_cached_tokens: int | None = None
+    total_cost_usd: float | None = None
+    total_steps: int | None = None
+
+
+class Step(AtifModel):
+    step_id: int
+    timestamp: str | None = None
+    source: Literal["user", "agent", "system"]
+    message: str | list[ContentPart] | None = None
+    reasoning_content: str | None = None
+    tool_calls: list[ToolCall] | None = None
+    observation: Observation | None = None
+    metrics: StepMetrics | None = None
+    llm_call_count: int | None = None
+    model_name: str | None = None
+
+
+class Trajectory(AtifModel):
+    schema_version: str
+    session_id: str | None = None
+    trajectory_id: str | None = None
+    agent: AtifAgent | None = None
+    steps: list[Step]
+    final_metrics: FinalMetrics | None = None
+    subagent_trajectories: list["Trajectory"] | None = None
+    extra: dict[str, Any] | None = None
+
+
+def validate_trajectory(data: dict) -> Trajectory:
+    version = data.get("schema_version")
+    if version not in ACCEPTED_SCHEMA_VERSIONS:
+        raise TrajectoryValidationError(
+            f"unsupported schema_version {version!r}; accepted: {sorted(ACCEPTED_SCHEMA_VERSIONS)}"
+        )
+    try:
+        return Trajectory.model_validate(data)
+    except ValidationError as exc:
+        missing = "; ".join(
+            f"{'.'.join(str(loc) for loc in err['loc'])}: {err['msg']}" for err in exc.errors()
+        )
+        raise TrajectoryValidationError(f"invalid ATIF document: {missing}") from exc
+
+
+def subagent_link_warnings(trajectory: Trajectory) -> list[str]:
+    missing = sum(1 for sub in trajectory.subagent_trajectories or [] if not sub.trajectory_id)
+    if not missing:
+        return []
+    return [f"{missing} embedded subagent(s) have no trajectory_id (required since ATIF-v1.7)"]