evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/core/promotion.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Any
+
+from evalkit.core.review import CaseReview, review_summary
+
+__all__ = [
+    "MissingTargetDataset",
+    "OverlayCopy",
+    "PromotionPlan",
+    "PromotionTarget",
+    "TwinState",
+    "twin_state",
+    "twin_summary",
+]
+
+
+class MissingTargetDataset(Exception):
+    def __init__(self, ref: str):
+        self.ref = ref
+        super().__init__(
+            f"{ref} does not exist; create it before approving (evalkit datasets create)"
+        )
+
+
+class PromotionTarget(Exception):
+    def __init__(self, workspace: str, source: str, hint: str):
+        self.workspace = workspace
+        self.source = source
+        super().__init__(f"{workspace} is the promotion target of {source}; {hint}")
+
+
+@dataclass(frozen=True)
+class OverlayCopy:
+    source_dataset: str
+    agents: tuple[str, ...]
+    tags: tuple[str, ...]
+    evaluators: dict[str, Any]
+    source_file: str | None
+
+    def key(self) -> tuple[str, tuple[str, ...], tuple[str, ...], str]:
+        return (self.source_dataset, self.agents, self.tags, repr(sorted(self.evaluators.items())))
+
+
+@dataclass(frozen=True)
+class PromotionPlan:
+    source_ref: str
+    target_ref: str
+    target_workspace_id: int
+    target_dataset_id: int
+    case_ref: int
+    case_id: str
+    content_hash: str
+    path_tags: list[str]
+    source: str | None
+    overlays: list[OverlayCopy]
+    overlay_targets: dict[str, int]
+
+
+class TwinState(StrEnum):
+    ABSENT = "absent"
+    SAME = "same"
+    DIFFERS = "differs"
+
+
+def twin_state(mine: str, other: str | None) -> TwinState:
+    if other is None:
+        return TwinState.ABSENT
+    return TwinState.SAME if mine == other else TwinState.DIFFERS
+
+
+def twin_summary(
+    *,
+    workspace: str,
+    dataset: str,
+    case_id: str,
+    mine: str,
+    other: str | None,
+    review: CaseReview | None,
+    my_overlays: list[OverlayCopy],
+    other_overlays: list[OverlayCopy],
+) -> dict[str, Any]:
+    state = twin_state(mine, other)
+    return {
+        "workspace": workspace,
+        "state": state.value,
+        "content_hash": other,
+        "review": review_summary(review) if review is not None else None,
+        "overlays_differ": state is not TwinState.ABSENT
+        and {o.key() for o in my_overlays} != {o.key() for o in other_overlays},
+        "href": f"/datasets/{dataset}@{workspace}/cases/{case_id}",
+    }

evalkit/core/pull.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Awaitable, Callable
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel
+
+from evalkit.contracts.enums import EvaluatorKind
+from evalkit.contracts.harness import RunListItem, RunListParams
+from evalkit.core.harness import HarnessClient
+from evalkit.store.base import AgentRunRecord, EvaluationRow, RunEvalContext
+
+UNSAFE = re.compile(r"[^A-Za-z0-9._-]")
+MAX_PULL = 100
+HARNESS_PAGE = 200
+MAX_HARNESS_SCAN = 10000
+
+Exclude = Callable[[list[str]], Awaitable[set[str]]]
+
+
+class HarnessRuns(BaseModel):
+    runs: list[RunListItem]
+    total: int
+    capped: bool = False
+
+
+SCAN_NOTE = (
+    f"stopped after scanning {MAX_HARNESS_SCAN} harness runs; narrow the window with since/until"
+)
+
+
+class RunBundle(BaseModel):
+    run_id: str
+    source: Literal["app", "eval"]
+    status: str | None = None
+    created_at: datetime | None = None
+    title: str | None = None
+    run: dict
+    trajectory: dict | None = None
+    eval: RunEvalContext | None = None
+    pull_error: str | None = None
+
+
+def in_window(run: RunListItem, since: datetime | None, until: datetime | None) -> bool:
+    if run.created_at is None:
+        return True
+    return (since is None or run.created_at >= since) and (until is None or run.created_at < until)
+
+
+async def list_harness_runs(
+    harness: HarnessClient,
+    *,
+    wanted: int,
+    since: datetime | None = None,
+    until: datetime | None = None,
+    query: str | None = None,
+    user_id: str | None = None,
+    client_id: str | None = None,
+    exclude: Exclude | None = None,
+) -> HarnessRuns:
+    kept: list[RunListItem] = []
+    sessions: set[str] = set()
+    dropped = offset = 0
+    size = HARNESS_PAGE
+
+    def new_session(run: RunListItem) -> bool:
+        if run.session_id is None:
+            return True
+        if run.session_id in sessions:
+            return False
+        sessions.add(run.session_id)
+        return True
+
+    while True:
+        page = await harness.list_runs(
+            RunListParams(
+                query=query,
+                user_id=user_id,
+                client_id=client_id,
+                limit=size,
+                offset=offset,
+                since=since,
+                until=until,
+            )
+        )
+        window = [run for run in page.runs if in_window(run, since, until)]
+        excluded = await exclude([run.run_id for run in window]) if exclude and window else set()
+        fresh = [run for run in window if run.run_id not in excluded and new_session(run)]
+        kept += fresh
+        dropped += len(page.runs) - len(fresh)
+        offset += len(page.runs)
+        oldest = page.runs[-1].created_at if page.runs else None
+        # A short page is the end only when the harness says so; a harness that
+        # caps its page below `size` keeps being paged at its own size.
+        size = min(size, len(page.runs)) or size
+        if (
+            not page.runs
+            or offset >= page.total
+            or (since is not None and oldest is not None and oldest < since)
+        ):
+            return HarnessRuns(runs=kept, total=len(kept))
+        if len(kept) >= wanted:
+            return HarnessRuns(runs=kept, total=max(page.total - dropped, len(kept)))
+        if offset >= MAX_HARNESS_SCAN:
+            return HarnessRuns(runs=kept, total=max(page.total - dropped, len(kept)), capped=True)
+
+
+def safe_dir(run_id: str) -> str:
+    return UNSAFE.sub("_", run_id)
+
+
+def run_body(
+    run_id: str,
+    source: Literal["app", "eval"],
+    record: AgentRunRecord | None,
+    eval_context: RunEvalContext | None,
+) -> dict:
+    body: dict = {"run_id": run_id, "source": source}
+    if eval_context is not None:
+        body["input"] = eval_context.case.input
+    if record is not None:
+        body |= {
+            "status": record.status.value,
+            "answer": record.answer,
+            "output": record.output,
+            "error": record.error,
+            "started_at": record.started_at.isoformat(),
+            "ended_at": record.ended_at.isoformat() if record.ended_at else None,
+            "metrics": record.metrics,
+        }
+    return body
+
+
+def scores(evaluations: list[EvaluationRow]) -> dict[str, float]:
+    return {
+        row.name: row.value
+        for row in evaluations
+        if row.kind is EvaluatorKind.SCORE and row.error is None and row.value is not None
+    }
+
+
+def index_row(bundle: RunBundle) -> dict:
+    context = bundle.eval
+    return {
+        "run_id": bundle.run_id,
+        "dir": f"runs/{safe_dir(bundle.run_id)}",
+        "source": bundle.source,
+        "status": bundle.status,
+        "created_at": bundle.created_at.isoformat() if bundle.created_at else None,
+        "title": bundle.title,
+        "experiment_id": context.experiment_id if context else None,
+        "case_id": context.case.case_id if context else None,
+        "repetition": context.repetition if context else None,
+        "scores": scores(context.evaluations) if context else None,
+        "error": bundle.run.get("error"),
+        "pull_error": bundle.pull_error,
+    }
+
+
+def _json(doc: object) -> bytes:
+    return json.dumps(doc, indent=2, ensure_ascii=False, default=str).encode()
+
+
+def bundle_files(bundle: RunBundle) -> dict[str, bytes]:
+    base = f"runs/{safe_dir(bundle.run_id)}"
+    run = (
+        bundle.run if bundle.pull_error is None else {**bundle.run, "pull_error": bundle.pull_error}
+    )
+    files = {f"{base}/run.json": _json(run)}
+    if bundle.trajectory is not None:
+        files[f"{base}/trajectory.json"] = _json(bundle.trajectory)
+    if bundle.eval is not None:
+        case = bundle.eval.case
+        files[f"{base}/case.json"] = _json(
+            {
+                "case_id": case.case_id,
+                "content_hash": case.content_hash,
+                "input": case.input,
+                "expected": case.expected,
+                "evaluators": case.effective_evaluators,
+                "tags": case.tags,
+            }
+        )
+        files[f"{base}/evaluations.json"] = _json(
+            [row.model_dump(mode="json", exclude_none=True) for row in bundle.eval.evaluations]
+        )
+    return files
+
+
+def merge_index(existing: dict | None, fresh: dict) -> dict:
+    if existing is None:
+        return fresh
+    replaced = {row["run_id"] for row in fresh["runs"]}
+    kept = [row for row in existing["runs"] if row["run_id"] not in replaced]
+    return {**fresh, "runs": kept + fresh["runs"]}

evalkit/core/python_eval.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from evalkit.contracts.custom_evaluator import EvalCase, EvalRun, ScoreItem
+from evalkit.core.custom_eval import EvalError
+
+LOGGER = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class PythonEvaluator:
+    name: str
+    kind: Literal["evaluator", "gauge"]
+    fn: Callable[[EvalCase, EvalRun], Any]
+    requires_trajectory: bool = False
+
+    def __call__(self, case: EvalCase, run: EvalRun) -> Any:
+        return self.fn(case, run)
+
+
+def _normalise(name: str, result: Any) -> list[ScoreItem]:
+    if isinstance(result, ScoreItem):
+        return [result]
+    if isinstance(result, int | float) and not isinstance(result, bool):
+        return [ScoreItem(name=name, score=float(result))]
+    if isinstance(result, list) and all(isinstance(item, ScoreItem) for item in result):
+        return result
+    raise TypeError(
+        f"evaluator {name} returned {type(result).__name__}; "
+        "expected float, ScoreItem or list[ScoreItem]"
+    )
+
+
+async def score_with_python_evaluator(
+    evaluator: PythonEvaluator,
+    *,
+    case: EvalCase,
+    answer: str,
+    output: dict | None,
+    trajectory: dict,
+) -> list[ScoreItem] | EvalError:
+    run = EvalRun(answer=answer, output=output, trajectory=trajectory)
+    try:
+        if inspect.iscoroutinefunction(evaluator.fn):
+            result = await evaluator.fn(case, run)
+        else:
+            result = await asyncio.to_thread(evaluator.fn, case, run)
+            if inspect.isawaitable(result):
+                result = await result
+        return _normalise(evaluator.name, result)
+    except Exception as exc:
+        LOGGER.warning("python evaluator %s errored: %s", evaluator.name, exc)
+        return EvalError(evaluator=evaluator.name, message=f"{type(exc).__name__}: {exc}")

evalkit/core/remote_evaluators.py

@@ -0,0 +1,32 @@
+from typing import Literal, Protocol
+
+from pydantic import BaseModel
+
+from evalkit.contracts.enums import EvaluatorKind
+from evalkit.contracts.settings import RemoteEvaluatorSettings
+
+
+class RemoteEvaluator(BaseModel):
+    name: str
+    workspace: str
+    kind: EvaluatorKind
+    settings: RemoteEvaluatorSettings
+    archived: bool = False
+    has_api_key: bool = False
+
+    @property
+    def ref(self) -> str:
+        return f"{self.name}@{self.workspace}"
+
+    @property
+    def wire_kind(self) -> Literal["evaluator", "gauge"]:
+        return "evaluator" if self.kind is EvaluatorKind.SCORE else "gauge"
+
+
+class EvaluatorLookup(Protocol):
+    async def in_workspace(self, workspace: str) -> dict[str, RemoteEvaluator]: ...
+
+
+class UnknownEvaluator(KeyError):
+    def __init__(self, ref: str):
+        super().__init__(f"unknown evaluator: {ref} — create it with evalkit evaluators create")

evalkit/core/review.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from datetime import datetime
+from enum import StrEnum
+from typing import Any, NamedTuple, Protocol
+
+from pydantic import BaseModel
+
+from evalkit.store.base import CaseView, DatasetRow
+
+__all__ = [
+    "DECISIVE_EVENT_KINDS",
+    "OPEN_CASE_STATES",
+    "CaseComment",
+    "CaseEvent",
+    "CaseEventKind",
+    "CaseReview",
+    "CaseState",
+    "CaseVersion",
+    "NewCaseComment",
+    "NewCaseEvent",
+    "NotReviewable",
+    "ReviewLogProtocol",
+    "ensure_reviewable",
+    "is_reviewable",
+    "group_threads",
+    "review_summary",
+]
+
+
+class CaseEventKind(StrEnum):
+    SYNCED = "synced"
+    REVIEW_REQUESTED = "review_requested"
+    APPROVED = "approved"
+    CHANGES_REQUESTED = "changes_requested"
+
+
+class CaseState(StrEnum):
+    UNREVIEWED = "unreviewed"
+    REVIEW_REQUESTED = "review_requested"
+    CHANGES_REQUESTED = "changes_requested"
+    APPROVED = "approved"
+
+
+DECISIVE_EVENT_KINDS = frozenset(
+    {
+        CaseEventKind.REVIEW_REQUESTED,
+        CaseEventKind.APPROVED,
+        CaseEventKind.CHANGES_REQUESTED,
+    }
+)
+
+_CASE_STATE_FOR = {
+    CaseEventKind.REVIEW_REQUESTED: CaseState.REVIEW_REQUESTED,
+    CaseEventKind.APPROVED: CaseState.APPROVED,
+    CaseEventKind.CHANGES_REQUESTED: CaseState.CHANGES_REQUESTED,
+}
+
+OPEN_CASE_STATES = frozenset({CaseState.REVIEW_REQUESTED, CaseState.CHANGES_REQUESTED})
+
+
+class CaseVersion(NamedTuple):
+    case_id: str
+    content_hash: str
+
+    @classmethod
+    def of(cls, case: CaseView) -> CaseVersion:
+        return cls(case.case_id, case.content_hash)
+
+
+class NewCaseEvent(BaseModel):
+    dataset: str
+    case_id: str
+    content_hash: str
+    kind: CaseEventKind
+    actor: str
+    target: str | None = None
+
+
+class CaseEvent(NewCaseEvent):
+    id: int
+    created_at: datetime
+
+
+class NewCaseComment(BaseModel):
+    dataset: str
+    case_id: str
+    content_hash: str
+    parent_id: int | None = None
+    author: str
+    body: str
+
+
+class CaseComment(NewCaseComment):
+    id: int
+    resolved: bool
+    created_at: datetime
+
+
+@dataclass
+class CaseReview:
+    dataset: str
+    case_id: str
+    content_hash: str
+    last_event: CaseEvent | None
+    requested_to: str | None
+
+    @classmethod
+    def unreviewed(cls, dataset: str, case_id: str, content_hash: str) -> CaseReview:
+        return cls(
+            dataset=dataset,
+            case_id=case_id,
+            content_hash=content_hash,
+            last_event=None,
+            requested_to=None,
+        )
+
+    @property
+    def version(self) -> CaseVersion:
+        return CaseVersion(self.case_id, self.content_hash)
+
+    def state(self) -> CaseState:
+        if self.last_event is None:
+            return CaseState.UNREVIEWED
+        return _CASE_STATE_FOR[self.last_event.kind]
+
+    def synced(self, actor: str) -> NewCaseEvent:
+        return self._event(CaseEventKind.SYNCED, actor)
+
+    def request_review(self, actor: str, target: str) -> NewCaseEvent:
+        return self._event(CaseEventKind.REVIEW_REQUESTED, actor, target)
+
+    def approve(self, actor: str) -> NewCaseEvent:
+        return self._event(CaseEventKind.APPROVED, actor)
+
+    def request_changes(self, actor: str) -> NewCaseEvent:
+        return self._event(CaseEventKind.CHANGES_REQUESTED, actor)
+
+    def carry_over(self, actor: str, new_hash: str) -> NewCaseEvent | None:
+        if self.last_event is None or self.state() not in OPEN_CASE_STATES:
+            return None
+        return NewCaseEvent(
+            dataset=self.dataset,
+            case_id=self.case_id,
+            content_hash=new_hash,
+            kind=CaseEventKind.REVIEW_REQUESTED,
+            actor=actor,
+            target=self.requested_to or self.last_event.actor,
+        )
+
+    def _event(self, kind: CaseEventKind, actor: str, target: str | None = None) -> NewCaseEvent:
+        return NewCaseEvent(
+            dataset=self.dataset,
+            case_id=self.case_id,
+            content_hash=self.content_hash,
+            kind=kind,
+            actor=actor,
+            target=target,
+        )
+
+
+def review_summary(review: CaseReview) -> dict[str, Any]:
+    state = review.state()
+    return {
+        "state": state.value,
+        "requested_to": review.requested_to if state is CaseState.REVIEW_REQUESTED else None,
+        "updated_at": review.last_event.created_at if review.last_event is not None else None,
+    }
+
+
+class ReviewLogProtocol(Protocol):
+    async def load(self, dataset: str, versions: list[CaseVersion]) -> dict[str, CaseReview]: ...
+
+    async def latest_hashes(self, dataset: str, case_ids: list[str]) -> dict[str, str]: ...
+
+    async def append(self, events: list[NewCaseEvent]) -> None: ...
+
+    async def decide(
+        self,
+        event: NewCaseEvent,
+        comment: NewCaseComment | None,
+        after: Callable[[Any, int], Awaitable[None]] | None = None,
+    ) -> None: ...
+
+    async def history(self, dataset: str, case_id: str) -> list[CaseEvent]: ...
+
+    async def comments(self, dataset: str, case_id: str) -> list[CaseComment]: ...
+
+    async def add_comment(self, comment: NewCaseComment) -> CaseComment: ...
+
+    async def get_comment(self, comment_id: int) -> CaseComment | None: ...
+
+    async def set_resolved(self, comment_id: int, resolved: bool) -> CaseComment: ...
+
+
+def group_threads(comments: list[CaseComment]) -> list[dict[str, Any]]:
+    ordered = sorted(comments, key=lambda c: (c.created_at, c.id))
+    roots = [c.model_dump() | {"replies": []} for c in ordered if c.parent_id is None]
+    by_id = {root["id"]: root for root in roots}
+    for comment in ordered:
+        if comment.parent_id is not None:
+            by_id[comment.parent_id]["replies"].append(comment.model_dump())
+    return roots
+
+
+class NotReviewable(Exception):
+    def __init__(self, dataset: str):
+        self.dataset = dataset
+        super().__init__(f"{dataset} is not a suite in a shared workspace; review does not apply")
+
+
+def is_reviewable(dataset: DatasetRow) -> bool:
+    return dataset.kind == "suite" and dataset.workspace_is_shared
+
+
+def ensure_reviewable(dataset: DatasetRow) -> None:
+    if not is_reviewable(dataset):
+        raise NotReviewable(dataset.ref)