evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/contracts/esf.py

@@ -0,0 +1,120 @@
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+EsfKind = Literal["llm", "tool", "agent", "chain", "retriever", "other"]
+MessageFormat = Literal["openinference", "gen_ai", "openai"]
+MAPPING_SCHEMA_VERSION = "1"
+
+
+class EsfToolCall(BaseModel):
+    id: str
+    name: str
+    arguments: dict[str, Any] | str
+
+
+class EsfMessage(BaseModel):
+    role: str
+    text: str | None = None
+    reasoning: str | None = None
+    tool_calls: list[EsfToolCall] | None = None
+    tool_call_id: str | None = None
+
+
+class EsfSpan(BaseModel):
+    span_id: str
+    trace_id: str
+    parent_id: str | None
+    name: str
+    kind: EsfKind
+    started_at: datetime
+    ended_at: datetime | None
+    error: str | None
+    model: str | None = None
+    input_messages: list[EsfMessage] | None = None
+    output_messages: list[EsfMessage] | None = None
+    input: Any | None = None
+    output: Any | None = None
+    tool_name: str | None = None
+    tool_call_id: str | None = None
+    agent_name: str | None = None
+    prompt_tokens: int | None = None
+    completion_tokens: int | None = None
+    cached_tokens: int | None = None
+    cache_write_tokens: int | None = None
+    cost_usd: float | None = None
+    unmapped: list[str] = Field(default_factory=list)
+
+
+class MappingModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+
+class AttributeRule(MappingModel):
+    attribute: str = Field(description="The span attribute key to read.")
+    json_: bool = Field(
+        default=False,
+        alias="json",
+        description="Parse the value with json.loads when it is a string; keep the string "
+        "on failure.",
+    )
+    values: dict[str, str] | None = Field(
+        default=None, description="Translate the read value through this table."
+    )
+    message_format: MessageFormat | None = Field(
+        default=None,
+        description="Collect messages laid out per this format under the attribute key.",
+    )
+
+
+class TokenRules(MappingModel):
+    prompt: AttributeRule | None = None
+    completion: AttributeRule | None = None
+    cached: AttributeRule | None = None
+    cache_write: AttributeRule | None = None
+
+
+class ToolRules(MappingModel):
+    name: AttributeRule | None = None
+    call_id: AttributeRule | None = None
+    arguments: AttributeRule | None = None
+    result: AttributeRule | None = None
+
+
+class MappingDocument(MappingModel):
+    schema_version: Literal["1"]
+    kind: AttributeRule | None = None
+    session_id: AttributeRule | None = None
+    model: AttributeRule | None = None
+    system_instructions: AttributeRule | None = Field(
+        default=None,
+        description="System prompt kept outside the input messages: a string or a list of "
+        "text parts. Read on llm spans and put first in the input messages.",
+    )
+    input_messages: AttributeRule | None = None
+    output_messages: AttributeRule | None = None
+    tokens: TokenRules = Field(default_factory=TokenRules)
+    cost_usd: AttributeRule | None = None
+    tool: ToolRules = Field(default_factory=ToolRules)
+    agent_name: AttributeRule | None = Field(
+        default=None,
+        description="The agent's name on agent spans; names the subagent trajectory. The span "
+        "name is used when it is absent.",
+    )
+    input: AttributeRule | None = None
+    output: AttributeRule | None = None
+    known_attributes: list[str] = Field(
+        default_factory=list,
+        description="Attributes no rule reads that are expected and need nothing: not reported "
+        "as unmapped. Glob patterns such as llm.tools.* are allowed.",
+    )
+    answer: str = Field(
+        default="root",
+        description="Where the run's answer comes from: root, a span name, or "
+        "<span name>.<output key>.",
+    )
+    exclude_spans: list[str] = Field(
+        default_factory=list,
+        description="Span names left out of the trajectory.",
+    )

evalkit/contracts/harness.py

@@ -0,0 +1,281 @@
+import json
+from datetime import datetime
+from typing import Literal
+
+from jsonschema.exceptions import SchemaError
+from jsonschema.validators import validator_for
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class HarnessModel(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+
+DEFAULT_INPUT_SCHEMA: dict = {
+    "type": "object",
+    "required": ["question"],
+    "properties": {"question": {"type": "string"}},
+}
+
+
+def input_errors(schema: dict, run_input: dict) -> list[str]:
+    validator_cls = validator_for(schema)
+    validator = validator_cls(schema)
+    errors = sorted(validator.iter_errors(run_input), key=lambda e: list(e.absolute_path))
+    return [
+        f"{'/'.join(str(part) for part in e.absolute_path) or '<root>'}: {e.message}"
+        for e in errors
+    ]
+
+
+def answer_text(answer: str | None, output: dict | None) -> str:
+    if answer is not None:
+        return answer
+    return json.dumps(output, sort_keys=True, ensure_ascii=False)
+
+
+class Provides(HarnessModel):
+    trajectory: bool = Field(
+        default=False, description="Can serve GET runs/{id}/trajectory at all."
+    )
+    tool_calls: bool = Field(default=False, description="Trajectory reliably contains tool calls.")
+    thinking: bool = Field(
+        default=False, description="Trajectory reliably contains reasoning content."
+    )
+    token_usage: bool = Field(default=False, description="Trajectory carries per-step token usage.")
+    artifacts: bool = Field(
+        default=False, description="Run status reports artifacts the agent produced."
+    )
+    workspace: bool = Field(default=False, description="Files endpoints are implemented.")
+    run_listing: bool = Field(
+        default=False,
+        description="GET /runs (list) is implemented. Powers the sessions browser.",
+    )
+
+
+class Accepts(HarnessModel):
+    files: bool = Field(default=False, description="POST /runs accepts multipart file attachments.")
+    file_urls: bool = Field(
+        default=False,
+        description="POST /runs (JSON body) accepts `files: [{name, url}]` and the harness "
+        "fetches each url itself. Preferred over multipart when both are declared. The url "
+        "is evalkit's blob store url plus the object key (e.g. s3://bucket/fixtures/<key>); "
+        "the harness must have access to it.",
+    )
+    model_override: bool = Field(default=False, description="POST /runs honours the 'model' field.")
+    stop: bool = Field(default=False, description="POST /runs/{id}/stop is implemented.")
+
+    @property
+    def file_input(self) -> bool:
+        return self.file_urls or self.files
+
+
+class Limits(HarnessModel):
+    max_concurrent_runs: int | None = Field(
+        default=None,
+        description="Harness-side cap. evalkit uses min(this, its own config).",
+    )
+
+
+class Capabilities(HarnessModel):
+    contract_version: str = Field(
+        default="v1", description="Harness API version this harness implements."
+    )
+    agent: str = Field(description="Harness's own name for itself. Informational.")
+    revision: str | None = Field(
+        default=None,
+        description="Optional. Git commit SHA of the deployed harness; evalkit "
+        "records it on the experiment so results are traceable to a build.",
+    )
+    branch: str | None = Field(
+        default=None, description="Optional. Git branch the deployed harness was built from."
+    )
+    input_schema: dict | None = Field(
+        default=None,
+        description="Optional. JSON Schema (draft 2020-12 unless `$schema` says otherwise) "
+        "of the run input this agent accepts. Excludes `files`, which evalkit sends "
+        "separately. When absent, evalkit assumes the default question-and-answer schema: "
+        "an object with a required string `question`. evalkit checks case inputs against "
+        "the declared or default schema before running an experiment and in "
+        "`evalkit lint --agent`.",
+    )
+    check_input: dict | None = Field(
+        default=None,
+        description="Optional. The run input `evalkit harness check` sends. Must pass "
+        "input_schema, or the default question-and-answer schema when none is declared.",
+    )
+    provides: Provides = Field(default_factory=Provides)
+    accepts: Accepts = Field(default_factory=Accepts)
+    limits: Limits = Field(default_factory=Limits)
+
+    @property
+    def effective_input_schema(self) -> dict:
+        return self.input_schema or DEFAULT_INPUT_SCHEMA
+
+    @model_validator(mode="after")
+    def _schema_and_check_input_agree(self) -> "Capabilities":
+        if self.input_schema is not None:
+            try:
+                validator_for(self.input_schema).check_schema(self.input_schema)
+            except SchemaError as exc:
+                raise ValueError(f"input_schema is not a valid JSON Schema: {exc.message}") from exc
+        if self.check_input is not None:
+            errors = input_errors(self.effective_input_schema, self.check_input)
+            if errors:
+                raise ValueError(f"check_input fails input_schema: {errors[0]}")
+        return self
+
+
+class RunMetadata(HarnessModel):
+    experiment_id: str | None = Field(
+        default=None,
+        description="Experiment id, so eval traffic is distinguishable and traceable.",
+    )
+    case_id: str | None = Field(default=None, description="Case id for the harness's records.")
+    repetition: int | None = Field(default=None, description="Repetition number, 1-based.")
+
+
+class FileRef(HarnessModel):
+    name: str = Field(description="File name to present to the agent — the fixture key.")
+    url: str | None = Field(
+        default=None,
+        description="Where the harness fetches the bytes from. Present on a JSON request "
+        "(accepts.file_urls); absent on multipart, where the bytes arrive as the part with "
+        "this name. Opaque to evalkit's experiment runner; the UI still serves the same "
+        "fixture via GET /api/fixtures/{name}.",
+    )
+    context: str | None = Field(
+        default=None,
+        description="What the file holds, from the suite's fixture registry. evalkit no "
+        "longer prepends it to input.question; show it to the agent however suits it.",
+    )
+
+
+class RunRequest(HarnessModel):
+    input: dict = Field(
+        description="The case input, verbatim; evalkit never rewrites it. Free-form "
+        "object (see capabilities.input_schema); the harness maps it "
+        "onto its own invocation."
+    )
+    model: str | None = Field(
+        default=None, description="Optional. Only sent if accepts.model_override."
+    )
+    metadata: RunMetadata | None = Field(
+        default=None,
+        description="Optional, for the harness's own records. The harness may ignore it.",
+    )
+    files: list[FileRef] | None = Field(
+        default=None,
+        description="Optional. The case's fixture files, each with its context. On a JSON "
+        "request (accepts.file_urls) each carries the url to fetch; on multipart it is in "
+        "the payload part without urls, next to one file part per entry.",
+    )
+
+
+class RunCreated(HarnessModel):
+    run_id: str = Field(
+        description="The harness's own identifier — typically its session id. "
+        "Key for every other endpoint."
+    )
+
+
+class ArtifactRef(HarnessModel):
+    name: str = Field(description="Artifact display name.")
+    path: str | None = Field(
+        default=None, description="Workspace path, fetchable via the files endpoint."
+    )
+    content_type: str | None = Field(default=None, description="MIME type, if known.")
+
+
+class RunStatus(HarnessModel):
+    status: Literal["queued", "running", "completed", "failed", "cancelled"] = Field(
+        description="Run state. evalkit polls until terminal or timeout."
+    )
+    answer: str | None = Field(
+        default=None,
+        description="Final user-visible answer. A completed run carries this, `output`, or both.",
+    )
+    output: dict | None = Field(
+        default=None,
+        description="Optional. The agent's structured result, for agents that return "
+        "typed data. A completed run carries this, `answer`, or both.",
+    )
+    model: str = Field(
+        description="Required. The model this run executes on — the requested "
+        "model when honoured, otherwise the harness's own default. evalkit "
+        "records it as the experiment's model and prefers it over the request.",
+    )
+    artifacts: list[ArtifactRef] | None = Field(
+        default=None, description="Optional. Files the agent produced as deliverables."
+    )
+    error: str | None = Field(
+        default=None,
+        description="Message when status=failed. A completed run may also carry a "
+        "non-fatal error (e.g. partial answer).",
+    )
+
+    @model_validator(mode="after")
+    def _completed_carries_answer_or_output(self) -> "RunStatus":
+        if self.status == "completed" and self.answer is None and self.output is None:
+            raise ValueError("a completed run must carry an answer or an output")
+        return self
+
+
+class RunListParams(BaseModel):
+    query: str | None = Field(
+        default=None,
+        description="Free-text filter over runs. Text that looks like an id (8 or more "
+        "hex characters; dashes and case ignored) matches runs whose id starts with it. "
+        "Any other text matches runs whose title contains it, case-insensitive.",
+    )
+    user_id: str | None = Field(default=None, description="Filter to one user's runs.")
+    client_id: str | None = Field(default=None, description="Filter to one client's runs.")
+    limit: int = Field(default=50, description="Page size.")
+    offset: int = Field(default=0, description="Page start.")
+    since: datetime | None = Field(
+        default=None, description="Only runs created at or after this time (ISO 8601)."
+    )
+    until: datetime | None = Field(
+        default=None, description="Only runs created before this time (ISO 8601)."
+    )
+
+    def as_query(self) -> dict[str, str | int]:
+        return {
+            k: v.isoformat().replace("+00:00", "Z") if isinstance(v, datetime) else v
+            for k, v in self.model_dump().items()
+            if v is not None
+        }
+
+
+class RunListItem(HarnessModel):
+    run_id: str = Field(description="Usable with the other /runs/{id} endpoints.")
+    title: str | None = Field(default=None, description="Display title for the run/session.")
+    user_id: str | None = Field(
+        default=None, description="Optional filter value; fixed dummy if no multi-tenancy."
+    )
+    client_id: str | None = Field(
+        default=None, description="Optional filter value; fixed dummy if no multi-tenancy."
+    )
+    created_at: datetime | None = Field(default=None, description="Run creation time.")
+    session_id: str | None = Field(
+        default=None,
+        description="The session this run belongs to, for a harness that keeps several of its "
+        "own runs per session (for example one per chat message) and answers each of them with "
+        "the whole session. evalkit's run is the session: items sharing a session_id are listed "
+        "once, as the newest of them.",
+    )
+
+
+class RunList(HarnessModel):
+    runs: list[RunListItem] = Field(description="Page of runs, newest first.")
+    total: int = Field(description="Total runs matching the query.")
+
+
+class FileEntry(HarnessModel):
+    path: str = Field(description="Workspace-relative path.")
+    size: int = Field(description="Size in bytes.")
+    content_type: str | None = Field(default=None, description="MIME type, if known.")
+
+
+class FileList(HarnessModel):
+    files: list[FileEntry] = Field(description="Workspace listing.")

evalkit/contracts/notes.json

@@ -0,0 +1,4 @@
+{
+  "endpoint_note": "Check this URL before saving. The server at this address receives the API key you enter here and the data evalkit sends it: cases, answers and trajectories.",
+  "personal_key_note": "Personal agents and evaluators read their API key from your evalkit.yaml (credentials:), never from the server."
+}

evalkit/contracts/notes.py

@@ -0,0 +1,7 @@
+import json
+from importlib.resources import files
+
+_NOTES = json.loads(files("evalkit.contracts").joinpath("notes.json").read_text())
+
+ENDPOINT_NOTE: str = _NOTES["endpoint_note"]
+PERSONAL_KEY_NOTE: str = _NOTES["personal_key_note"]

evalkit/contracts/settings.py

@@ -0,0 +1,41 @@
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class JudgeSettings(BaseModel):
+    """The LLM judge an experiment uses; exported from `evalkit.sdk` as `Judge`.
+
+    Its API key comes from `credentials.judges` in `evalkit.yaml`, keyed by `base_url`.
+
+    Attributes:
+        base_url: An OpenAI-compatible endpoint that supports structured output.
+        model: The model name at that endpoint.
+        reasoning_effort: Passed to the model as `reasoning_effort`.
+        max_tokens: The most tokens one judge reply may use.
+        concurrency: How many judge calls run at once.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    base_url: str
+    model: str
+    reasoning_effort: str = "high"
+    max_tokens: int = Field(32768, ge=1)
+    concurrency: int = Field(16, ge=1)
+
+
+class AgentSettings(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    max_concurrent_runs: int = Field(default=4, ge=1)
+    trajectory_wait_seconds: int = Field(default=60, ge=0)
+    run_timeout_seconds: int = Field(default=840, ge=0)
+    poll_interval_seconds: int = Field(default=5, ge=0)
+    stop_grace_seconds: int = Field(default=120, ge=0)
+    provides: dict[str, bool] | None = None
+
+
+class RemoteEvaluatorSettings(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    endpoint: str
+    timeout_seconds: int = Field(default=120, ge=1)

evalkit/core/access.py

@@ -0,0 +1,75 @@
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+from evalkit.contracts.enums import WorkspaceRole
+
+
+@dataclass(frozen=True)
+class WorkspaceRef:
+    id: int
+    name: str
+    owner_user_id: int | None
+    archived: bool
+    promotes_to: int | None = None
+    promoted_from: int | None = None
+
+    @property
+    def is_shared(self) -> bool:
+        return self.owner_user_id is None
+
+    @property
+    def is_target(self) -> bool:
+        return self.promoted_from is not None
+
+
+@dataclass(frozen=True)
+class Actor:
+    id: int
+    username: str
+    is_admin: bool
+    roles: Mapping[int, frozenset[WorkspaceRole]]
+
+    def has(self, ws: WorkspaceRef, role: WorkspaceRole) -> bool:
+        return role in self.roles.get(ws.id, frozenset())
+
+
+def can_read(actor: Actor, ws: WorkspaceRef) -> bool:
+    return True
+
+
+def can_write(actor: Actor, ws: WorkspaceRef) -> bool:
+    if not ws.is_shared:
+        return ws.owner_user_id == actor.id
+    return actor.is_admin or actor.has(ws, WorkspaceRole.WRITER)
+
+
+def can_review(actor: Actor, ws: WorkspaceRef) -> bool:
+    return ws.is_shared and actor.has(ws, WorkspaceRole.REVIEWER)
+
+
+def can_use_agent(actor: Actor, agent_ws: WorkspaceRef) -> bool:
+    if not agent_ws.is_shared:
+        return agent_ws.owner_user_id == actor.id
+    return True
+
+
+def can_change_experiment(actor: Actor, exp_ws: WorkspaceRef, created_by: int | None) -> bool:
+    return can_write(actor, exp_ws) or created_by == actor.id
+
+
+def can_create_experiment(actor: Actor, exp_ws: WorkspaceRef) -> bool:
+    if not exp_ws.is_shared:
+        return exp_ws.owner_user_id == actor.id
+    return (
+        actor.is_admin
+        or actor.has(exp_ws, WorkspaceRole.WRITER)
+        or actor.has(exp_ws, WorkspaceRole.REVIEWER)
+    )
+
+
+def experiment_workspace(
+    runner_personal: WorkspaceRef, dataset_ws: WorkspaceRef, agent_ws: WorkspaceRef
+) -> WorkspaceRef:
+    if dataset_ws.is_shared and agent_ws.is_shared:
+        return dataset_ws
+    return runner_personal

evalkit/core/agents.py

@@ -0,0 +1,65 @@
+from enum import StrEnum
+from typing import Protocol
+
+from pydantic import BaseModel
+
+from evalkit.contracts.enums import TrajectorySource
+from evalkit.contracts.settings import AgentSettings
+from evalkit.core.workspace import Reference
+
+
+class AgentDefinition(BaseModel):
+    id: int
+    name: str
+    workspace: str
+    workspace_is_shared: bool
+    harness_url: str | None
+    trajectory_source: TrajectorySource
+    repo_url: str | None
+    settings: AgentSettings
+    archived: bool = False
+    has_api_key: bool = False
+
+    @property
+    def ref(self) -> str:
+        return f"{self.name}@{self.workspace}"
+
+    @property
+    def reference(self) -> Reference:
+        return Reference(self.name, self.workspace)
+
+    @property
+    def is_otel(self) -> bool:
+        return self.trajectory_source is TrajectorySource.OTEL
+
+
+class Driver(StrEnum):
+    SERVER = "server"
+    CLI = "cli"
+    SDK = "sdk"
+
+
+def driver(agent: AgentDefinition) -> Driver:
+    if agent.harness_url is None:
+        return Driver.SDK
+    if agent.workspace_is_shared:
+        return Driver.SERVER
+    return Driver.CLI
+
+
+def server_may_drive(agent: AgentDefinition) -> bool:
+    return driver(agent) is Driver.SERVER
+
+
+class UnknownAgent(KeyError):
+    def __init__(self, ref: str):
+        super().__init__(f"unknown agent: {ref} — create it with evalkit agents create")
+        self.ref = ref
+
+
+class AgentLookup(Protocol):
+    async def get(self, ref: str) -> AgentDefinition: ...
+
+
+def explicit(ref: Reference) -> str:
+    return f"{ref.name}@{ref.workspace}"

evalkit/core/blob.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from typing import Protocol
+
+from evalkit.core.paths import require_safe_path, require_safe_run_id
+from evalkit.store.base import BlobStream
+
+
+class BlobStore(Protocol):
+    async def put_object(self, key: str, data: bytes) -> None: ...
+
+    async def put_stream(self, key: str, chunks: AsyncIterator[bytes]) -> None: ...
+
+    async def get_object(self, key: str) -> bytes: ...
+
+    async def get_stream(self, key: str) -> BlobStream: ...
+
+    async def object_exists(self, key: str) -> bool: ...
+
+    async def list_objects(self, prefix: str) -> list[tuple[str, int]]: ...
+
+    async def delete_object(self, key: str) -> None: ...
+
+    async def copy_object(self, source: str, destination: str) -> None: ...
+
+    async def object_url(self, key: str) -> str: ...
+
+
+def trajectory_key(agent_id: int, harness_run_id: str) -> str:
+    return f"trajectories/{agent_id}/{harness_run_id}.json"
+
+
+def artifact_key(agent_id: int, harness_run_id: str, path: str) -> str:
+    require_safe_run_id(harness_run_id)
+    if path:
+        path = require_safe_path(path)
+    return f"artifacts/{agent_id}/{harness_run_id}/{path}"
+
+
+def workspace_key(agent_id: int, harness_run_id: str) -> str:
+    return f"workspaces/{agent_id}/{harness_run_id}.json"
+
+
+def fixture_blob_key(sha256: str) -> str:
+    return f"fixtures/sha256/{sha256}"

evalkit/core/capabilities.py

@@ -0,0 +1,36 @@
+from evalkit.contracts.harness import Accepts, Capabilities, Limits, Provides
+from evalkit.core.agents import AgentDefinition
+
+SDK_CONTRACT_VERSION = "sdk"
+TRACED_FLAGS = ("trajectory", "tool_calls", "token_usage")
+
+
+def sdk_capabilities(agent: AgentDefinition) -> Capabilities:
+    provides = {
+        "trajectory": True,
+        "tool_calls": True,
+        "token_usage": True,
+        "thinking": False,
+        "artifacts": False,
+        "workspace": False,
+        "run_listing": True,
+        **(agent.settings.provides or {}),
+    }
+    return Capabilities(
+        contract_version=SDK_CONTRACT_VERSION,
+        agent=agent.name,
+        provides=Provides(**provides),
+        accepts=Accepts(files=True, model_override=True, file_urls=False, stop=False),
+        limits=Limits(max_concurrent_runs=agent.settings.max_concurrent_runs),
+    )
+
+
+def traced_capabilities(caps: Capabilities, agent: AgentDefinition) -> Capabilities:
+    if not agent.is_otel:
+        return caps
+    provides = {
+        **caps.provides.model_dump(),
+        **dict.fromkeys(TRACED_FLAGS, True),
+        **(agent.settings.provides or {}),
+    }
+    return caps.model_copy(update={"provides": Provides(**provides)})