evalkeep 0.1.0__py3-none-any.whl

evalkeep/__init__.py

@@ -0,0 +1,12 @@
+"""Evalkeep: turn production agent failures into a small, reviewed regression suite."""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__", "main"]
+
+
+def main() -> None:
+    """Console-script entry point (delegates to the Typer app)."""
+    from evalkeep.cli import main as _main
+
+    _main()

evalkeep/__main__.py

@@ -0,0 +1,6 @@
+"""Allow ``python -m evalkeep``."""
+
+from evalkeep.cli import main
+
+if __name__ == "__main__":
+    main()

evalkeep/adapters/__init__.py

@@ -0,0 +1,45 @@
+"""Trace adapters and the registry the CLI resolves ``--format`` against."""
+
+from __future__ import annotations
+
+from evalkeep.adapters.base import AdapterRecord, IssueKind, TraceAdapter, TraceIssue
+from evalkeep.adapters.jsonl import JsonlAdapter
+from evalkeep.adapters.langsmith import LangSmithAdapter
+from evalkeep.adapters.otlp import OtlpAdapter
+from evalkeep.errors import CommandError
+
+DEFAULT_ADAPTER = "jsonl"
+
+_ADAPTERS: dict[str, TraceAdapter] = {
+    JsonlAdapter.name: JsonlAdapter(),
+    LangSmithAdapter.name: LangSmithAdapter(),
+    OtlpAdapter.name: OtlpAdapter(),
+}
+
+
+def available_adapters() -> dict[str, TraceAdapter]:
+    return dict(_ADAPTERS)
+
+
+def get_adapter(name: str) -> TraceAdapter:
+    try:
+        return _ADAPTERS[name]
+    except KeyError:
+        known = ", ".join(sorted(_ADAPTERS))
+        raise CommandError(
+            f"Unknown trace format {name!r}.", hint=f"Available formats: {known}."
+        ) from None
+
+
+__all__ = [
+    "DEFAULT_ADAPTER",
+    "AdapterRecord",
+    "IssueKind",
+    "JsonlAdapter",
+    "LangSmithAdapter",
+    "OtlpAdapter",
+    "TraceAdapter",
+    "TraceIssue",
+    "available_adapters",
+    "get_adapter",
+]

evalkeep/adapters/base.py

@@ -0,0 +1,92 @@
+"""The adapter contract: every adapter normalizes or fails, never both.
+
+An adapter turns one provider's records into :class:`AdapterRecord` values,
+streaming them so that a 100k-trace file never has to fit in memory. A record
+either carries a validated trace or carries the issues explaining why it could
+not be validated -- an adapter never raises on bad input data. Exceptions are
+reserved for the file itself being unusable, which the CLI reports as a command
+error.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+from typing import Any, ClassVar, Protocol, runtime_checkable
+
+from evalkeep.trace import NormalizedTrace
+
+
+class IssueKind(StrEnum):
+    """Why a record was rejected. Written verbatim into the error JSONL."""
+
+    ENCODING = "encoding"
+    JSON = "json"
+    SCHEMA = "schema"
+    DUPLICATE_ID = "duplicate_id"
+    #: The trace ID is already stored with different content.
+    ID_CONFLICT = "id_conflict"
+
+
+@dataclass(frozen=True)
+class TraceIssue:
+    """One reason one record is not usable, addressed to whoever must fix it."""
+
+    line: int
+    kind: IssueKind
+    message: str
+    trace_id: str | None = None
+    field: str | None = None
+    hint: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """The structured error-JSONL record."""
+        record: dict[str, Any] = {
+            "line": self.line,
+            "kind": self.kind.value,
+            "message": self.message,
+        }
+        if self.trace_id is not None:
+            record["trace_id"] = self.trace_id
+        if self.field is not None:
+            record["field"] = self.field
+        if self.hint is not None:
+            record["hint"] = self.hint
+        return record
+
+
+@dataclass(frozen=True)
+class AdapterRecord:
+    """One source record: either a trace, or the issues that rejected it."""
+
+    line: int
+    trace: NormalizedTrace | None = None
+    issues: tuple[TraceIssue, ...] = field(default_factory=tuple)
+
+    @property
+    def ok(self) -> bool:
+        return self.trace is not None
+
+    @classmethod
+    def valid(cls, line: int, trace: NormalizedTrace) -> AdapterRecord:
+        return cls(line=line, trace=trace)
+
+    @classmethod
+    def rejected(cls, line: int, *issues: TraceIssue) -> AdapterRecord:
+        if not issues:
+            raise ValueError("a rejected record must carry at least one issue")
+        return cls(line=line, issues=tuple(issues))
+
+
+@runtime_checkable
+class TraceAdapter(Protocol):
+    """Converts a provider format into normalized traces."""
+
+    name: ClassVar[str]
+    description: ClassVar[str]
+
+    def read(self, path: Path) -> Iterator[AdapterRecord]:
+        """Stream records from ``path``, one per source record."""
+        ...

evalkeep/adapters/jsonl.py

@@ -0,0 +1,164 @@
+"""The generic JSONL adapter: one JSON object per line, already normalized.
+
+This is the format Evalkeep documents for users who export traces themselves.
+Provider-specific adapters (Langfuse, Opik, OpenTelemetry) arrive in 0.2 and
+map onto the same contract.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+from typing import Any, ClassVar
+
+from pydantic import ValidationError
+
+from evalkeep.adapters.base import AdapterRecord, IssueKind, TraceIssue
+from evalkeep.trace import NormalizedTrace
+
+EXTRA_FIELD_HINT = "Unknown fields belong under 'metadata.extra'."
+
+
+class JsonlAdapter:
+    """Reads newline-delimited JSON, one normalized trace per line."""
+
+    name: ClassVar[str] = "jsonl"
+    description: ClassVar[str] = "Generic newline-delimited JSON (one trace object per line)"
+
+    def read(self, path: Path) -> Iterator[AdapterRecord]:
+        """Stream ``path`` without loading it into memory.
+
+        Opened in binary so that a single undecodable line becomes one rejected
+        record instead of aborting the whole file.
+        """
+        with path.open("rb") as handle:
+            yield from self.read_binary_lines(handle)
+
+    def read_binary_lines(self, lines: Iterable[bytes]) -> Iterator[AdapterRecord]:
+        for line_number, raw in enumerate(lines, start=1):
+            try:
+                text = raw.decode("utf-8")
+            except UnicodeDecodeError as exc:
+                yield AdapterRecord.rejected(
+                    line_number,
+                    TraceIssue(
+                        line=line_number,
+                        kind=IssueKind.ENCODING,
+                        message=f"line is not valid UTF-8: {exc.reason}",
+                    ),
+                )
+                continue
+            record = self._parse(line_number, text)
+            if record is not None:
+                yield record
+
+    def read_lines(self, lines: Iterable[str]) -> Iterator[AdapterRecord]:
+        """Same contract as :meth:`read`, over already-decoded lines."""
+        for line_number, text in enumerate(lines, start=1):
+            record = self._parse(line_number, text)
+            if record is not None:
+                yield record
+
+    def _parse(self, line_number: int, text: str) -> AdapterRecord | None:
+        """Return a record, or ``None`` for a blank line (not a record at all)."""
+        stripped = text.lstrip("\ufeff").strip()
+        if not stripped:
+            return None
+
+        try:
+            payload: Any = json.loads(stripped)
+        except json.JSONDecodeError as exc:
+            return AdapterRecord.rejected(
+                line_number,
+                TraceIssue(
+                    line=line_number,
+                    kind=IssueKind.JSON,
+                    message=f"invalid JSON: {exc.msg} at column {exc.colno}",
+                ),
+            )
+
+        if not isinstance(payload, dict):
+            return AdapterRecord.rejected(
+                line_number,
+                TraceIssue(
+                    line=line_number,
+                    kind=IssueKind.JSON,
+                    message=f"expected a JSON object, got {type(payload).__name__}",
+                ),
+            )
+
+        try:
+            trace = NormalizedTrace.model_validate(payload)
+        except ValidationError as exc:
+            return AdapterRecord.rejected(
+                line_number, *_issues_from_validation_error(line_number, payload, exc)
+            )
+        return AdapterRecord.valid(line_number, trace)
+
+
+def _issues_from_validation_error(
+    line_number: int, payload: dict[str, Any], error: ValidationError
+) -> list[TraceIssue]:
+    """One issue per field error, so the error JSONL is directly actionable."""
+    trace_id = _best_effort_trace_id(payload)
+    issues: list[TraceIssue] = []
+    for detail in error.errors():
+        location = _format_location(detail["loc"], payload)
+        issues.append(
+            TraceIssue(
+                line=line_number,
+                kind=IssueKind.SCHEMA,
+                message=detail["msg"],
+                trace_id=trace_id,
+                field=location or None,
+                hint=EXTRA_FIELD_HINT if detail["type"] == "extra_forbidden" else None,
+            )
+        )
+    return issues
+
+
+def _format_location(location: tuple[int | str, ...], payload: Any) -> str:
+    """Render a Pydantic error location as a path into the user's own JSON.
+
+    Pydantic inserts the union tag for a discriminated union, so a bad tool name
+    arrives as ``('events', 0, 'tool_call', 'tool')``. There is no ``tool_call``
+    key in the document, and telling someone to look at a path that does not
+    exist wastes their time. Any non-final segment that cannot be resolved
+    against the payload is dropped; the final segment is always kept, since a
+    missing required field is exactly the one that will not resolve.
+    """
+    parts: list[str] = []
+    current: Any = payload
+    last_index = len(location) - 1
+    for index, part in enumerate(location):
+        resolved = _descend(current, part)
+        if resolved is _UNRESOLVED and index != last_index:
+            continue
+        parts.append(str(part))
+        current = None if resolved is _UNRESOLVED else resolved
+    return ".".join(parts)
+
+
+_UNRESOLVED = object()
+
+
+def _descend(container: Any, part: int | str) -> Any:
+    """One step into ``container``, or ``_UNRESOLVED`` if the step is not there."""
+    if isinstance(container, dict) and part in container:
+        return container[part]
+    if (
+        isinstance(container, list)
+        and isinstance(part, int)
+        and -len(container) <= part < len(container)
+    ):
+        return container[part]
+    return _UNRESOLVED
+
+
+def _best_effort_trace_id(payload: dict[str, Any]) -> str | None:
+    """Name the offending record even when the record failed validation."""
+    candidate = payload.get("trace_id")
+    if isinstance(candidate, str) and candidate.strip():
+        return candidate.strip()
+    return None