reactifact 0.11.2__tar.gz → 0.12.0__tar.gz

{reactifact-0.11.2/reactifact.egg-info → reactifact-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: reactifact
-Version: 0.11.2
+Version: 0.12.0
 Summary: Reactive, artifact-driven agent runtime: agents transform versioned, typed, provenance-aware artifacts inside an evolving context
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/bzdvdn/reactifact

{reactifact-0.11.2 → reactifact-0.12.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "reactifact"
-version = "0.11.2"
+version = "0.12.0"
 description = "Reactive, artifact-driven agent runtime: agents transform versioned, typed, provenance-aware artifacts inside an evolving context"
 readme = "README.md"
 requires-python = ">=3.11"

{reactifact-0.11.2 → reactifact-0.12.0}/reactifact/__init__.py

@@ -49,7 +49,7 @@ from .session import Session, SessionStore
 from .tools import FunctionTool, Tool, ToolOutput, tool
 from .triggers import Trigger
 
-__version__ = "0.11.2"
+__version__ = "0.12.0"
 
 __all__ = [
     "Agent",

{reactifact-0.11.2 → reactifact-0.12.0}/reactifact/resources.py

@@ -51,6 +51,7 @@ class RuntimeResources:
         context_builder: ContextBuilder | None = None,
         verification_threshold: float | None = None,
         redactor: Redactor | None = None,
+        trace_truncate: int | None = 1500,
         id_factory: IdFactory | None = None,
         **additional: Any,
     ):
@@ -67,6 +68,11 @@ class RuntimeResources:
         # persisted session. `None` (default) reproduces the pre-hook behavior.
         # See `reactifact.redaction`.
         self.redactor = redactor
+        # How many characters of an artifact's JSON (and an LLM response) are
+        # kept in a trace. `None` disables truncation. Larger values make
+        # configurable table columns / deep fields resolvable, at the cost of
+        # trace size.
+        self.trace_truncate = trace_truncate
         # Framework-wide pass/fail cutoff for `Verify` (verify.py): `None`
         # means "use Verify's own DEFAULT_THRESHOLD". A `Verify` instance's
         # own explicit `threshold=` still overrides this per agent.

{reactifact-0.11.2 → reactifact-0.12.0}/reactifact/testing/__init__.py

@@ -20,7 +20,7 @@ from .assertions import (
 from .exceptions import AssertionFailure, ScenarioError, ScenarioSkip
 from .fault import ToolCallRecord, ToolCallRecorder, ToolFault
 from .golden import GoldenRun, assert_golden, capture, prompt_hashes, replay_resources
-from .lab import Scenario, ScenarioLab, ScenarioResult
+from .lab import Scenario, ScenarioLab, ScenarioReport, ScenarioResult
 from .mock import ResourceFault
 from .record import Mode, mode_from_env
 from .registry import ScenarioCase, collect, scenario
@@ -38,6 +38,7 @@ __all__ = [
     "ScenarioCase",
     "ScenarioError",
     "ScenarioLab",
+    "ScenarioReport",
     "ScenarioResult",
     "ScenarioSkip",
     "ToolAssertions",

{reactifact-0.11.2 → reactifact-0.12.0}/reactifact/testing/assertions.py

@@ -272,9 +272,17 @@ class LLMAssertions:
     def calls(self) -> int:
         return len(self._calls())
 
+    @property
+    def prompt_tokens(self) -> int:
+        return sum(c.prompt_tokens for c in self._calls())
+
+    @property
+    def completion_tokens(self) -> int:
+        return sum(c.completion_tokens for c in self._calls())
+
     @property
     def tokens(self) -> int:
-        return sum(c.prompt_tokens + c.completion_tokens for c in self._calls())
+        return self.prompt_tokens + self.completion_tokens
 
     def max_calls(self, n: int) -> None:
         if self.calls > n:

{reactifact-0.11.2 → reactifact-0.12.0}/reactifact/testing/lab.py

@@ -24,7 +24,7 @@ from __future__ import annotations
 
 import asyncio
 from collections.abc import Callable, Sequence
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, TypeVar
 
@@ -81,6 +81,97 @@ class _CapturingTracer(Tracer):
         self.trace = trace
 
 
+@dataclass
+class ScenarioReport:
+    """Aggregate cost/shape of a scenario run: what it took, in one object.
+
+    The reporting counterpart of the assertion groups — token totals, LLM/tool
+    counts, the agents that ran and the outcome, with `to_dict()` for a CI/log
+    artifact and `render()` for a readable multi-line summary. A single
+    `ScenarioResult.report` covers one turn; `Scenario.report` aggregates every
+    turn run so far.
+    """
+
+    turns: int = 0
+    spans: int = 0
+    agents: list[str] = field(default_factory=list)
+    tool_calls: int = 0
+    llm_calls: int = 0
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    duration_ms: float = 0.0
+    errors: int = 0
+    outcomes: list[str] = field(default_factory=list)
+
+    @property
+    def total_tokens(self) -> int:
+        return self.prompt_tokens + self.completion_tokens
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "turns": self.turns,
+            "spans": self.spans,
+            "agents": list(self.agents),
+            "tool_calls": self.tool_calls,
+            "llm_calls": self.llm_calls,
+            "prompt_tokens": self.prompt_tokens,
+            "completion_tokens": self.completion_tokens,
+            "total_tokens": self.total_tokens,
+            "duration_ms": self.duration_ms,
+            "errors": self.errors,
+            "outcomes": list(self.outcomes),
+        }
+
+    def render(self) -> str:
+        rows = [
+            ("turns", str(self.turns)),
+            ("agents", ", ".join(self.agents) or "—"),
+            ("spans", str(self.spans)),
+            ("tool calls", str(self.tool_calls)),
+            ("llm calls", str(self.llm_calls)),
+            (
+                "tokens",
+                f"{self.prompt_tokens} in / {self.completion_tokens} out"
+                f" / {self.total_tokens} total",
+            ),
+            ("duration", f"{self.duration_ms:.1f} ms"),
+            ("errors", str(self.errors)),
+            ("outcomes", ", ".join(self.outcomes) or "—"),
+        ]
+        width = max(len(key) for key, _ in rows)
+        return "\n".join(
+            ["scenario report"] + [f"  {key:<{width}}  {value}" for key, value in rows]
+        )
+
+
+def _scenario_report(
+    traces: list[RunTrace],
+    calls: list[ToolCallRecord],
+    *,
+    turns: int | None = None,
+    errors: int | None = None,
+) -> ScenarioReport:
+    """Builds a `ScenarioReport` off one or more run traces + recorded calls."""
+    spans = [span for trace in traces for span in trace.spans]
+    llm = [call for trace in traces for call in trace.llm_calls]
+    outcomes: list[str] = []
+    for trace in traces:
+        if trace.outcome and trace.outcome not in outcomes:
+            outcomes.append(trace.outcome)
+    return ScenarioReport(
+        turns=len(traces) if turns is None else turns,
+        spans=len(spans),
+        agents=sorted({span.agent for span in spans}),
+        tool_calls=len(calls),
+        llm_calls=len(llm),
+        prompt_tokens=sum(call.prompt_tokens for call in llm),
+        completion_tokens=sum(call.completion_tokens for call in llm),
+        duration_ms=round(sum(trace.duration_ms for trace in traces), 1),
+        errors=(sum(1 for span in spans if span.error) if errors is None else errors),
+        outcomes=outcomes,
+    )
+
+
 @dataclass
 class ScenarioResult:
     """Everything a scenario assertion needs, read off one `ScenarioLab.run()`.
@@ -98,6 +189,15 @@ class ScenarioResult:
     def artifacts(self, artifact_type: type[T]) -> ArtifactAssertions[T]:
         return ArtifactAssertions(self.context, artifact_type)
 
+    @property
+    def report(self) -> ScenarioReport:
+        """Token/latency/tool totals for this one turn (see `ScenarioReport`)."""
+        traces = [self.trace] if self.trace is not None else []
+        errors = None
+        if not traces and self.stats is not None:
+            errors = self.stats.errors
+        return _scenario_report(traces, self.calls, turns=1, errors=errors)
+
     @property
     def tools(self) -> ToolAssertions:
         return ToolAssertions(self.calls)
@@ -389,3 +489,8 @@ class Scenario:
     def errors(self) -> ErrorAssertions:
         """Isolated agent errors across every turn run so far."""
         return ErrorAssertions(_combined_trace(self.all_traces), None)
+
+    @property
+    def report(self) -> ScenarioReport:
+        """Aggregate `ScenarioReport` across every turn run so far."""
+        return _scenario_report(self.all_traces, self.all_calls)

{reactifact-0.11.2 → reactifact-0.12.0}/reactifact/tracing/__init__.py

@@ -5,6 +5,7 @@ Package: models (`models`), delivery (`tracer`), storage (`store`), UI
 but without `web`, so that importing the package does not pull in FastAPI.
 """
 
+from .columns import TraceColumn
 from .langfuse import LangfuseTracer
 from .models import (
     AgentSpan,
@@ -41,6 +42,7 @@ __all__ = [
     "Tag",
     "TagAssignment",
     "TraceAnnotator",
+    "TraceColumn",
     "TraceReader",
     "TraceSink",
     "TraceStore",

reactifact-0.12.0/reactifact/tracing/columns.py

@@ -0,0 +1,169 @@
+"""Configurable trace-table columns: pull a field out of a run's artifacts.
+
+A `TraceColumn` names a column for the traces table and where its value comes
+from: which agent's span, which artifact type, reads or writes, and a dotted
+path into the artifact's data. A list step takes its first element unless an
+explicit index is given, so `sources.title` means `sources[0].title`.
+
+    TraceColumn(label="Q", agent="start", type=Question, field="query")
+    TraceColumn(label="Answer", agent="normalize",
+                type=FinalResponse, field="answer")
+    TraceColumn(label="Top source", agent="search", type=Evidence,
+                field="sources.title")
+
+Extraction runs over the artifact JSON persisted with each span. That JSON is
+clipped by `RuntimeResources.trace_truncate` (default 1500 chars); a value cut
+off mid-JSON cannot be parsed and falls back to the column's `default`.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from dataclasses import dataclass
+from dataclasses import field as dataclass_field
+from typing import Any, Literal
+
+from .models import ArtifactRef
+
+#: Which side of a span to read the artifact from.
+Direction = Literal["read", "write", "any"]
+
+#: How far to look for the artifact: this run alone, or its whole session
+#: (useful for a chat, where the question lives in the first run and the final
+#: answer in a later one — every row then shows the session's Q/A).
+Scope = Literal["run", "session"]
+
+
+@dataclass(frozen=True)
+class TraceColumn:
+    """One configurable column in the traces table.
+
+    `agent` / `type` narrow which artifacts are considered (both optional);
+    `direction` picks reads, writes or either; `index` selects among the
+    matches (default: the last one); `field` is a dotted path into that
+    artifact's JSON data (empty = the whole object). `scope="session"` looks
+    across every run of the row's session (chronological) instead of just this
+    run — so a chat's question (`index=0`) and final answer (`index=-1`) show on
+    every row. `default` is shown when nothing matches or the data is
+    unparseable.
+    """
+
+    label: str
+    field: str
+    agent: str | None = None
+    type: str | type | None = None
+    direction: Direction = "any"
+    index: int = -1
+    scope: Scope = "run"
+    default: str = "—"
+
+    def type_name(self) -> str | None:
+        if self.type is None:
+            return None
+        return getattr(self.type, "__name__", str(self.type))
+
+
+@dataclass(frozen=True)
+class SpanArtifacts:
+    """The minimum a column needs off a span: its agent + read/write refs."""
+
+    agent: str
+    reads: list[ArtifactRef] = dataclass_field(default_factory=list)
+    writes: list[ArtifactRef] = dataclass_field(default_factory=list)
+
+
+def _candidates(
+    spans: Sequence[SpanArtifacts], column: TraceColumn
+) -> list[ArtifactRef]:
+    """Artifacts across `spans` matching the column's agent/type/direction."""
+    want = column.type_name()
+    found: list[ArtifactRef] = []
+    for span in spans:
+        if column.agent is not None and span.agent != column.agent:
+            continue
+        # `Runtime._collect_reads` records the triggering artifact first, then
+        # the other inputs — so the trigger (the newest artifact of the run) is
+        # first. Reverse reads so it comes *last*, making the default
+        # `index=-1` mean "most recent" across reads and writes alike.
+        reads = list(reversed(span.reads))
+        if column.direction == "write":
+            refs = span.writes
+        elif column.direction == "read":
+            refs = reads
+        else:
+            refs = [*reads, *span.writes]
+        found.extend(ref for ref in refs if want is None or ref.data_type == want)
+    return found
+
+
+def _walk(value: Any, parts: Sequence[str]) -> Any:
+    """Descends `value` along `parts`; a list step takes element 0 by default."""
+    for part in parts:
+        if value is None:
+            return None
+        if isinstance(value, dict):
+            value = value.get(part)
+        elif isinstance(value, (list, tuple)):
+            if part.lstrip("-").isdigit():
+                index = int(part)
+                value = value[index] if -len(value) <= index < len(value) else None
+            elif value:
+                first = value[0]
+                value = first.get(part) if isinstance(first, dict) else None
+            else:
+                value = None
+        else:
+            return None
+    return value
+
+
+def _display(value: Any) -> str | None:
+    if isinstance(value, (list, tuple)):
+        value = value[0] if value else None
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, dict):
+        return json.dumps(value, ensure_ascii=False, default=str)
+    return str(value)
+
+
+def extract_columns(
+    spans: Sequence[SpanArtifacts],
+    columns: Sequence[TraceColumn],
+    *,
+    session_spans: Sequence[SpanArtifacts] | None = None,
+) -> dict[str, str]:
+    """Resolves every column to a display string for one run.
+
+    `spans` are the run's own spans; `session_spans` (when given) are every span
+    of the row's session in chronological order, used by `scope="session"`
+    columns. Best-effort by design (a dashboard convenience): a missing
+    artifact, an out-of-range index, or JSON clipped by the trace truncation
+    limit yields the column's `default` instead of raising.
+    """
+    out: dict[str, str] = {}
+    for column in columns:
+        source = session_spans if column.scope == "session" else spans
+        refs = _candidates(source if source is not None else spans, column)
+        ref: ArtifactRef | None = None
+        if refs:
+            try:
+                ref = refs[column.index]
+            except IndexError:
+                ref = None
+        value: Any = None
+        if ref is not None and ref.data:
+            try:
+                parsed = json.loads(ref.data)
+            except (ValueError, TypeError):
+                parsed = None
+            if isinstance(parsed, dict):
+                value = (
+                    _walk(parsed, column.field.split(".")) if column.field else parsed
+                )
+        display = _display(value)
+        out[column.label] = display if display is not None else column.default
+    return out

{reactifact-0.11.2 → reactifact-0.12.0}/reactifact/tracing/postgres.py

@@ -18,9 +18,11 @@ be shared across requests.
 
 from __future__ import annotations
 
+from collections.abc import Sequence
 from datetime import UTC, datetime
 from typing import Any
 
+from .columns import SpanArtifacts, TraceColumn, extract_columns
 from .models import (
     AgentSpan,
     ArtifactRef,
@@ -238,6 +240,7 @@ class PostgresStore:
         q: str | None = None,
         sort: str = "started_at",
         order: str = "desc",
+        columns: Sequence[TraceColumn] | None = None,
         limit: int = 50,
         offset: int = 0,
     ) -> dict[str, Any]:
@@ -287,6 +290,69 @@ class PostgresStore:
                         tags_by_run.setdefault(run_id, []).append(
                             {"tag": tag, "note": note, "color": color}
                         )
+                fields_by_run: dict[str, dict[str, str]] = {}
+                if run_ids and columns:
+                    placeholders = ",".join("%s" for _ in run_ids)
+                    await cur.execute(
+                        "SELECT run_id, agent, reads, writes FROM spans "
+                        f"WHERE run_id IN ({placeholders}) ORDER BY id",
+                        tuple(run_ids),
+                    )
+                    spans_by_run: dict[str, list[SpanArtifacts]] = {}
+                    for run_id, agent, reads, writes in await cur.fetchall():
+                        spans_by_run.setdefault(run_id, []).append(
+                            SpanArtifacts(
+                                agent=agent,
+                                reads=[ArtifactRef(**d) for d in reads],
+                                writes=[ArtifactRef(**d) for d in writes],
+                            )
+                        )
+                    needs_session = any(c.scope == "session" for c in columns)
+                    session_order: dict[str, list[tuple[str, list[SpanArtifacts]]]] = {}
+                    if needs_session:
+                        sessions = sorted({r[1] for r in rows if r[1]})
+                        for sid in sessions:
+                            await cur.execute(
+                                "SELECT id FROM runs WHERE session_id = %s "
+                                "ORDER BY started_at ASC, id ASC",
+                                (sid,),
+                            )
+                            ordered = [x[0] for x in await cur.fetchall()]
+                            if not ordered:
+                                continue
+                            sph = ",".join("%s" for _ in ordered)
+                            await cur.execute(
+                                "SELECT run_id, agent, reads, writes FROM spans "
+                                f"WHERE run_id IN ({sph}) ORDER BY id",
+                                tuple(ordered),
+                            )
+                            by_run: dict[str, list[SpanArtifacts]] = {}
+                            for run_id, agent, reads, writes in await cur.fetchall():
+                                by_run.setdefault(run_id, []).append(
+                                    SpanArtifacts(
+                                        agent=agent,
+                                        reads=[ArtifactRef(**d) for d in reads],
+                                        writes=[ArtifactRef(**d) for d in writes],
+                                    )
+                                )
+                            session_order[sid] = [
+                                (rid, by_run.get(rid, [])) for rid in ordered
+                            ]
+                    for row in rows:
+                        session_spans: list[SpanArtifacts] | None = None
+                        if needs_session:
+                            ordered = session_order.get(row[1])
+                            if ordered is not None:
+                                session_spans = []
+                                for rid, spans in ordered:
+                                    session_spans.extend(spans)
+                                    if rid == row[0]:
+                                        break
+                        fields_by_run[row[0]] = extract_columns(
+                            spans_by_run.get(row[0], []),
+                            columns,
+                            session_spans=session_spans,
+                        )
         finally:
             await conn.close()
 
@@ -301,6 +367,7 @@ class PostgresStore:
                 "completion_tokens": r[6],
                 "spans": r[7],
                 "tags": tags_by_run.get(r[0], []),
+                "fields": fields_by_run.get(r[0], {}),
             }
             for r in rows
         ]
@@ -374,6 +441,63 @@ class PostgresStore:
             annotations=annotations,
         )
 
+    async def field_values(
+        self, run_id: str, columns: Sequence[TraceColumn]
+    ) -> dict[str, str]:
+        if not columns:
+            return {}
+        await self._ensure_schema()
+        conn = await self._psycopg.AsyncConnection.connect(self.dsn)
+        try:
+            async with conn.cursor() as cur:
+                await cur.execute(
+                    "SELECT session_id FROM runs WHERE id = %s", (run_id,)
+                )
+                row = await cur.fetchone()
+                if row is None:
+                    return {}
+                session_id = row[0]
+
+                async def spans_of(
+                    run_ids: list[str],
+                ) -> dict[str, list[SpanArtifacts]]:
+                    placeholders = ",".join("%s" for _ in run_ids)
+                    await cur.execute(
+                        "SELECT run_id, agent, reads, writes FROM spans "
+                        f"WHERE run_id IN ({placeholders}) ORDER BY id",
+                        tuple(run_ids),
+                    )
+                    out: dict[str, list[SpanArtifacts]] = {}
+                    for rid, agent, reads, writes in await cur.fetchall():
+                        out.setdefault(rid, []).append(
+                            SpanArtifacts(
+                                agent=agent,
+                                reads=[ArtifactRef(**d) for d in reads],
+                                writes=[ArtifactRef(**d) for d in writes],
+                            )
+                        )
+                    return out
+
+                run_spans = (await spans_of([run_id])).get(run_id, [])
+                session_spans: list[SpanArtifacts] | None = None
+                if session_id and any(c.scope == "session" for c in columns):
+                    await cur.execute(
+                        "SELECT id FROM runs WHERE session_id = %s "
+                        "ORDER BY started_at ASC, id ASC",
+                        (session_id,),
+                    )
+                    ordered = [r[0] for r in await cur.fetchall()]
+                    if ordered:
+                        by_run = await spans_of(ordered)
+                        session_spans = []
+                        for rid in ordered:
+                            session_spans.extend(by_run.get(rid, []))
+                            if rid == run_id:
+                                break
+        finally:
+            await conn.close()
+        return extract_columns(run_spans, columns, session_spans=session_spans)
+
     async def sessions(
         self,
         *,