reactifact 0.6.0__py3-none-any.whl

reactifact/effects.py

@@ -0,0 +1,232 @@
+"""reactifact.effects — the produce-scoped effect set (§24, §41, §67).
+
+The *authoring surface* of a produce. Instead of hand-assembling a `Patch` on
+every turn, a produce writes what should change into its effect slot, and the
+runtime compiles the slot into one atomic patch (commit + events + trace):
+
+    class Scout(Produce[SourceRef]):
+        async def produce(self, context, inputs, event=None) -> None:
+            refs = await fan_out_sources(context, query, owner_id=...)
+            self.effects.create(SearchDone(...), id=f"scouted:{qid}")
+            return None
+
+`Effects` is produce-scoped and concurrency-safe: the runtime pushes a fresh
+slot via a contextvar before invoking the produce and pops it afterwards, so
+parallel produces never see each other's effects. Nothing is applied until the
+runtime commits — atomicity stays structural (§41), no diff/rollback.
+
+Users rarely touch `Patch` in an ordinary produce: `Effects` is the language,
+`Patch` is the compiled transport. Advanced assembly (recipes, tool loops) can
+still build a `Patch` explicitly and return it — the runtime merges both.
+"""
+
+from __future__ import annotations
+
+from contextvars import ContextVar, Token
+from typing import TYPE_CHECKING, Any
+
+from .patches import (
+    Create,
+    Link,
+    Operation,
+    Patch,
+    Unlink,
+    Update,
+    _auto_id,
+    _id_of,
+)
+
+if TYPE_CHECKING:
+    from .artifacts import Artifact
+    from .context import Context
+
+
+class Handle:
+    """A patch-local handle for a *planned* artifact — link/unlink without ids (§38).
+
+    Returned by `Effects.create`; the id is pinned, so the handle is a valid
+    link target (`answer.link("supported_by", evidence)`), and the reader sees
+    which artifact a provenance edge comes from.
+    """
+
+    __slots__ = ("_effects", "id", "_data")
+
+    def __init__(self, effects: Effects, artifact_id: str, data: Any):
+        self._effects = effects
+        self.id = artifact_id
+        self._data = data
+
+    def link(self, relation: str, target: Any) -> Handle:
+        """Appends `Link` from this planned artifact to `target` (id/Artifact/Handle)."""
+        self._effects.link(self.id, relation, _id_of(target))
+        return self
+
+    def unlink(self, relation: str | None = None, target: Any | None = None) -> Handle:
+        self._effects.unlink(
+            self.id, relation, _id_of(target) if target is not None else None
+        )
+        return self
+
+    @property
+    def type(self) -> str:
+        return type(self._data).__name__
+
+    def __repr__(self) -> str:
+        return f"Handle(id={self.id!r}, {self.type})"
+
+
+class Effects:
+    """The current produce's effect set (creates/updates/links to commit once)."""
+
+    __slots__ = ("_context", "operations")
+
+    def __init__(self, context: Context):
+        self._context = context
+        self.operations: list[Operation] = []
+
+    @property
+    def context(self) -> Context:
+        return self._context
+
+    def add(self, op: Operation) -> Effects:
+        self.operations.append(op)
+        return self
+
+    def create(self, data: Any, *, id: str | None = None) -> Handle:
+        """Plans a new artifact; returns a linkable/updatable handle (§38).
+
+        If `id` names an artifact that already exists when this effect is
+        applied, the runtime treats it as a refresh (a new version of the
+        same logical entity, §42/§43) rather than creating a duplicate — the
+        same rule `Runtime._apply_patch` documents. Use `upsert` at the call
+        site when that "may already exist" intent should be explicit instead
+        of implicit in a plain `create(..., id=...)`.
+        """
+        artifact_id = id or _auto_id(type(data).__name__)
+        self.operations.append(Create(data, id=artifact_id))
+        return Handle(self, artifact_id, data)
+
+    def create_once(self, data: Any, *, id: str) -> Handle | None:
+        """Idempotent create (§42): `None` if `id` already exists in the
+        context, otherwise the same as `create(data, id=id)`.
+
+        Folds the "already done" guard every produce needs for a re-derived
+        id (`f"answer:{qid}"`) into the call itself:
+
+            handle = self.effects.create_once(Answer(...), id=f"answer:{qid}")
+            if handle is None:
+                return None  # already answered — nothing to do
+
+        instead of a separate `if context.get(f"answer:{qid}") is not None:
+        return None` above the call — one less place to get the id string
+        wrong between the guard and the create.
+        """
+        if self._context.get(id) is not None:
+            return None
+        return self.create(data, id=id)
+
+    def upsert(self, data: Any, *, id: str) -> Handle:
+        """Explicit create-or-refresh: same effect as `create(data, id=id)`.
+
+        Prefer this over `create(..., id=...)` when the artifact may already
+        exist (e.g. a re-derived id like `f"answer:{qid}"`) — it says at the
+        call site that an update is an expected outcome, not a surprise.
+        """
+        return self.create(data, id=id)
+
+    def update(self, artifact: Artifact[Any], **fields: Any) -> Effects:
+        """Bumps fields of an *existing* artifact (a new version)."""
+        new_data = artifact.data.model_copy(update=fields)
+        self.operations.append(Update(artifact.id, new_data))
+        return self
+
+    def delete(self, artifact: Any) -> Effects:
+        from .patches import Delete
+
+        self.operations.append(Delete(_id_of(artifact)))
+        return self
+
+    def link(self, source: Any, relation: str, target: Any) -> Effects:
+        self.operations.append(Link(_id_of(source), relation, _id_of(target)))
+        return self
+
+    def ask(
+        self,
+        question: str,
+        *,
+        kind: str = "general",
+        notes: dict[str, Any] | None = None,
+        id: str | None = None,
+    ) -> Handle:
+        """Poses a question to a human (HITL, §60): creates a `PendingQuestion`.
+
+        Returns a handle you can link later; the human answer is recorded via
+        `effects.resume(question_art, resolution)` (§60). Pass `id` for a
+        stable, re-derivable question (e.g. `f"steer:{qid}:{round}"`) so a
+        guard like `if context.get(id) is not None: return None` can stop the
+        produce from asking again while the question is still unanswered.
+        """
+        from .interrupt import PendingQuestion
+
+        return self.create(
+            PendingQuestion(question=question, kind=kind, notes=dict(notes or {})),
+            id=id,
+        )
+
+    def resume(self, question: Any, resolution: str) -> Effects:
+        """Records the human answer on a `PendingQuestion` (HITL, §60)."""
+        from datetime import UTC, datetime
+
+        self.update(
+            question,
+            answered=True,
+            resolution=resolution,
+            resolved_at=datetime.now(UTC),
+        )
+        return self
+
+    def unlink(
+        self,
+        source: Any,
+        relation: str | None = None,
+        target: Any | None = None,
+    ) -> Effects:
+        self.operations.append(
+            Unlink(
+                _id_of(source), relation, _id_of(target) if target is not None else None
+            )
+        )
+        return self
+
+    def is_empty(self) -> bool:
+        return len(self.operations) == 0
+
+    def to_patch(self) -> Patch:
+        """Compiles the effects into a `Patch` (the runtime's transport)."""
+        patch = Patch()
+        for op in self.operations:
+            patch.add(op)
+        return patch
+
+
+# --------------------------------------------------------------------------- #
+# Produce-scoped slot — the runtime pushes/pops a fresh Effects per produce run
+# --------------------------------------------------------------------------- #
+
+_ACTIVE: ContextVar[Effects | None] = ContextVar("reactifact_effects", default=None)
+
+
+def current_effects() -> Effects | None:
+    """The produce-scoped effect slot, or None outside a running produce."""
+    return _ACTIVE.get()
+
+
+def set_effects(effects: Effects | None) -> Token[Any]:
+    return _ACTIVE.set(effects)
+
+
+def reset_effects(token: Token[Any]) -> None:
+    _ACTIVE.reset(token)
+
+
+__all__ = ["Effects", "Handle", "current_effects", "reset_effects", "set_effects"]

reactifact/eval.py

@@ -0,0 +1,319 @@
+"""reactifact.eval — multi-level evaluation harness (§56).
+
+Because state is structured, evaluation is not only `answer == expected`: it
+separates *evidence quality*, *claim verification*, *provenance grounding*,
+*calculation correctness*, *answer coverage* and *source coverage*. Each metric
+is a pure function over the final `Context` (+ optional ground truth); the
+harness runs a case, collects its metrics, and renders a weighted report.
+
+    cases = [EvalCase("calc-question", run=_run_knowledge_calc)]
+    report = run_suite(cases, metrics={**core_metrics, "calc": calculation_correctness()})
+    print(report.render())
+
+All metrics are deterministic and LLM-free. They match artifact *classes by
+name* (`Answer`, `Evidence`, …) so the harness needs no domain imports — the
+domain stays out of the framework.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from .context import Context
+
+
+@dataclass(frozen=True)
+class Metric:
+    """One measured quantity with a 0..1 score and a reporting weight (§56)."""
+
+    name: str
+    score: float = 0.0
+    weight: float = 1.0
+    note: str = ""
+
+
+MetricFn = Callable[[Context, Mapping[str, Any] | None], float | None]
+
+
+@dataclass
+class EvalCase:
+    """A runnable evaluation case: executes the pipeline and returns its Context."""
+
+    name: str
+    run: Callable[[], Context]
+    expected: Mapping[str, Any] | None = None
+
+
+@dataclass
+class EvalResult:
+    """Scores of one case; `overall` is the weighted mean of its metrics."""
+
+    case: str
+    metrics: list[Metric] = field(default_factory=list)
+    skipped: list[str] = field(default_factory=list)
+
+    def overall(self) -> float:
+        metrics = self.metrics or ()
+        weights = [m.weight for m in metrics]
+        total_weight = sum(weights) or 1.0
+        return sum(m.score * m.weight for m in metrics) / total_weight
+
+
+@dataclass
+class EvalReport:
+    """The whole suite: one `EvalResult` per case."""
+
+    results: list[EvalResult] = field(default_factory=list)
+
+    def overall(self) -> float:
+        if not self.results:
+            return 0.0
+        return sum(r.overall() for r in self.results) / len(self.results)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "overall": round(self.overall(), 4),
+            "cases": [
+                {
+                    "case": r.case,
+                    "overall": round(r.overall(), 4),
+                    "metrics": [
+                        {
+                            "name": m.name,
+                            "score": round(m.score, 4),
+                            "weight": m.weight,
+                            "note": m.note,
+                        }
+                        for m in r.metrics
+                    ],
+                    "skipped": r.skipped,
+                }
+                for r in self.results
+            ],
+        }
+
+    def render(self) -> str:
+        lines = ["eval · multi-level report (§56)", ""]
+        for r in self.results:
+            lines.append(f"[{r.case}] overall {r.overall():.3f}")
+            for m in r.metrics:
+                entry = f"    {m.name:<28} {m.score:6.3f}"
+                if m.note:
+                    entry += f"  {m.note}"
+                lines.append(entry)
+            for name in r.skipped:
+                lines.append(f"    {name:<28}      — skipped (no ground truth)")
+        lines.append(f"\nsuite overall: {self.overall():.3f}")
+        return "\n".join(lines)
+
+
+def run_case(
+    case: EvalCase,
+    metrics: Mapping[str, MetricFn],
+) -> EvalResult:
+    """Executes the case and scores it against every metric."""
+    context = case.run()
+    result = EvalResult(case=case.name)
+    for name, metric_fn in metrics.items():
+        score = metric_fn(context, case.expected)
+        if score is None:
+            result.skipped.append(name)
+            continue
+        result.metrics.append(
+            Metric(name=name, score=round(max(0.0, min(1.0, score)), 4))
+        )
+    return result
+
+
+def run_suite(
+    cases: list[EvalCase],
+    metrics: Mapping[str, MetricFn],
+) -> EvalReport:
+    return EvalReport(results=[run_case(case, metrics) for case in cases])
+
+
+# --------------------------------------------------------------------------- #
+# Generic helpers — artifact classes are matched *by name* (no domain imports)
+# --------------------------------------------------------------------------- #
+
+
+def _named(context: Context, *class_names: str) -> list[Any]:
+    return [a for a in context.list_artifacts() if type(a.data).__name__ in class_names]
+
+
+def answer_present(
+    context: Context, expected: Mapping[str, Any] | None = None
+) -> float:
+    """1.0 when at least one `Answer` artifact exists."""
+    return 1.0 if _named(context, "Answer") else 0.0
+
+
+def provenance_grounded(
+    context: Context, expected: Mapping[str, Any] | None = None
+) -> float:
+    """Share of answers backed by at least one existing `supported_by` link.
+
+    Provenance correctness (§34): an answer without a resolvable supporting link
+    is ungrounded — the graph is the source of truth, not the text.
+    """
+    answers = _named(context, "Answer")
+    if not answers:
+        return 0.0
+    grounded = sum(1 for a in answers if context.related(a.id, relation="supported_by"))
+    return grounded / len(answers)
+
+
+def evidence_quality(
+    context: Context,
+    expected: Mapping[str, Any] | None = None,
+    *,
+    threshold: float = 0.5,
+) -> float:
+    """Share of `Evidence` artifacts scoring at or above `threshold` (0 if none)."""
+    evidences = _named(context, "Evidence")
+    if not evidences:
+        return 0.0
+    good = sum(
+        1 for e in evidences if (getattr(e.data, "score", 0.0) or 0.0) >= threshold
+    )
+    return good / len(evidences)
+
+
+def claim_verification(
+    context: Context,
+    expected: Mapping[str, Any] | None = None,
+    *,
+    valid: tuple[str, ...] = ("verified",),
+) -> float:
+    """Share of `Claim` artifacts in a `valid` status (1.0 if no claims)."""
+    claims = _named(context, "Claim")
+    if not claims:
+        return 1.0
+    good = sum(1 for c in claims if getattr(c.data, "status", "") in valid)
+    return good / len(claims)
+
+
+def confidence_calibration(
+    expected_key: str = "claim_correctness",
+) -> Callable[[Context, Mapping[str, Any] | None], float | None]:
+    """How well `Claim.confidence` predicts actual correctness (Brier score, §56).
+
+    `expected[expected_key]` maps claim artifact id -> bool (was the claim
+    actually correct). Score is `1 - mean((confidence - label) ** 2)` over
+    claims with ground truth, so 1.0 is perfectly calibrated. This is
+    distinct from `claim_verification`: a claim can have `status="verified"`
+    with a poorly-calibrated confidence and still pass that metric.
+    """
+
+    def _score(context: Context, expected: Mapping[str, Any] | None) -> float | None:
+        if expected is None or expected.get(expected_key) is None:
+            return None
+        labels: Mapping[str, bool] = expected[expected_key]
+        claims = _named(context, "Claim")
+        scored = [
+            (float(getattr(c.data, "confidence", 0.0)), 1.0 if labels[c.id] else 0.0)
+            for c in claims
+            if c.id in labels
+        ]
+        if not scored:
+            return None
+        brier = sum((conf - label) ** 2 for conf, label in scored) / len(scored)
+        return 1.0 - brier
+
+    return _score
+
+
+def answer_coverage(
+    expected_key: str = "answer",
+) -> Callable[[Context, Mapping[str, Any] | None], float | None]:
+    """Coverage of the expected answer text by the actual answer (0..1)."""
+
+    def _score(context: Context, expected: Mapping[str, Any] | None) -> float | None:
+        from .recipes import keyword_score
+
+        if expected is None or expected.get(expected_key) is None:
+            return None
+        answers = _named(context, "Answer")
+        if not answers:
+            return 0.0
+        return keyword_score(
+            str(getattr(answers[0].data, "text", "")), str(expected[expected_key])
+        )
+
+    return _score
+
+
+def calculation_correctness(
+    *,
+    values: tuple[float, ...] | None = None,
+    expected_key: str = "calculations",
+) -> Callable[[Context, Mapping[str, Any] | None], float | None]:
+    """Share of `Calculation` artifacts whose value matches the ground truth.
+
+    Expected values come from `values` and/or the case metadata `calculations`.
+    """
+
+    def _score(context: Context, expected: Mapping[str, Any] | None) -> float | None:
+        allowed = set(values or ())
+        if expected is not None and expected.get(expected_key) is not None:
+            allowed |= {float(v) for v in expected[expected_key]}
+        calcs = _named(context, "Calculation")
+        if not calcs:
+            return 0.0
+        if not allowed:
+            return None  # no ground truth yet
+        good = sum(1 for c in calcs if float(getattr(c.data, "value", 0.0)) in allowed)
+        return good / len(calcs)
+
+    return _score
+
+
+def source_coverage(
+    expected_key: str = "sources",
+) -> Callable[[Context, Mapping[str, Any] | None], float | None]:
+    """Share of the answer's sources matched by the expected source markers."""
+
+    def _score(context: Context, expected: Mapping[str, Any] | None) -> float | None:
+        if expected is None or expected.get(expected_key) is None:
+            return None
+        markers = [str(m) for m in expected[expected_key]]
+        answers = _named(context, "Answer")
+        if not answers:
+            return 0.0
+        sources = [s for s in getattr(answers[0].data, "sources", []) if s]
+        if not sources:
+            return 0.0
+        covered = sum(1 for s in sources if any(m in s for m in markers))
+        return covered / len(sources)
+
+    return _score
+
+
+#: The non-generative core metrics, ready to reuse (§56).
+core_metrics: dict[str, MetricFn] = {
+    "answer_present": answer_present,
+    "provenance_grounded": provenance_grounded,
+    "evidence_quality": evidence_quality,
+    "claim_verification": claim_verification,
+}
+
+
+__all__ = [
+    "EvalCase",
+    "EvalReport",
+    "EvalResult",
+    "Metric",
+    "answer_coverage",
+    "answer_present",
+    "calculation_correctness",
+    "claim_verification",
+    "confidence_calibration",
+    "core_metrics",
+    "evidence_quality",
+    "provenance_grounded",
+    "run_case",
+    "run_suite",
+    "source_coverage",
+]

reactifact/events.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from enum import StrEnum
+
+
+class EventType(StrEnum):
+    ARTIFACT_CREATED = "artifact_created"
+    ARTIFACT_UPDATED = "artifact_updated"
+    ARTIFACT_DELETED = "artifact_deleted"
+    ARTIFACT_STALE = "artifact_stale"
+
+
+class Event:
+    """A lightweight event referencing an artifact by id and type."""
+
+    def __init__(
+        self,
+        type: EventType,
+        artifact_type: type | str,
+        artifact_id: str,
+    ):
+        self.type = type
+        self.artifact_type = artifact_type
+        self.artifact_id = artifact_id
+        self.timestamp = datetime.now(UTC)
+
+    def __repr__(self) -> str:
+        type_name = (
+            self.artifact_type.__name__
+            if isinstance(self.artifact_type, type)
+            else self.artifact_type
+        )
+        return f"<Event {self.type.value} {type_name} id={self.artifact_id}>"

reactifact/interrupt.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class PendingQuestion(BaseModel):
+    """Artifact awaiting a human response (HITL, constitution §60).
+
+    Created by an agent (or directly) to block a step until user input.
+    A human answer is recorded via `self.effects.resume(question, answer)` (§60),
+    after which agents subscribed to `PendingQuestion(answered=True)` continue.
+    """
+
+    question: str
+    kind: str = "general"
+    notes: dict[str, Any] = Field(default_factory=dict)
+    answered: bool = False
+    resolution: str | None = None
+    resolved_at: datetime | None = None