sengol 0.8.0__py3-none-any.whl

sengol/__init__.py

@@ -0,0 +1,44 @@
+"""sengol — AI governance SDK."""
+
+from sengol.build.gate import PromotionGate
+from sengol.runtime.guardrails import GuardrailPolicy, GuardrailRuntime
+from sengol.runtime.sampling import FeedbackSampler, RandomSampler
+
+__version__ = "0.8.0"
+
+
+# Auto-intercept (ADR-0084) — lazy imports to keep anthropic/openai optional
+def configure(**kwargs):
+    """Configure sengol auto-intercept. See sengol.instrument.configure for args."""
+    from sengol.instrument import configure as _configure
+
+    return _configure(**kwargs)
+
+
+def instrument():
+    """Patch Anthropic and OpenAI clients with sengol guardrails."""
+    from sengol.instrument import instrument as _instrument
+
+    return _instrument()
+
+
+def uninstrument():
+    """Reverse patches applied by instrument()."""
+    from sengol.instrument import uninstrument as _uninstrument
+
+    return _uninstrument()
+
+
+from sengol.instrument.exceptions import BlockedError  # noqa: E402
+
+__all__ = [
+    "GuardrailPolicy",
+    "GuardrailRuntime",
+    "PromotionGate",
+    "RandomSampler",
+    "FeedbackSampler",
+    "configure",
+    "instrument",
+    "uninstrument",
+    "BlockedError",
+]

sengol/adapters/__init__.py

@@ -0,0 +1,38 @@
+"""Backend export adapters — push Sengol evaluation results to external systems.
+
+Adapters here are **optional, bank-configurable export targets**. They are never on
+the agent code path and never the source of truth — the Postgres ``AuditStore``
+(HMAC-signed, append-only) always is. An adapter is a one-way exporter the operator
+opts into; removing one changes nothing about what Sengol records.
+
+Each adapter soft-imports its vendor SDK so the dependency is required **only** when
+the adapter is used (see the matching ``[project.optional-dependencies]`` extra in
+``pyproject.toml``). Importing this package never imports a vendor SDK.
+
+Free / Apache 2.0.
+"""
+
+from sengol.adapters.base import BaseExporter, ResultExporter
+from sengol.adapters.registry import (
+    DISABLE_PLUGINS_ENV,
+    ENTRY_POINT_GROUP,
+    ExporterRegistry,
+    register_exporter,
+)
+
+# Importing these modules registers the built-in exporters (via the
+# @register_exporter decorator) — keep them last so the registry symbols above are
+# defined when the decorators run.
+from sengol.adapters.jsonl import JsonlFileExporter
+from sengol.adapters.mlflow import MLflowExporter
+
+__all__ = [
+    "BaseExporter",
+    "ResultExporter",
+    "ExporterRegistry",
+    "register_exporter",
+    "ENTRY_POINT_GROUP",
+    "DISABLE_PLUGINS_ENV",
+    "MLflowExporter",
+    "JsonlFileExporter",
+]

sengol/adapters/base.py

@@ -0,0 +1,125 @@
+"""Exporter plugin contract — the structural type and base class third parties extend.
+
+An *exporter* is an optional, downstream sink for evaluation results: it pushes an
+:class:`~sengol.core.types.EvalResult` to a system of the operator's choosing (MLflow,
+Weights & Biases, Datadog, an in-house warehouse, …). Exporters are **never** on the
+agent code path and **never** the source of truth — the Postgres ``AuditStore`` always
+is. Removing or breaking an exporter changes nothing about what Sengol records.
+
+Two surfaces:
+
+* :class:`ResultExporter` — a ``runtime_checkable`` Protocol describing the minimal shape
+  the run-suite exporter loop depends on (``name`` + ``export``). Type against this.
+* :class:`BaseExporter` — the ABC built-ins and third-party plugins subclass. It supplies
+  ``from_config`` (build from a ``sengol.yaml`` options mapping) and a default
+  ``export_batch``, so a plugin only has to implement ``export`` and set ``name``.
+
+Register a subclass with :func:`sengol.adapters.registry.register_exporter` (built-ins) or
+declare it under the ``sengol.exporters`` entry-point group (third-party packages) — see
+:mod:`sengol.adapters.registry`.
+
+Free / Apache 2.0 — no enterprise gate on this code path.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import (
+    Any,
+    ClassVar,
+    Mapping,
+    Optional,
+    Protocol,
+    Sequence,
+    runtime_checkable,
+)
+
+from sengol.core.types import EvalResult
+
+
+@runtime_checkable
+class ResultExporter(Protocol):
+    """Structural type the exporter loop depends on.
+
+    Anything carrying a ``name`` and an
+    ``export(result, *, gold_score=None, tenant_id=None)`` method satisfies it —
+    subclassing :class:`BaseExporter` is the supported way to get there, but the
+    Protocol keeps the run-suite loop decoupled from the base class.
+
+    ``export`` is **synchronous by design** even though most targets are network
+    I/O. The run-suite loop runs each exporter in a worker thread with a timeout
+    (so a slow/unreachable backend can't stall the run), which keeps the plugin
+    contract trivial for authors — no event loop, no ``async def``. An exporter
+    that wants concurrency internally is free to use it.
+    """
+
+    name: str
+
+    def export(
+        self,
+        result: EvalResult,
+        *,
+        gold_score: Optional[float] = None,
+        tenant_id: Optional[str] = None,
+    ) -> Optional[str]: ...
+
+
+class BaseExporter(ABC):
+    """Base class for built-in and third-party exporters.
+
+    Subclasses set the class-level :attr:`name` and implement :meth:`export`. The
+    default :meth:`from_config` maps a ``sengol.yaml`` options mapping straight to
+    keyword arguments; override it when the exporter needs custom parsing/validation.
+    """
+
+    #: Stable short name used to select the exporter in ``sengol.yaml`` / ``--export``.
+    #: Must be non-empty and unique across registered exporters.
+    name: ClassVar[str] = ""
+
+    @classmethod
+    def from_config(cls, config: Mapping[str, Any]) -> "BaseExporter":
+        """Build an instance from a ``sengol.yaml`` ``exporters.<name>`` options mapping.
+
+        The default forwards the mapping as keyword arguments, so an exporter whose
+        ``__init__`` keywords match the config keys needs no override. Raise
+        ``TypeError``/``ValueError`` for invalid options — the run-suite loop reports
+        it and skips the exporter rather than failing the run.
+        """
+        return cls(**dict(config))
+
+    @abstractmethod
+    def export(
+        self,
+        result: EvalResult,
+        *,
+        gold_score: Optional[float] = None,
+        tenant_id: Optional[str] = None,
+    ) -> Optional[str]:
+        """Push one result downstream. Return an opaque id (e.g. a run id) or ``None``.
+
+        ``tenant_id`` is the run's tenant (``EvalResult`` itself carries none) so the
+        exported record stays attributable in a multi-tenant deployment — log it as a
+        tag/label/column. ``gold_score`` is the period score when the caller computed
+        one (the ``run-suite`` path does not, so it is ``None`` there).
+
+        Must not raise for *expected* downstream conditions the caller can do nothing
+        about — but any exception is caught by the run-suite loop and logged, never
+        propagated into the gate verdict.
+        """
+        raise NotImplementedError
+
+    def export_batch(
+        self,
+        results: Sequence[EvalResult],
+        *,
+        gold_score: Optional[float] = None,
+        tenant_id: Optional[str] = None,
+    ) -> list[Optional[str]]:
+        """Export each result in order. Override for a more efficient bulk path
+        (e.g. grouping the batch under one parent run)."""
+        return [
+            self.export(r, gold_score=gold_score, tenant_id=tenant_id) for r in results
+        ]
+
+
+__all__ = ["ResultExporter", "BaseExporter"]

sengol/adapters/jsonl.py

@@ -0,0 +1,139 @@
+"""JSONL file exporter — the lightweight, dependency-free built-in (and worked example).
+
+Writes one JSON line per :class:`~sengol.core.types.EvalResult` to a local file. It needs
+no extra dependency (unlike ``[mlflow]``), so it is the cheapest way to get results out of
+Sengol — useful on its own and as the **reference implementation** a third party copies to
+build their own exporter.
+
+Unlike :class:`sengol.adapters.mlflow.MLflowExporter` (content-free by construction), this
+exporter *can* emit free text — evaluator ``reason`` strings — which may carry regulated
+content. It therefore demonstrates the **redaction duty of care**: pass ``redact: true`` (or
+custom ``redact_patterns``) and every emitted ``reason`` is scrubbed through a
+:class:`~sengol.capture.redaction.RegexRedactor` before it touches disk. Exporters that ship
+content to a *third-party SaaS* should always redact.
+
+Free / Apache 2.0.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, ClassVar, Mapping, Optional
+
+from sengol.adapters.base import BaseExporter
+from sengol.adapters.registry import register_exporter
+from sengol.capture.redaction import NoOpRedactor, Redactor, RegexRedactor
+from sengol.core.types import EvalResult
+
+# Default PII patterns applied when ``redact: true`` is set without custom patterns.
+# Conservative, illustrative set — emails and US-style SSNs. Operators with stricter
+# needs pass their own ``redact_patterns``.
+_DEFAULT_PII_PATTERNS = [
+    r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}",  # email
+    r"\b\d{3}-\d{2}-\d{4}\b",  # SSN
+]
+
+
+@register_exporter
+class JsonlFileExporter(BaseExporter):
+    """Append each result as a JSON line to ``{path}``.
+
+    Parameters
+    ----------
+    path:
+        Output file. Parent directories are created. Lines are appended, so reruns
+        accumulate (rotate/clear it yourself if you want a fresh file per run).
+    include_reasons:
+        When ``True`` (default), each score's ``reason`` is included. Set ``False`` to
+        emit only structured fields (ids, pass/fail) — the safest, content-free shape.
+    redactor:
+        A :class:`~sengol.capture.redaction.Redactor` applied to every emitted ``reason``.
+        Defaults to :class:`NoOpRedactor`. Build one from config with ``redact`` /
+        ``redact_patterns`` (see :meth:`from_config`).
+    """
+
+    name: ClassVar[str] = "jsonl"
+
+    @classmethod
+    def from_config(cls, config: Mapping[str, Any]) -> "JsonlFileExporter":
+        """Build from a ``sengol.yaml`` ``exporters.jsonl`` mapping.
+
+        Keys: ``path`` (required), ``include_reasons`` (bool, default True),
+        ``redact`` (bool — turn on the default PII patterns) and/or ``redact_patterns``
+        (list[str] — custom regexes; implies redaction). Any pattern list takes
+        precedence over the default set.
+        """
+        path = config.get("path")
+        if not path:
+            raise ValueError("jsonl exporter requires a `path` option")
+        patterns = config.get("redact_patterns")
+        redactor: Redactor
+        if patterns:
+            redactor = RegexRedactor(list(patterns))
+        elif config.get("redact"):
+            redactor = RegexRedactor(_DEFAULT_PII_PATTERNS)
+        else:
+            redactor = NoOpRedactor()
+        return cls(
+            path=path,
+            include_reasons=bool(config.get("include_reasons", True)),
+            redactor=redactor,
+        )
+
+    def __init__(
+        self,
+        *,
+        path: str | os.PathLike[str],
+        include_reasons: bool = True,
+        redactor: Optional[Redactor] = None,
+    ) -> None:
+        self._path = Path(path)
+        self._include_reasons = include_reasons
+        self._redactor: Redactor = redactor if redactor is not None else NoOpRedactor()
+
+    def export(
+        self,
+        result: EvalResult,
+        *,
+        gold_score: Optional[float] = None,
+        tenant_id: Optional[str] = None,
+    ) -> str:
+        """Append one JSON line for ``result``; return the path written to."""
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        line = json.dumps(self._row(result, gold_score, tenant_id), sort_keys=True)
+        with self._path.open("a", encoding="utf-8") as fh:
+            fh.write(line + "\n")
+        return str(self._path)
+
+    def _row(
+        self,
+        result: EvalResult,
+        gold_score: Optional[float],
+        tenant_id: Optional[str],
+    ) -> dict[str, Any]:
+        scores = []
+        for s in result.scores:
+            entry: dict[str, Any] = {"evaluator": s.evaluator, "passed": s.passed}
+            if s.failure_mode is not None:
+                entry["failure_mode"] = s.failure_mode
+            if self._include_reasons and s.reason:
+                # The one free-text field — scrub it before it lands on disk.
+                entry["reason"] = self._redactor.redact(s.reason)
+            scores.append(entry)
+        row: dict[str, Any] = {
+            "result_id": str(result.result_id),
+            "agent_id": result.agent_id,
+            "agent_version": result.agent_version,
+            "tenant_id": tenant_id,
+            "overall_passed": result.overall_passed,
+            "policies": list(result.policies),
+            "scores": scores,
+        }
+        if gold_score is not None:
+            row["gold_score"] = gold_score
+        return row
+
+
+__all__ = ["JsonlFileExporter"]

sengol/adapters/mlflow.py

@@ -0,0 +1,229 @@
+"""MLflow backend adapter — export Sengol eval results to MLflow tracking.
+
+MLflow is reachable two ways in Sengol: implicitly over OTLP (ADR-0059, Session 5c)
+when a collector fans spans out to it, and explicitly through *this* adapter. This is
+the explicit path — a deliberate ``adapter.export(result)`` call that logs an MLflow run
+with the eval's params, metrics, and tags, for teams that track model/agent quality in
+MLflow experiments alongside their training runs.
+
+The adapter is an **optional export target**, never on the agent code path and never the
+source of truth — the Postgres ``AuditStore`` always is. ``mlflow`` is a soft import
+(``pip install 'sengol[mlflow]'`` / ``uv sync --extra mlflow``): importing this module
+costs nothing, and the SDK is only required when an adapter is actually constructed.
+
+**Authentication.** This adapter brokers no credentials. ``tracking_uri`` is the endpoint
+only; auth is handled by the mlflow SDK's own environment-variable mechanism in the
+runtime — ``MLFLOW_TRACKING_USERNAME``/``MLFLOW_TRACKING_PASSWORD`` (basic),
+``MLFLOW_TRACKING_TOKEN`` (bearer), or ``DATABRICKS_HOST``/``DATABRICKS_TOKEN``. Keep
+secrets in the environment, not in ``sengol.yaml`` (see ``docs/reference/env.md``).
+
+Free / Apache 2.0 — no enterprise gate on this code path.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, ClassVar, Mapping, Optional
+
+from sengol.adapters.base import BaseExporter
+from sengol.adapters.registry import register_exporter
+from sengol.core.types import EvalResult
+
+# Message shown when the adapter is used without the optional dependency. Mirrors
+# the soft-import contract used by the ``otlp`` and ``s3`` extras.
+_MISSING_MLFLOW = (
+    "MLflowExporter requires the 'mlflow' optional dependency. "
+    "Install it with `pip install 'sengol[mlflow]'` or `uv sync --extra mlflow`."
+)
+
+
+def _import_mlflow() -> Any:
+    try:
+        import mlflow  # type: ignore
+    except ImportError as exc:  # pragma: no cover - exercised via injected module
+        raise ImportError(_MISSING_MLFLOW) from exc
+    return mlflow
+
+
+def _pass_rate(result: EvalResult) -> float:
+    """Fraction of this result's scores that passed. No scores → 0.0."""
+    if not result.scores:
+        return 0.0
+    return sum(1 for s in result.scores if s.passed) / len(result.scores)
+
+
+# MLflow metric keys allow only these characters; anything else raises on log.
+_METRIC_KEY_OK = re.compile(r"[^A-Za-z0-9_\-./ ]")
+
+
+def _sanitize_metric_key(name: str) -> str:
+    """Replace characters MLflow forbids in a metric key with ``_``.
+
+    Keeps the key stable and unique-enough for the common case (most evaluator
+    names are already valid); a name that is *only* invalid chars collapses to
+    underscores, which is acceptable for a derived per-evaluator metric.
+    """
+    return _METRIC_KEY_OK.sub("_", name)
+
+
+@register_exporter
+class MLflowExporter(BaseExporter):
+    """Export :class:`EvalResult` batches to an MLflow experiment.
+
+    Registered as the built-in ``mlflow`` exporter — select it via the ``sengol.yaml``
+    ``exporters:`` block or ``sengol run-suite --export mlflow``.
+
+    **Content-free by construction.** This exporter logs only *structured, non-content*
+    fields — ids, counts, pass/fail metrics, policy ids. It never logs prompts, responses,
+    or evaluator ``reason`` free text, so it cannot leak regulated content to an external
+    tracking server. An exporter that *does* need to emit free text must scrub it first
+    (see :class:`sengol.adapters.jsonl.JsonlFileExporter` and its ``redact`` option for the
+    reference pattern).
+
+    Parameters
+    ----------
+    experiment:
+        MLflow experiment name. Created on first use by ``set_experiment``.
+    tracking_uri:
+        Optional MLflow tracking URI (``http://…``, ``databricks``, a local
+        path, …). When omitted, MLflow's own default/config resolution applies.
+        Authentication is handled by the mlflow SDK's environment variables
+        (``MLFLOW_TRACKING_*`` / ``DATABRICKS_*``) — never passed here.
+    mlflow_module:
+        Injection seam for tests — pass a stand-in exposing
+        ``set_tracking_uri`` / ``set_experiment`` / ``start_run`` /
+        ``log_params`` / ``log_metrics`` / ``set_tags``. Production callers
+        leave this ``None`` and the real ``mlflow`` SDK is soft-imported.
+    """
+
+    name: ClassVar[str] = "mlflow"
+
+    @classmethod
+    def from_config(cls, config: Mapping[str, Any]) -> "MLflowExporter":
+        """Build from a ``sengol.yaml`` ``exporters.mlflow`` mapping.
+
+        Recognized keys: ``experiment``, ``tracking_uri``. The ``mlflow_module``
+        test seam is intentionally not configurable from YAML.
+        """
+        return cls(
+            experiment=config.get("experiment", "sengol"),
+            tracking_uri=config.get("tracking_uri"),
+        )
+
+    def __init__(
+        self,
+        *,
+        experiment: str = "sengol",
+        tracking_uri: Optional[str] = None,
+        mlflow_module: Any = None,
+    ) -> None:
+        self._mlflow = mlflow_module if mlflow_module is not None else _import_mlflow()
+        if tracking_uri:
+            self._mlflow.set_tracking_uri(tracking_uri)
+        self._experiment = experiment
+        self._mlflow.set_experiment(experiment)
+
+    def export(
+        self,
+        result: EvalResult,
+        *,
+        gold_score: Optional[float] = None,
+        tenant_id: Optional[str] = None,
+        run_name: Optional[str] = None,
+        nested: bool = False,
+    ) -> str:
+        """Log one MLflow run for ``result`` and return its ``run_id``.
+
+        Params capture the run's identity (agent, version, policies, tenant); metrics
+        capture the verdict (pass-rate, per-evaluator pass/fail, latency/token/cost
+        rollups, optional ``gold_score``); tags carry cross-references back to the Sengol
+        record. ``gold_score`` is logged only when supplied — the ``run-suite`` path does
+        not compute one. ``nested`` opens the run under the active parent run (used by
+        :meth:`export_batch`).
+        """
+        run_ctx = self._mlflow.start_run(
+            run_name=run_name or result.agent_id, nested=nested
+        )
+        with run_ctx as run:
+            self._mlflow.log_params(self._params(result, tenant_id))
+            self._mlflow.log_metrics(self._metrics(result, gold_score))
+            self._mlflow.set_tags(self._tags(result, tenant_id))
+            return run.info.run_id
+
+    def export_batch(
+        self,
+        results,
+        *,
+        gold_score: Optional[float] = None,
+        tenant_id: Optional[str] = None,
+    ) -> list[Optional[str]]:
+        """Group the batch under one parent MLflow run, each result a nested child.
+
+        Keeps a multi-input suite legible in the MLflow UI: one parent run per
+        ``export_batch`` call, ``len(results)`` nested children. Empty input → no run.
+        """
+        results = list(results)
+        if not results:
+            return []
+        parent_name = f"sengol-suite-{results[0].agent_id}"
+        with self._mlflow.start_run(run_name=parent_name):
+            if tenant_id is not None:
+                self._mlflow.set_tags({"sengol.tenant_id": tenant_id})
+            return [
+                self.export(r, gold_score=gold_score, tenant_id=tenant_id, nested=True)
+                for r in results
+            ]
+
+    # ------------------------------------------------------------------ #
+    # Field mapping (pure — no SDK calls, so it is unit-testable directly)
+    # ------------------------------------------------------------------ #
+
+    @staticmethod
+    def _params(result: EvalResult, tenant_id: Optional[str]) -> dict[str, Any]:
+        params: dict[str, Any] = {
+            "agent_id": result.agent_id,
+            "agent_version": result.agent_version,
+            "policies": ",".join(result.policies),
+            "result_id": str(result.result_id),
+            "evaluator_count": len(result.scores),
+            "saturation_warning": result.saturation_warning,
+        }
+        if tenant_id is not None:
+            params["tenant_id"] = tenant_id
+        return params
+
+    @staticmethod
+    def _metrics(result: EvalResult, gold_score: Optional[float]) -> dict[str, float]:
+        metrics: dict[str, float] = {
+            "pass_rate": _pass_rate(result),
+            "overall_passed": 1.0 if result.overall_passed else 0.0,
+            "total_latency_ms": sum(s.latency_ms for s in result.scores),
+            "total_tokens": float(sum(s.tokens_used for s in result.scores)),
+            "total_cost_usd": sum(s.cost_usd for s in result.scores),
+        }
+        for s in result.scores:
+            # MLflow metric keys allow only alnum + _-./ space. The evaluator
+            # registry permits any non-empty name, so a third-party evaluator may
+            # carry chars MLflow rejects (e.g. ':') — sanitize so one such name
+            # can't raise and abort the whole export.
+            metrics[f"eval.{_sanitize_metric_key(s.evaluator)}.passed"] = (
+                1.0 if s.passed else 0.0
+            )
+        if gold_score is not None:
+            metrics["gold_score"] = gold_score
+        return metrics
+
+    @staticmethod
+    def _tags(result: EvalResult, tenant_id: Optional[str]) -> dict[str, str]:
+        tags = {
+            "sengol.source": "sengol",
+            "sengol.result_id": str(result.result_id),
+            "sengol.agent_id": result.agent_id,
+            "sengol.agent_version": result.agent_version,
+        }
+        if tenant_id is not None:
+            tags["sengol.tenant_id"] = tenant_id
+        return tags
+
+
+__all__ = ["MLflowExporter"]