toolframe-audit 0.1.0__py3-none-any.whl

toolframe_audit/__init__.py

@@ -0,0 +1,8 @@
+"""Audit's public collector identifier and transactional producer helpers."""
+
+from .outbox import drain, enqueue, outbox_migration
+
+__version__ = "0.1.0"
+SERVICE_NAME = "audit.events"
+
+__all__ = ["SERVICE_NAME", "drain", "enqueue", "outbox_migration"]

toolframe_audit/docs/usage.md

@@ -0,0 +1,77 @@
+# Audit event history
+
+The Audit plugin keeps a durable, read-only event history for an application.
+Use the `audit.events@1` service or the read-only `audit__list_events` MCP tool
+to inspect events; both require the `audit.read` scope. The plugin does not
+expose public caller fields for ingestion. Producers enqueue events through the
+package's transactional outbox helper while committing their business mutation,
+then use the bounded drain to deliver them to Audit.
+
+## Reading events
+
+Call `list_events` with an optional filter object:
+
+```python
+service = context.services.require("audit.events", version=1)
+page = service.call(
+    "list_events",
+    {
+        "limit": 50,
+        "source_owner": "billing",
+        "action": "invoice.paid",
+        "entity_type": "invoice",
+        "entity_id": "inv_123",
+        "since": "2026-01-01T00:00:00Z",
+        "until": "2026-01-31T23:59:59Z",
+    },
+)
+```
+
+`limit` accepts 1–100 and defaults to 50. The Dashboard uses 25, 50, or 100
+rows per page. `source_owner`, `action`, `entity_type`, and `entity_id` are
+exact filters. `since` and `until` are UTC ISO 8601 timestamps. Omit empty
+filters. Results are ordered newest first and contain `events`, the returned
+row `count`, the effective `limit`, and `next_before`. The service may return
+fewer than `limit` rows when its event-byte budget is reached. Pass a non-null
+`next_before` back as `before` to continue toward older rows; stop when it is
+`null`.
+
+Each event includes its sequence, event ID, application and source identity,
+occurred and received timestamps, action, entity identity, bounded metadata,
+`source_attested_actor`, and `delivery_actor`. The source-attested actor is
+what the producer attested when it created the event; it is not a cryptographic
+verification of that historical actor. The delivery actor comes from the
+host-admitted delivery context that accepted the event. Replay is safe for a
+previously accepted event ID, while a conflicting payload for the same logical
+ID is rejected.
+
+The Dashboard's JSONL button exports only the currently displayed page (up to
+100 events) and labels the file as a partial export. Continue through the
+`next_before` cursor when a complete review is required.
+
+Operators with the Manage plugins permission (`system.plugins.manage`) can use the
+Dashboard Retention panel. First
+preview a cutoff and batch limit. The preview records a `through_sequence`
+high-water mark; the Dashboard preserves those inputs and requires explicit
+confirmation before applying. Direct service callers supply the cutoff, limit
+and `through_sequence` cap explicitly; the service does not store or bind preview
+tokens. It never substitutes a newer sequence cap on apply. The persisted cutoff rejects later events whose
+occurred timestamp is older than the cutoff, so pending producer outbox items
+need attention.
+
+## Producer and outbox integration
+
+Use the public outbox helper exported by the package and the fixture under
+`examples/outbox-producer/` as the integration reference. The business write
+and enqueue must use the same owner SQL transaction. A finite drain delivers
+each pending item to Audit and acknowledges it only after Audit accepts or
+deduplicates it. If the process crashes between those two steps, replay uses
+the event ID and does not create a second row. The helper captures the original
+actor from trusted context; a replaying drain cannot turn that historical actor
+into a new delivery identity, and callers cannot override the source owner.
+
+Keep event metadata to the minimum needed for investigation. Redact tokens,
+passwords, credentials, session material, and full request or response bodies
+before enqueueing. Audit provides durable operational history with source and
+delivery attribution; it is not a tamper-proof ledger or cryptographic proof
+of the historical actor.

toolframe_audit/events.py

@@ -0,0 +1,210 @@
+"""Bounded event normalization and non-secret actor snapshots."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from datetime import UTC, datetime
+from typing import Any
+
+from toolframe_sdk import PluginError, Principal
+
+MAX_METADATA_BYTES = 16_384
+MAX_EVENT_BYTES = 20_480
+MAX_PAGE = 100
+MAX_SEQUENCE = 2**63 - 1
+ACTOR_KINDS = (
+    "local",
+    "operator",
+    "password",
+    "mcp_key",
+    "user",
+    "delegated",
+    "anonymous",
+    "public",
+    "background",
+)
+EVENT_FIELDS = {
+    "event_id",
+    "occurred_at",
+    "action",
+    "entity_type",
+    "entity_id",
+    "metadata",
+    "source_attested_actor",
+}
+_SECRET_KEYS = {
+    "password",
+    "secret",
+    "token",
+    "authorization",
+    "cookie",
+    "credentials",
+    "credential",
+    "apikey",
+    "accesskey",
+    "privatekey",
+    "clientsecret",
+    "accesstoken",
+    "refreshtoken",
+    "idtoken",
+    "bearer",
+    "sessiontoken",
+}
+
+
+def invalid(message: str) -> PluginError:
+    return PluginError("audit_validation", message)
+
+
+def text_field(value: Any, field: str, maximum: int) -> str:
+    if (
+        not isinstance(value, str)
+        or not value.strip()
+        or len(value) > maximum
+        or any(ord(char) < 32 for char in value)
+    ):
+        raise invalid(f"{field} must be nonempty text of at most {maximum} characters.")
+    return value
+
+
+def timestamp(value: Any, field: str = "occurred_at") -> str:
+    text_field(value, field, 40)
+    try:
+        parsed = datetime.fromisoformat(value)
+        if parsed.tzinfo is None:
+            raise ValueError
+        return (
+            parsed.astimezone(UTC)
+            .isoformat(timespec="microseconds")
+            .replace("+00:00", "Z")
+        )
+    except ValueError, OverflowError:
+        raise invalid(
+            f"{field} must be an ISO 8601 timestamp with a timezone."
+        ) from None
+
+
+def now() -> str:
+    return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
+
+
+def canonical(value: Any) -> str:
+    try:
+        result = json.dumps(
+            value,
+            sort_keys=True,
+            separators=(",", ":"),
+            allow_nan=False,
+            ensure_ascii=False,
+        )
+        result.encode("utf-8")
+        return result
+    except TypeError, ValueError, RecursionError, UnicodeEncodeError:
+        raise invalid("Event values must be finite JSON data.") from None
+
+
+def actor(value: Any) -> dict:
+    if not isinstance(value, dict) or set(value) - {"kind", "user_id", "client_id"}:
+        raise invalid("Actor snapshots allow only kind, user_id and client_id.")
+    if value.get("kind") not in ACTOR_KINDS:
+        raise invalid("Actor kind is not supported.")
+    result = {"kind": value["kind"]}
+    for key in ("user_id", "client_id"):
+        if key in value:
+            result[key] = text_field(value[key], key, 256)
+    return result
+
+
+def admitted(context, *, producer: bool = False) -> tuple[dict, dict]:
+    """Trust only the SDK's host-delivered call context, never event parameters."""
+    info = context.call_info
+    owner = info.get("owner")
+    if (
+        not isinstance(owner, str)
+        or not re.fullmatch(r"[a-z][a-z0-9-]{0,63}", owner)
+        or not all(
+            isinstance(info.get(key), str) and 0 < len(info[key]) <= 256
+            for key in ("owner_generation", "application_id", "root_id")
+        )
+        or "principal" not in info
+        or (producer and owner == context.plugin_id)
+    ):
+        raise PluginError(
+            "audit_untrusted_context", "A host-admitted producer context is required."
+        )
+    principal = info["principal"]
+    if principal is None:
+        return info, {"kind": "background"}
+    try:
+        principal = Principal.model_validate(principal)
+        snapshot = {"kind": principal.kind}
+        for key in ("user_id", "client_id"):
+            if getattr(principal, key) is not None:
+                snapshot[key] = getattr(principal, key)
+        return info, actor(snapshot)
+    except Exception as error:
+        raise PluginError(
+            "audit_untrusted_context", "The admitted principal is invalid."
+        ) from error
+
+
+def _metadata(value: Any, depth: int = 0, budget: list[int] | None = None) -> None:
+    budget = [1024] if budget is None else budget
+    budget[0] -= 1
+    if budget[0] < 0 or depth > 8:
+        raise invalid(
+            "Metadata must have at most 1024 values including containers and the root, and 8 nesting levels."
+        )
+    if isinstance(value, dict):
+        for key, item in value.items():
+            if not isinstance(key, str) or len(key) > 128:
+                raise invalid("Metadata keys must be text of at most 128 characters.")
+            normalized = re.sub(r"[^a-z0-9]", "", key.lower())
+            if normalized in _SECRET_KEYS:
+                raise invalid(
+                    "Metadata contains a credential or secret field; redact it at the source."
+                )
+            _metadata(item, depth + 1, budget)
+    elif isinstance(value, list):
+        for item in value:
+            _metadata(item, depth + 1, budget)
+    elif value is not None and not isinstance(value, (str, int, float, bool)):
+        raise invalid("Metadata must contain JSON values.")
+
+
+def normalize_event(value: Any) -> tuple[dict, str, str]:
+    if not isinstance(value, dict) or set(value) != EVENT_FIELDS:
+        raise invalid(
+            "Event requires exactly the documented event fields; source overrides are forbidden."
+        )
+    event = {
+        "event_id": text_field(value["event_id"], "event_id", 128),
+        "occurred_at": timestamp(value["occurred_at"]),
+        "action": text_field(value["action"], "action", 128),
+        "entity_type": text_field(value["entity_type"], "entity_type", 128),
+        "entity_id": text_field(value["entity_id"], "entity_id", 256),
+        "source_attested_actor": actor(value["source_attested_actor"]),
+    }
+    metadata = value["metadata"]
+    if not isinstance(metadata, dict):
+        raise invalid("metadata must be a JSON object.")
+    _metadata(metadata)
+    if len(canonical(metadata).encode()) > MAX_METADATA_BYTES:
+        raise invalid("Metadata exceeds 16384 UTF-8 bytes.")
+    event["metadata"] = metadata
+    payload = canonical(event)
+    if len(payload.encode()) > MAX_EVENT_BYTES:
+        raise invalid("Event exceeds 20480 UTF-8 bytes.")
+    return event, payload, hashlib.sha256(payload.encode()).hexdigest()
+
+
+def bounded_integer(value: Any, field: str, maximum: int) -> int:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, int)
+        or not 1 <= value <= maximum
+    ):
+        raise invalid(f"{field} must be an integer between 1 and {maximum}.")
+    return value

toolframe_audit/migrations/0001_init.postgresql.sql

@@ -0,0 +1,22 @@
+CREATE TABLE audit_events (
+    sequence BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    application_id TEXT NOT NULL,
+    source_owner TEXT NOT NULL,
+    event_id TEXT NOT NULL,
+    source_generation TEXT NOT NULL,
+    occurred_at TEXT NOT NULL,
+    received_at TEXT NOT NULL,
+    action TEXT NOT NULL,
+    entity_type TEXT NOT NULL,
+    entity_id TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    content_hash TEXT NOT NULL,
+    delivery_actor TEXT NOT NULL,
+    UNIQUE(application_id, source_owner, event_id)
+);
+CREATE INDEX audit_source_sequence ON audit_events(source_owner, sequence);
+CREATE INDEX audit_action_sequence ON audit_events(action, sequence);
+CREATE INDEX audit_entity_sequence ON audit_events(entity_type, entity_id, sequence);
+CREATE INDEX audit_occurred_sequence ON audit_events(occurred_at, sequence);
+CREATE TABLE audit_capacity (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), event_count BIGINT NOT NULL, retention_cutoff TEXT);
+INSERT INTO audit_capacity(singleton, event_count) VALUES (1, 0);

toolframe_audit/migrations/0001_init.sql

@@ -0,0 +1,22 @@
+CREATE TABLE audit_events (
+    sequence INTEGER PRIMARY KEY AUTOINCREMENT,
+    application_id TEXT NOT NULL,
+    source_owner TEXT NOT NULL,
+    event_id TEXT NOT NULL,
+    source_generation TEXT NOT NULL,
+    occurred_at TEXT NOT NULL,
+    received_at TEXT NOT NULL,
+    action TEXT NOT NULL,
+    entity_type TEXT NOT NULL,
+    entity_id TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    content_hash TEXT NOT NULL,
+    delivery_actor TEXT NOT NULL,
+    UNIQUE(application_id, source_owner, event_id)
+);
+CREATE INDEX audit_source_sequence ON audit_events(source_owner, sequence);
+CREATE INDEX audit_action_sequence ON audit_events(action, sequence);
+CREATE INDEX audit_entity_sequence ON audit_events(entity_type, entity_id, sequence);
+CREATE INDEX audit_occurred_sequence ON audit_events(occurred_at, sequence);
+CREATE TABLE audit_capacity (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), event_count BIGINT NOT NULL, retention_cutoff TEXT);
+INSERT INTO audit_capacity(singleton, event_count) VALUES (1, 0);

toolframe_audit/migrations/outbox.sql

@@ -0,0 +1,9 @@
+CREATE TABLE audit_outbox (
+    event_id TEXT PRIMARY KEY,
+    payload TEXT NOT NULL,
+    content_hash TEXT NOT NULL,
+    enqueued_at TEXT NOT NULL
+);
+CREATE INDEX audit_outbox_order ON audit_outbox(enqueued_at, event_id);
+CREATE TABLE audit_outbox_capacity (singleton INTEGER PRIMARY KEY CHECK(singleton = 1), pending_count BIGINT NOT NULL);
+INSERT INTO audit_outbox_capacity(singleton, pending_count) VALUES (1, 0);

toolframe_audit/outbox.py

@@ -0,0 +1,162 @@
+"""Reusable same-owner transactional enqueue and finite at-least-once delivery."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from importlib.resources import files
+
+from toolframe_sdk import Migration, PluginError, StorageClient
+
+from .events import admitted, bounded_integer, normalize_event, now
+
+
+def outbox_migration(*, revision="audit_outbox_v1", parent=None) -> Migration:
+    """Append this migration to the producer's own linear store history."""
+    sql = files("toolframe_audit").joinpath("migrations/outbox.sql").read_text()
+    return Migration(
+        revision=revision, parent=parent, sql=sql, dialects={"postgresql": sql}
+    )
+
+
+async def enqueue(
+    context,
+    transaction: StorageClient,
+    *,
+    event_id: str,
+    action: str,
+    entity_type: str,
+    entity_id: str,
+    metadata: dict | None = None,
+    occurred_at: str | None = None,
+    max_pending: int = 1000,
+) -> dict:
+    """Enqueue inside the SAME live StorageClient transaction as the business write.
+
+    Identity and actor are captured from the admitted context. This function never
+    opens a second transaction and cannot override the producer owner or actor.
+    """
+    if (
+        not isinstance(transaction, StorageClient)
+        or not transaction.token
+        or transaction.context is not context
+    ):
+        raise PluginError(
+            "audit_outbox_transaction", "Pass the producer's live transaction client."
+        )
+    info = context.call_info
+    if (
+        "owner" not in info
+        and "application_id" not in info
+        and "principal" not in info
+        and isinstance(info.get("root_id"), str)
+        and info["root_id"]
+        and info.get("chain") == [context.plugin_id]
+    ):
+        # The SDK's newly admitted task root has no visitor principal. Source
+        # fencing still happens at Core.invoke on the later collector call.
+        original_actor = {"kind": "background"}
+    else:
+        _, original_actor = admitted(context)
+    maximum = bounded_integer(max_pending, "max_pending", 100_000)
+    event, payload, content_hash = normalize_event(
+        {
+            "event_id": event_id,
+            "occurred_at": occurred_at if occurred_at is not None else now(),
+            "action": action,
+            "entity_type": entity_type,
+            "entity_id": entity_id,
+            "metadata": metadata if metadata is not None else {},
+            "source_attested_actor": original_actor,
+        }
+    )
+    result = await transaction.execute_returning(
+        "INSERT INTO audit_outbox(event_id, payload, content_hash, enqueued_at) VALUES (?, ?, ?, ?) "
+        "ON CONFLICT(event_id) DO NOTHING RETURNING event_id",
+        (event_id, payload, content_hash, now()),
+    )
+    if not result.get("rows"):
+        rows = await transaction.query(
+            "SELECT content_hash FROM audit_outbox WHERE event_id = ?", (event_id,)
+        )
+        if len(rows) != 1 or rows[0]["content_hash"] != content_hash:
+            raise PluginError(
+                "audit_event_conflict",
+                "The pending event ID already has different content.",
+            )
+        return event
+    capacity = await transaction.execute_returning(
+        "UPDATE audit_outbox_capacity SET pending_count = pending_count + 1 "
+        "WHERE singleton = 1 AND pending_count < ? RETURNING pending_count",
+        (maximum,),
+    )
+    if not capacity.get("rows"):
+        raise PluginError(
+            "audit_outbox_capacity",
+            "Outbox capacity is reached; the business transaction must roll back.",
+        )
+    return event
+
+
+async def drain(
+    context, *, alias="audit", storage_alias="storage", limit=25, budget_seconds=10
+) -> dict:
+    """Deliver one short batch; acknowledge only accepted/deduplicated events.
+
+    Failures propagate without logging payloads. Successfully acknowledged prior
+    events remain settled; retry the remaining outbox in another admitted root.
+    """
+    # context.task roots carry only root_id/deadline/chain. Core attaches complete
+    # source identity when this producer crosses the collector service boundary.
+    if (
+        not isinstance(context.call_info.get("root_id"), str)
+        or not context.call_info["root_id"]
+    ):
+        raise PluginError("audit_untrusted_context", "Drain requires an admitted root.")
+    limit = bounded_integer(limit, "limit", 100)
+    budget_seconds = bounded_integer(budget_seconds, "budget_seconds", 30)
+    storage = context.storage(storage_alias)
+    deadline = time.monotonic() + budget_seconds
+    delivered = acknowledged = 0
+    async with asyncio.timeout(budget_seconds):
+        rows = await storage.query(
+            "SELECT event_id FROM audit_outbox ORDER BY enqueued_at, event_id LIMIT ?",
+            (limit,),
+        )
+        for row in rows:
+            pending = await storage.query(
+                "SELECT event_id, payload FROM audit_outbox WHERE event_id = ?",
+                (row["event_id"],),
+            )
+            if not pending:  # Another finite drain may have acknowledged it.
+                continue
+            row = pending[0]
+            response = await context.call(
+                alias,
+                "ingest",
+                {"event": json.loads(row["payload"])},
+                timeout=max(0.01, deadline - time.monotonic()),
+            )
+            if (
+                not isinstance(response, dict)
+                or response.get("status") not in {"accepted", "duplicate"}
+                or response.get("event_id") != row["event_id"]
+                or response.get("source_owner") != context.plugin_id
+            ):
+                raise PluginError(
+                    "audit_outbox_delivery",
+                    "Collector did not acknowledge this producer event.",
+                )
+            delivered += 1
+            async with storage.transaction() as transaction:
+                deleted = await transaction.execute_returning(
+                    "DELETE FROM audit_outbox WHERE event_id = ? AND payload = ? RETURNING event_id",
+                    (row["event_id"], row["payload"]),
+                )
+                if deleted.get("rows"):
+                    await transaction.execute(
+                        "UPDATE audit_outbox_capacity SET pending_count = pending_count - 1 WHERE singleton = 1"
+                    )
+                    acknowledged += 1
+    return {"delivered": delivered, "acknowledged": acknowledged, "limit": limit}