primarypath-server 1.0.0__py3-none-any.whl

app/sla.py

@@ -0,0 +1,234 @@
+"""SLAs: every pending approval gets a deadline; remind before it,
+escalate at breach. Escalation is notification, not auto-decision — the
+human still owns the call (auto-deny is an explicit opt-in that writes an
+ordinary approval_denied by "system:sla").
+
+Notification reach, honestly stated: reminders and escalations post to
+Slack (plus the escalation webhook); an auto-DENY is a decision, so it is
+delivered like one: the agent's callback_url is called and the card is
+reflected on every configured chat surface, Teams included.
+
+Reminders/escalations are idempotent on the log itself: the append happens
+in a transaction that holds the append lock and re-checks for an existing
+event, so overlapping scans and restarts can never double-fire. Their
+notifications are outbox rows written in that same transaction (B10): the
+marker and "will be announced" commit together, and a Slack outage delays
+the message instead of losing it. Delivery is at-least-once, like a
+webhook: a post whose success was not recorded is posted again.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+
+from .ledger import AlreadyDecided, Ledger, UnknownApproval
+from .ledger.store import APPEND_LOCK
+
+SLACK_SENDER = "chat:slack"
+
+log = logging.getLogger("primarypath.sla")
+
+
+def resolve_deadline_ms(settings, tool: str, requested_deadline_ms: int | None,
+                        now_ms: int | None = None) -> int | None:
+    if requested_deadline_ms is not None:
+        return requested_deadline_ms
+    sla_s = settings.sla_for_tool(tool)
+    if sla_s is None:
+        return None
+    return (now_ms if now_ms is not None else int(time.time() * 1000)) + sla_s * 1000
+
+
+class SLAMonitor:
+    def __init__(self, ledger: Ledger, pool, settings, slack=None,
+                 webhooks=None, chat_routers=None, now=time.time):
+        self.ledger = ledger
+        self.pool = pool
+        self.settings = settings
+        self.slack = slack
+        self.webhooks = webhooks
+        # every chat surface gets the auto-deny reflected (a Teams card
+        # stuck on Approve/Reject after the system already denied would
+        # invite decisions on a dead approval)
+        self.chat_routers = (chat_routers if chat_routers is not None
+                             else [r for r in (slack,) if r is not None])
+        self.now = now
+        self._task: asyncio.Task | None = None
+        register = getattr(webhooks, "register_sender", None)
+        if slack is not None and register is not None:
+            register(SLACK_SENDER, self._deliver_chat)
+
+    # -- lifecycle ------------------------------------------------------------
+
+    def start(self) -> None:
+        if self.settings.sla_scan_interval_s > 0:
+            self._task = asyncio.get_running_loop().create_task(self._loop())
+
+    async def stop(self) -> None:
+        if self._task is not None:
+            self._task.cancel()
+            try:
+                await self._task
+            except asyncio.CancelledError:
+                pass
+
+    async def _loop(self) -> None:
+        while True:
+            try:
+                await self.scan()
+            except Exception:
+                log.exception("sla scan failed; will retry")
+            await asyncio.sleep(self.settings.sla_scan_interval_s)
+
+    # -- one scan pass -----------------------------------------------------------
+
+    async def scan(self) -> None:
+        now_ms = int(self.now() * 1000)
+        remind_ms = self.settings.sla_remind_before_seconds * 1000
+        async with self.pool.connection() as conn:
+            cur = await conn.execute(
+                "SELECT tenant_id, approval_id, run_id, tool, deadline_ms, "
+                "agent_id, unit FROM approvals_view "
+                "WHERE status = 'pending' AND deadline_ms IS NOT NULL "
+                "AND deadline_ms <= %s", (now_ms + remind_ms,))
+            rows = await cur.fetchall()
+
+        for tenant, approval_id, run_id, tool, deadline_ms, agent, unit in rows:
+            if now_ms >= deadline_ms:
+                await self._append_once("approval_escalated", approval_id, {
+                    "approval_id": approval_id, "run_id": run_id,
+                    "to": self.settings.sla_escalate_to,
+                    "reason": "sla_breach"}, tenant, agent, unit,
+                    notify=self._escalation_rows(
+                        approval_id, run_id, tool, deadline_ms, tenant))
+                # the CONTROL on every scan: expiry must not hang off the
+                # one-shot marker or the notification. The row is still
+                # pending, so a scan that died after committing the marker
+                # is finished by the next one; expire() is idempotent on a
+                # decided case.
+                if self.settings.sla_breach_auto_deny:
+                    await self._auto_deny(approval_id, deadline_ms, tenant)
+            else:
+                await self._append_once("approval_reminded", approval_id, {
+                    "approval_id": approval_id, "run_id": run_id}, tenant,
+                    agent, unit,
+                    notify=self._reminder_rows(approval_id, tool,
+                                               deadline_ms, tenant))
+
+    async def _append_once(self, type: str, approval_id: str,
+                           payload: dict, tenant: str, agent: str,
+                           unit: str | None, notify=None) -> bool:
+        """Append unless an event of this type already exists for the
+        approval; check + append share the approval's per-chain append lock
+        (F24), so it is race-free. SLA events land on the agent's chain,
+        beside the approval they concern. `notify` is a list of outbox rows
+        (url, payload, kind) enqueued in the same transaction."""
+        async with self.pool.connection() as conn:
+            async with conn.transaction():
+                await conn.execute(
+                    "SELECT pg_advisory_xact_lock(%s, hashtext(%s))",
+                    (APPEND_LOCK, f"{tenant}:{agent}"))
+                cur = await conn.execute(
+                    "SELECT 1 FROM events WHERE tenant_id = %s AND type = %s "
+                    "AND payload_json->>'approval_id' = %s LIMIT 1",
+                    (tenant, type, approval_id))
+                if await cur.fetchone() is not None:
+                    return False
+                event = await self.ledger._append_in_tx(
+                    conn, type, payload, tenant, agent_id=agent, unit=unit)
+                if self.webhooks is not None:
+                    for url, body, kind in notify or ():
+                        await self.webhooks.enqueue_in_tx(
+                            conn, url, body, tenant=tenant, kind=kind,
+                            approval_id=approval_id,
+                            event_hash=event.get("hash"))
+                return True
+
+    # -- notifications ---------------------------------------------------------------
+
+    def _reminder_rows(self, approval_id: str, tool: str, deadline_ms: int,
+                       tenant: str) -> list[tuple[str, dict, str]]:
+        if self.slack is None:
+            return []
+        return [(SLACK_SENDER, {
+            "message": "sla_reminder", "approval_id": approval_id,
+            "tool": tool, "deadline_ms": deadline_ms, "tenant": tenant,
+            "channel": self.slack.channel}, "sla_reminder")]
+
+    def _escalation_rows(self, approval_id: str, run_id: str, tool: str,
+                         deadline_ms: int,
+                         tenant: str) -> list[tuple[str, dict, str]]:
+        rows = []
+        if self.settings.sla_escalation_webhook:
+            # the page: signed, retried, dead-lettered, replayable
+            rows.append((self.settings.sla_escalation_webhook, {
+                "kind": "sla_escalation", "approval_id": approval_id,
+                "run_id": run_id, "tool": tool, "deadline_ms": deadline_ms,
+                "to": self.settings.sla_escalate_to}, "sla_escalation"))
+        if self.slack is not None:
+            rows.append((SLACK_SENDER, {
+                "message": "sla_escalation", "approval_id": approval_id,
+                "run_id": run_id, "tool": tool, "deadline_ms": deadline_ms,
+                "tenant": tenant, "to": self.settings.sla_escalate_to,
+                "channel": (self.settings.sla_escalation_channel
+                            or self.slack.channel)}, "sla_escalation_chat"))
+        return rows
+
+    async def _deliver_chat(self, p: dict) -> None:
+        """The outbox's sender for chat:slack rows. Worded at delivery
+        time, from the case as it stands then: a message delayed by an
+        outage must not invite a decision the case can no longer take.
+        Raising leaves the row pending for a retry."""
+        a = await self.ledger.approval(p["approval_id"],
+                                       tenant=p.get("tenant", "default"))
+        status = a["status"] if a else None
+        aid, tool = p["approval_id"], p["tool"]
+        case = f"<{self.slack.public_url}/?approval={aid}|"
+        if p["message"] == "sla_reminder":
+            if status != "pending":
+                return  # decided while Slack was down: nothing to remind
+            left_s = max(0, (p["deadline_ms"] - int(self.now() * 1000))
+                         // 1000)
+            await self.slack.api.post_message(
+                p["channel"],
+                [{"type": "section", "text": {"type": "mrkdwn", "text":
+                    f":hourglass_flowing_sand: *Reminder:* `{tool}` approval "
+                    f"`{aid}` is still pending — {left_s}s to its SLA. "
+                    f"{case}decide now>"}}],
+                text=f"Reminder: {aid} pending")
+            return
+        # a breach happened and is announced even if the case has since
+        # been decided; only the call to action follows the case
+        if status == "pending":
+            link = f"{case}decide now>"
+        elif status == "expired":
+            link = f"{case}open the case> (expired; the agent must re-propose)"
+        else:
+            link = f"{case}open the case> (now {status})"
+        await self.slack.api.post_message(
+            p["channel"],
+            [{"type": "section", "text": {"type": "mrkdwn", "text":
+                f":rotating_light: *SLA breached:* `{tool}` approval "
+                f"`{aid}` (run `{p['run_id']}`) blew its deadline and "
+                f"was escalated to *{p['to']}*. {link}"}}],
+            text=f"SLA breached: {aid}")
+
+    async def _auto_deny(self, approval_id: str, deadline_ms: int,
+                         tenant: str) -> None:
+        """The deadline passed with no decision: the proposal EXPIRES
+        (status `expired`, event approval_expired) — final for the waiting
+        agent like a denial, but honest about what happened: nobody said
+        no, nobody was there."""
+        try:
+            a = await self.ledger.expire(
+                approval_id,
+                reason=f"SLA breached: no decision by deadline {deadline_ms}",
+                tenant=tenant)
+        except (AlreadyDecided, UnknownApproval):
+            return
+        # the agent waiting on its callback_url gets called back (outbox
+        # row, written by the ledger's hook in the expiry's own
+        # transaction), and every chat surface's card flips (not just Slack)
+        for chat in self.chat_routers:
+            chat.reflect(a)

app/slack.py

@@ -0,0 +1,282 @@
+"""Slack routing: post approvals to a channel, decide from Slack.
+
+A Slack decision is an ordinary approval_granted/approval_denied in the
+same chain as a web decision; the only Slack-specific trace in the audit
+is the reviewer string carrying the Slack identity. Message refs live in
+their own operational table, never in the chain.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import logging
+import time
+
+import httpx
+
+from .chat import ChatRouter
+from .ledger import Ledger
+
+log = logging.getLogger("primarypath.slack")
+
+SIGNATURE_TOLERANCE_S = 300
+
+
+def verify_slack_signature(secret: str, timestamp: str, body: bytes,
+                           signature: str, now: float | None = None) -> bool:
+    """v0= HMAC over "v0:{timestamp}:{body}"; stale timestamps rejected."""
+    try:
+        ts = int(timestamp)
+    except (TypeError, ValueError):
+        return False
+    if abs((now if now is not None else time.time()) - ts) > SIGNATURE_TOLERANCE_S:
+        return False
+    base = b"v0:" + timestamp.encode() + b":" + body
+    expected = "v0=" + hmac.new(secret.encode(), base, hashlib.sha256).hexdigest()
+    return hmac.compare_digest(expected, signature or "")
+
+
+def _args_summary(arguments: dict, max_items: int = 4) -> str:
+    items = list(arguments.items())
+    shown = ", ".join(f"{k}={json.dumps(v, ensure_ascii=False)}"
+                      for k, v in items[:max_items])
+    return shown + (", …" if len(items) > max_items else "")
+
+
+def case_blocks(approval: dict, public_url: str) -> list[dict]:
+    reasoning = approval.get("reasoning") or "(no reasoning recorded)"
+    if len(reasoning) > 600:
+        reasoning = reasoning[:600] + "…"
+    return [
+        {"type": "section", "text": {"type": "mrkdwn", "text":
+            f"*Approval requested:* `{approval['tool']}"
+            f"({_args_summary(approval.get('arguments', {}))})`"}},
+        {"type": "section", "text": {"type": "mrkdwn", "text": reasoning}},
+        {"type": "context", "elements": [{"type": "mrkdwn", "text":
+            f"run `{approval['run_id']}` · "
+            f"<{public_url}/?approval={approval['approval_id']}|open in inbox>"}]},
+        {"type": "actions", "block_id": "primarypath_decision", "elements": [
+            {"type": "button", "action_id": "primarypath_approve",
+             "text": {"type": "plain_text", "text": "Approve"},
+             "style": "primary", "value": approval["approval_id"]},
+            {"type": "button", "action_id": "primarypath_reject",
+             "text": {"type": "plain_text", "text": "Reject"},
+             "style": "danger", "value": approval["approval_id"]},
+        ]},
+    ]
+
+
+def decided_blocks(approval: dict, public_url: str) -> list[dict]:
+    icon = "✅" if approval["status"] == "granted" else "⛔"
+    line = (f"{icon} *{approval['status']}* by {approval.get('reviewer')}"
+            + (f": {approval['reason']}" if approval.get("reason") else ""))
+    return [
+        {"type": "section", "text": {"type": "mrkdwn", "text":
+            f"`{approval['tool']}` — {line}"}},
+        {"type": "context", "elements": [{"type": "mrkdwn", "text":
+            f"run `{approval['run_id']}` · "
+            f"<{public_url}/?approval={approval['approval_id']}|audit in inbox>"}]},
+    ]
+
+
+def reject_modal(approval_id: str) -> dict:
+    return {
+        "type": "modal",
+        "callback_id": "primarypath_reject_reason",
+        "private_metadata": approval_id,
+        "title": {"type": "plain_text", "text": "Reject approval"},
+        "submit": {"type": "plain_text", "text": "Reject"},
+        "blocks": [
+            {"type": "input", "block_id": "reason_block",
+             "label": {"type": "plain_text",
+                       "text": "Reason (recorded in the audit trail)"},
+             "element": {"type": "plain_text_input", "action_id": "reason",
+                         "multiline": True}},
+        ],
+    }
+
+
+class SlackAPI:
+    """Thin real client; tests swap in a stub with the same surface."""
+
+    def __init__(self, token: str):
+        self._client = httpx.AsyncClient(
+            base_url="https://slack.com/api",
+            headers={"Authorization": f"Bearer {token}"}, timeout=10)
+
+    async def _call(self, method: str, payload: dict) -> dict:
+        r = await self._client.post(f"/{method}", json=payload)
+        out = r.json()
+        if not out.get("ok"):
+            log.warning("slack %s failed: %s", method, out.get("error"))
+        return out
+
+    async def post_message(self, channel: str, blocks: list, text: str) -> dict:
+        return await self._call("chat.postMessage",
+                                {"channel": channel, "blocks": blocks, "text": text})
+
+    async def update_message(self, channel: str, ts: str, blocks: list,
+                             text: str) -> dict:
+        return await self._call("chat.update", {"channel": channel, "ts": ts,
+                                                "blocks": blocks, "text": text})
+
+    async def open_view(self, trigger_id: str, view: dict) -> dict:
+        return await self._call("views.open",
+                                {"trigger_id": trigger_id, "view": view})
+
+    async def post_ephemeral(self, channel: str, user: str, text: str) -> dict:
+        return await self._call("chat.postEphemeral",
+                                {"channel": channel, "user": user, "text": text})
+
+
+class SlackRouter(ChatRouter):
+    def __init__(self, api, pool, ledger: Ledger, channel: str, public_url: str,
+                 approver_ids: set[str] | None = None,
+                 tenant: str = "default"):
+        super().__init__(ledger, tenant=tenant)
+        self.api = api
+        self.pool = pool
+        self.channel = channel
+        self.public_url = public_url
+        # empty set = nobody may decide from Slack (fail closed): the HMAC
+        # proves the request came from Slack, not that the user is an approver
+        self.approver_ids = approver_ids or set()
+
+    # -- notify + reflect ------------------------------------------------------
+
+    def notify(self, approval: dict) -> None:
+        self._spawn(self._notify(approval))
+
+    async def _notify(self, approval: dict) -> None:
+        out = await self.api.post_message(
+            self.channel, case_blocks(approval, self.public_url),
+            text=f"Approval requested: {approval['tool']}")
+        if out.get("ok"):
+            async with self.pool.connection() as conn:
+                await conn.execute(
+                    "INSERT INTO slack_messages (tenant_id, approval_id, "
+                    "channel, ts) VALUES (%s, %s, %s, %s) "
+                    "ON CONFLICT (tenant_id, approval_id) DO NOTHING",
+                    (approval.get("tenant_id", "default"),
+                     approval["approval_id"], out["channel"], out["ts"]))
+
+    def reflect(self, approval: dict) -> None:
+        """Update the Slack message after a decision from any surface."""
+        self._spawn(self._reflect(approval))
+
+    async def _reflect(self, approval: dict) -> None:
+        ref = await self._message_ref(approval["approval_id"],
+                                      approval.get("tenant_id", "default"))
+        if ref:
+            await self.api.update_message(
+                ref[0], ref[1], decided_blocks(approval, self.public_url),
+                text=f"{approval['status']}: {approval['tool']}")
+
+    async def _message_ref(self, approval_id: str,
+                           tenant: str = "default") -> tuple[str, str] | None:
+        async with self.pool.connection() as conn:
+            cur = await conn.execute(
+                "SELECT channel, ts FROM slack_messages "
+                "WHERE tenant_id = %s AND approval_id = %s",
+                (tenant, approval_id))
+            return await cur.fetchone()
+
+    # -- interactions ----------------------------------------------------------
+
+    async def handle_interaction(self, payload: dict) -> dict | None:
+        kind = payload.get("type")
+        if kind == "block_actions":
+            return await self._handle_block_action(payload)
+        if kind == "view_submission":
+            return await self._handle_view_submission(payload)
+        return None
+
+    @staticmethod
+    def _reviewer(payload: dict) -> str:
+        user = payload.get("user", {})
+        display = user.get("name") or user.get("username") or "unknown"
+        return f"{display} (slack:{user.get('id', '?')})"
+
+    def _authorized(self, payload: dict) -> bool:
+        return payload.get("user", {}).get("id") in self.approver_ids
+
+    async def _refuse(self, payload: dict, approval_id: str, verb: str) -> None:
+        """Record the unauthorized attempt in the audit log and tell the
+        clicker why nothing happened. Signature-valid != authorized."""
+        reviewer = self._reviewer(payload)
+        await self.record_refusal(approval_id, reviewer, "Slack", verb)
+        channel = (payload.get("channel") or {}).get("id") or self.channel
+        user = payload.get("user", {}).get("id", "")
+        if user:
+            await self.api.post_ephemeral(
+                channel, user,
+                "You are not an authorized approver for this workspace; "
+                "the attempt was recorded in the audit log.")
+
+    async def _handle_block_action(self, payload: dict) -> dict | None:
+        actions = payload.get("actions") or []
+        if not actions:
+            return None
+        action = actions[0]
+        approval_id = action.get("value", "")
+        verb = {"primarypath_approve": "approve", "primarypath_reject": "reject"}.get(
+            action.get("action_id", ""))
+        if verb is None:
+            return None
+        if not self._authorized(payload):
+            await self._refuse(payload, approval_id, verb)
+            return None
+        if verb == "reject":
+            await self.api.open_view(payload.get("trigger_id", ""),
+                                     reject_modal(approval_id))
+            return None
+        await self._decide(approval_id, True, self._reviewer(payload), "",
+                            channel=(payload.get("channel") or {}).get("id"),
+                            user=payload.get("user", {}).get("id"))
+        return None
+
+    async def _handle_view_submission(self, payload: dict) -> dict | None:
+        view = payload.get("view", {})
+        if view.get("callback_id") != "primarypath_reject_reason":
+            return None
+        approval_id = view.get("private_metadata", "")
+        if not self._authorized(payload):
+            # the modal click was already gated, but submissions are forgeable
+            await self._refuse(payload, approval_id, "reject")
+            return {"response_action": "errors",
+                    "errors": {"reason_block":
+                               "You are not an authorized approver."}}
+        values = view.get("state", {}).get("values", {})
+        reason = (values.get("reason_block", {}).get("reason", {})
+                  .get("value") or "").strip()
+        if not reason:
+            return {"response_action": "errors",
+                    "errors": {"reason_block": "A reason is required to reject."}}
+        try:
+            await self.decide_from_chat(approval_id, False,
+                                        self._reviewer(payload), reason)
+        except self.ChatNotPermitted as e:
+            await self.record_refusal(approval_id, self._reviewer(payload),
+                                      "Slack", "reject")
+            return {"response_action": "errors",
+                    "errors": {"reason_block": str(e)}}
+        approval = await self.ledger.approval(approval_id, tenant=self.tenant)
+        if approval is not None:
+            await self._reflect(approval)
+        return {"response_action": "clear"}
+
+    async def _decide(self, approval_id: str, approved: bool, reviewer: str,
+                      reason: str, channel: str | None = None,
+                      user: str | None = None) -> None:
+        try:
+            approval = await self.decide_from_chat(approval_id, approved,
+                                                   reviewer, reason)
+        except self.ChatNotPermitted as e:
+            await self.record_refusal(approval_id, reviewer, "Slack",
+                                      "approve" if approved else "reject")
+            if channel and user:
+                await self.api.post_ephemeral(channel, user, str(e))
+            return
+        if approval is not None:
+            await self._reflect(approval)