pinecall-runtime 0.1.6__py3-none-any.whl

pinecall/log/private.py

@@ -0,0 +1,135 @@
+"""What a call's log keeps of the values its agent declared private: masked, and sealed aside."""
+
+import json
+import logging
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from cryptography.fernet import InvalidToken, MultiFernet
+
+from pinecall.domain.agent import AgentConfig
+from pinecall.domain.names import Json, JsonObject
+from pinecall.log.store import Store
+from pinecall.wire.frames import Entry
+
+logger = logging.getLogger(__name__)
+
+
+# The key stays, so a reader knows a value exists; the value goes whole, so its type does not leak.
+MASK = "***"
+
+
+# The entries whose data carries the app's state, masked against the state fields declared pii.
+CARRY_STATE = frozenset({"state.changed", "call.attached"})
+
+
+# A tool's arguments, masked against the names its declaration lists under pii.
+CARRY_ARGUMENTS = "tool.call"
+
+
+# One row per entry that held a private value: the values, sealed under the box's vault key.
+KEEP = """
+INSERT INTO call_private (log, seq, sealed)
+SELECT %(log)s, seq, sealed FROM unnest(%(seqs)s::bigint[], %(sealed)s::text[]) AS kept(seq, sealed)
+ON CONFLICT (log, seq) DO NOTHING
+"""
+
+OF_LOG = "SELECT seq, sealed FROM call_private WHERE log = %(log)s"
+
+
+@dataclass(frozen=True)
+class Privacy:
+    """What a call's writer masks by: the vault it seals with, the declaration it reads."""
+
+    vault: MultiFernet
+    config: AgentConfig
+
+
+@dataclass(frozen=True)
+class Split:
+    """An entry's data as the log keeps it, and the values masked in it, by their field."""
+
+    kept: JsonObject
+    private: JsonObject
+
+
+def split(kind: str, data: JsonObject, config: AgentConfig) -> Split:
+    """The entry with every value its agent declared private masked, and those values apart."""
+    marked = _marked(kind, data, config)
+    if marked is None:
+        return Split(data, {})
+    field, names = marked
+    values = data.get(field)
+    if not isinstance(values, dict):
+        return Split(data, {})
+    private: JsonObject = {name: value for name, value in values.items() if name in names}
+    if not private:
+        return Split(data, {})
+    return Split({**data, field: {**values, **dict.fromkeys(private, MASK)}}, {field: private})
+
+
+def masked_state(app_state: JsonObject, config: AgentConfig) -> JsonObject:
+    """The app's state with every field declared pii masked."""
+    return {
+        name: MASK if config.visibility_of(name) == "pii" else value
+        for name, value in app_state.items()
+    }
+
+
+def restored(entry: Entry, private: JsonObject | None) -> Entry:
+    """The entry as its writer wrote it: the private values back where they were masked."""
+    if not private:
+        return entry
+    data = dict(entry.data)
+    for field, values in private.items():
+        kept = data.get(field)
+        if isinstance(kept, dict) and isinstance(values, dict):
+            data[field] = {**kept, **values}
+    return entry.model_copy(update={"data": data})
+
+
+async def kept_aside(
+    store: Store, vault: MultiFernet, log: str, private: Sequence[tuple[int, JsonObject]]
+) -> None:
+    """Seal the private values of these entries of the log, by seq; one kept already stays."""
+    if not private:
+        return
+    rows = {
+        "log": log,
+        "seqs": [seq for seq, _ in private],
+        "sealed": [vault.encrypt(json.dumps(values).encode()).decode() for _, values in private],
+    }
+    async with store.pool.connection() as connection:
+        await connection.execute(KEEP, rows)
+
+
+# Read only where a call is taken up: an app taking it over, a written call after a restart.
+async def opened_whole(store: Store, vault: MultiFernet, log: str) -> list[Entry]:
+    """Every entry of the log as its writer wrote it, the private values opened."""
+    entries = await store.whole(log)
+    async with store.pool.connection() as connection:
+        rows = await (await connection.execute(OF_LOG, {"log": log})).fetchall()
+    sealed = {int(row["seq"]): str(row["sealed"]) for row in rows}
+    return [restored(entry, _opened(vault, sealed.get(entry.seq))) for entry in entries]
+
+
+# A value sealed under a key PINECALL_VAULT_KEY no longer lists stays masked: the call goes on.
+def _opened(vault: MultiFernet, token: str | None) -> JsonObject | None:
+    if token is None:
+        return None
+    try:
+        values: Json = json.loads(vault.decrypt(token.encode()))
+    except InvalidToken:
+        logger.warning("a private value sealed under a key the vault no longer lists stays masked")
+        return None
+    return values if isinstance(values, dict) else None
+
+
+def _marked(kind: str, data: JsonObject, config: AgentConfig) -> tuple[str, frozenset[str]] | None:
+    if kind == CARRY_ARGUMENTS:
+        spec = config.tools_by_name.get(str(data.get("name")))
+        return None if spec is None else ("arguments", spec.pii)
+    if kind in CARRY_STATE:
+        declared = frozenset(name for name, seen in config.state_fields.items() if seen == "pii")
+        return "state", declared
+    return None

pinecall/log/queries.py

@@ -0,0 +1,229 @@
+"""One call's facts as a gateway asks them: where it opened, its tool calls, whether it sealed."""
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import LiteralString
+
+from pinecall.domain.agent import Versions
+from pinecall.domain.call import Opener
+from pinecall.domain.names import Env, parse_env
+from pinecall.domain.scope import Scope
+from pinecall.log.facts import CallFacts, facts_of
+from pinecall.log.store import entry_of
+from pinecall.postgres.pool import Pool, box_wide
+from pinecall.wire.frames import Entry
+from pinecall.wire.parts import ToolResult
+
+# Head rows older than the env and holder columns read as production, the org's own.
+CORNER_OF_CALL = """
+select org, coalesce(env, 'production') as env, coalesce(holder, '') as holder, agent,
+       config_version, lexicon_version, sealed, started_at, written, opened_by
+from call_log_head where log = %(call)s and call is not null
+"""
+
+
+ENVS_OF = """
+select log as call, coalesce(env, 'production') as env
+from call_log_head where log = any(%(calls)s) and call is not null
+"""
+
+
+FACTS_OF = """
+select f.*, head.agent
+from call_facts f join call_log_head head on head.log = f.call
+where f.call = any(%(calls)s)
+"""
+
+
+# call_log is joined on log, its primary key's first column, never on call, which scans it.
+# A call that never reached call.started may have no facts row, hence the left join.
+# A head claimed at the open, whose first entry waits on the writer, moved last when it opened; a
+# head with no entry, no start and no opening (a seal's lease) is never quiet. Under load this is
+# the whole window: at 300 calls held the reaper sealed calls 36 ms after they opened (2026-10-01).
+UNSEALED_SPOKEN = """
+select head.log as call, head.agent,
+       coalesce(head.started_at, 0) as started_at,
+       coalesce(max(entry.ts), head.started_at, extract(epoch from opening.opened_at)) as last_at,
+       null as channel, head.env
+from call_log_head head
+left join call_facts f on f.call = head.log
+left join call_openings opening on opening.call = head.log
+left join call_log entry on entry.log = head.log
+where head.call is not null and not head.sealed
+  and (coalesce(f.spoken, false)
+       or not exists (select 1 from call_log began
+                       where began.log = head.log and began.type = 'call.started'))
+group by head.log, head.agent, head.started_at, head.env, opening.opened_at
+having coalesce(max(entry.ts), head.started_at, extract(epoch from opening.opened_at))
+       < %(quiet_since)s
+order by last_at
+limit %(limit)s
+"""
+
+
+UNSEALED_WRITTEN = """
+select head.log as call, head.agent,
+       coalesce(head.started_at, 0) as started_at,
+       coalesce(max(entry.ts), head.started_at, 0) as last_at, f.channel, head.env
+from call_log_head head
+join call_facts f on f.call = head.log
+left join call_log entry on entry.log = head.log
+where head.call is not null and not head.sealed and not coalesce(f.spoken, false)
+  and exists (select 1 from call_log began
+               where began.log = head.log and began.type = 'call.started')
+group by head.log, head.agent, head.started_at, head.env, f.channel
+having coalesce(max(entry.ts), head.started_at, 0) < %(quiet_since)s
+order by last_at
+limit %(limit)s
+"""
+
+
+TOOL_ANSWERED = """
+select data from call_log
+where log = %(log)s and type = 'tool.result' and data->>'call_id' = %(call_id)s
+order by seq
+limit 1
+"""
+
+# A tool call id the log asked already: a retry that reached another gateway waits for its answer.
+TOOL_CALLED = """
+select call, seq, ts, agent, type, ephemeral, data from call_log
+where log = %(log)s and type = 'tool.call' and data->>'call_id' = %(call_id)s
+order by seq
+limit 1
+"""
+
+# The head's word on who opened a call, read back as the type; anything else is nobody's.
+OPENERS: dict[str, Opener] = {"fleet": "fleet", "app": "app", "gateway": "gateway"}
+
+
+# Which of the calls a gateway serves another gateway sealed: one read of their heads.
+SEALED_AMONG = """
+select log from call_log_head where log = any(%(calls)s) and sealed
+"""
+
+
+@dataclass(frozen=True, slots=True)
+class CallScope:
+    """Where a call was opened: its scope, its agent, and what its head row says."""
+
+    scope: Scope | None
+    agent: str
+    versions: Versions
+    sealed: bool
+    started_at: float | None
+    # The entries its worker's writer sent: where a writer that takes the call over follows on.
+    written: int
+    # Who opened it; None for a call opened before the head said.
+    opened_by: Opener | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class Unsealed:
+    """A call still open, for the reaper: its start, its last move, its channel and its world."""
+
+    call: str
+    agent: str
+    started_at: float
+    last_at: float
+    channel: str | None
+    # None for a log no scope ever claimed.
+    env: Env | None
+
+
+async def sealed_among(pool: Pool, calls: list[str]) -> set[str]:
+    """The ones of these calls whose log is sealed, wherever it was sealed."""
+    if not calls:
+        return set()
+    async with pool.connection() as connection:
+        rows = await (await connection.execute(SEALED_AMONG, {"calls": calls})).fetchall()
+    return {str(row["log"]) for row in rows}
+
+
+async def scope_of_call(pool: Pool, call: str) -> CallScope | None:
+    """Return where the call was opened, or None for a call nobody wrote to or claimed."""
+    # Whose a call is decides who may act on it: asked box-wide, so another org's call is that
+    # org's (a 404 to the asker), never a call nobody opened that the asker could claim.
+    with box_wide():
+        async with pool.connection() as connection:
+            found = await connection.execute(CORNER_OF_CALL, {"call": call})
+            row = await found.fetchone()
+    if row is None:
+        return None
+    env: Env = "sandbox" if row["env"] == "sandbox" else "production"
+    return CallScope(
+        scope=None if row["org"] is None else Scope(row["org"], env, row["holder"]),
+        agent=row["agent"] or "",
+        versions=Versions(config=row["config_version"], lexicon=row["lexicon_version"]),
+        sealed=row["sealed"],
+        started_at=row["started_at"],
+        written=row["written"],
+        opened_by=_opener(row["opened_by"]),
+    )
+
+
+# Read only by a tool's round trip: the log's primary key starts with `log`, so it reads one call.
+async def tool_answered(pool: Pool, call: str, call_id: str) -> ToolResult | None:
+    """The `tool.result` the call's log holds for this tool call id, or None."""
+    wanted = {"log": call, "call_id": call_id}
+    async with pool.connection() as connection:
+        row = await (await connection.execute(TOOL_ANSWERED, wanted)).fetchone()
+    return None if row is None else ToolResult.model_validate(row["data"])
+
+
+async def tool_called(pool: Pool, call: str, call_id: str) -> Entry | None:
+    """The `tool.call` entry the call's log holds for this tool call id, or None."""
+    wanted = {"log": call, "call_id": call_id}
+    async with pool.connection() as connection:
+        row = await (await connection.execute(TOOL_CALLED, wanted)).fetchone()
+    return None if row is None else entry_of(row)
+
+
+async def envs_of_calls(pool: Pool, calls: Sequence[str]) -> dict[str, Env]:
+    """Return the world each of the calls was opened in, for the calls that have a log."""
+    async with pool.connection() as connection:
+        rows = await (await connection.execute(ENVS_OF, {"calls": list(calls)})).fetchall()
+    return {str(row["call"]): parse_env(str(row["env"])) for row in rows}
+
+
+async def facts_of_calls(pool: Pool, calls: Sequence[str]) -> dict[str, CallFacts]:
+    """Return the facts of each of the calls that has a row."""
+    async with pool.connection() as connection:
+        rows = await (await connection.execute(FACTS_OF, {"calls": list(calls)})).fetchall()
+    return {str(row["call"]): facts_of(row) for row in rows}
+
+
+# Across every org: the reaper's question. A call that never reached call.started is reaped
+# whatever its channel.
+async def unsealed_spoken(pool: Pool, quiet_since: float, *, limit: int) -> list[Unsealed]:
+    """Return the spoken or never-started calls still open and quiet since then, oldest first."""
+    return await _unsealed(pool, UNSEALED_SPOKEN, quiet_since, limit)
+
+
+# A written call idles out where it runs; this finds the ones a restart left open.
+async def unsealed_written(pool: Pool, quiet_since: float, *, limit: int) -> list[Unsealed]:
+    """Return the written calls still open and quiet since then, with their channel."""
+    return await _unsealed(pool, UNSEALED_WRITTEN, quiet_since, limit)
+
+
+def _opener(named: object) -> Opener | None:
+    return OPENERS.get(named) if isinstance(named, str) else None
+
+
+async def _unsealed(
+    pool: Pool, query: LiteralString, quiet_since: float, limit: int
+) -> list[Unsealed]:
+    async with pool.connection() as connection:
+        cursor = await connection.execute(query, {"quiet_since": quiet_since, "limit": limit})
+        rows = await cursor.fetchall()
+    return [
+        Unsealed(
+            call=str(row["call"]),
+            agent=str(row["agent"] or ""),
+            started_at=float(row["started_at"]),
+            last_at=float(row["last_at"]),
+            channel=row["channel"],
+            env=None if row["env"] is None else parse_env(str(row["env"])),
+        )
+        for row in rows
+    ]

pinecall/log/readers.py

@@ -0,0 +1,207 @@
+"""What each reader receives: the filter it asked for, and the projection its credential allows."""
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+from pinecall.domain.agent import AgentConfig
+from pinecall.domain.errors import DeclarationRefused
+from pinecall.domain.names import Json, JsonObject
+from pinecall.log.private import masked_state, split
+from pinecall.wire.events import TERMINAL_EVENT, AgentTurnEnded, LogGap, StateChanged, event_of
+from pinecall.wire.frames import Entry, WireModel
+from pinecall.wire.parts import Projection
+from pinecall.wire.state import AgentTurn, State, Turn
+
+MAX_TYPES = 32
+
+
+_A_TYPE_NAME = re.compile(r"^[a-z0-9_.]+$")
+
+
+# A reader that filtered these out could wait forever on a log that already ended.
+ALWAYS_PASS = frozenset({"log.gap", "log.caught_up", "call.ended", TERMINAL_EVENT})
+
+
+NEEDS_READING = frozenset({"turn.agent", "state.changed", "log.gap"})
+
+
+# The contract's table (docs/protocol/projections.md), row by row: a type absent is dropped
+# whole, a field absent is dropped.
+PUBLIC_ENTRY_FIELDS: Mapping[str, tuple[str, ...]] = {
+    "call.ringing": ("channel",),
+    "call.dialing": ("channel",),
+    "call.started": ("channel", "direction", "started_at"),
+    "call.ended": ("reason", "ended_by", "ended_at", "duration_s"),
+    "user.state": ("state",),
+    "agent.state": ("state",),
+    "user.transcript": ("text", "final"),
+    "agent.transcript": ("speech_id", "text", "final"),
+    "turn.user": ("speech_id", "text"),
+    "turn.agent": ("speech_id", "text", "interrupted", "metrics"),
+    "room.opened": ("name", "sid"),
+    "participant.joined": ("identity", "kind", "name"),
+    "participant.left": ("identity",),
+    "participant.speaking": ("identity", "speaking"),
+    "confirm.request": ("phrase", "ttl_s"),
+    "confirm.granted": (),
+    "confirm.declined": (),
+    "call.transferred": ("to", "mode", "ok", "error"),
+    "call.line": ("held",),
+    "event.received": ("name", "data", "source", "identity"),
+    "state.changed": ("state", "changed"),
+    "log.gap": ("from_seq", "to_seq", "snapshot"),
+    "log.caught_up": ("seq",),
+}
+
+
+# agent and call are left out so a participant learns nothing of the tenant's setup.
+PUBLIC_ENVELOPE = ("seq", "ts", "type", "ephemeral")
+
+
+PUBLIC_TURN = frozenset({"role", "speech_id", "text", "interrupted"})
+
+
+PUBLIC_SEAT = frozenset({"identity", "kind", "name", "joined_at", "speaking"})
+
+
+PUBLIC_CONFIRM = frozenset({"phrase", "status"})
+
+
+@dataclass(frozen=True, slots=True)
+class Filter:
+    """One reader's filter: the types it wants (None for every one), and whether only durable."""
+
+    types: frozenset[str] | None = None
+    durable: bool = False
+
+    def passes(self, entry: Entry) -> bool:
+        """Return whether the entry reaches this reader."""
+        if entry.type in ALWAYS_PASS:
+            return True
+        if self.durable and entry.ephemeral:
+            return False
+        return self.types is None or entry.type in self.types
+
+
+EVERYTHING = Filter()
+
+
+def parse_filter(types: str | None, *, durable: bool) -> Filter:
+    """Return the filter a query string asks for (`types=a.b,c.d`), refusing a bad type name."""
+    if types is None:
+        return Filter(durable=durable)
+    wanted = [name.strip() for name in types.split(",") if name.strip()]
+    if len(wanted) > MAX_TYPES:
+        raise DeclarationRefused(f"a filter names at most {MAX_TYPES} types, not {len(wanted)}")
+    if bad := [name for name in wanted if not _A_TYPE_NAME.match(name)]:
+        raise DeclarationRefused(f"an event type is lowercase words joined by dots: {bad}")
+    return Filter(types=frozenset(wanted), durable=durable)
+
+
+def project_state(
+    state: State, projection: Projection, config: AgentConfig | None, viewer: str | None = None
+) -> JsonObject:
+    """Return the reduced state as this audience may read it."""
+    if projection == "tenant":
+        app_state = state.app_state if config is None else masked_state(state.app_state, config)
+        return {**state.written(), "app_state": app_state}
+    written = state.written()
+    room: Json = None
+    if state.room is not None:
+        seats: list[Json] = [_kept(seat.written(), PUBLIC_SEAT) for seat in state.room.participants]
+        room = {**state.room.written(), "participants": seats}
+    return {
+        "seq": state.seq,
+        "status": state.status,
+        "user_state": written["user_state"],
+        "agent_state": written["agent_state"],
+        "live": written["live"],
+        "turns": [_public_turn(turn) for turn in state.turns],
+        "app_state": _only_public(state.app_state, config),
+        "room": room,
+        "confirms": [_kept(confirm.written(), PUBLIC_CONFIRM) for confirm in state.confirms],
+        "transfer": written["transfer"],
+        "held": state.held,
+        "events": [
+            event.written()
+            for event in state.events
+            if event.source == "participant" and event.identity == viewer
+        ],
+    }
+
+
+def project_entry(
+    entry: Entry, projection: Projection, config: AgentConfig | None, viewer: str | None = None
+) -> JsonObject | None:
+    """Return the entry as this audience may read it, or None when it must not reach them."""
+    if projection == "tenant":
+        return {**entry.written(), "data": _tenant_data(entry, config)}
+    fields = PUBLIC_ENTRY_FIELDS.get(entry.type)
+    if fields is None:
+        return None
+    data: JsonObject = {name: entry.data[name] for name in fields if name in entry.data}
+    if entry.type == "event.received" and not (
+        data.get("source") == "participant" and data.get("identity") == viewer
+    ):
+        return None
+    match _readable(entry):
+        case AgentTurnEnded() as turn:
+            data["metrics"] = _public_metrics(turn.metrics.e2e_latency)
+        case StateChanged() as changed:
+            public = _only_public(changed.state, config)
+            kept: list[Json] = [name for name in changed.changed if name in public]
+            data = {"state": public, "changed": kept}
+        case LogGap(snapshot=State() as snapshot):
+            data["snapshot"] = project_state(snapshot, "public", config, viewer)
+        case None if entry.type in NEEDS_READING:
+            return None
+        case _:
+            pass
+    envelope = entry.written()
+    return {**{name: envelope[name] for name in PUBLIC_ENVELOPE}, "data": data}
+
+
+# A log written since private values are masked when written holds them masked already; the
+# rows written before are masked here, by the declaration the reader reads with.
+def _tenant_data(entry: Entry, config: AgentConfig | None) -> JsonObject:
+    gap = _readable(entry)
+    if isinstance(gap, LogGap) and gap.snapshot is not None:
+        return {**entry.data, "snapshot": project_state(gap.snapshot, "tenant", config)}
+    return entry.data if config is None else split(entry.type, entry.data, config).kept
+
+
+# Only the entries a projection reaches into are read; an old shape of one is withheld from the
+# public, since what it would leak cannot be told, and passed whole to the tenant.
+def _readable(entry: Entry) -> WireModel | None:
+    if entry.type not in NEEDS_READING:
+        return None
+    try:
+        return event_of(entry)
+    except DeclarationRefused:
+        return None
+
+
+def _public_turn(turn: Turn) -> JsonObject:
+    kept = _kept(turn.written(), PUBLIC_TURN)
+    if isinstance(turn, AgentTurn):
+        kept["metrics"] = _public_metrics(turn.metrics.e2e_latency)
+    return kept
+
+
+# How long the caller waited is the one number about them; the rest is the tenant's cost and tuning.
+def _public_metrics(e2e_latency: float | None) -> JsonObject:
+    return {} if e2e_latency is None else {"e2e_latency": e2e_latency}
+
+
+# Without a declaration every field is the tenant's: nothing is public, nothing is masked.
+def _only_public(app_state: JsonObject, config: AgentConfig | None) -> JsonObject:
+    if config is None:
+        return {}
+    return {
+        name: value for name, value in app_state.items() if config.visibility_of(name) == "public"
+    }
+
+
+def _kept(written: JsonObject, fields: frozenset[str]) -> dict[str, Json]:
+    return {name: value for name, value in written.items() if name in fields}