pinecall-runtime 0.1.6__py3-none-any.whl

pinecall/log/refold.py

@@ -0,0 +1,217 @@
+"""The facts folded again from the log: rebuilt a call at a time, and a sample checked by doctor."""
+
+from dataclasses import asdict, dataclass, fields
+from uuid import uuid4
+
+from pinecall.domain.call import A_CALL
+from pinecall.domain.errors import DeclarationRefused
+from pinecall.log import facts
+from pinecall.log.facts import CallFacts
+from pinecall.log.store import entry_of
+from pinecall.postgres.pool import Connection, Pool
+from pinecall.wire.events import EVENTS, event_of
+from pinecall.wire.frames import Entry
+
+# Calls named per read of the heads; each is refolded in a transaction of its own, so a rebuild
+# never holds a lock longer than one call's fold.
+A_PAGE = 500
+
+
+# What doctor refolds and compares, and how many of the heads behind their rows it names.
+A_SAMPLE = 20
+
+
+HEADS_EXAMINED = 20
+
+
+CALLS_PAGE = """
+select log from call_log_head
+where call is not null and log > %(after)s
+  and (%(call)s::text is null or log = %(call)s)
+  and (%(org)s::text is null or org = %(org)s)
+  and (%(since)s::float8 is null or started_at >= %(since)s)
+order by log
+limit %(limit)s
+"""
+
+
+# The append's own lock: an append waits for the refold, and the refold reads what it wrote.
+HEAD_LOCKED = "select log from call_log_head where log = %(call)s for update"
+
+
+# Snapshot reads: an append writes its entries and its facts in one transaction, so one snapshot
+# sees both or neither.
+SNAPSHOT = "set transaction isolation level repeatable read, read only"
+
+
+ENTRIES = """
+select call, seq, ts, agent, type, ephemeral, data
+from call_log where log = %(call)s order by seq
+"""
+
+
+FACTS = "select * from call_facts where call = %(call)s"
+
+
+# The heads doctor examines: the newest calls by the log's own order (the index of an entry's
+# type and position finds the last call.started without reading the rest), and as many from a
+# random point of the heads' key, round to the start. Each is one probe of the log's primary key:
+# a head is never behind a row it numbered. Every head would be a probe per call on the box.
+HEADS_BEHIND = """
+with newest as (
+    select log from call_log where type = 'call.started' order by position desc limit %(limit)s
+), sampled as (
+    (select log from call_log_head where log >= %(start)s order by log limit %(limit)s)
+    union all
+    (select log from call_log_head where log < %(start)s order by log limit %(limit)s)
+    limit %(limit)s
+), examined as (select log from newest union select log from sampled)
+select examined.log,
+       head.seq < (select max(entry.seq) from call_log entry where entry.log = examined.log)
+           as behind
+from examined join call_log_head head on head.log = examined.log
+order by examined.log
+"""
+
+
+# From a random point of the call ids, round to the start: a sample the primary key finds.
+SAMPLE = """
+(select log from call_log_head where call is not null and sealed and log >= %(start)s
+ order by log limit %(limit)s)
+union all
+(select log from call_log_head where call is not null and sealed and log < %(start)s
+ order by log limit %(limit)s)
+limit %(limit)s
+"""
+
+
+@dataclass(frozen=True)
+class Refolding:
+    """Which calls a rebuild refolds: one, an org's, those started since; every call when none."""
+
+    call: str | None = None
+    org: str | None = None
+    since: float | None = None
+
+
+@dataclass(frozen=True)
+class Rebuilt:
+    """What a rebuild read, what it rewrote, and what it left: logs this release cannot read."""
+
+    calls: int
+    rewritten: int
+    left: int
+
+
+@dataclass(frozen=True)
+class Heads:
+    """The heads doctor examined, and those among them whose rows are past their head's seq."""
+
+    examined: int
+    behind: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class Differs:
+    """A call whose stored facts are not what its log folds to, and the columns that differ."""
+
+    call: str
+    columns: tuple[str, ...]
+
+
+async def rebuild(pool: Pool, refolding: Refolding, *, page: int = A_PAGE) -> Rebuilt:
+    """Fold each call's facts again from its log and write the row where it differs."""
+    calls = rewritten = left = 0
+    after = ""
+    while names := await _calls_after(pool, refolding, after, page):
+        for call in names:
+            async with pool.connection() as connection, connection.transaction():
+                await connection.execute(HEAD_LOCKED, {"call": call})
+                stored, folded = await _refolded(connection, call)
+                if folded is None:
+                    left += 1
+                elif folded != (stored or CallFacts(call=call)):
+                    await connection.execute(facts.FACTS_WRITTEN, _row(folded))
+                    rewritten += 1
+            calls += 1
+        after = names[-1]
+    return Rebuilt(calls=calls, rewritten=rewritten, left=left)
+
+
+async def heads_behind(pool: Pool) -> Heads:
+    """The newest heads and a sample of the rest, and those that gave out fewer seqs than rows."""
+    params = {"limit": HEADS_EXAMINED, "start": f"{A_CALL}{uuid4().hex}"}
+    async with pool.connection() as connection:
+        rows = await (await connection.execute(HEADS_BEHIND, params)).fetchall()
+    return Heads(examined=len(rows), behind=tuple(str(row["log"]) for row in rows if row["behind"]))
+
+
+async def differing(pool: Pool, *, sample: int = A_SAMPLE) -> list[Differs]:
+    """Refold a sample of sealed calls and name each whose stored facts differ, and where."""
+    start = f"{A_CALL}{uuid4().hex}"
+    async with pool.connection() as connection:
+        rows = await (
+            await connection.execute(SAMPLE, {"start": start, "limit": sample})
+        ).fetchall()
+    found: list[Differs] = []
+    for call in (str(row["log"]) for row in rows):
+        async with pool.connection() as connection, connection.transaction():
+            await connection.execute(SNAPSHOT)
+            stored, folded = await _refolded(connection, call)
+        if folded is None:
+            continue
+        columns = _columns_apart(stored or CallFacts(call=call), folded)
+        if columns:
+            found.append(Differs(call=call, columns=columns))
+    return found
+
+
+async def _calls_after(pool: Pool, refolding: Refolding, after: str, page: int) -> list[str]:
+    params = {**asdict(refolding), "after": after, "limit": page}
+    async with pool.connection() as connection:
+        rows = await (await connection.execute(CALLS_PAGE, params)).fetchall()
+    return [str(row["log"]) for row in rows]
+
+
+# The fold record() runs as entries land, run over the whole log from nothing. A log holding an
+# entry this release's wire refuses folds to None: the fold would pass the entry by and lose what
+# the release that wrote it folded from it, so its stored row is the better account.
+async def _refolded(connection: Connection, call: str) -> tuple[CallFacts | None, CallFacts | None]:
+    row = await (await connection.execute(FACTS, {"call": call})).fetchone()
+    stored = None if row is None else facts.facts_of(row)
+    folded = CallFacts(call=call)
+    for read in await (await connection.execute(ENTRIES, {"call": call})).fetchall():
+        entry = entry_of(read)
+        if _refused(entry):
+            return stored, None
+        folded = facts.fold(folded, entry)
+    return stored, folded
+
+
+def _row(folded: CallFacts) -> dict[str, object]:
+    written: dict[str, object] = {
+        **asdict(folded),
+        "e2e": list(folded.e2e),
+        "heard_at": list(folded.heard_at),
+    }
+    written.pop("agent")
+    return written
+
+
+def _columns_apart(stored: CallFacts, folded: CallFacts) -> tuple[str, ...]:
+    return tuple(
+        field.name
+        for field in fields(CallFacts)
+        if field.name in facts.COLUMNS
+        and getattr(stored, field.name) != getattr(folded, field.name)
+    )
+
+
+def _refused(entry: Entry) -> bool:
+    if entry.ephemeral or entry.type not in EVENTS:
+        return False
+    try:
+        event_of(entry)
+    except DeclarationRefused:
+        return True
+    return False

pinecall/log/store.py

@@ -0,0 +1,370 @@
+"""The log in Postgres: a call's or an agent's entries, the seq each one is born with, its owner."""
+
+import logging
+import time
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass, field, replace
+from uuid import uuid4
+
+from psycopg.rows import DictRow
+
+from pinecall.domain.agent import Versions
+from pinecall.domain.call import Opener
+from pinecall.domain.errors import DeclarationRefused
+from pinecall.domain.names import Env, JsonObject, parse_env
+from pinecall.domain.scope import Scope
+from pinecall.log._writer import FED_TYPES, Append, Batch, Unnumbered, Writer
+from pinecall.log.reduce import Metered
+from pinecall.postgres.pool import Pool, box_wide
+from pinecall.wire.frames import Entry
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_LIMIT = 500
+
+# An agent's own log is named "@<agent>", a name the CHECK on call_log keeps from any call id.
+AGENT_LOG_PREFIX = "@"
+
+# The most entries a worker's batch may carry.
+MOST_IN_A_BATCH = 256
+
+PAGE = """
+select call, seq, ts, agent, type, ephemeral, data
+from call_log
+where log = %(log)s and seq > %(after)s
+order by seq
+limit %(limit)s
+"""
+
+# SealCallRequest a log nothing was written to is allowed: recovery may seal before the first entry.
+SEAL = """
+insert into call_log_head (log, call, sealed) values (%(call)s, %(call)s, true)
+on conflict (log) do update set sealed = true
+"""
+
+HEAD = "select seq, sealed, written, written_seq from call_log_head where log = %(log)s"
+
+# Taken by one gateway at a time, and only while the call is open; it runs out by itself. A call
+# that wrote nothing yet has no head: the lease makes it, as the seal does.
+LEASED = """
+insert into call_log_head as head (log, call, sealing_until, sealing_by)
+values (%(call)s, %(call)s, now() + make_interval(secs => %(seconds)s), %(by)s)
+on conflict (log) do update
+    set sealing_until = excluded.sealing_until, sealing_by = excluded.sealing_by
+where not head.sealed and (head.sealing_until is null or head.sealing_until < now())
+returning log
+"""
+
+# The holder alone gives the lease back or renews it: a gateway that lost the race to the lease
+# cannot keep the winner's alive, and one that died stops renewing.
+UNLEASED = """
+update call_log_head set sealing_until = null, sealing_by = null
+where log = %(call)s and not sealed and sealing_by = %(by)s
+"""
+
+RENEWED = """
+update call_log_head set sealing_until = now() + make_interval(secs => %(seconds)s)
+where log = %(call)s and not sealed and sealing_by = %(by)s
+"""
+
+
+# The first claim wins, and it may come before the first entry.
+CLAIM = """
+insert into call_log_head as head
+    (log, agent, call, org, env, holder, config_version, lexicon_version, opened_by)
+values (%(log)s, %(agent)s, %(call)s, %(org)s, %(env)s, %(holder)s, %(config)s, %(lexicon)s,
+        %(opened_by)s)
+on conflict (log) do update
+    set org             = coalesce(head.org, excluded.org),
+        env             = coalesce(head.env, excluded.env),
+        holder          = coalesce(head.holder, excluded.holder),
+        config_version  = coalesce(head.config_version, excluded.config_version),
+        lexicon_version = coalesce(head.lexicon_version, excluded.lexicon_version),
+        opened_by       = coalesce(head.opened_by, excluded.opened_by)
+"""
+
+OWNER = "select org from call_log_head where log = %(log)s"
+
+CLAIMANT = "select org, env from call_log_head where log = %(log)s"
+
+MOVED = """
+with moved as (update call_log_head set org = %(org)s where agent = %(agent)s returning log)
+select count(*) as moved from moved
+"""
+
+NOT_FED = "{types}: only the fed types are read across every log (log/_writer.py FED_TYPES)"
+
+# Served by call_log_metered (type, position).
+ACROSS = """
+select entry.position, head.org, entry.call, entry.seq, entry.ts, entry.agent, entry.type,
+       entry.ephemeral, entry.data
+from call_log entry
+join call_log_head head on head.log = entry.log
+where entry.type = any(%(types)s) and entry.position > %(after)s
+order by entry.position
+limit %(limit)s
+"""
+
+NEWEST_CALLS = """
+select call from call_log_head
+where call is not null and (%(agent)s::text is null or agent = %(agent)s)
+order by started_at desc nulls last, log desc
+limit %(limit)s
+"""
+
+NEWEST_LIVE_CALL = """
+select call from call_log_head
+where call is not null and not sealed
+order by started_at desc nulls last, log desc
+limit 1
+"""
+
+
+@dataclass(frozen=True, slots=True)
+class Claimant:
+    """Whose a log is and the world it was claimed in; no world on an agent's own log."""
+
+    org: str
+    env: Env | None
+
+
+@dataclass(frozen=True, slots=True)
+class Claim:
+    """What a call log is claimed with: its scope, the versions it was built on, who opened it."""
+
+    scope: Scope
+    versions: Versions = field(default_factory=Versions)
+    # None for a dial placed before its worker opens it: the open says who.
+    opened_by: Opener | None = None
+
+
+class Store:
+    """The log tables on a pool the store is given and never closes."""
+
+    def __init__(
+        self, pool: Pool, *, clock: Callable[[], float] = time.time, writing: Pool | None = None
+    ) -> None:
+        """Keep the pools and the clock entries are stamped with; appends go through a writer."""
+        self.pool = pool
+        # ts is the runtime's clock, when it saw the event, never the database's.
+        self.clock = clock
+        # This process, as the holder of the seals it leases.
+        self.lease_id = uuid4().hex
+        # A gateway's writer has connections of its own; a verb's and a test's share the pool.
+        self.writer = Writer(pool if writing is None else writing)
+
+    async def append(
+        self, call: str | None, agent: str, kind: str, data: JsonObject, *, ephemeral: bool
+    ) -> Entry:
+        """Write the entry with the next seq of its log, folding the call's facts with it."""
+        entry = Unnumbered(type=kind, data=data, ephemeral=ephemeral, ts=self.clock())
+        append = Append("entry", log_name(call, agent), call, agent, [entry])
+        [written] = (await self.writer.written(append)).entries
+        return written
+
+    # Two writers deciding the same end (a webhook delivered twice) write it once: the writer looks
+    # under the head's lock, in the transaction that would write it.
+    async def append_first(
+        self, call: str, agent: str, kind: str, data: JsonObject
+    ) -> Entry | None:
+        """Write a durable entry unless the call's log holds one of its type or is sealed."""
+        entry = Unnumbered(type=kind, data=data, ephemeral=False, ts=self.clock())
+        written = (await self.writer.written(Append("first", call, call, agent, [entry]))).entries
+        return written[0] if written else None
+
+    # Of two gateways answering one tool call (the app's answer, a timeout) the first stands.
+    async def append_answer(self, call: str, agent: str, data: JsonObject) -> Entry | None:
+        """Write a tool.result unless the log never asked its call id or answered it; None then."""
+        entry = Unnumbered(type="tool.result", data=data, ephemeral=False, ts=self.clock())
+        written = (await self.writer.written(Append("answer", call, call, agent, [entry]))).entries
+        return written[0] if written else None
+
+    # One writer, in order: `after` is how many entries the log took from it before this batch.
+    async def append_many(
+        self, call: str | None, agent: str, entries: Sequence[Unnumbered], *, after: int
+    ) -> Batch:
+        """Write a worker's batch once under the next contiguous seqs, or answer a retry of it."""
+        if not entries or len(entries) > MOST_IN_A_BATCH:
+            raise DeclarationRefused(
+                f"a batch carries 1 to {MOST_IN_A_BATCH} entries, not {len(entries)}"
+            )
+        stamped = [
+            replace(item, ts=stamp)
+            for item, stamp in zip(entries, _stamps(entries, self.clock()), strict=True)
+        ]
+        append = Append("batch", log_name(call, agent), call, agent, stamped, after=after)
+        return await self.writer.written(append)
+
+    async def rescored(self, call: str, agent: str, data: JsonObject) -> Entry:
+        """Write a call.score to a sealed log, the one entry a sealed log still takes."""
+        score = Unnumbered(type="call.score", data=data, ephemeral=False, ts=self.clock())
+        [written] = (await self.writer.written(Append("score", call, call, agent, [score]))).entries
+        return written
+
+    async def since(self, log: str, *, after: int = 0, limit: int = DEFAULT_LIMIT) -> list[Entry]:
+        """Return one page of the log's durable entries above the cursor, in seq order."""
+        async with self.pool.connection() as connection:
+            cursor = await connection.execute(PAGE, {"log": log, "after": after, "limit": limit})
+            return [entry_of(row) for row in await cursor.fetchall()]
+
+    async def whole(self, log: str, *, after: int = 0) -> list[Entry]:
+        """Return every durable entry of the log above the cursor, oldest first."""
+        found: list[Entry] = []
+        while True:
+            page = await self.since(log, after=after, limit=DEFAULT_LIMIT)
+            found += page
+            if len(page) < DEFAULT_LIMIT:
+                return found
+            after = page[-1].seq
+
+    async def latest_seq(self, log: str) -> int:
+        """Return the last seq the log gave out, ephemerals counted; 0 for a log never written."""
+        head = await self._head(log)
+        return 0 if head is None else int(head["seq"])
+
+    async def written(self, log: str) -> int:
+        """Return how many entries the log took from its writer's batches; 0 for none."""
+        head = await self._head(log)
+        return 0 if head is None else int(head["written"])
+
+    async def sealed(self, log: str) -> bool:
+        """Return whether the log is sealed."""
+        head = await self._head(log)
+        return head is not None and bool(head["sealed"])
+
+    async def seal(self, call: str) -> None:
+        """Seal the call's log; sealing twice changes nothing."""
+        async with self.pool.connection() as connection:
+            await connection.execute(SEAL, {"call": call})
+
+    async def lease_seal(self, call: str, seconds: float) -> bool:
+        """Take the right to seal the call for so long; False when another holds it, or it ended."""
+        async with self.pool.connection() as connection:
+            taken = await (
+                await connection.execute(
+                    LEASED, {"call": call, "seconds": seconds, "by": self.lease_id}
+                )
+            ).fetchone()
+        return taken is not None
+
+    async def renew_seal(self, call: str, seconds: float) -> None:
+        """Hold the right to seal the call for so long again, while sealing it."""
+        async with self.pool.connection() as connection:
+            await connection.execute(
+                RENEWED, {"call": call, "seconds": seconds, "by": self.lease_id}
+            )
+
+    async def release_seal(self, call: str) -> None:
+        """Give the right to seal back, after a seal that broke: the next knock takes it."""
+        async with self.pool.connection() as connection:
+            await connection.execute(UNLEASED, {"call": call, "by": self.lease_id})
+
+    async def claim(
+        self, call: str | None, agent: str, org: str, claim: Claim | None = None
+    ) -> None:
+        """Record the log's org, and for a call its scope and versions; the first claim stands."""
+        scope = None if call is None or claim is None else claim.scope
+        versions = Versions() if call is None or claim is None else claim.versions
+        async with self.pool.connection() as connection:
+            await connection.execute(
+                CLAIM,
+                {
+                    "log": log_name(call, agent),
+                    "agent": agent,
+                    "call": call,
+                    "org": org,
+                    "env": None if scope is None else scope.env,
+                    "holder": None if scope is None else scope.holder,
+                    "config": versions.config,
+                    "lexicon": versions.lexicon,
+                    "opened_by": None if call is None or claim is None else claim.opened_by,
+                },
+            )
+
+    # Whose a log is decides who may read it: asked box-wide, so another org's log reads as that
+    # org's (a 404 to the asker) and never as nobody's (postgres/pool.py, migration 0096).
+    async def owner(self, agent: str) -> str | None:
+        """Return the org the agent's own log belongs to, or None while nobody claimed it."""
+        with box_wide():
+            async with self.pool.connection() as connection:
+                found = await connection.execute(OWNER, {"log": log_name(None, agent)})
+                row = await found.fetchone()
+        return None if row is None or row["org"] is None else str(row["org"])
+
+    async def claimant(self, call: str | None, agent: str) -> Claimant | None:
+        """Return whose the log is and in which world, or None while nobody claimed it."""
+        with box_wide():
+            async with self.pool.connection() as connection:
+                found = await connection.execute(CLAIMANT, {"log": log_name(call, agent)})
+                row = await found.fetchone()
+        if row is None or row["org"] is None:
+            return None
+        env = None if row["env"] is None else parse_env(str(row["env"]))
+        return Claimant(org=str(row["org"]), env=env)
+
+    # The operator's way back from a claim: an agent registered with the wrong key.
+    async def moved(self, agent: str, org: str) -> int:
+        """Move every log of the agent to another org and return how many moved."""
+        async with self.pool.connection() as connection:
+            row = await (await connection.execute(MOVED, {"agent": agent, "org": org})).fetchone()
+        return 0 if row is None else int(row["moved"])
+
+    async def across(
+        self, types: Sequence[str], *, after: int = 0, limit: int = DEFAULT_LIMIT
+    ) -> list[Metered]:
+        """Return one page of the fed entries of every log, by position, with their owners."""
+        unfed = sorted(set(types) - FED_TYPES)
+        if unfed:
+            raise DeclarationRefused(NOT_FED.format(types=", ".join(unfed)))
+        params = {"types": list(types), "after": after, "limit": limit}
+        async with self.pool.connection() as connection:
+            rows = await (await connection.execute(ACROSS, params)).fetchall()
+        return [
+            Metered(
+                position=int(row["position"]),
+                org=None if row["org"] is None else str(row["org"]),
+                entry=entry_of(row),
+            )
+            for row in rows
+        ]
+
+    async def newest_calls(self, limit: int, *, agent: str | None = None) -> list[str]:
+        """Return the newest calls of the box, or of one agent."""
+        async with self.pool.connection() as connection:
+            cursor = await connection.execute(NEWEST_CALLS, {"limit": limit, "agent": agent})
+            return [str(row["call"]) for row in await cursor.fetchall()]
+
+    async def newest_live_call(self) -> str | None:
+        """Return the newest call whose log is not sealed, or None."""
+        async with self.pool.connection() as connection:
+            row = await (await connection.execute(NEWEST_LIVE_CALL)).fetchone()
+        return None if row is None else str(row["call"])
+
+    async def _head(self, log: str) -> DictRow | None:
+        async with self.pool.connection() as connection:
+            return await (await connection.execute(HEAD, {"log": log})).fetchone()
+
+
+def log_name(call: str | None, agent: str) -> str:
+    """Return the name a log goes by: the call id, or the agent's with the prefix."""
+    return f"{AGENT_LOG_PREFIX}{agent}" if call is None else call
+
+
+# Built, not validated: a row is what an Entry wrote, and its columns are typed (text, bigint,
+# float8, boolean, jsonb), so validating it again re-checked our own write on every read, a tenth
+# of a gateway's core at 300 calls (a seal reads its whole log; 2026-10-01).
+def entry_of(row: DictRow) -> Entry:
+    """Return the entry a call_log row holds."""
+    return Entry.model_construct(
+        **{name: row[name] for name in ("call", "seq", "ts", "agent", "type", "ephemeral", "data")}
+    )
+
+
+# The worker's clock says when each thing happened; the gateway's bounds it, and a batch never
+# steps back inside itself.
+def _stamps(entries: Sequence[Unnumbered], now: float) -> list[float]:
+    stamps: list[float] = []
+    for item in entries:
+        stamp = min(item.ts, now)
+        stamps.append(stamp if not stamps else max(stamp, stamps[-1]))
+    return stamps