handcode 0.3.0rc1__py3-none-any.whl

agentctl/kernel/ledger/store.py

@@ -0,0 +1,596 @@
+"""Durable effect ledger. Local, single-writer, fsynced.
+
+Spec: `docs/0012` §2.1, §3.1.
+
+The one rule that matters: **a write returns only when it is durable.**
+`synchronous = FULL` is deliberate — `NORMAL` can lose the last commit on power
+failure, and that is precisely the record whose absence causes a double effect.
+"""
+from __future__ import annotations
+
+import sqlite3
+import time
+from pathlib import Path
+
+from .models import (
+    TRANSITIONS,
+    EffectClass,
+    EffectRecord,
+    EffectState,
+    IllegalTransition,
+    ToolCall,
+)
+
+SCHEMA = (Path(__file__).parent / "schema.sql").read_text(encoding="utf-8")
+
+
+class LeaseHeld(RuntimeError):
+    """Another live holder owns this conversation. Refuse to act."""
+
+
+class StaleFence(RuntimeError):
+    """A superseded holder tried to write. Its lease was taken over."""
+
+
+class AmbiguousPrefix(LookupError):
+    """More than one effect starts with this. Carries the candidates."""
+
+    def __init__(self, prefix: str, matches: list[str]):
+        self.prefix, self.matches = prefix, matches
+        super().__init__(f"{prefix!r} matches {len(matches)} effects")
+
+
+#: Enough of a `tool_call_id` to show a human.
+#:
+#: The id is minted by the *model*, and Gemini smuggles a **thought signature**
+#: through it (`research/phase-10-3` V4). A real one from a live run is over
+#: 300 characters of base64. The ledger keys on that id, so `agentctl resolve`
+#: was asking a human to retype 300 characters to answer a blocked effect --
+#: which makes the human half of fail-closed unusable at exactly the moment it
+#: is needed (`docs/0013` §3, Panel 3).
+#:
+#: So ids print short and are accepted by prefix, the way git takes a SHA. The
+#: distinguishing part is early (`call_2524396__thought__...`), and an
+#: ambiguous prefix is reported rather than guessed at.
+SHORT_ID = 12
+
+
+def short_id(tool_call_id: str) -> str:
+    """The displayable form, marked when it is not the whole id."""
+    if len(tool_call_id) <= SHORT_ID:
+        return tool_call_id
+    return tool_call_id[:SHORT_ID] + "..."
+
+
+#: Opening a ledger right after its previous holder was KILLED can fail on
+#: Windows with "disk I/O error": the dead process's locks on the WAL's -shm
+#: file are released asynchronously, and SQLite's recovery trips over them.
+#: Measured (`docs/0046`): 5 of 8 kill-then-open trials failed on the first
+#: attempt and every one succeeded ~50 ms later. That is the crash -> resume
+#: path, so it is retried, briefly, and never for any other error.
+_TRANSIENT = ("disk i/o error", "database is locked")
+_OPEN_DEADLINE_S = 3.0
+
+
+def _open(path: Path) -> sqlite3.Connection:
+    import logging
+
+    deadline, delay, attempt = time.monotonic() + _OPEN_DEADLINE_S, 0.02, 0
+    while True:
+        attempt += 1
+        db = sqlite3.connect(str(path), isolation_level=None)
+        try:
+            db.row_factory = sqlite3.Row
+            db.executescript(SCHEMA)
+            if attempt > 1:
+                logging.getLogger("agentctl.ledger").info(
+                    "opened %s on attempt %d", path, attempt)
+            return db
+        except sqlite3.OperationalError as e:
+            db.close()
+            if (not any(t in str(e).lower() for t in _TRANSIENT)
+                    or time.monotonic() + delay > deadline):
+                raise
+            time.sleep(delay)
+            delay = min(delay * 2, 0.5)
+
+
+class LedgerStore:
+    """SQLite-backed effect ledger.
+
+    Not thread-safe by design: one conversation, one writer, one lease.
+    """
+
+    def __init__(self, path: str | Path, holder: str = "local"):
+        self.path = Path(path)
+        self.holder = holder
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._db = _open(self.path)
+        self._fences: dict[str, int] = {}   # conversation_id -> our fence token
+
+    def close(self) -> None:
+        self._db.close()
+
+    def __enter__(self) -> "LedgerStore":
+        return self
+
+    def __exit__(self, *_) -> None:
+        self.close()
+
+    # ── lease / fencing ────────────────────────────────────────────────
+    def acquire(
+        self,
+        conversation_id: str,
+        ttl_s: float = 60.0,
+        takeover: bool = False,
+    ) -> int:
+        """Take the writer lease. Returns a monotonic fence token.
+
+        An expired lease is taken silently. A *live* one raises `LeaseHeld`,
+        because two processes must never act on the same pending effect.
+
+        `takeover=True` steals a live lease. That is safe only because every
+        write is fenced (see `_assert_fence`): the superseded holder — a
+        process that crashed, or hung and is about to wake up — is rejected on
+        its next write rather than corrupting the ledger.
+
+        A crashed process cannot release its own lease, so recovery needs
+        either a short TTL or an explicit takeover. Keep `ttl_s` short and
+        renew it while working.
+        """
+        now = time.time()
+        row = self._db.execute(
+            "SELECT holder, fence_token, expires_at FROM lease WHERE conversation_id=?",
+            (conversation_id,),
+        ).fetchone()
+
+        if (row and row["expires_at"] > now and row["holder"] != self.holder
+                and not takeover):
+            raise LeaseHeld(
+                f"conversation {conversation_id} held by {row['holder']} "
+                f"for another {row['expires_at'] - now:.1f}s "
+                f"(pass takeover=True to steal it; fencing makes that safe)"
+            )
+
+        token = (row["fence_token"] + 1) if row else 1
+        self._db.execute(
+            "INSERT INTO lease(conversation_id, holder, fence_token, expires_at) "
+            "VALUES(?,?,?,?) ON CONFLICT(conversation_id) DO UPDATE SET "
+            "holder=excluded.holder, fence_token=excluded.fence_token, "
+            "expires_at=excluded.expires_at",
+            (conversation_id, self.holder, token, now + ttl_s),
+        )
+        self._fences[conversation_id] = token
+        return token
+
+    def renew(self, conversation_id: str, ttl_s: float = 60.0) -> None:
+        """Extend our lease. Call periodically while working."""
+        self._db.execute(
+            "UPDATE lease SET expires_at=? WHERE conversation_id=? AND holder=?",
+            (time.time() + ttl_s, conversation_id, self.holder),
+        )
+
+    def release(self, conversation_id: str) -> None:
+        self._db.execute(
+            "UPDATE lease SET expires_at=0 WHERE conversation_id=? AND holder=?",
+            (conversation_id, self.holder),
+        )
+
+    def lease(self, conversation_id: str) -> dict | None:
+        """The lease as it stands: holder, fence_token, expires_at."""
+        row = self._db.execute(
+            "SELECT holder, fence_token, expires_at FROM lease "
+            "WHERE conversation_id=?", (conversation_id,)).fetchone()
+        return dict(row) if row else None
+
+    # ── reads ──────────────────────────────────────────────────────────
+    def lookup(self, tool_call_id: str) -> EffectRecord | None:
+        row = self._db.execute(
+            "SELECT * FROM effect_record WHERE tool_call_id=?", (tool_call_id,)
+        ).fetchone()
+        return _to_record(row) if row else None
+
+    def resolve_id(self, prefix: str) -> str | None:
+        """An exact id, or the one effect whose id starts with `prefix`.
+
+        Exact match wins outright — a full id must never be re-interpreted as
+        a prefix, or a short id that happens to prefix a longer one would
+        resolve to the wrong effect.
+
+        Raises `AmbiguousPrefix` rather than picking. This is the command that
+        decides whether an effect is recorded as having happened; guessing
+        between two candidates is the one thing it must not do.
+        """
+        if self.lookup(prefix) is not None:
+            return prefix
+        rows = self._db.execute(
+            "SELECT tool_call_id FROM effect_record WHERE tool_call_id LIKE ? "
+            "ESCAPE '\\' ORDER BY started_at",
+            (prefix.replace("\\", "\\\\").replace("%", "\\%")
+                   .replace("_", "\\_") + "%",),
+        ).fetchall()
+        ids = [r["tool_call_id"] for r in rows]
+        if not ids:
+            return None
+        if len(ids) > 1:
+            raise AmbiguousPrefix(prefix, ids)
+        return ids[0]
+
+    def find_by_intent(self, conversation_id: str, intent_hash: str,
+                       exclude_tool_call_id: str | None = None) -> EffectRecord | None:
+        """The same logical effect under a different `tool_call_id`.
+
+        WHY THIS EXISTS (`docs/0023` §4). The ledger is keyed on
+        `tool_call_id`, which the *model* mints. Ask a different model the same
+        question -- as any multi-model pool will on a retry or resume -- and it
+        returns a different id for the identical call. Keyed lookup misses, the
+        gate sees a first sighting, and the effect repeats.
+
+        `intent_hash` is ours: tool name plus canonicalised arguments. Same
+        logical effect, same hash, whoever asked.
+
+        Only the MOST RECENT identical call is a candidate, and not when it is
+        OBSERVED (`docs/0045`). Aliasing exists for a call the model re-mints
+        because it never saw the first one's result: a crash before the
+        observation was persisted. Once the result is in the history the model
+        reads -- success or reported failure -- an identical call is the model
+        choosing to repeat it, which is the ordinary edit -> test -> re-test
+        loop. Matching that as "the same effect" handed back stale output or
+        blocked a fixed retry, in the first two tasks a new user ran
+        (`docs/0044` N15). The crash cases are untouched: an INTENT twin, or a
+        COMMITTED one whose observation was never delivered, still matches.
+        """
+        rows = self._db.execute(
+            "SELECT * FROM effect_record WHERE conversation_id=? AND intent_hash=? "
+            "ORDER BY started_at DESC", (conversation_id, intent_hash)).fetchall()
+        for r in rows:
+            if exclude_tool_call_id and r["tool_call_id"] == exclude_tool_call_id:
+                continue
+            if r["state"] == EffectState.OBSERVED.value:
+                return None
+            return _to_record(r)
+        return None
+
+    def pending(self, conversation_id: str) -> list[EffectRecord]:
+        """Records stuck in INTENT — the ambiguous set after a crash."""
+        rows = self._db.execute(
+            "SELECT * FROM effect_record WHERE conversation_id=? AND state=? "
+            "ORDER BY started_at",
+            (conversation_id, EffectState.INTENT.value),
+        ).fetchall()
+        return [_to_record(r) for r in rows]
+
+    def committed_since(self, conversation_id: str, since: float,
+                        exclude_tool_call_id: str) -> list[EffectRecord]:
+        """Effects in this conversation that landed at or after `since`.
+
+        WHY THIS EXISTS (`docs/0038` §3). A probe that reasons from world state
+        — HEAD moved, therefore my commit landed — is only sound while this
+        call is the sole writer. A sibling call that executed after this one
+        was fingerprinted breaks that, and the probe cannot see siblings: it is
+        handed one record and asked about the world.
+
+        The gate can see them. This is the query that lets it check the
+        assumption before trusting a verdict built on it.
+
+        The comparison is against `committed_at`, not `started_at`: what
+        threatens a fingerprint is a sibling whose effect *landed* after it was
+        taken, not one that was merely decided earlier in the same batch. A
+        sibling that committed *before* this call was fingerprinted is already
+        part of the world that fingerprint describes, and is no threat at all.
+
+        `>=` rather than `>`: two records can share a timestamp, and counting a
+        simultaneous sibling as concurrent costs a human decision, while
+        missing one costs a dropped effect.
+
+        OBSERVED counts as landed here. It is where a delivered effect now
+        ends up, so leaving it out would hide every ordinary sibling from the
+        sole-writer check (`docs/0045`). A delivered *reported failure* is
+        OBSERVED too and counts as well: whether it touched the world is
+        unknown, and unknown is treated as "it might have".
+        """
+        rows = self._db.execute(
+            "SELECT * FROM effect_record WHERE conversation_id=? "
+            "AND state IN (?, ?) "
+            "AND COALESCE(committed_at, started_at)>=? AND tool_call_id!=? "
+            "ORDER BY committed_at",
+            (conversation_id, EffectState.COMMITTED.value,
+             EffectState.OBSERVED.value, since, exclude_tool_call_id),
+        ).fetchall()
+        return [_to_record(r) for r in rows]
+
+    def blocked(self, conversation_id: str | None = None) -> list[EffectRecord]:
+        """Effects awaiting a human decision (`docs/0013` §3, Panel 3)."""
+        if conversation_id:
+            rows = self._db.execute(
+                "SELECT * FROM effect_record WHERE state=? AND conversation_id=?",
+                (EffectState.BLOCKED.value, conversation_id),
+            ).fetchall()
+        else:
+            rows = self._db.execute(
+                "SELECT * FROM effect_record WHERE state=?",
+                (EffectState.BLOCKED.value,),
+            ).fetchall()
+        return [_to_record(r) for r in rows]
+
+    # ── the write-ahead protocol ───────────────────────────────────────
+    def _assert_fence(self, conversation_id: str, record_fence: int | None) -> None:
+        """Reject a write from a superseded holder.
+
+        This is what makes lease takeover safe: a zombie process that wakes up
+        after its lease was stolen holds an older token and is refused here.
+        """
+        ours = self._fences.get(conversation_id)
+        if ours is None or record_fence is None:
+            return                       # no lease taken; single-process use
+        if record_fence > ours:
+            raise StaleFence(
+                f"fence {ours} superseded by {record_fence} for {conversation_id}"
+            )
+
+    def _assert_current_lease(self, conversation_id: str) -> None:
+        """Refuse to START an effect if another holder has taken the lease."""
+        ours = self._fences.get(conversation_id)
+        if ours is None:
+            return                       # no lease taken; single-process use
+        row = self.lease(conversation_id)
+        if row and row["fence_token"] > ours:
+            raise StaleFence(
+                f"fence {ours} superseded by {row['fence_token']} "
+                f"({row['holder']}) for {conversation_id}")
+
+    def write_intent(
+        self, call: ToolCall, cls: EffectClass, fence: int,
+        pre_state: str | None = None,
+    ) -> EffectRecord:
+        """Record intent *before* the tool runs. Durable on return.
+
+        Two fence checks, because they catch different zombies. The record's
+        own fence catches a superseded holder touching an effect the new one
+        has re-claimed. The LEASE's fence catches the one that check cannot
+        see: a superseded holder starting a *new* effect, under an id nobody
+        has recorded yet. Without it a zombie's fresh call went straight
+        through and committed (`docs/0042` I-02, probe P2).
+        """
+        self._assert_current_lease(call.conversation_id)
+        prev = self.lookup(call.tool_call_id)
+        if prev is not None:
+            self._assert_fence(call.conversation_id, prev.fence_token)
+        _check(prev.state if prev else None, EffectState.INTENT)
+        attempt = (prev.attempt + 1) if prev else 1
+        now = time.time()
+
+        self._db.execute(
+            "INSERT INTO effect_record(tool_call_id, conversation_id, turn_id, "
+            "action_event_id, tool_name, intent_hash, effect_class, state, "
+            "fence_token, attempt, started_at, pre_state) "
+            "VALUES(?,?,?,?,?,?,?,?,?,?,?,?) "
+            "ON CONFLICT(tool_call_id) DO UPDATE SET state=excluded.state, "
+            "attempt=excluded.attempt, started_at=excluded.started_at, "
+            "fence_token=excluded.fence_token, error=NULL, "
+            "pre_state=COALESCE(excluded.pre_state, effect_record.pre_state)",
+            (call.tool_call_id, call.conversation_id, call.turn_id, None,
+             call.tool_name, call.intent_hash(), cls.value,
+             EffectState.INTENT.value, fence, attempt, now, pre_state),
+        )
+        rec = self.lookup(call.tool_call_id)
+        assert rec is not None
+        return rec
+
+    def refresh_pre_state(self, tool_call_id: str, pre_state: str | None) -> None:
+        """Re-fingerprint the world immediately before this call executes.
+
+        WHY THIS EXISTS (`docs/0038` §3). The gate decides at Seam B, which the
+        harness runs for *every* call in an assistant message before it
+        executes *any* of them. So a batch of calls all carry a fingerprint of
+        the world as it was before the first one ran — and the git probe's
+        LANDED rule ("HEAD moved, and there is a single writer, so it was our
+        commit") is then wrong about every call but the first.
+
+        Seam C is the only place that sees a call at its own execution moment.
+        Refreshing here makes each fingerprint describe the world that call
+        actually acts on.
+
+        Deliberately narrow: `pre_state` and `started_at` only. The state
+        machine is untouched and `attempt` is not incremented, because this is
+        not a new attempt — it is the same attempt, finally about to run.
+        A record that has left INTENT is not refreshed: it has already
+        executed, and its fingerprint is evidence rather than a prediction.
+        """
+        if pre_state is None:
+            return
+        self._db.execute(
+            "UPDATE effect_record SET pre_state=?, started_at=? "
+            "WHERE tool_call_id=? AND state=?",
+            (pre_state, time.time(), tool_call_id, EffectState.INTENT.value),
+        )
+
+    def commit(self, tool_call_id: str, observation: bytes | None = None) -> None:
+        """The effect landed. Durable on return."""
+        self._transition(
+            tool_call_id, EffectState.COMMITTED,
+            extra={"committed_at": time.time(), "observation": observation},
+        )
+
+    def fail(self, tool_call_id: str, error: str) -> None:
+        """The effect provably did not land."""
+        self._transition(tool_call_id, EffectState.FAILED, extra={"error": error[:2000]})
+
+    def block(self, tool_call_id: str, reason: str) -> None:
+        """Fail closed. A human must resolve this."""
+        self._transition(tool_call_id, EffectState.BLOCKED, extra={"error": reason[:2000]})
+
+    def observed(self, tool_call_id: str) -> None:
+        """A COMMITTED effect's recorded result has now reached the model."""
+        self._transition(tool_call_id, EffectState.OBSERVED)
+
+    def deliver(self, tool_call_id: str, observation: bytes | None = None,
+                error: str | None = None) -> None:
+        """INTENT -> OBSERVED in ONE write: the tool returned, and its result
+        is already persisted in the history the model reads. Durable on return.
+
+        `error` is set when the tool reported failure. Whether such an effect
+        touched the world is unknown, and the record says so; what IS known is
+        that the model saw the failure, so a repeat is its decision.
+        """
+        self._transition(tool_call_id, EffectState.OBSERVED, extra={
+            "committed_at": time.time(), "observation": observation,
+            "error": error[:2000] if error else None,
+        })
+
+    # ── approvals (`docs/0042` I-06) ──────────────────────────────────────
+    def decide(self, rec: EffectRecord, decision: str, summary: str = "") -> None:
+        """A human's answer to an action that asked first.
+
+        The record leaves BLOCKED for FAILED -- "did not run" is the truth for
+        an action that was queued rather than executed -- and the answer is
+        kept by intent hash, so the identical call on resume is recognised.
+        """
+        if decision not in ("approve", "deny"):
+            raise ValueError(decision)
+        self._db.execute(
+            "INSERT INTO approval(conversation_id, intent_hash, tool_call_id, "
+            "summary, decision, decided_at) VALUES(?,?,?,?,?,?) "
+            "ON CONFLICT(conversation_id, intent_hash) DO UPDATE SET "
+            "decision=excluded.decision, decided_at=excluded.decided_at, "
+            "summary=excluded.summary, told_at=NULL, used_at=NULL",
+            (rec.conversation_id, rec.intent_hash, rec.tool_call_id, summary,
+             decision, time.time()))
+        self.fail(rec.tool_call_id,
+                  "approved by the operator; not yet run" if decision == "approve"
+                  else "denied by the operator; not run")
+
+    def approval(self, conversation_id: str, intent_hash: str) -> str | None:
+        """'approve' | 'deny' for an unused decision on this action, or None."""
+        row = self._db.execute(
+            "SELECT decision FROM approval WHERE conversation_id=? AND "
+            "intent_hash=? AND used_at IS NULL", (conversation_id, intent_hash)
+        ).fetchone()
+        return row["decision"] if row else None
+
+    def use_approval(self, conversation_id: str, intent_hash: str) -> None:
+        self._db.execute("UPDATE approval SET used_at=? WHERE conversation_id=? "
+                         "AND intent_hash=?", (time.time(), conversation_id,
+                                               intent_hash))
+
+    def untold(self, conversation_id: str) -> list[dict]:
+        """Decisions the agent has not been told about yet, oldest first."""
+        rows = self._db.execute(
+            "SELECT * FROM approval WHERE conversation_id=? AND told_at IS NULL "
+            "ORDER BY decided_at", (conversation_id,)).fetchall()
+        return [dict(r) for r in rows]
+
+    def told(self, conversation_id: str) -> None:
+        self._db.execute("UPDATE approval SET told_at=? WHERE conversation_id=? "
+                         "AND told_at IS NULL", (time.time(), conversation_id))
+
+    def reconcile(self, tool_call_id: str, verdict: str, landed: bool) -> None:
+        """Resolve an ambiguous INTENT using out-of-band evidence."""
+        target = EffectState.COMMITTED if landed else EffectState.FAILED
+        self._transition(tool_call_id, target, extra={
+            "probe_verdict": verdict,
+            **({"committed_at": time.time()} if landed else {}),
+        })
+
+    # ── internals ──────────────────────────────────────────────────────
+    def _transition(self, tool_call_id: str, target: EffectState, extra: dict | None = None) -> None:
+        rec = self.lookup(tool_call_id)
+        if rec is None:
+            raise IllegalTransition(f"no record for {tool_call_id}")
+        self._assert_fence(rec.conversation_id, rec.fence_token)
+        _check(rec.state, target)
+
+        cols, vals = ["state=?"], [target.value]
+        for k, v in (extra or {}).items():
+            cols.append(f"{k}=?")
+            vals.append(v)
+        vals.append(tool_call_id)
+        self._db.execute(
+            f"UPDATE effect_record SET {', '.join(cols)} WHERE tool_call_id=?", vals
+        )
+
+
+class LeaseHeartbeat:
+    """Keep a lease live for as long as the process holding it is.
+
+    `renew()` had no callers, and the TTL is 60 s (`docs/0042` I-02). Any tool
+    call longer than that -- a test suite, a build -- let the lease lapse, and
+    the next `acquire` from another terminal took it silently.
+
+    A thread rather than a renew-per-decision: nothing decides while a long
+    tool runs, which is exactly when the lease runs out. It holds its own
+    connection, because `LedgerStore` is single-threaded by design.
+
+    A crash stops the heartbeat with the process, so the lease expires within
+    one TTL -- which is what lets a resume recover without being told to. It
+    renews only while the holder still matches: after a takeover it renews
+    nothing, so it cannot resurrect a lease it lost.
+    """
+
+    def __init__(self, path: str | Path, holder: str, conversation_id: str,
+                 ttl_s: float = 60.0, interval_s: float | None = None):
+        import threading
+
+        self._args = (Path(path), holder, conversation_id, ttl_s)
+        self._interval = interval_s if interval_s is not None else ttl_s / 4
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, daemon=True,
+                                        name=f"lease-heartbeat-{conversation_id}")
+
+    def start(self) -> "LeaseHeartbeat":
+        self._thread.start()
+        return self
+
+    def stop(self) -> None:
+        self._stop.set()
+        if self._thread.is_alive():
+            self._thread.join(timeout=self._interval + 5)
+
+    def _run(self) -> None:
+        import logging
+
+        path, holder, cid, ttl = self._args
+        try:
+            store = LedgerStore(path, holder=holder)
+        except Exception:                               # noqa: BLE001
+            logging.getLogger("agentctl.lease").exception("heartbeat could not open %s", path)
+            return
+        try:
+            while not self._stop.wait(self._interval):
+                try:
+                    store.renew(cid, ttl_s=ttl)
+                except Exception:                       # noqa: BLE001
+                    # A missed beat costs at most a lapsed lease, which the
+                    # fence turns into a refused write -- never a duplicate.
+                    logging.getLogger("agentctl.lease").exception("renew failed")
+        finally:
+            store.close()
+
+
+def _check(current: EffectState | None, target: EffectState) -> None:
+    allowed = TRANSITIONS.get(current, set())
+    if target not in allowed:
+        raise IllegalTransition(
+            f"{current.value if current else 'None'} -> {target.value} is not legal"
+        )
+
+
+def _to_record(row: sqlite3.Row) -> EffectRecord:
+    return EffectRecord(
+        tool_call_id=row["tool_call_id"],
+        conversation_id=row["conversation_id"],
+        turn_id=row["turn_id"],
+        tool_name=row["tool_name"],
+        intent_hash=row["intent_hash"],
+        effect_class=EffectClass(row["effect_class"]),
+        state=EffectState(row["state"]),
+        fence_token=row["fence_token"],
+        attempt=row["attempt"],
+        started_at=row["started_at"],
+        committed_at=row["committed_at"],
+        observation=row["observation"],
+        probe_verdict=row["probe_verdict"],
+        pre_state=row["pre_state"],
+        error=row["error"],
+        action_event_id=row["action_event_id"],
+    )