semora-store-pg 0.1.0__tar.gz

semora_store_pg-0.1.0/.gitignore

@@ -0,0 +1,28 @@
+.venv/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.coverage
+htmlcov/
+.env
+.env.*
+!.env.example
+
+
+# Local tool/editor state — machine-specific, never pushed.
+.claude/
+.codecanvas/
+.vscode/
+
+
+# Superpowers design/spec scratch — working notes, not project documentation.
+docs/superpowers/
+
+# 로컬 자격증명 — 절대 커밋 금지.
+a.txt
+*.token

semora_store_pg-0.1.0/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.5
+Name: semora-store-pg
+Version: 0.1.0
+Summary: PostgreSQL adapters for Semora storage-semantic contracts.
+Project-URL: Homepage, https://github.com/donggyun112/semora
+Project-URL: Source, https://github.com/donggyun112/semora
+Project-URL: Changelog, https://github.com/donggyun112/semora/blob/main/CHANGELOG.md
+Author: donggyun112
+License-Expression: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: psycopg[binary,pool]<4,>=3.3
+Requires-Dist: semora-store==0.1.0

semora_store_pg-0.1.0/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "semora-store-pg"
+version = "0.1.0"
+description = "PostgreSQL adapters for Semora storage-semantic contracts."
+requires-python = ">=3.12"
+license = "MIT"
+authors = [{ name = "donggyun112" }]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.12",
+  "Typing :: Typed",
+]
+urls = { Homepage = "https://github.com/donggyun112/semora", Source = "https://github.com/donggyun112/semora", Changelog = "https://github.com/donggyun112/semora/blob/main/CHANGELOG.md" }
+dependencies = [
+  "semora-store==0.1.0",
+  "psycopg[binary,pool]>=3.3,<4",
+]
+# Not `semora`. This implements the ledger protocol and nothing above it, which is what keeps a
+# database driver out of the default install.
+
+[tool.uv.sources]
+semora-store = { workspace = true }
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/semora_store_pg"]

semora_store_pg-0.1.0/src/semora_store_pg/__init__.py

@@ -0,0 +1,17 @@
+"""Expose PostgreSQL-backed step ledger and transcript implementations."""
+
+from .postgres import (
+    SCHEMA,
+    PostgresSteps,
+)
+from .transcript import (
+    TRANSCRIPT_SCHEMA,
+    PostgresTranscript,
+)
+
+__all__ = [
+    "SCHEMA",
+    "TRANSCRIPT_SCHEMA",
+    "PostgresSteps",
+    "PostgresTranscript",
+]

semora_store_pg-0.1.0/src/semora_store_pg/postgres.py

@@ -0,0 +1,369 @@
+"""Implement the durable step ledger with PostgreSQL."""
+
+from typing import Any, Self
+
+from psycopg import AsyncConnection
+from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
+from psycopg_pool import AsyncConnectionPool
+from semora_store import (
+    EffectConflict,
+    ExecutionContext,
+    ExecutionTransition,
+    Fenced,
+    InputRecord,
+    Step,
+)
+
+__all__ = ["SCHEMA", "PostgresSteps"]
+
+SCHEMA = """
+create table if not exists ledger_step (
+    run_id      text        not null,
+    key         text        not null,
+    status      text        not null check (status in ('running', 'done')),
+    value       jsonb,
+    attempt     integer     not null default 1,
+    started_at  timestamptz not null default now(),
+    finished_at timestamptz,
+    primary key (run_id, key)
+);
+
+create table if not exists ledger_run_lease (
+    run_id     text        primary key,
+    owner      text        not null,
+    token      bigint      not null default 1,
+    expires_at timestamptz not null
+);
+
+create table if not exists ledger_input (
+    sequence     bigserial   primary key,
+    run_id       text        not null,
+    input_id     text        not null,
+    status       text        not null,
+    value        jsonb       not null,
+    submitted_at timestamptz not null default now(),
+    admitted_at  timestamptz,
+    unique (run_id, input_id)
+);
+
+alter table ledger_input drop constraint if exists ledger_input_status_check;
+alter table ledger_input add constraint ledger_input_status_check
+    check (status in ('pending', 'claimed', 'admitted', 'discarded'));
+
+drop index if exists ledger_input_pending;
+create index if not exists ledger_input_pending
+    on ledger_input (run_id, sequence) where status in ('pending', 'claimed');
+
+create index if not exists ledger_step_running
+    on ledger_step (started_at) where status = 'running';
+"""
+"""The three tables. The partial indexes serve recovery and operator stuck-work queries."""
+
+
+class PostgresSteps:
+    """Persist steps, leases, and queued inputs in PostgreSQL."""
+
+    def __init__(self, pool: AsyncConnectionPool[AsyncConnection[Any]]) -> None:
+        """Initialize the ledger with a caller-owned connection pool."""
+        self._pool = pool
+
+    def for_execution(self, context: ExecutionContext) -> Self:
+        """Return this adapter because its default schema is scope-neutral."""
+        del context
+        return self
+
+    async def read(self, run_id: str, key: str) -> Step:
+        """Return the persisted state of a step."""
+        async with (
+            self._pool.connection() as connection,
+            connection.cursor(row_factory=dict_row) as cursor,
+        ):
+            await cursor.execute(
+                "select status, value from ledger_step where run_id = %s and key = %s",
+                (run_id, key),
+            )
+            row = await cursor.fetchone()
+        if row is None:
+            return Step("absent")
+        if row["status"] == "running":
+            return Step("running")
+        return Step("done", row["value"])
+
+    async def start(self, run_id: str, key: str, token: int = 0) -> bool:
+        """Atomically record new running intent after validating the fencing token."""
+        async with self._pool.connection() as connection:
+            await self._fence(connection, run_id, token)
+            async with connection.cursor() as cursor:
+                await cursor.execute(
+                    """
+                insert into ledger_step (run_id, key, status)
+                values (%s, %s, 'running')
+                on conflict (run_id, key) do nothing
+                returning 1
+                """,
+                    (run_id, key),
+                )
+                return await cursor.fetchone() is not None
+
+    async def finish_effect(self, run_id: str, key: str, value: Any, token: int = 0) -> None:
+        """Complete a running effect without replacing a committed result."""
+        async with self._pool.connection() as connection:
+            await self._fence(connection, run_id, token)
+            await self._complete_effect(connection, run_id, key, value, "running")
+
+    async def _complete_effect(
+        self,
+        connection: AsyncConnection[Any],
+        run_id: str,
+        key: str,
+        value: Any,
+        expected: str,
+    ) -> None:
+        """Apply one conditional effect completion inside the caller's transaction."""
+        async with connection.cursor(row_factory=dict_row) as cursor:
+            if expected == "absent":
+                await cursor.execute(
+                    """
+                    insert into ledger_step (run_id, key, status, value, finished_at)
+                    values (%s, %s, 'done', %s, now())
+                    on conflict (run_id, key) do nothing
+                    returning 1
+                    """,
+                    (run_id, key, Jsonb(value)),
+                )
+            else:
+                await cursor.execute(
+                    """
+                    update ledger_step
+                    set status = 'done', value = %s, finished_at = now()
+                    where run_id = %s and key = %s and status = 'running'
+                    returning 1
+                    """,
+                    (Jsonb(value), run_id, key),
+                )
+            if await cursor.fetchone() is not None:
+                return
+            await cursor.execute(
+                "select status, value from ledger_step where run_id = %s and key = %s",
+                (run_id, key),
+            )
+            row = await cursor.fetchone()
+        if row is not None and row["status"] == "done":
+            if row["value"] == value:
+                return
+            raise EffectConflict(run_id, key, "a different result is already committed")
+        status = "absent" if row is None else str(row["status"])
+        raise EffectConflict(run_id, key, f"expected {expected!r}, found {status!r}")
+
+    async def write_control(self, run_id: str, key: str, value: Any, token: int = 0) -> None:
+        """Upsert mutable framework control state."""
+        async with self._pool.connection() as connection:
+            await self._fence(connection, run_id, token)
+            async with connection.cursor() as cursor:
+                await cursor.execute(
+                    """
+                insert into ledger_step (run_id, key, status, value, finished_at)
+                values (%s, %s, 'done', %s, now())
+                on conflict (run_id, key) do update
+                    set status = 'done', value = excluded.value, finished_at = now()
+                """,
+                    (run_id, key, Jsonb(value)),
+                )
+
+    async def finish(self, run_id: str, key: str, value: Any, token: int = 0) -> None:
+        """Compatibility alias for ``write_control``."""
+        await self.write_control(run_id, key, value, token)
+
+    async def _fence(self, connection: AsyncConnection[Any], run_id: str, token: int) -> None:
+        """Reject stale lease holders within the caller's transaction."""
+        if not token:
+            return
+        async with connection.cursor(row_factory=dict_row) as cursor:
+            await cursor.execute("select token from ledger_run_lease where run_id = %s", (run_id,))
+            row = await cursor.fetchone()
+        issued = row["token"] if row else 0
+        if token < issued:
+            raise Fenced(run_id, token, issued)
+
+    async def forget(self, run_id: str, key: str, token: int = 0) -> None:
+        """Remove an unfinished step after validating the fencing token."""
+        async with self._pool.connection() as connection:
+            await self._fence(connection, run_id, token)
+            async with connection.cursor() as cursor:
+                await cursor.execute(
+                    "delete from ledger_step where run_id = %s and key = %s and status = 'running'",
+                    (run_id, key),
+                )
+
+    async def acquire(self, run_id: str, owner: str, ttl_seconds: float = 60.0) -> int:
+        """Acquire or renew a run lease and return its fencing token, or zero on contention."""
+        async with (
+            self._pool.connection() as connection,
+            connection.cursor(row_factory=dict_row) as cursor,
+        ):
+            await cursor.execute(
+                """
+                insert into ledger_run_lease (run_id, owner, token, expires_at)
+                values (%s, %s, 1, now() + make_interval(secs => %s))
+                on conflict (run_id) do update
+                    set owner = excluded.owner,
+                        expires_at = excluded.expires_at,
+                        token = ledger_run_lease.token
+                                + (ledger_run_lease.owner <> excluded.owner)::int
+                    where ledger_run_lease.owner = excluded.owner
+                       or ledger_run_lease.expires_at < now()
+                returning token
+                """,
+                (run_id, owner, ttl_seconds),
+            )
+            row = await cursor.fetchone()
+        return int(row["token"]) if row else 0
+
+    async def release(self, run_id: str, owner: str) -> None:
+        """Expire an owned lease without resetting its fencing token.
+
+        Args:
+            run_id: Durable run identifier.
+            owner: Current lease owner.
+        """
+        async with self._pool.connection() as connection, connection.cursor() as cursor:
+            await cursor.execute(
+                """
+                update ledger_run_lease
+                set owner = '', expires_at = to_timestamp(0)
+                where run_id = %s and owner = %s
+                """,
+                (run_id, owner),
+            )
+
+    async def enqueue_input(self, run_id: str, input_id: str, value: dict[str, Any]) -> bool:
+        """Append an input idempotently and report whether it was inserted."""
+        async with self._pool.connection() as connection, connection.cursor() as cursor:
+            await cursor.execute(
+                """
+                insert into ledger_input (run_id, input_id, status, value)
+                values (%s, %s, 'pending', %s)
+                on conflict (run_id, input_id) do nothing
+                """,
+                (run_id, input_id, Jsonb(value)),
+            )
+            inserted = cursor.rowcount > 0
+        return inserted
+
+    async def list_inputs(self, run_id: str) -> list[InputRecord]:
+        """Return a run's inputs in submission order."""
+        async with (
+            self._pool.connection() as connection,
+            connection.cursor(row_factory=dict_row) as cursor,
+        ):
+            await cursor.execute(
+                """
+                select input_id, status, value, sequence
+                from ledger_input
+                where run_id = %s
+                order by sequence
+                """,
+                (run_id,),
+            )
+            rows = await cursor.fetchall()
+        return [
+            InputRecord(row["input_id"], row["status"], row["value"], int(row["sequence"]))
+            for row in rows
+        ]
+
+    async def claim_input(self, run_id: str, input_id: str, token: int = 0) -> None:
+        """Mark an input as claimed unless it is already terminal."""
+        async with self._pool.connection() as connection:
+            await self._fence(connection, run_id, token)
+            async with connection.cursor() as cursor:
+                await cursor.execute(
+                    """
+                update ledger_input
+                set status = 'claimed'
+                where run_id = %s and input_id = %s
+                  and status not in ('admitted', 'discarded')
+                """,
+                    (run_id, input_id),
+                )
+
+    async def admit_inputs(self, run_id: str, input_ids: list[str], token: int = 0) -> None:
+        """Mark the selected inputs as admitted."""
+        if not input_ids:
+            return
+        async with self._pool.connection() as connection:
+            await self._fence(connection, run_id, token)
+            async with connection.cursor() as cursor:
+                await cursor.execute(
+                    """
+                update ledger_input
+                set status = 'admitted', admitted_at = now()
+                where run_id = %s and input_id = any(%s) and status <> 'discarded'
+                """,
+                    (run_id, input_ids),
+                )
+
+    async def discard_inputs(self, run_id: str, input_ids: list[str], token: int = 0) -> None:
+        """Make screened-out inputs terminal without deleting their idempotency keys."""
+        if not input_ids:
+            return
+        async with self._pool.connection() as connection:
+            await self._fence(connection, run_id, token)
+            async with connection.cursor() as cursor:
+                await cursor.execute(
+                    """
+                update ledger_input
+                set status = 'discarded'
+                where run_id = %s and input_id = any(%s)
+                """,
+                    (run_id, input_ids),
+                )
+
+    async def commit_transition(
+        self,
+        run_id: str,
+        transition: ExecutionTransition,
+        token: int = 0,
+    ) -> set[str]:
+        """Atomically apply typed effect, control, and input changes."""
+        inserted: set[str] = set()
+        async with self._pool.connection() as connection:
+            await self._fence(connection, run_id, token)
+            seen: set[str] = set()
+            for effect in transition.effects:
+                if effect.key in seen:
+                    raise ValueError(f"effect {effect.key!r} appears twice in one transition")
+                seen.add(effect.key)
+            overlap = seen.intersection(transition.controls)
+            if overlap:
+                raise ValueError(
+                    f"transition classifies keys as both effect and control: {sorted(overlap)}"
+                )
+            for effect in transition.effects:
+                await self._complete_effect(
+                    connection, run_id, effect.key, effect.value, effect.expected
+                )
+            async with connection.cursor() as cursor:
+                for key, value in transition.controls.items():
+                    await cursor.execute(
+                        """
+                        insert into ledger_step (run_id, key, status, value, finished_at)
+                        values (%s, %s, 'done', %s, now())
+                        on conflict (run_id, key) do update
+                            set status = 'done', value = excluded.value, finished_at = now()
+                        """,
+                        (run_id, key, Jsonb(value)),
+                    )
+                for input_id, value in transition.inputs:
+                    await cursor.execute(
+                        """
+                        insert into ledger_input (run_id, input_id, status, value)
+                        values (%s, %s, 'pending', %s)
+                        on conflict (run_id, input_id) do nothing
+                        returning input_id
+                        """,
+                        (run_id, input_id, Jsonb(value)),
+                    )
+                    if await cursor.fetchone() is not None:
+                        inserted.add(input_id)
+        return inserted

semora_store_pg-0.1.0/src/semora_store_pg/transcript.py

@@ -0,0 +1,195 @@
+"""PostgreSQL implementation of the append-only transcript store.
+
+Entries are persisted verbatim as JSONB. Generated columns provide indexed transcript metadata,
+while run and model-usage tables store operational accounting data.
+"""
+
+from typing import Any, Self
+
+from psycopg import AsyncConnection
+from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
+from psycopg_pool import AsyncConnectionPool
+from semora_store import MODEL_USAGE_FIELDS, RUN_FIELDS, ExecutionContext, check_fields
+
+__all__ = ["TRANSCRIPT_SCHEMA", "PostgresTranscript"]
+
+TRANSCRIPT_SCHEMA = """
+create table if not exists ledger_transcript (
+    entry           jsonb       not null,
+    seq             bigserial   primary key,
+    ts              timestamptz not null,
+    conversation_id text        generated always as (entry ->> 'conversation_id') stored,
+    uuid            text        generated always as (entry ->> 'uuid') stored,
+    parent_uuid     text        generated always as (entry ->> 'parent_uuid') stored,
+    type            text        generated always as (entry ->> 'type') stored,
+    schema_version  text        generated always as (entry ->> 'schema_version') stored,
+    run_id          text        generated always as (entry -> 'metadata' ->> 'run_id') stored,
+    unique (conversation_id, uuid)
+);
+
+create index if not exists ledger_transcript_replay
+    on ledger_transcript (conversation_id, seq);
+
+create index if not exists ledger_transcript_run
+    on ledger_transcript (run_id) where run_id is not null;
+
+create table if not exists ledger_run (
+    run_id               text primary key,
+    conversation_id      text,
+    stop_reason          text,
+    tool_calls           integer,
+    interrupted_mid_turn boolean,
+    started_at           timestamptz,
+    ended_at             timestamptz
+);
+
+create index if not exists ledger_run_conversation
+    on ledger_run (conversation_id, started_at desc);
+
+create table if not exists ledger_run_model (
+    run_id             text   not null,
+    model              text   not null,
+    prompt_tokens      bigint,
+    completion_tokens  bigint,
+    total_tokens       bigint,
+    cached_tokens      bigint,
+    cache_write_tokens bigint,
+    cost_usd           numeric(12, 6),
+    primary key (run_id, model)
+);
+
+create index if not exists ledger_run_model_by_model
+    on ledger_run_model (model);
+"""
+"""Schema for transcript entries, run metadata, and per-model usage records."""
+
+
+class PostgresTranscript:
+    """Persist conversation entries and per-run cost in PostgreSQL."""
+
+    def __init__(self, pool: AsyncConnectionPool[AsyncConnection[Any]]) -> None:
+        """Initialize the store with a caller-owned connection pool."""
+        self._pool = pool
+
+    def for_execution(self, context: ExecutionContext) -> Self:
+        """Return this adapter because its default schema is scope-neutral."""
+        del context
+        return self
+
+    async def append(self, entry: dict[str, Any]) -> bool:
+        """Append an entry verbatim and report whether it was new."""
+        async with self._pool.connection() as connection, connection.cursor() as cursor:
+            await cursor.execute(
+                """
+                insert into ledger_transcript (entry, ts)
+                values (%s, coalesce((%s)::timestamptz, now()))
+                on conflict (conversation_id, uuid) do nothing
+                returning seq
+                """,
+                (Jsonb(entry), entry.get("timestamp")),
+            )
+            inserted = await cursor.fetchone()
+        return inserted is not None
+
+    async def read(self, conversation_id: str, *, limit: int | None = None) -> list[dict[str, Any]]:
+        """Return entries in arrival order, optionally limited to the newest tail."""
+        async with (
+            self._pool.connection() as connection,
+            connection.cursor(row_factory=dict_row) as cursor,
+        ):
+            if limit is None:
+                await cursor.execute(
+                    """
+                    select entry from ledger_transcript
+                    where conversation_id = %s
+                    order by seq
+                    """,
+                    (conversation_id,),
+                )
+            else:
+                await cursor.execute(
+                    """
+                    select entry from (
+                        select entry, seq from ledger_transcript
+                        where conversation_id = %s
+                        order by seq desc
+                        limit %s
+                    ) tail
+                    order by seq
+                    """,
+                    (conversation_id, max(limit, 0)),
+                )
+            rows = await cursor.fetchall()
+        return [row["entry"] for row in rows]
+
+    async def record_run(self, run_id: str, fields: dict[str, Any]) -> None:
+        """Merge validated fields into a run record, creating it when absent."""
+        await self._upsert("ledger_run", ("run_id",), (run_id,), fields, RUN_FIELDS)
+
+    async def record_model_usage(self, run_id: str, model: str, counts: dict[str, Any]) -> None:
+        """Merge validated token counts for one model of one run."""
+        await self._upsert(
+            "ledger_run_model", ("run_id", "model"), (run_id, model), counts, MODEL_USAGE_FIELDS
+        )
+
+    async def read_run(self, run_id: str) -> dict[str, Any] | None:
+        """Return a run record with unwritten null fields omitted."""
+        async with (
+            self._pool.connection() as connection,
+            connection.cursor(row_factory=dict_row) as cursor,
+        ):
+            await cursor.execute(
+                f"select {', '.join(sorted(RUN_FIELDS))} from ledger_run where run_id = %s",
+                (run_id,),
+            )
+            row = await cursor.fetchone()
+        return None if row is None else {k: v for k, v in row.items() if v is not None}
+
+    async def read_model_usage(self, run_id: str) -> dict[str, dict[str, Any]]:
+        """This run's token counts keyed by the model that spent them."""
+        async with (
+            self._pool.connection() as connection,
+            connection.cursor(row_factory=dict_row) as cursor,
+        ):
+            await cursor.execute(
+                f"""
+                select model, {", ".join(sorted(MODEL_USAGE_FIELDS))}
+                from ledger_run_model where run_id = %s
+                """,
+                (run_id,),
+            )
+            rows = await cursor.fetchall()
+        return {
+            row["model"]: {k: v for k, v in row.items() if k != "model" and v is not None}
+            for row in rows
+        }
+
+    async def _upsert(
+        self,
+        table: str,
+        keys: tuple[str, ...],
+        key_values: tuple[Any, ...],
+        fields: dict[str, Any],
+        allowed: frozenset[str],
+    ) -> None:
+        """Merge allow-listed fields into a keyed record."""
+        check_fields(table, fields, allowed)
+        columns = sorted(fields)
+        named = ", ".join((*keys, *columns))
+        placeholders = ", ".join(["%s"] * (len(keys) + len(columns)))
+        # No column to merge means the row only has to exist — an insert that keeps what is there.
+        merge = (
+            ", ".join(f"{column} = excluded.{column}" for column in columns)
+            if columns
+            else f"{keys[0]} = {table}.{keys[0]}"
+        )
+        async with self._pool.connection() as connection, connection.cursor() as cursor:
+            await cursor.execute(
+                f"""
+                insert into {table} ({named})
+                values ({placeholders})
+                on conflict ({", ".join(keys)}) do update set {merge}
+                """,
+                (*key_values, *(fields[column] for column in columns)),
+            )