shikiru-cli 0.2.0__py3-none-any.whl

shikiru/store/database.py

@@ -0,0 +1,297 @@
+"""Connections, migrations and JSON/timestamp column helpers."""
+
+from __future__ import annotations
+
+import sqlite3  # noqa: TID251
+from contextlib import contextmanager, suppress
+from datetime import UTC, datetime
+from importlib import resources
+from typing import TYPE_CHECKING
+
+from shikiru.errors import DatabaseError
+from shikiru.paths import SCHEMA_BACKUPS_DIR_NAME
+from shikiru.private import create_private_file, make_private_dir
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+    from pathlib import Path
+
+    from pydantic import TypeAdapter
+
+BUSY_TIMEOUT_MS = 5000
+SCHEMA_BACKUPS_KEPT = 3
+OperationalDatabaseError = sqlite3.OperationalError
+"""A database fault of the environment: locked, full or unreadable."""
+_MIGRATIONS_PACKAGE = "shikiru.store.migrations"
+_MIGRATION_SUFFIX = ".sql"
+_MIGRATION_NUMBER_SEPARATOR = "_"
+_BACKUP_PREFIX = "state-schema"
+_BACKUP_SUFFIX = ".db"
+_BEGIN = "BEGIN"
+_BEGIN_IMMEDIATE = "BEGIN IMMEDIATE"
+_READ_VERSION = "PRAGMA user_version"
+
+
+class Database:
+    """Opens `.shikiru/state.db`, keeping its schema migrated to the packaged latest version."""
+
+    def __init__(self, path: Path) -> None:
+        """Point at the SQLite file `path`; no connection is opened until asked for one."""
+        self._path = path
+
+    @property
+    def path(self) -> Path:
+        """Return the database file path used in actionable read errors."""
+        return self._path
+
+    def connect(self) -> sqlite3.Connection:
+        """Open a read-write connection and migrate the schema up to date.
+
+        The engine uses this for every state table; clients (CLI, MCP) use it only to append
+        `commands` rows and, for `Submit`, to allocate the task id from `sequences`.
+        """
+        make_private_dir(self._path.parent)
+        create_private_file(self._path)
+        connection = _guard_schema(self._connect(mode="rwc"), self._path)
+        connection.execute("PRAGMA journal_mode = WAL")
+        try:
+            _migrate(connection, self._path)
+        except BaseException:
+            connection.close()
+            raise
+        return connection
+
+    def connect_readonly(self) -> sqlite3.Connection:
+        """Open a connection SQLite itself refuses to let write."""
+        try:
+            connection = self._connect(mode="ro")
+        except sqlite3.Error as error:
+            message = f"could not open read-only database {self._path}: {error}"
+            raise DatabaseError(message) from error
+        return _guard_schema(connection, self._path)
+
+    def schema_version(self, connection: sqlite3.Connection) -> int:
+        """Return the migrated schema version for an open database connection."""
+        return _schema_version(connection)
+
+    def health(self) -> int:
+        """Open and migrate the database, returning its schema version."""
+        connection: sqlite3.Connection | None = None
+        try:
+            connection = self.connect()
+            return self.schema_version(connection)
+        except sqlite3.Error as error:
+            raise DatabaseError(str(error)) from error
+        finally:
+            if connection is not None:
+                connection.close()
+
+    def _uri(self, mode: str) -> str:
+        return f"{self._path.absolute().as_uri()}?mode={mode}"
+
+    def _connect(self, *, mode: str) -> sqlite3.Connection:
+        connection = sqlite3.connect(
+            self._uri(mode),
+            uri=True,
+            isolation_level=None,
+            check_same_thread=False,
+        )
+        connection.row_factory = sqlite3.Row
+        connection.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_MS}")
+        connection.execute("PRAGMA foreign_keys = ON")
+        return connection
+
+
+@contextmanager
+def transaction(
+    connection: sqlite3.Connection, *, immediate: bool = False
+) -> Generator[sqlite3.Connection]:
+    """Run a block as one transaction: commits on success, rolls back on any exception.
+
+    With `immediate` the write lock is taken at once, so what the block reads first cannot
+    change before it writes.
+
+    Composes with an already-open transaction by joining it, so multiple store operations can
+    participate in one atomic unit without attempting a nested `BEGIN`.
+    """
+    if connection.in_transaction:
+        yield connection
+        return
+    connection.execute(_BEGIN_IMMEDIATE if immediate else _BEGIN)
+    try:
+        yield connection
+    except BaseException:
+        connection.execute("ROLLBACK")
+        raise
+    else:
+        connection.execute("COMMIT")
+
+
+def next_sequence(connection: sqlite3.Connection, name: str) -> int:
+    """Atomically increment and return `sequences.value` for `name`, inside a transaction."""
+    row = connection.execute(
+        "INSERT INTO sequences (name, value) VALUES (?, 1) "
+        "ON CONFLICT (name) DO UPDATE SET value = value + 1 RETURNING value",
+        (name,),
+    ).fetchone()
+    return row[0]
+
+
+def dump_json[T](adapter: TypeAdapter[T], value: T) -> str:
+    """Serialise `value` to JSON text via `adapter`, for a JSON-typed column."""
+    return adapter.dump_json(value).decode()
+
+
+def load_json[T](adapter: TypeAdapter[T], text: str) -> T:
+    """Parse `text` into `adapter`'s type, for a JSON-typed column."""
+    return adapter.validate_json(text)
+
+
+def format_timestamp(value: datetime) -> str:
+    """Format `value` as the fixed-width UTC ISO-8601 text every timestamp column stores.
+
+    `timespec="microseconds"` always renders the fractional part, so every stored timestamp has
+    the same width and plain string comparison (as `agent_blocks.until > ?` relies on) orders
+    them the same way their instants do; without it, a whole-second value's missing ".ffffff"
+    only sorts correctly by accident.
+    """
+    return value.astimezone(UTC).isoformat(timespec="microseconds")
+
+
+def parse_timestamp(text: str | None) -> datetime | None:
+    """Parse a nullable timestamp column back into a UTC-aware `datetime`."""
+    return datetime.fromisoformat(text) if text is not None else None
+
+
+def parse_required_timestamp(text: str) -> datetime:
+    """Parse a `NOT NULL` timestamp column back into a UTC-aware `datetime`."""
+    return datetime.fromisoformat(text)
+
+
+def require_after_insert[T](value: T | None, message: str) -> T:
+    """Unwrap `value`, or raise `RuntimeError(message)` if it's `None`.
+
+    Guards the two ways a same-transaction insert-then-read can't come back empty in practice:
+    sqlite refusing a rowid, or an immediate re-fetch by that rowid finding no row.
+    """
+    if value is None:
+        raise RuntimeError(message)
+    return value
+
+
+def _migration_files() -> list[tuple[int, str]]:
+    package = resources.files(_MIGRATIONS_PACKAGE)
+    migrations: list[tuple[int, str]] = []
+    for entry in package.iterdir():
+        if entry.name.endswith(_MIGRATION_SUFFIX):
+            number = int(entry.name.split(_MIGRATION_NUMBER_SEPARATOR, 1)[0])
+            migrations.append((number, entry.read_text(encoding="utf-8")))
+    return sorted(migrations, key=lambda migration: migration[0])
+
+
+def _statements(sql: str) -> list[str]:
+    return [statement.strip() for statement in sql.split(";") if statement.strip()]
+
+
+def _schema_version(connection: sqlite3.Connection) -> int:
+    return connection.execute(_READ_VERSION).fetchone()[0]
+
+
+def _guard_schema(connection: sqlite3.Connection, path: Path) -> sqlite3.Connection:
+    stored = _schema_version(connection)
+    known = _migration_files()[-1][0]
+    if stored > known:
+        connection.close()
+        message = (
+            f"state database {path} has schema version {stored}, newer than the version "
+            f"{known} this Shikiru knows; use the Shikiru release that wrote it"
+            f"{_restore_hint(path)}"
+        )
+        raise DatabaseError(message)
+    return connection
+
+
+def _restore_hint(path: Path) -> str:
+    newest = _newest_backup(_backups_dir(path))
+    if newest is None:
+        return ""
+    return f"; the newest backup is {newest}, copy it back over {path} to restore it"
+
+
+def _migrate(connection: sqlite3.Connection, path: Path) -> None:
+    migrations = _migration_files()
+    latest = migrations[-1][0]
+    if _schema_version(connection) >= latest:
+        return
+    with transaction(connection, immediate=True):
+        stored = _schema_version(connection)
+        if 0 < stored < latest:
+            _back_up(path, stored)
+    for number, sql in migrations:
+        with transaction(connection, immediate=True):
+            if number <= _schema_version(connection):
+                continue
+            for statement in _statements(sql):
+                connection.execute(statement)
+            connection.execute(f"PRAGMA user_version = {number}")
+
+
+def _backups_dir(path: Path) -> Path:
+    return path.parent / SCHEMA_BACKUPS_DIR_NAME
+
+
+def _back_up(path: Path, schema: int) -> None:
+    directory = _backups_dir(path)
+    target = directory / f"{_BACKUP_PREFIX}{schema}{_BACKUP_SUFFIX}"
+    try:
+        make_private_dir(directory)
+        target.unlink(missing_ok=True)
+        create_private_file(target)
+        _copy_database(path, target)
+        _prune_backups(directory)
+    except (OSError, sqlite3.Error) as error:
+        with suppress(OSError):
+            target.unlink(missing_ok=True)
+        message = (
+            f"could not back up state database {path} before migrating from schema {schema}: "
+            f"{error}"
+        )
+        raise DatabaseError(message) from error
+
+
+def _copy_database(source_path: Path, target: Path) -> None:
+    source = sqlite3.connect(source_path)
+    destination = sqlite3.connect(target)
+    try:
+        source.backup(destination)
+    finally:
+        destination.close()
+        source.close()
+
+
+def _prune_backups(directory: Path) -> None:
+    for _schema, stale in _backups_by_schema(directory)[:-SCHEMA_BACKUPS_KEPT]:
+        stale.unlink()
+
+
+def _newest_backup(directory: Path) -> Path | None:
+    backups = _backups_by_schema(directory)
+    return backups[-1][1] if backups else None
+
+
+def _backups_by_schema(directory: Path) -> list[tuple[int, Path]]:
+    if not directory.is_dir():
+        return []
+    backups: list[tuple[int, Path]] = []
+    for entry in directory.iterdir():
+        schema = _backup_schema(entry.name)
+        if schema is not None:
+            backups.append((schema, entry))
+    return sorted(backups, key=lambda backup: backup[0])
+
+
+def _backup_schema(name: str) -> int | None:
+    if not (name.startswith(_BACKUP_PREFIX) and name.endswith(_BACKUP_SUFFIX)):
+        return None
+    digits = name.removeprefix(_BACKUP_PREFIX).removesuffix(_BACKUP_SUFFIX)
+    return int(digits) if digits.isdecimal() else None

shikiru/store/escalations.py

@@ -0,0 +1,240 @@
+"""Escalation rows: open, list and resolve."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from pydantic import TypeAdapter
+
+from shikiru.domain.commands import ResolutionKind
+from shikiru.domain.escalation import Escalation, EscalationOption, EscalationStatus
+from shikiru.domain.events import EventKind
+from shikiru.domain.tasks import EscalationReason, Step
+from shikiru.store.database import (
+    dump_json,
+    format_timestamp,
+    load_json,
+    next_sequence,
+    parse_required_timestamp,
+    parse_timestamp,
+    require_after_insert,
+    transaction,
+)
+
+if TYPE_CHECKING:
+    import sqlite3  # noqa: TID251
+
+    from shikiru.clock import Clock
+
+ESCALATION_ID_PREFIX = "E-"
+ESCALATION_SEQUENCE = "escalation"
+ANSWER_RESOLUTION_PREFIX = f"{ResolutionKind.ANSWER.value} "
+ANSWER_ESCALATION_ID_JSON_PATH = "$.escalation_id"
+TURN_STEP_JSON_PATH = "$.step"
+
+_OPTIONS_ADAPTER: TypeAdapter[tuple[EscalationOption, ...]] = TypeAdapter(
+    tuple[EscalationOption, ...]
+)
+
+_ESCALATION_COLUMNS = (
+    "id, task_id, step, reason, headline, packet_path, options, status, resolution, "
+    "created_at, resolved_at"
+)
+
+
+def _row_to_escalation(row: sqlite3.Row) -> Escalation:
+    return Escalation(
+        id=row["id"],
+        task_id=row["task_id"],
+        step=Step(row["step"]) if row["step"] is not None else None,
+        reason=EscalationReason(row["reason"]),
+        headline=row["headline"],
+        packet_path=row["packet_path"],
+        options=load_json(_OPTIONS_ADAPTER, row["options"]),
+        status=EscalationStatus(row["status"]),
+        resolution=row["resolution"],
+        created_at=parse_required_timestamp(row["created_at"]),
+        resolved_at=parse_timestamp(row["resolved_at"]),
+    )
+
+
+def fetch_escalation(connection: sqlite3.Connection, escalation_id: str) -> Escalation | None:
+    """Read one escalation by id, or `None` if it doesn't exist."""
+    row = connection.execute(
+        f"SELECT {_ESCALATION_COLUMNS} FROM escalations WHERE id = ?", (escalation_id,)
+    ).fetchone()
+    return _row_to_escalation(row) if row is not None else None
+
+
+def fetch_open_escalations(connection: sqlite3.Connection) -> list[Escalation]:
+    """Read every open escalation, oldest first."""
+    rows = connection.execute(
+        f"SELECT {_ESCALATION_COLUMNS} FROM escalations WHERE status = ? ORDER BY id",
+        (EscalationStatus.OPEN.value,),
+    ).fetchall()
+    return [_row_to_escalation(row) for row in rows]
+
+
+class EscalationStore:
+    """Opens and resolves escalations, `E-n` ids allocated from `sequences`."""
+
+    def __init__(self, connection: sqlite3.Connection, clock: Clock) -> None:
+        """Use `connection` (the engine's writer) and `clock` for every write below."""
+        self._connection = connection
+        self._clock = clock
+
+    def open(
+        self,
+        task_id: str,
+        *,
+        escalation_id: str | None = None,
+        step: Step | None,
+        reason: EscalationReason,
+        headline: str,
+        packet_path: str,
+        options: tuple[EscalationOption, ...],
+    ) -> Escalation:
+        """Record a new escalation, allocating an id unless one was reserved for its packet."""
+        with transaction(self._connection):
+            if escalation_id is None:
+                escalation_id = self._next_id()
+            self._insert(
+                escalation_id,
+                task_id=task_id,
+                step=step,
+                reason=reason,
+                headline=headline,
+                packet_path=packet_path,
+                options=options,
+            )
+        return self._require_opened(escalation_id)
+
+    def reserve_id(self) -> str:
+        """Reserve the next escalation id before creating its packet file."""
+        with transaction(self._connection):
+            return self._next_id()
+
+    def _next_id(self) -> str:
+        seq = next_sequence(self._connection, ESCALATION_SEQUENCE)
+        return f"{ESCALATION_ID_PREFIX}{seq}"
+
+    def _insert(
+        self,
+        escalation_id: str,
+        *,
+        task_id: str,
+        step: Step | None,
+        reason: EscalationReason,
+        headline: str,
+        packet_path: str,
+        options: tuple[EscalationOption, ...],
+    ) -> None:
+        self._connection.execute(
+            """
+            INSERT INTO escalations (
+              id, task_id, step, reason, headline, packet_path, options, status, created_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                escalation_id,
+                task_id,
+                step.value if step is not None else None,
+                reason.value,
+                headline,
+                packet_path,
+                dump_json(_OPTIONS_ADAPTER, options),
+                EscalationStatus.OPEN.value,
+                format_timestamp(self._clock.now()),
+            ),
+        )
+
+    def _require_opened(self, escalation_id: str) -> Escalation:
+        escalation = fetch_escalation(self._connection, escalation_id)
+        return require_after_insert(
+            escalation, f"escalation {escalation_id} vanished immediately after insert"
+        )
+
+    def list_open(self) -> list[Escalation]:
+        """Read every open escalation, oldest first."""
+        return fetch_open_escalations(self._connection)
+
+    def get(self, escalation_id: str) -> Escalation | None:
+        """Read one escalation by id, or `None` if it doesn't exist."""
+        return fetch_escalation(self._connection, escalation_id)
+
+    def latest_answer(self, task_id: str) -> str | None:
+        """Return the Operator answer that the task's unfinished consult still acts on.
+
+        Retries of a consult that the answer started are skipped, so the answer survives
+        them; any other escalation ends the search.
+        """
+        rows = self._connection.execute(
+            """
+            SELECT id, step, resolution FROM escalations
+            WHERE task_id = ? AND status = ?
+            ORDER BY resolved_at DESC, CAST(substr(id, 3) AS INTEGER) DESC
+            """,
+            (
+                task_id,
+                EscalationStatus.RESOLVED.value,
+            ),
+        )
+        for row in rows:
+            resolution = row["resolution"]
+            if resolution is not None and resolution.startswith(ANSWER_RESOLUTION_PREFIX):
+                if self._answer_was_consumed(task_id, row["id"]):
+                    return None
+                return resolution[len(ANSWER_RESOLUTION_PREFIX) :]
+            if row["step"] != Step.CONSULT.value:
+                return None
+        return None
+
+    def _answer_was_consumed(self, task_id: str, escalation_id: str) -> bool:
+        resolution_event = self._connection.execute(
+            """
+            SELECT id FROM events
+            WHERE task_id = ? AND kind = ? AND json_extract(data, ?) = ?
+            ORDER BY id DESC LIMIT 1
+            """,
+            (
+                task_id,
+                EventKind.ESCALATION_RESOLVED.value,
+                ANSWER_ESCALATION_ID_JSON_PATH,
+                escalation_id,
+            ),
+        ).fetchone()
+        if resolution_event is None:
+            return False
+        consult_turn = self._connection.execute(
+            """
+            SELECT 1 FROM events
+            WHERE task_id = ? AND kind = ? AND id > ? AND json_extract(data, ?) = ?
+            LIMIT 1
+            """,
+            (
+                task_id,
+                EventKind.TURN_COMPLETED.value,
+                resolution_event["id"],
+                TURN_STEP_JSON_PATH,
+                Step.CONSULT.value,
+            ),
+        ).fetchone()
+        return consult_turn is not None
+
+    def settle_open(self, task_id: str, resolution: str) -> None:
+        """Resolve every open escalation of `task_id`, so none can act on its later state."""
+        for escalation in fetch_open_escalations(self._connection):
+            if escalation.task_id == task_id:
+                self.resolve(escalation.id, resolution)
+
+    def resolve(self, escalation_id: str, resolution: str) -> None:
+        """Mark `escalation_id` resolved, recording `resolution`'s summary text."""
+        self._connection.execute(
+            "UPDATE escalations SET status = ?, resolution = ?, resolved_at = ? WHERE id = ?",
+            (
+                EscalationStatus.RESOLVED.value,
+                resolution,
+                format_timestamp(self._clock.now()),
+                escalation_id,
+            ),
+        )