phlo-postgres 0.14.0__tar.gz → 0.15.2__tar.gz

{phlo_postgres-0.14.0 → phlo_postgres-0.15.2}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: phlo-postgres
-Version: 0.14.0
+Version: 0.15.2
 Summary: Postgres service plugin for Phlo
 Author-email: Phlo Team <team@phlo.dev>
 License: MIT
 Requires-Python: >=3.11
 Description-Content-Type: text/plain
-Requires-Dist: phlo<0.15,>=0.14.0
+Requires-Dist: phlo<0.17,>=0.16.0
 Requires-Dist: psycopg2-binary>=2.9.11
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"

{phlo_postgres-0.14.0 → phlo_postgres-0.15.2}/pyproject.toml

@@ -7,13 +7,13 @@ requires = [
 
 [project]
 dependencies = [
-    "phlo>=0.14.0,<0.15",
+    "phlo>=0.16.0,<0.17",
     "psycopg2-binary>=2.9.11",
 ]
 description = "Postgres service plugin for Phlo"
 name = "phlo-postgres"
 requires-python = ">=3.11"
-version = "0.14.0"
+version = "0.15.2"
 
 [[project.authors]]
 email = "team@phlo.dev"

{phlo_postgres-0.14.0 → phlo_postgres-0.15.2}/src/phlo_postgres/__init__.py

@@ -15,6 +15,9 @@ Example:
 
 """
 
+from importlib.metadata import version
+
+from phlo_postgres.checkpoints import PostgresIngestionCheckpointStore
 from phlo_postgres.plugin import PostgresServicePlugin
 from phlo_postgres.publish_target import PostgresPublishTarget
 from phlo_postgres.resource import PostgresResource
@@ -22,6 +25,7 @@ from phlo_postgres.settings import PostgresSettings, get_settings
 from phlo_postgres.settings_store import PostgresSettingsStore
 
 __all__ = [
+    "PostgresIngestionCheckpointStore",
     "PostgresPublishTarget",
     "PostgresResource",
     "PostgresServicePlugin",
@@ -29,4 +33,6 @@ __all__ = [
     "PostgresSettingsStore",
     "get_settings",
 ]
-__version__ = "0.14.0"
+
+
+__version__ = version("phlo-postgres")

{phlo_postgres-0.14.0 → phlo_postgres-0.15.2}/src/phlo_postgres/authorization.py

@@ -1,4 +1,8 @@
-"""phlo_postgres CLI authorization table."""
+"""Authorization surface table for the phlo-postgres CLI.
+
+Declares which postgres commands mutate state plus their dataset resources and
+required actions; the shared CLI surface adapter enforces these mappings.
+"""
 
 from __future__ import annotations
 
@@ -38,4 +42,5 @@ PostgresCliSurfaceAdapter = cli_surface_adapter_class(
 
 
 def get_postgres_cli_adapter() -> CliSurfaceAdapter:
+    """Return the shared Postgres CLI surface adapter instance."""
     return PostgresCliSurfaceAdapter.get_instance()

phlo_postgres-0.15.2/src/phlo_postgres/checkpoints.py

@@ -0,0 +1,309 @@
+"""Durable ingestion checkpoints backed by Phlo PostgreSQL.
+
+Implements the neutral :class:`~phlo.capabilities.interfaces.IngestionCheckpointStore`
+contract on the platform's existing PostgreSQL service so stream consumers
+(Kafka today) can persist claimed offset ranges, bind them to output Iceberg
+snapshots, and commit only after the snapshot is audited and promoted.
+
+Claims are serialized per idempotency key with a PostgreSQL advisory
+transaction lock plus a unique partial index, so two workers racing on the
+same range cannot both hold an open claim.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from phlo.capabilities.interfaces import (
+    CheckpointRecord,
+    IngestionCheckpointStore,
+    SourceOffsetRange,
+)
+from phlo.logging import get_logger
+from phlo_postgres.resource import PostgresResource
+
+logger = get_logger(__name__)
+
+_SCHEMA_NAME = "phlo"
+_TABLE_NAME = "ingestion_checkpoints"
+_OPEN_STATUSES = ("claimed", "staged", "failed")
+_RESTAGABLE_STATUSES = ("claimed", "staged", "failed")
+
+_DDL_STATEMENTS = (
+    f"CREATE SCHEMA IF NOT EXISTS {_SCHEMA_NAME}",
+    f"""
+    CREATE TABLE IF NOT EXISTS {_SCHEMA_NAME}.{_TABLE_NAME} (
+        checkpoint_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+        source_id TEXT NOT NULL,
+        target_table TEXT NOT NULL,
+        status TEXT NOT NULL,
+        ranges JSONB NOT NULL DEFAULT '[]'::jsonb,
+        snapshot_id BIGINT,
+        release_id TEXT,
+        idempotency_key TEXT,
+        failure_reason TEXT,
+        created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    )
+    """,
+    f"""
+    CREATE UNIQUE INDEX IF NOT EXISTS {_TABLE_NAME}_idempotency_key_idx
+    ON {_SCHEMA_NAME}.{_TABLE_NAME} (idempotency_key)
+    WHERE idempotency_key IS NOT NULL
+    """,
+    f"""
+    CREATE INDEX IF NOT EXISTS {_TABLE_NAME}_source_status_idx
+    ON {_SCHEMA_NAME}.{_TABLE_NAME} (source_id, status)
+    """,
+)
+
+_SELECT_COLUMNS = (
+    "checkpoint_id, source_id, target_table, status, ranges, snapshot_id, "
+    "release_id, idempotency_key, failure_reason, updated_at"
+)
+
+
+def _ranges_to_json(ranges: list[SourceOffsetRange]) -> list[dict[str, Any]]:
+    return [
+        {
+            "topic": item.topic,
+            "partition": item.partition,
+            "start_offset": item.start_offset,
+            "end_offset": item.end_offset,
+        }
+        for item in ranges
+    ]
+
+
+def _ranges_from_json(raw: Any) -> tuple[SourceOffsetRange, ...]:
+    if not raw:
+        return ()
+    return tuple(
+        SourceOffsetRange(
+            topic=item["topic"],
+            partition=int(item["partition"]),
+            start_offset=int(item["start_offset"]),
+            end_offset=int(item["end_offset"]),
+        )
+        for item in raw
+    )
+
+
+def _record_from_row(row: tuple) -> CheckpointRecord:
+    (
+        checkpoint_id,
+        source_id,
+        target_table,
+        status,
+        ranges,
+        snapshot_id,
+        release_id,
+        idempotency_key,
+        failure_reason,
+        updated_at,
+    ) = row
+    return CheckpointRecord(
+        checkpoint_id=str(checkpoint_id),
+        source_id=source_id,
+        target_table=target_table,
+        status=status,
+        ranges=_ranges_from_json(ranges),
+        snapshot_id=snapshot_id,
+        release_id=release_id,
+        idempotency_key=idempotency_key,
+        failure_reason=failure_reason,
+        updated_at=updated_at,
+    )
+
+
+class PostgresIngestionCheckpointStore(IngestionCheckpointStore):
+    """PostgreSQL-backed durable checkpoint store for stream ingestion."""
+
+    def __init__(
+        self,
+        *,
+        resource: PostgresResource | None = None,
+        table_ensured: bool = False,
+    ) -> None:
+        self._resource = resource
+        self._table_ensured = table_ensured
+
+    def _ensure_resource(self) -> PostgresResource:
+        if self._resource is None:
+            self._resource = PostgresResource()
+        return self._resource
+
+    def _ensure_table(self) -> None:
+        if self._table_ensured:
+            return
+        resource = self._ensure_resource()
+        with resource.transactional_cursor() as cursor:
+            for statement in _DDL_STATEMENTS:
+                cursor.execute(statement)
+        self._table_ensured = True
+
+    def claim(
+        self,
+        *,
+        source_id: str,
+        target_table: str,
+        ranges: list[SourceOffsetRange],
+        idempotency_key: str | None = None,
+    ) -> CheckpointRecord:
+        """Record an exclusive claim on the supplied ranges.
+
+        Existing claims with the same idempotency key are returned unchanged
+        so a retry after a crash resumes the same checkpoint instead of
+        double-claiming the range.
+        """
+        self._ensure_table()
+        resource = self._ensure_resource()
+        lock_key = idempotency_key or f"{source_id}:{target_table}"
+        with resource.transactional_cursor() as cursor:
+            # The advisory lock serialises concurrent claims for the same key
+            # before the existence check, closing the check-then-insert race.
+            cursor.execute("SELECT pg_advisory_xact_lock(hashtext(%s))", (lock_key,))
+            if idempotency_key:
+                cursor.execute(
+                    f"SELECT {_SELECT_COLUMNS} FROM {_SCHEMA_NAME}.{_TABLE_NAME} "
+                    "WHERE idempotency_key = %s",
+                    (idempotency_key,),
+                )
+                row = cursor.fetchone()
+                if row:
+                    return _record_from_row(row)
+            cursor.execute(
+                f"""
+                INSERT INTO {_SCHEMA_NAME}.{_TABLE_NAME}
+                    (source_id, target_table, status, ranges, idempotency_key)
+                VALUES (%s, %s, 'claimed', %s, %s)
+                RETURNING {_SELECT_COLUMNS}
+                """,
+                (
+                    source_id,
+                    target_table,
+                    json.dumps(_ranges_to_json(ranges)),
+                    idempotency_key,
+                ),
+            )
+            return _record_from_row(cursor.fetchone())
+
+    def record_snapshot(
+        self,
+        *,
+        checkpoint_id: str,
+        snapshot_id: int | str,
+        release_id: str | None = None,
+    ) -> CheckpointRecord:
+        """Bind the claimed ranges to the output snapshot that represents them.
+
+        A ``failed`` checkpoint may re-stage once its blocking issue (for
+        example a schema migration) is resolved; the failure reason is
+        cleared on the transition.
+        """
+        self._ensure_table()
+        resource = self._ensure_resource()
+        with resource.transactional_cursor() as cursor:
+            cursor.execute(
+                f"""
+                UPDATE {_SCHEMA_NAME}.{_TABLE_NAME}
+                SET status = 'staged', snapshot_id = %s, release_id = %s,
+                    failure_reason = NULL, updated_at = NOW()
+                WHERE checkpoint_id = %s AND status IN %s
+                RETURNING {_SELECT_COLUMNS}
+                """,
+                (int(snapshot_id), release_id, checkpoint_id, _RESTAGABLE_STATUSES),
+            )
+            row = cursor.fetchone()
+        if row is None:
+            raise ValueError(
+                f"Checkpoint {checkpoint_id!r} is not open; refusing to bind a snapshot."
+            )
+        return _record_from_row(row)
+
+    def commit(self, *, checkpoint_id: str) -> CheckpointRecord:
+        """Mark a snapshot-bound checkpoint as durably committed."""
+        self._ensure_table()
+        resource = self._ensure_resource()
+        with resource.transactional_cursor() as cursor:
+            cursor.execute(
+                f"""
+                UPDATE {_SCHEMA_NAME}.{_TABLE_NAME}
+                SET status = 'committed', updated_at = NOW()
+                WHERE checkpoint_id = %s AND status = 'staged' AND snapshot_id IS NOT NULL
+                RETURNING {_SELECT_COLUMNS}
+                """,
+                (checkpoint_id,),
+            )
+            row = cursor.fetchone()
+        if row is None:
+            raise ValueError(
+                f"Checkpoint {checkpoint_id!r} has no audited snapshot; refusing to commit."
+            )
+        return _record_from_row(row)
+
+    def fail(self, *, checkpoint_id: str, reason: str) -> CheckpointRecord:
+        """Mark a checkpoint failed while retaining its claimed ranges."""
+        self._ensure_table()
+        resource = self._ensure_resource()
+        with resource.transactional_cursor() as cursor:
+            cursor.execute(
+                f"""
+                UPDATE {_SCHEMA_NAME}.{_TABLE_NAME}
+                SET status = 'failed', failure_reason = %s, updated_at = NOW()
+                WHERE checkpoint_id = %s AND status IN %s
+                RETURNING {_SELECT_COLUMNS}
+                """,
+                (reason, checkpoint_id, _OPEN_STATUSES),
+            )
+            row = cursor.fetchone()
+        if row is None:
+            raise ValueError(f"Checkpoint {checkpoint_id!r} is not open; cannot mark failed.")
+        return _record_from_row(row)
+
+    def latest_committed(self, *, source_id: str, target_table: str) -> CheckpointRecord | None:
+        """Return the newest committed checkpoint for one source/table pair."""
+        self._ensure_table()
+        resource = self._ensure_resource()
+        with resource.cursor() as cursor:
+            cursor.execute(
+                f"""
+                SELECT {_SELECT_COLUMNS} FROM {_SCHEMA_NAME}.{_TABLE_NAME}
+                WHERE source_id = %s AND target_table = %s AND status = 'committed'
+                ORDER BY updated_at DESC
+                LIMIT 1
+                """,
+                (source_id, target_table),
+            )
+            row = cursor.fetchone()
+        return _record_from_row(row) if row else None
+
+    def find_by_idempotency_key(self, *, idempotency_key: str) -> CheckpointRecord | None:
+        """Resolve an existing claim by its deterministic idempotency key."""
+        self._ensure_table()
+        resource = self._ensure_resource()
+        with resource.cursor() as cursor:
+            cursor.execute(
+                f"SELECT {_SELECT_COLUMNS} FROM {_SCHEMA_NAME}.{_TABLE_NAME} "
+                "WHERE idempotency_key = %s",
+                (idempotency_key,),
+            )
+            row = cursor.fetchone()
+        return _record_from_row(row) if row else None
+
+    def list_open(self, *, source_id: str) -> list[CheckpointRecord]:
+        """List claimed-but-uncommitted checkpoints needing reconciliation."""
+        self._ensure_table()
+        resource = self._ensure_resource()
+        with resource.cursor() as cursor:
+            cursor.execute(
+                f"""
+                SELECT {_SELECT_COLUMNS} FROM {_SCHEMA_NAME}.{_TABLE_NAME}
+                WHERE source_id = %s AND status IN %s
+                ORDER BY updated_at ASC
+                """,
+                (source_id, _OPEN_STATUSES),
+            )
+            rows = cursor.fetchall()
+        return [_record_from_row(row) for row in rows]

{phlo_postgres-0.14.0 → phlo_postgres-0.15.2}/src/phlo_postgres/cli.py

@@ -43,26 +43,10 @@ from phlo_postgres.settings import get_settings
 
 
 def _read_sql(*, query: str | None, file: Path | None) -> str:
-    """Read SQL from inline query string or file path.
-
-    Validates that exactly one of query or file is provided and returns the
-    SQL content. Handles empty file detection and encoding issues.
-
-    Args:
-        query: Inline SQL query string.
-        file: Path to SQL file to read.
-
-    Returns:
-        str: The SQL content to execute.
-
-    Raises:
-        click.ClickException: If both query and file are provided, neither is
-            provided, or the file is empty/cannot be read.
-
-    Example:
-        >>> sql = _read_sql(query="SELECT 1")
-        >>> sql = _read_sql(file=Path("query.sql"))
+    """Read SQL from an inline query string or file path.
 
+    Exactly one of ``query`` and ``file`` must be provided. Raise ClickException
+    when both or neither are given, or the file is empty or unreadable.
     """
     if query and file:
         raise exclusive_options_error("an inline query", "--file")
@@ -85,22 +69,14 @@ def _require_container_backend() -> None:
 
 
 def _postgres_exec_base(*, tty: bool) -> list[str]:
-    """Build the docker compose exec base command for PostgreSQL container.
-
-    Constructs the initial portion of the docker compose exec command including
-    project name and service name. Used as a base for all container operations.
+    """Build the docker compose exec base command for the PostgreSQL container.
 
-    Args:
-        tty: Whether to allocate a TTY (-t flag). Disable for non-interactive
-            commands that capture output.
-
-    Returns:
-        list[str]: Base command as a list of strings ready for subprocess.
+    Includes project and service names; ``tty=False`` drops the ``-t`` flag for
+    commands that capture output.
 
     Example:
         >>> cmd = _postgres_exec_base(tty=True)
         >>> # Returns: ['docker', 'compose', '-p', 'phlo', '-f', '...', 'exec', '-t', 'postgres']
-
     """
     phlo_dir = ensure_compose_project()
     project_name = get_project_name()
@@ -113,25 +89,13 @@ def _postgres_exec_base(*, tty: bool) -> list[str]:
 
 
 def _postgres_identity(*, user: str | None, database: str | None) -> tuple[str, str]:
-    """Resolve PostgreSQL connection identity (user and database).
-
-    Returns explicit values if provided, otherwise falls back to settings
-    defaults. This allows CLI commands to use configured defaults while
-    permitting overrides.
+    """Resolve PostgreSQL user and database, falling back to settings defaults.
 
-    Args:
-        user: Database username override, or None to use settings default.
-        database: Database name override, or None to use settings default.
-
-    Returns:
-        tuple[str, str]: Tuple of (resolved_user, resolved_database).
+    Returns a ``(resolved_user, resolved_database)`` tuple.
 
     Example:
-        >>> user, db = _postgres_identity(user=None, database=None)
-        >>> # Uses settings.postgres_user and settings.postgres_db
         >>> user, db = _postgres_identity(user="admin", database=None)
         >>> # Uses "admin" for user, settings default for database
-
     """
     settings = get_settings()
     return user or settings.postgres_user, database or settings.postgres_db
@@ -146,21 +110,15 @@ def _postgres_identity(*, user: str | None, database: str | None) -> tuple[str,
 def postgres_group(ctx: click.Context, postgres_args: tuple[str, ...]) -> None:
     """Run psql or PostgreSQL helper commands against the project database.
 
-    This is the main entry point for PostgreSQL CLI operations. It supports:
-    - Interactive psql sessions (default if no subcommand)
-    - Subcommands: query, dump, restore, vacuum
-    - Direct psql arguments passthrough
-
-    Args:
-        ctx: Click context object.
-        postgres_args: Additional arguments passed to psql or subcommands.
+    With no subcommand, opens an interactive psql session; subcommands cover
+    query, dump, restore, and vacuum, and remaining arguments pass through to
+    psql.
 
     Example:
         $ phlo postgres                    # Interactive psql
         $ phlo postgres -c "SELECT 1"       # One-off query via psql
         $ phlo postgres query "SELECT * FROM users"
         $ phlo postgres dump --file backup.sql.gz
-
     """
     if postgres_args and postgres_args[0] == "query":
         postgres_query.main(
@@ -221,23 +179,12 @@ def postgres_query(
 ) -> None:
     """Execute a SQL query against the running PostgreSQL service.
 
-    Executes a SQL query inside the PostgreSQL container and prints results
-    to stdout. Supports inline queries or reading from a file.
-
-    Args:
-        query: SQL query string to execute.
-        query_file: Path to file containing SQL query.
-        user: Database user (default from settings).
-        database: Database name (default from settings).
-        timeout_seconds: Maximum time to wait for query completion.
-
-    Raises:
-        click.ClickException: If the query fails or times out.
+    Accepts inline SQL or a query file and prints results to stdout. Raise
+    ClickException when the query fails or times out.
 
     Example:
         $ phlo postgres query "SELECT * FROM users"
         $ phlo postgres query --file query.sql --timeout 60
-
     """
     _require_container_backend()
     enforce_surface_mutation_authorization("postgres.query", get_postgres_cli_adapter)
@@ -287,22 +234,12 @@ def postgres_dump(
 ) -> None:
     """Create a PostgreSQL logical backup (pg_dump) to a local file.
 
-    Dumps the entire database using pg_dump, with optional gzip compression
-    if the output file has a .gz extension.
-
-    Args:
-        output_file: Path to write the dump. Use .gz extension for compression.
-        user: Database user (default from settings).
-        database: Database name (default from settings).
-        timeout_seconds: Maximum time to wait for dump completion.
-
-    Raises:
-        click.ClickException: If the dump fails or times out.
+    Dumps the entire database, gzip-compressing when the output file ends in
+    ``.gz``. Raise ClickException when the dump fails or times out.
 
     Example:
         $ phlo postgres dump --file backup.sql
         $ phlo postgres dump --file backup.sql.gz --timeout 300
-
     """
     _require_container_backend()
     enforce_surface_mutation_authorization("postgres.dump", get_postgres_cli_adapter)
@@ -360,25 +297,16 @@ def postgres_restore(
 ) -> None:
     """Restore a PostgreSQL database from a logical backup file.
 
-    Restores the database from a SQL dump file (plain or gzip-compressed).
-    Uses psql internally to execute the dump SQL.
+    Uses psql to execute a plain or gzip-compressed SQL dump.
 
     Warning:
         This may overwrite existing data. Use with caution on production databases.
 
-    Args:
-        input_file: Path to the dump file (.sql or .sql.gz).
-        user: Database user (default from settings).
-        database: Database name (default from settings).
-        timeout_seconds: Maximum time to wait for restore completion.
-
-    Raises:
-        click.ClickException: If the restore fails or times out.
+    Raise ClickException when the restore fails or times out.
 
     Example:
         $ phlo postgres restore --file backup.sql
         $ phlo postgres restore --file backup.sql.gz --db mydb --timeout 600
-
     """
     _require_container_backend()
     enforce_surface_mutation_authorization("postgres.restore", get_postgres_cli_adapter)
@@ -427,23 +355,13 @@ def postgres_vacuum(
 ) -> None:
     """Run vacuumdb for PostgreSQL maintenance inside the container.
 
-    Executes vacuumdb to reclaim storage and optionally update statistics.
-    This is useful for routine database maintenance after large operations.
-
-    Args:
-        user: Database user (default from settings).
-        database: Database name (default from settings).
-        analyze: Whether to run ANALYZE after vacuum (updates statistics).
-        timeout_seconds: Maximum time to wait for vacuum completion.
-
-    Raises:
-        click.ClickException: If vacuum fails or times out.
+    Reclaims storage and optionally ANALYZEs afterwards to refresh statistics.
+    Raise ClickException when vacuum fails or times out.
 
     Example:
         $ phlo postgres vacuum
         $ phlo postgres vacuum --no-analyze
         $ phlo postgres vacuum --db analytics --timeout 300
-
     """
     _require_container_backend()
     enforce_surface_mutation_authorization("postgres.vacuum", get_postgres_cli_adapter)

phlo_postgres-0.15.2/src/phlo_postgres/cli_plugin.py

@@ -0,0 +1,24 @@
+"""Postgres CLI plugin registration.
+
+Exposes the postgres command group as a cli-command plugin so the
+phlo-postgres package contributes its commands through plugin discovery
+rather than a core-CLI import.
+Loaded through the phlo plugin entry-point mechanism at startup rather than
+imported directly; exposes the command group from phlo_postgres.cli.
+"""
+
+from __future__ import annotations
+
+
+from phlo.plugins.base import cli_command_plugin_class
+
+from phlo_postgres.cli import postgres_group
+
+
+PostgresCliPlugin = cli_command_plugin_class(
+    "PostgresCliPlugin",
+    name="postgres",
+    version="0.1.0",
+    description="CLI commands for PostgreSQL service access",
+    commands=[postgres_group],
+)