interloper-db 0.2.0__py3-none-any.whl

interloper_db/__init__.py

@@ -0,0 +1,57 @@
+from interloper_db.engine import get_engine, init_engine
+from interloper_db.models import (
+    Asset,
+    AssetDependency,
+    AssetDestination,
+    AssetResource,
+    Backfill,
+    Destination,
+    DestinationResource,
+    Event,
+    Invitation,
+    Job,
+    JobAsset,
+    JobSource,
+    Organisation,
+    Profile,
+    Resource,
+    Run,
+    Session,
+    Source,
+    SourceDestination,
+    SourceResource,
+    UserOrganisation,
+)
+from interloper_db.provision import create_all, downgrade, ensure_database, upgrade
+from interloper_db.store import Store
+
+__all__ = [
+    "Asset",
+    "AssetDependency",
+    "AssetDestination",
+    "AssetResource",
+    "Backfill",
+    "Destination",
+    "DestinationResource",
+    "Event",
+    "Invitation",
+    "Job",
+    "JobAsset",
+    "JobSource",
+    "Organisation",
+    "Profile",
+    "Resource",
+    "Run",
+    "Session",
+    "Source",
+    "SourceDestination",
+    "SourceResource",
+    "Store",
+    "UserOrganisation",
+    "create_all",
+    "downgrade",
+    "ensure_database",
+    "upgrade",
+    "get_engine",
+    "init_engine",
+]

interloper_db/engine.py

@@ -0,0 +1,46 @@
+"""Database engine singleton."""
+
+from __future__ import annotations
+
+import os
+
+from sqlalchemy import Engine, create_engine
+
+_engine: Engine | None = None
+
+
+def init_engine(dsn: str | None = None, **kwargs: object) -> Engine:
+    """Initialize the global database engine.
+
+    Args:
+        dsn: PostgreSQL connection string. Falls back to ``DATABASE_URL`` env var.
+        **kwargs: Additional kwargs forwarded to ``create_engine``.
+
+    Returns:
+        The SQLAlchemy engine.
+
+    Raises:
+        ValueError: If no DSN is provided and ``DATABASE_URL`` is not set.
+    """
+    global _engine  # noqa: PLW0603
+    dsn = dsn or os.getenv("DATABASE_URL")
+    if not dsn:
+        from interloper.errors import ConfigError
+
+        raise ConfigError("Database DSN required: pass dsn= or set DATABASE_URL")
+    _engine = create_engine(dsn, **kwargs)
+    return _engine
+
+
+def get_engine() -> Engine:
+    """Return the global database engine.
+
+    Returns:
+        The SQLAlchemy engine.
+
+    Raises:
+        RuntimeError: If ``init_engine`` has not been called.
+    """
+    if _engine is None:
+        raise RuntimeError("Database engine not initialized. Call init_engine() first.")
+    return _engine

interloper_db/hydration.py

@@ -0,0 +1,335 @@
+"""Hydration: translates DB rows into ``ComponentSpec`` trees.
+
+This module is a pure transformation layer.  It reads rows from the
+database and builds the ``ComponentSpec`` tree that ``Component.from_spec``
+expects.  No framework classes are instantiated here — reconstruction
+happens at the call site via ``spec.reconstruct()``::
+
+    hydrator = Hydrator(catalog, decrypt=decrypt_fn)
+    with Session(engine) as session:
+        db_source = session.get(Source, source_id, options=[...])
+        spec = hydrator.build_source_spec(session, db_source)
+    source = spec.reconstruct()
+
+The Store wraps this pattern in thin ``load_*`` convenience methods, but
+any caller can use the hydrator directly to assemble a spec (for example,
+to serialize it to JSON and send it across a process boundary).
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Iterable
+from typing import Any
+from uuid import UUID
+
+from interloper.catalog.base import Catalog
+from interloper.component.base import ComponentSpec
+from interloper.errors import CatalogKeyError
+from sqlmodel import Session, select
+
+from interloper_db.models import (
+    Asset,
+    AssetDependency,
+    AssetResource,
+    Destination,
+    DestinationResource,
+    Resource,
+    Source,
+    SourceResource,
+)
+
+
+class Hydrator:
+    """Builds ``ComponentSpec`` trees from DB rows.
+
+    The hydrator holds a catalog (for import-path lookups) and an optional
+    decrypt callable (for resource data).  All methods are pure
+    transformations — they read rows and return specs without ever
+    instantiating framework classes.  Reconstruction is the caller's job.
+    """
+
+    def __init__(
+        self,
+        catalog: Catalog,
+        decrypt: Callable[[bytes], bytes] | None = None,
+    ) -> None:
+        """Initialize the hydrator.
+
+        Args:
+            catalog: Catalog used to resolve ``key → import path``.
+            decrypt: Optional ``(bytes) -> bytes`` callable for decrypting
+                resource data blobs marked ``encrypted=True``.
+        """
+        self._catalog = catalog
+        self._decrypt = decrypt
+
+    # ------------------------------------------------------------------
+    # Resource
+    # ------------------------------------------------------------------
+
+    def build_resource_spec(self, db_resource: Resource) -> ComponentSpec:
+        """Build a spec from a Resource row.
+
+        Resources are leaves — they carry their full state in ``data``
+        and have no nested specs.  No session is needed.
+
+        Args:
+            db_resource: The Resource row.
+
+        Returns:
+            A ``ComponentSpec`` with the row's ``id`` and decoded data.
+        """
+        path = self._resolve_path(db_resource.key, kind="resource")
+        init = self.decode_resource_data(db_resource)
+        return ComponentSpec(
+            path=path,
+            id=str(db_resource.id) if db_resource.id else "",
+            init=init or None,
+        )
+
+    def decode_resource_data(self, db_resource: Resource) -> dict[str, Any]:
+        """Decrypt (when needed) and JSON-decode a resource's data blob.
+
+        Args:
+            db_resource: The Resource row whose ``data`` bytes should be
+                decoded.
+
+        Returns:
+            The decoded configuration dict, or ``{}`` if the row carries
+            no data.
+        """
+        if db_resource.data is None:
+            return {}
+        raw = db_resource.data
+        if db_resource.encrypted and self._decrypt:
+            raw = self._decrypt(raw)
+        return json.loads(raw)
+
+    # ------------------------------------------------------------------
+    # Destination
+    # ------------------------------------------------------------------
+
+    def build_destination_spec(
+        self,
+        session: Session,
+        db_destination: Destination,
+    ) -> ComponentSpec:
+        """Build a spec from a Destination row.
+
+        Captures the destination's ``config`` dict and resolves its
+        resource bindings as nested resource specs under ``init.resources``.
+
+        Args:
+            session: Active DB session.
+            db_destination: The Destination row.
+
+        Returns:
+            A ``ComponentSpec`` with the row's ``id`` and resolved bindings.
+        """
+        path = self._resolve_path(db_destination.key, kind="destination")
+
+        init: dict[str, Any] = dict(db_destination.config) if db_destination.config else {}
+
+        bindings = session.exec(
+            select(DestinationResource).where(DestinationResource.destination_id == db_destination.id)
+        ).all()
+        resources = self._resource_specs_from_bindings(session, bindings)
+        if resources:
+            init["resources"] = resources
+
+        return ComponentSpec(
+            path=path,
+            id=str(db_destination.id) if db_destination.id else "",
+            init=init or None,
+        )
+
+    # ------------------------------------------------------------------
+    # Asset
+    # ------------------------------------------------------------------
+
+    def build_asset_spec(
+        self,
+        session: Session,
+        db_asset: Asset,
+    ) -> ComponentSpec:
+        """Build a spec from a standalone Asset row.
+
+        Source-owned assets should not be built through this method —
+        they're embedded in the parent source's spec via
+        :meth:`build_source_spec`, which handles the
+        ``"source_path:asset_key"`` path convention.
+
+        Args:
+            session: Active DB session.
+            db_asset: A standalone Asset row (``source_id`` is ``None``).
+
+        Returns:
+            A ``ComponentSpec`` with the row's ``id`` and all per-asset
+            state (materializable, config, resources, destinations, deps).
+        """
+        path = self._resolve_path(db_asset.key, kind="asset")
+        init = self._build_asset_init(session, db_asset)
+        return ComponentSpec(
+            path=path,
+            id=str(db_asset.id) if db_asset.id else "",
+            init=init or None,
+        )
+
+    def _build_asset_init(
+        self,
+        session: Session,
+        db_asset: Asset,
+    ) -> dict[str, Any]:
+        """Build the ``init`` dict for an asset spec.
+
+        Captures materializable, per-field config overrides, resource
+        bindings, destination bindings, and cross-asset dependencies.
+
+        Returns:
+            A dict suitable for use as a ``ComponentSpec.init``.
+        """
+        init: dict[str, Any] = {"materializable": db_asset.materializable}
+
+        if db_asset.config:
+            init.update(db_asset.config)
+
+        if db_asset.id:
+            resource_bindings = session.exec(
+                select(AssetResource).where(AssetResource.asset_id == db_asset.id)
+            ).all()
+            resources = self._resource_specs_from_bindings(session, resource_bindings)
+            if resources:
+                init["resources"] = resources
+
+            destination_specs = [
+                self.build_destination_spec(session, db_dest).model_dump(mode="json")
+                for db_dest in db_asset.destinations
+            ]
+            if destination_specs:
+                init["destination"] = (
+                    destination_specs if len(destination_specs) > 1 else destination_specs[0]
+                )
+
+            deps = self._deps_for_asset(session, db_asset.id)
+            if deps:
+                init["deps"] = deps
+
+        return init
+
+    def _deps_for_asset(
+        self,
+        session: Session,
+        asset_id: UUID,
+    ) -> dict[str, str]:
+        """Return ``{param_name: upstream_uuid}`` from ``AssetDependency`` rows."""
+        dependency_rows = session.exec(
+            select(AssetDependency).where(AssetDependency.asset_id == asset_id)
+        ).all()
+        return {d.param_name: str(d.upstream_asset_id) for d in dependency_rows}
+
+    # ------------------------------------------------------------------
+    # Source
+    # ------------------------------------------------------------------
+
+    def build_source_spec(
+        self,
+        session: Session,
+        db_source: Source,
+    ) -> ComponentSpec:
+        """Build a spec from a Source row including per-asset overrides.
+
+        The returned spec is self-contained: it carries the source's own
+        config, resource bindings, destination bindings, plus an
+        ``assets`` override map keyed by asset key.  Each entry is the
+        sparse init payload for one asset (materializable, config,
+        resources, destinations, deps).  Reconstruction via
+        ``ComponentSpec.reconstruct()`` produces a live ``Source`` whose
+        ``_apply_asset_overrides`` validator materialises each asset from
+        ``asset_types`` with the overrides applied.
+
+        Args:
+            session: Active DB session.
+            db_source: The Source row (relationships should be eagerly
+                loaded via ``selectinload``).
+
+        Returns:
+            A ``ComponentSpec`` capturing the source and its assets.
+        """
+        path = self._resolve_path(db_source.key, kind="source")
+
+        init: dict[str, Any] = {}
+
+        if db_source.config:
+            init.update(db_source.config)
+
+        resource_bindings = session.exec(
+            select(SourceResource).where(SourceResource.source_id == db_source.id)
+        ).all()
+        resources = self._resource_specs_from_bindings(session, resource_bindings)
+        if resources:
+            init["resources"] = resources
+
+        destination_specs = [
+            self.build_destination_spec(session, db_dest).model_dump(mode="json")
+            for db_dest in db_source.destinations
+        ]
+        if destination_specs:
+            init["destination"] = (
+                destination_specs if len(destination_specs) > 1 else destination_specs[0]
+            )
+
+        asset_overrides: dict[str, dict[str, Any]] = {
+            db_asset.key: {"id": str(db_asset.id), **self._build_asset_init(session, db_asset)}
+            for db_asset in db_source.assets
+        }
+        if asset_overrides:
+            init["assets"] = asset_overrides
+
+        return ComponentSpec(
+            path=path,
+            id=str(db_source.id) if db_source.id else "",
+            init=init or None,
+        )
+
+    # ------------------------------------------------------------------
+    # Shared helpers
+    # ------------------------------------------------------------------
+
+    def _resolve_path(self, key: str, *, kind: str) -> str:
+        """Look up a component's import path via the catalog.
+
+        Args:
+            key: The catalog key.
+            kind: Component kind, used only in the error message.
+
+        Returns:
+            The resolved import path.
+
+        Raises:
+            CatalogKeyError: If the catalog has no entry for *key*.
+        """
+        definition = self._catalog.get(key)
+        if not definition:
+            raise CatalogKeyError(f"Unknown {kind} key: {key}")
+        return definition.path
+
+    def _resource_specs_from_bindings(
+        self,
+        session: Session,
+        bindings: Iterable[Any],
+    ) -> dict[str, dict[str, Any]]:
+        """Build a ``{slot: resource_spec_dict}`` map from binding rows.
+
+        Each binding row must expose ``resource_id`` and ``key`` attributes.
+
+        Returns:
+            A dict mapping slot name → resource spec (as a JSON-safe dict).
+        """
+        result: dict[str, dict[str, Any]] = {}
+        for binding in bindings:
+            db_resource = session.get(Resource, binding.resource_id)
+            if db_resource:
+                spec = self.build_resource_spec(db_resource)
+                result[binding.key] = spec.model_dump(mode="json")
+        return result

interloper_db/migrations/env.py

@@ -0,0 +1,34 @@
+"""Alembic environment configuration.
+
+Reads the database URL from the engine singleton initialised by
+``init_engine()``, so migrations share the exact same connection as the
+rest of the application.
+"""
+
+from __future__ import annotations
+
+from alembic import context
+from sqlmodel import SQLModel
+
+import interloper_db.models as _models  # noqa: F401 — register all models
+
+target_metadata = SQLModel.metadata
+
+
+def run_migrations_online() -> None:
+    """Run migrations against a live database."""
+    from interloper_db.engine import get_engine
+
+    connectable = get_engine()
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=True,
+        )
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+run_migrations_online()

interloper_db/migrations/script.py.mako

@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+
+from __future__ import annotations
+
+from alembic import op
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: str | None = ${repr(down_revision)}
+branch_labels: str | None = None
+depends_on: str | None = None
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

interloper_db/migrations/versions/001_pgcrypto.py

@@ -0,0 +1,23 @@
+"""Enable the pgcrypto extension.
+
+Revision ID: 001
+Revises: None
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "001"
+down_revision: str | None = None
+branch_labels: str | None = None
+depends_on: str | None = None
+
+
+def upgrade() -> None:
+    op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
+
+
+def downgrade() -> None:
+    op.execute("DROP EXTENSION IF EXISTS pgcrypto")

interloper_db/migrations/versions/002_asset_executions.py

@@ -0,0 +1,79 @@
+"""Create asset_executions view.
+
+Revision ID: 002
+Revises: 001
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "002"
+down_revision: str | None = "001"
+branch_labels: str | None = None
+depends_on: str | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+CREATE OR REPLACE VIEW asset_executions AS
+WITH ranked AS (
+    SELECT
+        e.run_id,
+        e.org_id,
+        e.asset_id,
+        e.asset_key,
+        e.event_type,
+        e.timestamp,
+        row_number() OVER (
+            PARTITION BY e.run_id, e.asset_id
+            ORDER BY
+                CASE e.event_type
+                    WHEN 'asset_failed' THEN 1
+                    WHEN 'asset_canceled' THEN 2
+                    WHEN 'asset_completed' THEN 3
+                    WHEN 'asset_started' THEN 4
+                    WHEN 'asset_skipped' THEN 5
+                    WHEN 'asset_queued' THEN 6
+                END,
+                e.timestamp DESC
+        ) AS rn,
+        min(CASE WHEN e.event_type = 'asset_queued' THEN e.timestamp END) OVER (
+            PARTITION BY e.run_id, e.asset_id
+        ) AS queued_at,
+        min(CASE WHEN e.event_type = 'asset_started' THEN e.timestamp END) OVER (
+            PARTITION BY e.run_id, e.asset_id
+        ) AS started_at,
+        max(CASE WHEN e.event_type IN ('asset_completed', 'asset_failed', 'asset_canceled')
+            THEN e.timestamp END) OVER (
+            PARTITION BY e.run_id, e.asset_id
+        ) AS completed_at
+    FROM events e
+    WHERE e.asset_id IS NOT NULL
+)
+SELECT
+    r.run_id,
+    r.org_id,
+    r.asset_id,
+    r.asset_key,
+    CASE r.event_type
+        WHEN 'asset_failed' THEN 'failed'
+        WHEN 'asset_canceled' THEN 'canceled'
+        WHEN 'asset_completed' THEN 'success'
+        WHEN 'asset_started' THEN 'running'
+        WHEN 'asset_skipped' THEN 'skipped'
+        WHEN 'asset_queued' THEN 'queued'
+    END AS status,
+    r.started_at,
+    r.completed_at,
+    r.queued_at AS created_at
+FROM ranked r
+WHERE r.rn = 1
+"""
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP VIEW IF EXISTS asset_executions CASCADE")

interloper_db/migrations/versions/003_realtime_notify.py

@@ -0,0 +1,60 @@
+"""Add notify_table_change() function and triggers for realtime.
+
+Revision ID: 003
+Revises: 002
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "003"
+down_revision: str | None = "002"
+branch_labels: str | None = None
+depends_on: str | None = None
+
+_TRIGGERS = [
+    ("trg_runs_notify", "runs", "INSERT OR UPDATE"),
+    ("trg_events_notify", "events", "INSERT"),
+    ("trg_backfills_notify", "backfills", "INSERT OR UPDATE"),
+]
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        CREATE OR REPLACE FUNCTION notify_table_change()
+        RETURNS trigger
+        LANGUAGE plpgsql
+        AS $$
+        DECLARE
+            payload jsonb;
+            rec RECORD;
+        BEGIN
+            rec := COALESCE(NEW, OLD);
+            payload := jsonb_build_object(
+                'table', TG_TABLE_NAME,
+                'op', TG_OP,
+                'org_id', rec.org_id,
+                'record', row_to_json(rec)::jsonb
+            );
+            PERFORM pg_notify('table_changes', payload::text);
+            RETURN rec;
+        END;
+        $$
+        """
+    )
+
+    for trigger_name, table, events in _TRIGGERS:
+        op.execute(
+            f"CREATE OR REPLACE TRIGGER {trigger_name} "
+            f"AFTER {events} ON {table} "
+            f"FOR EACH ROW EXECUTE FUNCTION notify_table_change()"
+        )
+
+
+def downgrade() -> None:
+    for trigger_name, table, _events in _TRIGGERS:
+        op.execute(f"DROP TRIGGER IF EXISTS {trigger_name} ON {table}")
+    op.execute("DROP FUNCTION IF EXISTS notify_table_change()")

interloper_db/migrations/versions/004_event_traceback.py

@@ -0,0 +1,28 @@
+"""Add traceback column to events table.
+
+Stores the formatted Python traceback alongside the error message so
+that failure diagnostics are available in the UI without parsing logs.
+
+Revision ID: 004
+Revises: 003
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "004"
+down_revision: str | None = "003"
+branch_labels: str | None = None
+depends_on: str | None = None
+
+
+def upgrade() -> None:
+    """Add the traceback column to the events table."""
+    op.execute("ALTER TABLE events ADD COLUMN IF NOT EXISTS traceback TEXT;")
+
+
+def downgrade() -> None:
+    """Remove the traceback column from the events table."""
+    op.execute("ALTER TABLE events DROP COLUMN IF EXISTS traceback;")