interloper-api 0.2.0__py3-none-any.whl

interloper_api/routes/sources.py

@@ -0,0 +1,187 @@
+"""Sources API: CRUD for source instances."""
+
+from __future__ import annotations
+
+from typing import Any
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, HTTPException
+from interloper.errors import NotFoundError
+from interloper_db import Profile, Store
+from interloper_db.models import Destination, DestinationResource, Source, SourceResource
+from pydantic import BaseModel
+from sqlalchemy.orm import selectinload
+from sqlmodel import Session, select
+
+from interloper_api.dependencies import get_org_id, get_store, require_editor, require_viewer
+
+router = APIRouter()
+
+
+class SourceCreateRequest(BaseModel):
+    """Request body for creating or updating a source."""
+
+    key: str
+    name: str
+    config: dict[str, Any] | None = None
+    resources: dict[str, str] | None = None
+    asset_keys: list[str] | None = None
+    destination_ids: list[str] | None = None
+    cross_deps: dict[str, dict[str, str]] | None = None
+
+
+class AssetResponse(BaseModel):
+    """Nested asset in source response."""
+
+    id: UUID
+    key: str
+    materializable: bool
+
+
+class DestinationResponse(BaseModel):
+    """Nested destination in source response."""
+
+    id: UUID
+    key: str
+    name: str | None = None
+    config: dict[str, Any] | None = None
+    resources: dict[str, str] = {}
+    created_at: str | None = None
+
+
+class SourceResponse(BaseModel):
+    """Response body for a source."""
+
+    id: UUID
+    org_id: UUID
+    key: str
+    name: str
+    config: dict[str, Any] | None = None
+    resources: dict[str, str] = {}
+    destinations: list[DestinationResponse] = []
+    assets: list[AssetResponse] = []
+    created_at: str | None = None
+
+
+def _resource_map(session: Session, junction_cls: type, fk_column: str, fk_value: UUID) -> dict[str, str]:
+    """Build a {slot_key: resource_id} map from junction rows."""
+    col = getattr(junction_cls, fk_column)
+    rows = session.exec(select(junction_cls).where(col == fk_value)).all()
+    return {r.key: str(r.resource_id) for r in rows}
+
+
+def _build_source_response(session: Session, source: Source) -> SourceResponse:
+    """Convert a DB Source to a SourceResponse within a session."""
+    return SourceResponse(
+        id=source.id,  # type: ignore[arg-type]
+        org_id=source.org_id,
+        key=source.key,
+        name=source.name,
+        config=source.config,
+        resources=_resource_map(session, SourceResource, "source_id", source.id),  # type: ignore[arg-type]
+        destinations=[
+            DestinationResponse(
+                id=d.id,  # type: ignore[arg-type]
+                key=d.key,
+                name=d.name,
+                config=d.config,
+                resources=_resource_map(session, DestinationResource, "destination_id", d.id),  # type: ignore[arg-type]
+                created_at=str(d.created_at) if d.created_at else None,
+            )
+            for d in source.destinations
+        ],
+        assets=[
+            AssetResponse(id=a.id, key=a.key, materializable=a.materializable)  # type: ignore[arg-type]
+            for a in source.assets
+        ],
+        created_at=str(source.created_at) if source.created_at else None,
+    )
+
+
+def _load_source_for_response(source_id: UUID) -> SourceResponse:
+    """Load a source with relations and build the response."""
+    from interloper_db.engine import get_engine
+
+    with Session(get_engine()) as session:
+        source = session.get(
+            Source,
+            source_id,
+            options=[
+                selectinload(Source.assets),  # type: ignore[arg-type]
+                selectinload(Source.resources),  # type: ignore[arg-type]
+                selectinload(Source.destinations).selectinload(Destination.resources),  # type: ignore[arg-type]
+            ],
+        )
+        if not source:
+            raise NotFoundError(f"Source {source_id} not found")
+        return _build_source_response(session, source)
+
+
+@router.get("/")
+def list_sources(
+    user: Profile = Depends(require_viewer),
+    org_id: UUID = Depends(get_org_id),
+    store: Store = Depends(get_store),
+) -> list[SourceResponse]:
+    """List all sources for the current organisation."""
+    from interloper_db.engine import get_engine
+
+    sources = store.list_sources(org_id)
+    with Session(get_engine()) as session:
+        return [_build_source_response(session, s) for s in sources]
+
+
+@router.post("/")
+def create_source(
+    body: SourceCreateRequest,
+    user: Profile = Depends(require_editor),
+    org_id: UUID = Depends(get_org_id),
+    store: Store = Depends(get_store),
+) -> SourceResponse:
+    """Create a new source."""
+    source = store.create_source(
+        org_id,
+        key=body.key,
+        name=body.name,
+        config=body.config,
+        resources=body.resources,
+        asset_keys=body.asset_keys,
+        destination_ids=body.destination_ids,
+        cross_deps=body.cross_deps,
+    )
+    return _load_source_for_response(source.id)  # type: ignore[arg-type]
+
+
+@router.put("/{source_id}")
+def update_source(
+    source_id: UUID,
+    body: SourceCreateRequest,
+    user: Profile = Depends(require_editor),
+    org_id: UUID = Depends(get_org_id),
+    store: Store = Depends(get_store),
+) -> SourceResponse:
+    """Update a source."""
+    try:
+        store.update_source(
+            source_id,
+            name=body.name,
+            config=body.config,
+            resources=body.resources,
+            asset_keys=body.asset_keys,
+            destination_ids=body.destination_ids,
+            cross_deps=body.cross_deps,
+        )
+    except NotFoundError:
+        raise HTTPException(status_code=404, detail=f"Source {source_id} not found")
+    return _load_source_for_response(source_id)
+
+
+@router.delete("/{source_id}")
+def delete_source(
+    source_id: UUID,
+    user: Profile = Depends(require_editor),
+    store: Store = Depends(get_store),
+) -> dict[str, str]:
+    """Delete a source. Assets cascade via FK."""
+    store.delete_source(source_id)
+    return {"status": "deleted"}

interloper_api/routes/ws.py

@@ -0,0 +1,164 @@
+"""WebSocket realtime endpoint — bridges PostgreSQL NOTIFY to connected clients.
+
+Architecture:
+- A background thread LISTENs on the ``table_changes`` PostgreSQL channel.
+- When a notification arrives, the payload (table, op, org_id, record) is
+  broadcast to all WebSocket clients belonging to that org.
+- Clients subscribe by connecting to ``/api/ws`` with a valid session cookie.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import select as _select
+import threading
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+
+import psycopg2
+import psycopg2.extensions
+from fastapi import APIRouter, Cookie, FastAPI, WebSocket, WebSocketDisconnect
+
+from interloper_api.dependencies import get_store
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+class ConnectionManager:
+    """Manages WebSocket connections grouped by organisation ID."""
+
+    def __init__(self) -> None:
+        self._connections: dict[str, set[WebSocket]] = {}
+        self._lock = asyncio.Lock()
+
+    async def connect(self, ws: WebSocket, org_id: str) -> None:
+        """Register a WebSocket for an org."""
+        async with self._lock:
+            if org_id not in self._connections:
+                self._connections[org_id] = set()
+            self._connections[org_id].add(ws)
+
+    async def disconnect(self, ws: WebSocket, org_id: str) -> None:
+        """Unregister a WebSocket."""
+        async with self._lock:
+            if org_id in self._connections:
+                self._connections[org_id].discard(ws)
+                if not self._connections[org_id]:
+                    del self._connections[org_id]
+
+    async def broadcast(self, org_id: str, message: dict[str, object]) -> None:
+        """Send a message to all connections for an org."""
+        async with self._lock:
+            connections = list(self._connections.get(org_id, []))
+        for ws in connections:
+            try:
+                await ws.send_json(message)
+            except Exception:
+                pass
+
+
+_manager = ConnectionManager()
+
+
+def _start_notify_listener(dsn: str, loop: asyncio.AbstractEventLoop) -> None:
+    """Background thread: listens for PostgreSQL NOTIFY and dispatches to WebSocket clients.
+
+    Args:
+        dsn: PostgreSQL connection string.
+        loop: The asyncio event loop to schedule coroutines on.
+    """
+    while True:
+        try:
+            conn = psycopg2.connect(dsn)
+            conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+
+            with conn.cursor() as cur:
+                cur.execute("LISTEN table_changes")
+
+            logger.info("[Realtime] NOTIFY listener started")
+
+            while True:
+                if _select.select([conn], [], [], 1.0):
+                    conn.poll()
+                    while conn.notifies:
+                        notify = conn.notifies.pop(0)
+                        try:
+                            payload = json.loads(notify.payload)
+                            org_id = str(payload["org_id"])
+                            message = {
+                                "table": payload["table"],
+                                "event": payload["op"],
+                                "record": payload.get("record"),
+                            }
+                            asyncio.run_coroutine_threadsafe(
+                                _manager.broadcast(org_id, message), loop
+                            )
+                        except (json.JSONDecodeError, KeyError) as e:
+                            logger.warning("[Realtime] Invalid NOTIFY payload: %s", e)
+        except Exception as e:
+            logger.error("[Realtime] NOTIFY listener error: %s, reconnecting in 5s...", e)
+            import time
+            time.sleep(5)
+
+
+@asynccontextmanager
+async def realtime_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
+    """Start the NOTIFY listener thread on app startup.
+
+    Reads the DSN from the engine already initialized by ``init_engine()``.
+    """
+    from interloper_db import get_engine
+
+    engine = get_engine()
+    # str(engine.url) masks the password as '***' — psycopg2 needs the real one.
+    dsn = engine.url.render_as_string(hide_password=False)
+
+    loop = asyncio.get_running_loop()
+    thread = threading.Thread(
+        target=_start_notify_listener,
+        args=(dsn, loop),
+        daemon=True,
+        name="NotifyListener",
+    )
+    thread.start()
+    yield
+
+
+@router.websocket("/ws")
+async def websocket_endpoint(
+    ws: WebSocket,
+    session_token: str | None = Cookie(default=None),
+) -> None:
+    """Authenticate via session cookie, then stream table change events."""
+    store = get_store()
+
+    if not session_token:
+        await ws.close(code=4001, reason="Unauthorized")
+        return
+
+    result = store.resolve_session(session_token)
+    if not result:
+        await ws.close(code=4001, reason="Unauthorized")
+        return
+
+    _, session_row = result
+    if not session_row.organisation_id:
+        await ws.close(code=4002, reason="No organisation selected")
+        return
+
+    org_id = str(session_row.organisation_id)
+    logger.info("[WS] Authenticated. org=%s — accepting connection", org_id)
+
+    await ws.accept()
+    await _manager.connect(ws, org_id)
+
+    try:
+        while True:
+            await ws.receive_text()
+    except WebSocketDisconnect:
+        pass
+    finally:
+        await _manager.disconnect(ws, org_id)

interloper_api-0.2.0.dist-info/METADATA

@@ -0,0 +1,18 @@
+Metadata-Version: 2.3
+Name: interloper-api
+Version: 0.2.0
+Summary: Interloper FastAPI routes
+Author: Guillaume Onfroy
+Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
+Requires-Dist: interloper-core
+Requires-Dist: interloper-db
+Requires-Dist: fastapi>=0.115.0
+Requires-Dist: httpx>=0.28.0
+Requires-Dist: psycopg2-binary>=2.9.0
+Requires-Dist: uvicorn[standard]>=0.41.0
+Requires-Dist: wsproto>=1.2.0
+Requires-Dist: interloper-agent ; extra == 'agent'
+Requires-Python: >=3.10
+Provides-Extra: agent
+Description-Content-Type: text/markdown
+

interloper_api-0.2.0.dist-info/RECORD

@@ -0,0 +1,27 @@
+interloper_api/__init__.py,sha256=rAB0arSfA2ngJ8YcD3CbOpE5p9_mM5dFsRZzg-J78cE,68
+interloper_api/app.py,sha256=bG406EnP9RPjkM9zFzDHzl0Pt5inocOzn-6SLzkfsOg,3272
+interloper_api/dependencies.py,sha256=-8GsFhQydL4m983sUzsHevPS9VJsdEZFVQwhoJB5o5Y,7536
+interloper_api/email.py,sha256=VTw_j476yOHamcdFMZtrE7vfri6NtugLkqUDDppCb9A,2697
+interloper_api/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+interloper_api/routes/agent.py,sha256=kL-kimx7UWqR8kcWw4VeZ4pL9hvxeExvfotBdQ-56Mc,7351
+interloper_api/routes/assets.py,sha256=4PjaXieLam59itFlK0oSvj-pXjx33bBJ_yWo1xLgnxM,9184
+interloper_api/routes/auth.py,sha256=OsYaDSGYUkgjRd-BDaLRkz__RzQk2wIQSW0rEdyUjNU,7943
+interloper_api/routes/backfills.py,sha256=MMG08H1Xtw5gKdiF3MrpgNcGavrbXd72ANMA2gor5zk,2555
+interloper_api/routes/catalog.py,sha256=WsQByqYV6bNgWvsukNvGnz30brjYLvwRZnzRhjhgJr0,1297
+interloper_api/routes/destinations.py,sha256=MPAympSigDa5jfCSQm-qnk2PzxveFkTUd_4LOjVbxl0,3436
+interloper_api/routes/external/__init__.py,sha256=uLSboVkjjLienHNKUIyDYv7xqcHX1McIcsRqGOSv-zA,1830
+interloper_api/routes/external/amazon_ads.py,sha256=2A0TRaGadVGPPfHj1S5IwzMzIb5F1KEb3GX-XDXXYqo,2700
+interloper_api/routes/external/facebook_ads.py,sha256=SrCEcwJwZWP7Z-jKmLYP2gzvZ-NCC96ZN_hu4BLNM7k,1665
+interloper_api/routes/external/google_ads.py,sha256=HXTzzwl6lgxYsKgUJHEEM2ahuTNC_EcIOuvnFcKawWk,3810
+interloper_api/routes/external/pinterest_ads.py,sha256=NRasE18Q6CvUCJBvIr3XRDDrNy0-JmuztFPOBdffcBc,2453
+interloper_api/routes/external/snapchat_ads.py,sha256=wEWd0E9UmUPvC5-N0EDh9DjHuuJSmSqUiB7fwn96GiM,2959
+interloper_api/routes/jobs.py,sha256=Q2Wmffp4uODHCZRoUL4KZc_TIMMuIctKbJC9R0kqrO4,5887
+interloper_api/routes/oauth.py,sha256=0oG-1FXy-MX_IHDBsGfDlNmo1wI1a_N9hS334grrEO4,11439
+interloper_api/routes/organisations.py,sha256=YIZbWnqDgRyQ7PTpwIlFx6jq0Q8nO2YNEEtgsQZXN70,8546
+interloper_api/routes/resources.py,sha256=VgsA8gcfomfvJGOgrHVxGl95nrsqpfwLVoHZfPDEq7Y,5352
+interloper_api/routes/runs.py,sha256=-FljEfdasRDCmcSEC55lJnHkBc9rr4xmseV4i19wPL8,4811
+interloper_api/routes/sources.py,sha256=NbZohsruzt-fpfzIQyrZup_2HQHtcUr3igZ0iw7_K10,5957
+interloper_api/routes/ws.py,sha256=h4u1CADw2OpxjtOpUCmjiJVqbfiCR9k1wzXTYbNFlyU,5468
+interloper_api-0.2.0.dist-info/WHEEL,sha256=f5fWSvWsg5Knq5GWa6t1nJIug0Tqo69GqAWD_9LbBKw,81
+interloper_api-0.2.0.dist-info/METADATA,sha256=DEGBsv3FUXUM529-CpdctRcz0yxFotsAOiN6HW8L9hA,545
+interloper_api-0.2.0.dist-info/RECORD,,

interloper_api-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.11.16
+Root-Is-Purelib: true
+Tag: py3-none-any