topos-node 0.1.0__py3-none-any.whl

topos/filter_lab/worker.py

@@ -0,0 +1,250 @@
+"""Background processing for Filter Lab job groups (serial runs, Ollama pull/cleanup)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import time
+from typing import Any, Dict, List, Set
+
+from topos.config.sanitization_ollama import resolve_sanitization_ollama_effective
+from topos.config.settings import settings
+from topos.core.state import get_db_connection
+from topos.engine.backends.ollama import OllamaAdapter
+from topos.sanitization.ollama_transforms import apply_text_transform_with_ollama
+
+from . import bundles as bundles_mod
+from . import store
+
+logger = logging.getLogger("topos.filter_lab.worker")
+
+
+def _protection_tags(eff: Any) -> Set[str]:
+    s = {eff.default_model}
+    s.update(v for v in eff.models.values() if v)
+    return {str(x).strip() for x in s if x and str(x).strip()}
+
+
+def _input_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()[:24]
+
+
+def _ensure_model_pulled(
+    adapter: OllamaAdapter,
+    model_tag: str,
+    baseline: Set[str],
+    pulled: List[str],
+    conn: Any,
+    group_id: str,
+) -> None:
+    if model_tag in baseline:
+        return
+    if model_tag in pulled:
+        return
+    adapter.pull_model(model_tag)
+    pulled.append(model_tag)
+    store.set_group_pulled_models(conn, group_id, list(pulled))
+    store.insert_model_event(conn, group_id, "pull", model_tag)
+
+
+def _cleanup_ephemeral(
+    adapter: OllamaAdapter,
+    pulled: List[str],
+    baseline: Set[str],
+    protected: Set[str],
+    conn: Any,
+    group_id: str,
+) -> None:
+    for tag in pulled:
+        if tag in baseline:
+            continue
+        if tag in protected:
+            continue
+        try:
+            adapter.delete_model(tag)
+            store.insert_model_event(conn, group_id, "delete", tag)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("filter_lab cleanup delete failed for %s: %s", tag, exc)
+
+
+def process_job_group_sync(group_id: str) -> None:
+    """Execute all queued runs for a group serially; policy B cleanup on terminal state."""
+    conn = get_db_connection()
+    if not conn:
+        logger.error("filter_lab: no DB for group %s", group_id)
+        return
+
+    row = store.get_group(conn, group_id)
+    if not row:
+        return
+    group = dict(row)
+    status = group["status"]
+    if status in ("completed", "failed", "cancelled"):
+        return
+
+    adapter: OllamaAdapter | None = None
+    pulled: List[str] = []
+    baseline_set: Set[str] = set()
+    protected: Set[str] = set()
+
+    try:
+        try:
+            eff = resolve_sanitization_ollama_effective(settings, conn)
+        except Exception as exc:  # noqa: BLE001
+            logger.error("filter_lab: effective config failed: %s", exc)
+            store.update_group_status(conn, group_id, "failed")
+            return
+
+        # Pipeline "sanitization Ollama enabled" can be off while the user still wants Lab eval.
+        # Use the same host / models / limits; only skip blocking on eff.enabled.
+        if not eff.enabled:
+            logger.info(
+                "filter_lab: sanitization pipeline is disabled in config; running Lab eval to Ollama anyway (group=%s)",
+                group_id,
+            )
+        eff_for_lab = eff.model_copy(update={"enabled": True})
+
+        opts: Dict[str, Any] = {}
+        try:
+            opts = json.loads(group.get("options_json") or "{}")
+        except json.JSONDecodeError:
+            opts = {}
+
+        eff_merged = eff_for_lab.model_copy(
+            update={
+                "timeout_sec": float(opts["timeout_sec"])
+                if isinstance(opts.get("timeout_sec"), (int, float))
+                else eff.timeout_sec,
+                "max_input_chars": int(opts["max_input_chars"])
+                if isinstance(opts.get("max_input_chars"), int)
+                else eff.max_input_chars,
+            }
+        )
+
+        bundle = bundles_mod.get_bundle(group["bundle_id"])
+        if not bundle:
+            store.update_group_status(conn, group_id, "failed")
+            return
+
+        baseline_list = list(json.loads(group.get("baseline_models_json") or "[]"))
+        baseline_set = set(baseline_list)
+        pulled = list(json.loads(group.get("pulled_models_json") or "[]"))
+        protected = _protection_tags(eff_merged)
+        adapter = OllamaAdapter(base_url=eff_merged.host)
+
+        store.update_group_status(conn, group_id, "running")
+
+        for run_row in store.list_runs(conn, group_id):
+            run = dict(run_row)
+            if run["status"] != "queued":
+                continue
+
+            g2 = dict(store.get_group(conn, group_id) or {})
+            if g2.get("status") == "cancelled":
+                store.update_run(
+                    conn,
+                    run["id"],
+                    status="cancelled",
+                    finished_at=store.utc_now_iso(),
+                )
+                continue
+
+            rec_id = run["record_id"]
+            model_tag = run["model_tag"]
+            record = next(
+                (r for r in (bundle.get("records") or []) if str(r.get("id")) == rec_id),
+                None,
+            )
+            if not record:
+                store.update_run(
+                    conn,
+                    run["id"],
+                    status="failed",
+                    finished_at=store.utc_now_iso(),
+                    error_code="UNKNOWN_RECORD",
+                )
+                continue
+
+            text = bundles_mod.record_text(record)
+            max_c = eff_merged.max_input_chars
+            if max_c > 0 and len(text) > max_c:
+                store.update_run(
+                    conn,
+                    run["id"],
+                    status="failed",
+                    finished_at=store.utc_now_iso(),
+                    error_code="INPUT_TOO_LARGE",
+                    input_hash=_input_hash(text),
+                    input_text=text[: max_c + 50],
+                )
+                continue
+
+            store.update_run(
+                conn,
+                run["id"],
+                status="running",
+                started_at=store.utc_now_iso(),
+                input_hash=_input_hash(text),
+                input_text=text[:8000],
+            )
+
+            try:
+                assert adapter is not None
+                _ensure_model_pulled(adapter, model_tag, baseline_set, pulled, conn, group_id)
+                t0 = time.perf_counter()
+                out = apply_text_transform_with_ollama(
+                    text,
+                    group["filter_id"],
+                    None,
+                    effective=eff_merged,
+                    model_override=model_tag,
+                )
+                ms = int((time.perf_counter() - t0) * 1000)
+                store.update_run(
+                    conn,
+                    run["id"],
+                    status="succeeded",
+                    finished_at=store.utc_now_iso(),
+                    latency_ms=ms,
+                    output_text=out,
+                    metrics_json=json.dumps({"input_chars": len(text), "output_chars": len(out)}),
+                )
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("filter_lab run failed: %s", exc)
+                store.update_run(
+                    conn,
+                    run["id"],
+                    status="failed",
+                    finished_at=store.utc_now_iso(),
+                    error_code="RUN_ERROR",
+                    output_text=str(exc)[:2000],
+                )
+
+        g3 = dict(store.get_group(conn, group_id) or {})
+        if g3.get("status") == "cancelled":
+            for r in store.list_runs(conn, group_id):
+                rd = dict(r)
+                if rd["status"] == "queued":
+                    store.update_run(
+                        conn,
+                        rd["id"],
+                        status="cancelled",
+                        finished_at=store.utc_now_iso(),
+                    )
+        else:
+            store.update_group_status(conn, group_id, "completed")
+
+    finally:
+        if adapter and conn:
+            try:
+                row_f = store.get_group(conn, group_id)
+                if row_f:
+                    group_f = dict(row_f)
+                    pulled_f = list(json.loads(group_f.get("pulled_models_json") or "[]"))
+                    baseline_f = set(json.loads(group_f.get("baseline_models_json") or "[]"))
+                    eff2 = resolve_sanitization_ollama_effective(settings, conn)
+                    prot2 = _protection_tags(eff2)
+                    _cleanup_ephemeral(adapter, pulled_f, baseline_f, prot2, conn, group_id)
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("filter_lab cleanup skipped: %s", exc)

topos/hosted_pool_lease.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+from urllib.parse import urlparse
+
+import httpx
+
+from .config.settings import settings
+
+logger = logging.getLogger("topos.hosted_pool_lease")
+
+
+def _parse_iso_datetime(value: Any) -> Optional[datetime]:
+    raw = str(value or "").strip()
+    if not raw:
+        return None
+    try:
+        parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+        if parsed.tzinfo is None:
+            parsed = parsed.replace(tzinfo=timezone.utc)
+        return parsed.astimezone(timezone.utc)
+    except Exception:  # noqa: BLE001
+        return None
+
+
+def _control_plane_http_base(ws_url: str) -> str:
+    parsed = urlparse(ws_url)
+    if parsed.scheme not in {"ws", "wss"}:
+        raise ValueError(f"Unsupported control plane websocket URL scheme: {parsed.scheme}")
+    http_scheme = "https" if parsed.scheme == "wss" else "http"
+    netloc = parsed.netloc
+    if not netloc:
+        raise ValueError("Control plane websocket URL missing host")
+    return f"{http_scheme}://{netloc}"
+
+
+def _metadata_identity_token(audience: str) -> str:
+    metadata_url = (
+        "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity"
+    )
+    params = {"audience": audience, "format": "full"}
+    headers = {"Metadata-Flavor": "Google"}
+    with httpx.Client(timeout=5.0) as client:
+        resp = client.get(metadata_url, params=params, headers=headers)
+        resp.raise_for_status()
+        token = str(resp.text or "").strip()
+        if not token:
+            raise RuntimeError("Metadata server returned an empty identity token")
+        return token
+
+
+@dataclass
+class HostedPoolLease:
+    connector_key: str
+    lease_expires_at: Optional[datetime]
+    lease_ttl_seconds: int
+
+
+class HostedPoolLeaseClient:
+    def __init__(self, *, control_plane_ws_url: str) -> None:
+        self.control_plane_ws_url = control_plane_ws_url
+        self.control_plane_http_base = _control_plane_http_base(control_plane_ws_url)
+        self.issue_path = str(settings.hosted_pool_lease_issue_path or "/v1/system/pool-connectors/lease/issue")
+        self.renew_path = str(settings.hosted_pool_lease_renew_path or "/v1/system/pool-connectors/lease/renew")
+        self.revoke_path = str(settings.hosted_pool_lease_revoke_path or "/v1/system/pool-connectors/lease/revoke")
+        self.pool_group = str(settings.hosted_pool_lease_pool_group or "default")
+        self.instance_id = str(os.getenv("HOSTNAME") or "unknown-instance").strip()
+        self.service_name = str(os.getenv("K_SERVICE") or "unknown-service").strip()
+        self.revision = str(os.getenv("K_REVISION") or "").strip() or None
+        self.lease: Optional[HostedPoolLease] = None
+
+    def _build_url(self, path: str) -> str:
+        normalized = f"/{str(path or '').lstrip('/')}"
+        return f"{self.control_plane_http_base}{normalized}"
+
+    def _audience(self) -> str:
+        configured = str(settings.hosted_pool_lease_audience or "").strip()
+        return configured or self.control_plane_http_base
+
+    def _identity_token(self) -> str:
+        return _metadata_identity_token(self._audience())
+
+    async def issue(self) -> HostedPoolLease:
+        url = self._build_url(self.issue_path)
+        token = await asyncio.to_thread(self._identity_token)
+        payload: Dict[str, Any] = {
+            "service_name": self.service_name,
+            "revision": self.revision,
+            "instance_id": self.instance_id,
+            "pool_group": self.pool_group,
+        }
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            resp = await client.post(url, headers={"Authorization": f"Bearer {token}"}, json=payload)
+            resp.raise_for_status()
+            body = resp.json()
+        connector_key = str(body.get("connector_key") or "").strip()
+        if not connector_key:
+            raise RuntimeError("Lease issue response missing connector_key")
+        ttl = int(body.get("lease_ttl_seconds") or 300)
+        lease = HostedPoolLease(
+            connector_key=connector_key,
+            lease_expires_at=_parse_iso_datetime(body.get("lease_expires_at")),
+            lease_ttl_seconds=max(30, ttl),
+        )
+        self.lease = lease
+        return lease
+
+    async def renew(self) -> HostedPoolLease:
+        if not self.lease:
+            return await self.issue()
+        url = self._build_url(self.renew_path)
+        token = await asyncio.to_thread(self._identity_token)
+        payload = {
+            "connector_key": self.lease.connector_key,
+            "service_name": self.service_name,
+            "revision": self.revision,
+            "instance_id": self.instance_id,
+        }
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            resp = await client.post(url, headers={"Authorization": f"Bearer {token}"}, json=payload)
+            resp.raise_for_status()
+            body = resp.json()
+        ttl = int(body.get("lease_ttl_seconds") or self.lease.lease_ttl_seconds)
+        self.lease = HostedPoolLease(
+            connector_key=self.lease.connector_key,
+            lease_expires_at=_parse_iso_datetime(body.get("lease_expires_at")),
+            lease_ttl_seconds=max(30, ttl),
+        )
+        return self.lease
+
+    async def revoke(self) -> None:
+        if not self.lease:
+            return
+        url = self._build_url(self.revoke_path)
+        token = await asyncio.to_thread(self._identity_token)
+        payload = {
+            "connector_key": self.lease.connector_key,
+            "service_name": self.service_name,
+            "instance_id": self.instance_id,
+        }
+        try:
+            async with httpx.AsyncClient(timeout=10.0) as client:
+                resp = await client.post(url, headers={"Authorization": f"Bearer {token}"}, json=payload)
+                resp.raise_for_status()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Hosted pool lease revoke failed: %s", exc)
+        finally:
+            self.lease = None

topos/ingestion/__init__.py

@@ -0,0 +1 @@
+"""Ingestion layer for Topos."""

topos/ingestion/checkpoints/__init__.py

@@ -0,0 +1,6 @@
+"""Checkpoint storage abstractions."""
+
+from .checkpoint_store import CheckpointStore, IngestionCheckpoint
+from .sqlite_checkpoint_store import SqliteCheckpointStore, ensure_table
+
+__all__ = ["CheckpointStore", "IngestionCheckpoint", "SqliteCheckpointStore", "ensure_table"]

topos/ingestion/checkpoints/checkpoint_store.py

@@ -0,0 +1,24 @@
+"""Checkpoint store contract for ingestion."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Optional
+
+
+@dataclass(frozen=True)
+class IngestionCheckpoint:
+    dataset_id: str
+    schema_id: str
+    last_record_id: str
+    metadata: Dict[str, str]
+
+
+class CheckpointStore:
+    """Persist and retrieve ingestion checkpoints."""
+
+    def get_checkpoint(self, dataset_id: str, schema_id: str) -> Optional[IngestionCheckpoint]:
+        raise NotImplementedError
+
+    def save_checkpoint(self, checkpoint: IngestionCheckpoint) -> None:
+        raise NotImplementedError

topos/ingestion/checkpoints/sqlite_checkpoint_store.py

@@ -0,0 +1,82 @@
+"""SQLite-backed checkpoint store for ingestion (e.g. iMessage/Signal sync)."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Optional
+
+from .checkpoint_store import CheckpointStore, IngestionCheckpoint
+
+logger = logging.getLogger("topos.ingestion.checkpoints.sqlite")
+
+TABLE = "ingestion_checkpoints"
+
+
+def ensure_table(conn) -> None:
+    """Create ingestion_checkpoints table if not exists."""
+    conn.execute(f"""
+        CREATE TABLE IF NOT EXISTS {TABLE} (
+            dataset_id TEXT NOT NULL,
+            schema_id TEXT NOT NULL,
+            last_record_id TEXT NOT NULL,
+            metadata_json TEXT,
+            updated_at TEXT DEFAULT (datetime('now')),
+            PRIMARY KEY (dataset_id, schema_id)
+        )
+    """)
+    conn.commit()
+
+
+class SqliteCheckpointStore(CheckpointStore):
+    """Persist checkpoints to SQLite (same DB as app)."""
+
+    def __init__(self, conn) -> None:
+        self.conn = conn
+        if conn:
+            ensure_table(conn)
+
+    def get_checkpoint(self, dataset_id: str, schema_id: str) -> Optional[IngestionCheckpoint]:
+        if not self.conn:
+            return None
+        try:
+            ensure_table(self.conn)
+            row = self.conn.execute(
+                f"SELECT last_record_id, metadata_json FROM {TABLE} WHERE dataset_id = ? AND schema_id = ?",
+                (dataset_id, schema_id),
+            ).fetchone()
+            if not row:
+                return None
+            last_record_id, metadata_json = row
+            metadata = {}
+            if metadata_json:
+                try:
+                    metadata = json.loads(metadata_json)
+                except (TypeError, json.JSONDecodeError):
+                    pass
+            return IngestionCheckpoint(
+                dataset_id=dataset_id,
+                schema_id=schema_id,
+                last_record_id=last_record_id or "0",
+                metadata=metadata,
+            )
+        except Exception as e:
+            logger.warning("get_checkpoint failed: %s", e)
+            return None
+
+    def save_checkpoint(self, checkpoint: IngestionCheckpoint) -> None:
+        if not self.conn:
+            return
+        try:
+            ensure_table(self.conn)
+            metadata_json = json.dumps(checkpoint.metadata, ensure_ascii=False) if checkpoint.metadata else None
+            self.conn.execute(
+                f"""
+                INSERT OR REPLACE INTO {TABLE} (dataset_id, schema_id, last_record_id, metadata_json, updated_at)
+                VALUES (?, ?, ?, ?, datetime('now'))
+                """,
+                (checkpoint.dataset_id, checkpoint.schema_id, checkpoint.last_record_id, metadata_json),
+            )
+            self.conn.commit()
+        except Exception as e:
+            logger.warning("save_checkpoint failed: %s", e)