topos-node 0.1.0__py3-none-any.whl

topos/storage/canonical/mapping_store.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Optional
+
+
+@dataclass(frozen=True)
+class MappingRecord:
+    source_id: str
+    source_record_id: str
+    canonical_id: str
+
+
+class MappingStore:
+    def get_mapping(self, source_id: str, source_record_id: str) -> Optional[MappingRecord]:
+        raise NotImplementedError
+
+    def save_mapping(self, record: MappingRecord) -> None:
+        raise NotImplementedError
+
+
+class InMemoryMappingStore(MappingStore):
+    def __init__(self):
+        self._records: Dict[str, MappingRecord] = {}
+
+    def get_mapping(self, source_id: str, source_record_id: str) -> Optional[MappingRecord]:
+        return self._records.get(f"{source_id}:{source_record_id}")
+
+    def save_mapping(self, record: MappingRecord) -> None:
+        self._records[f"{record.source_id}:{record.source_record_id}"] = record

topos/storage/canonical/postgres.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+
+class PostgresCanonicalStore:
+    def __init__(self, conn):
+        self.conn = conn
+
+    def upsert(self, record):
+        _ = record
+        raise NotImplementedError("PostgresCanonicalStore not implemented yet")

topos/storage/db/__init__.py

@@ -0,0 +1 @@
+"""Database helpers for Topos storage."""

topos/storage/db/client.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+
+class DBClient:
+    """Database client placeholder."""
+
+    def __init__(self, conn):
+        self.conn = conn

topos/storage/db/migrations/__init__.py

@@ -0,0 +1 @@
+"""Migration package placeholder."""

topos/storage/db/migrations/stage9_column_renames.py

@@ -0,0 +1,78 @@
+"""
+Stage 9 column renames: apply standardized names to existing DBs.
+SQLite 3.25.1+ RENAME COLUMN. Idempotent: skips if new name already exists or old missing.
+Source: docs/SCHEMA_CONVENTIONS.md §7.0.
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from typing import Any, List, Optional, Tuple
+
+logger = logging.getLogger("topos.storage.db.migrations.stage9")
+
+# (table, old_column, new_column) in dependency order
+RENAME_TARGETS: List[Tuple[str, str, str]] = [
+    ("ai_chat_conversations", "source", "source_id"),
+    ("conversation_messages", "ts", "event_at"),
+    ("conversation_messages", "from_self", "is_from_self"),
+    ("ai_chat_messages", "ts", "event_at"),
+    ("ai_chat_messages", "seq", "sequence"),
+    ("message_emotions", "model", "model_name"),
+    ("message_emotions", "all_emotions", "all_emotions_json"),
+    ("browser_url_classification", "source_table", "enriched_from_table"),
+]
+
+
+def _column_exists(conn: sqlite3.Connection, table: str, column: str) -> bool:
+    cursor = conn.execute(f"PRAGMA table_info({table})")
+    rows = cursor.fetchall()
+    # PRAGMA table_info returns (cid, name, type, notnull, dflt_value, pk)
+    names = [row[1] for row in rows]
+    return column in names
+
+
+def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
+    cursor = conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
+        (table,),
+    )
+    return cursor.fetchone() is not None
+
+
+def apply_stage9_renames(conn: sqlite3.Connection) -> dict[str, Any]:
+    """
+    Apply Stage 9 column renames. Idempotent.
+    Returns dict with applied=[(table, old, new), ...], skipped=[...], errors=[...].
+    """
+    applied: List[Tuple[str, str, str]] = []
+    skipped: List[Tuple[str, str, str]] = []
+    errors: List[str] = []
+
+    for table, old_col, new_col in RENAME_TARGETS:
+        if not _table_exists(conn, table):
+            skipped.append((table, old_col, new_col))
+            continue
+        if not _column_exists(conn, table, old_col):
+            skipped.append((table, old_col, new_col))
+            continue
+        if _column_exists(conn, table, new_col):
+            skipped.append((table, old_col, new_col))
+            continue
+        try:
+            conn.execute(f'ALTER TABLE "{table}" RENAME COLUMN "{old_col}" TO "{new_col}"')
+            conn.commit()
+            applied.append((table, old_col, new_col))
+            logger.info("Stage9 migration: %s.%s -> %s", table, old_col, new_col)
+        except Exception as e:
+            conn.rollback()
+            errors.append(f"{table}.{old_col}->{new_col}: {e}")
+            logger.warning("Stage9 migration failed: %s.%s -> %s: %s", table, old_col, new_col, e)
+
+    return {"applied": applied, "skipped": skipped, "errors": errors}
+
+
+def run_stage9_migrations(conn: sqlite3.Connection) -> dict[str, Any]:
+    """Run Stage 9 migrations (currently only column renames)."""
+    return apply_stage9_renames(conn)

topos/storage/db/paths.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import platform
+import shutil
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("topos.storage.db.paths")
+
+
+def get_database_path(base_path: Optional[str]) -> Path:
+    if base_path:
+        return Path(base_path)
+    return Path.home() / ".topos" / "database.db"
+
+
+def get_config_file() -> Path:
+    return get_data_directory() / "config.json"
+
+
+def load_config() -> dict:
+    config_file = get_config_file()
+    if config_file.exists():
+        try:
+            with open(config_file, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except Exception as exc:
+            logger.warning("Failed to load config from %s: %s", config_file, exc)
+    return {}
+
+
+def save_config(config: dict) -> None:
+    config_file = get_config_file()
+    config_file.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        with open(config_file, "w", encoding="utf-8") as f:
+            json.dump(config, f, indent=2)
+    except Exception as exc:
+        logger.error("Failed to save config to %s: %s", config_file, exc)
+        raise
+
+
+def discover_databases() -> list[Path]:
+    databases: list[Path] = []
+    config = load_config()
+    if "database_path" in config:
+        db_path = Path(config["database_path"])
+        if db_path.exists() and db_path.is_file():
+            databases.append(db_path)
+
+    data_dir = get_data_directory()
+    default_db = data_dir / "database.db"
+    if default_db.exists() and default_db not in databases:
+        databases.append(default_db)
+
+    legacy_db = Path.home() / ".topos_engine" / "database.db"
+    if legacy_db.exists() and legacy_db not in databases:
+        databases.append(legacy_db)
+
+    env_path = os.getenv("TOPOS_DATABASE_PATH")
+    if env_path:
+        env_db = Path(env_path)
+        if env_db.exists() and env_db.is_file() and env_db not in databases:
+            databases.append(env_db)
+
+    return databases
+
+
+def get_data_directory() -> Path:
+    system = platform.system()
+    if system == "Windows":
+        base = Path(os.getenv("APPDATA", Path.home() / "AppData" / "Roaming"))
+        return base / "ToposControlPlane"
+    if system == "Darwin":
+        return Path.home() / "Library" / "Application Support" / "ToposControlPlane"
+    xdg_data = os.getenv("XDG_DATA_HOME")
+    if xdg_data:
+        return Path(xdg_data) / "topos-control-plane"
+    return Path.home() / ".local" / "share" / "topos-control-plane"
+
+
+def migrate_legacy_database() -> Optional[Path]:
+    legacy_path = Path.home() / ".topos_engine" / "database.db"
+    if not legacy_path.exists():
+        return None
+
+    new_path = get_data_directory() / "database.db"
+    if new_path.exists():
+        logger.warning("Both legacy (%s) and new (%s) database exist. Using new location.", legacy_path, new_path)
+        return new_path
+
+    try:
+        new_path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(legacy_path, new_path)
+        config = load_config()
+        config["database_path"] = str(new_path)
+        save_config(config)
+        logger.info("Migrated database from %s to %s", legacy_path, new_path)
+        return new_path
+    except Exception as exc:
+        logger.error("Failed to migrate database: %s", exc)
+        return None
+
+
+def validate_database(db_path: Path) -> bool:
+    if not db_path.exists() or not db_path.is_file():
+        return False
+    try:
+        import sqlite3
+
+        conn = sqlite3.connect(str(db_path))
+        cursor = conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        existing_tables = {row[0] for row in cursor}
+        conn.close()
+        required_tables = {"oplog", "messages", "engine_config"}
+        return required_tables.issubset(existing_tables)
+    except Exception as exc:
+        logger.warning("Failed to validate database %s: %s", db_path, exc)
+        return False

topos/storage/db/postgres.py

@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+import logging
+from typing import Any, Iterable, Sequence, Tuple
+
+from ...config.settings import settings
+
+logger = logging.getLogger("topos.storage.db.postgres")
+
+
+class PostgresConfigurationError(RuntimeError):
+    pass
+
+
+def _build_postgres_dsn() -> str:
+    if settings.topos_postgres_dsn:
+        return settings.topos_postgres_dsn
+    if settings.topos_database_service_url:
+        return settings.topos_database_service_url
+
+    required = {
+        "TOPOS_POSTGRES_HOST": settings.topos_postgres_host,
+        "TOPOS_POSTGRES_PORT": settings.topos_postgres_port,
+        "TOPOS_POSTGRES_DB": settings.topos_postgres_db,
+        "TOPOS_POSTGRES_USER": settings.topos_postgres_user,
+        "TOPOS_POSTGRES_PASSWORD": settings.topos_postgres_password,
+    }
+    missing = [key for key, value in required.items() if value in (None, "")]
+    if missing:
+        raise PostgresConfigurationError(
+            "Missing Postgres settings: " + ", ".join(sorted(missing))
+        )
+    return (
+        f"postgresql://{settings.topos_postgres_user}:{settings.topos_postgres_password}"
+        f"@{settings.topos_postgres_host}:{int(settings.topos_postgres_port)}/{settings.topos_postgres_db}"
+    )
+
+
+def _is_sqlite_connection(conn: Any) -> bool:
+    module_name = conn.__class__.__module__.lower()
+    return "sqlite" in module_name
+
+
+def _normalize_query_for_connection(conn: Any, query: str) -> str:
+    if _is_sqlite_connection(conn):
+        return query.replace("%s", "?")
+    return query
+
+
+def _table_exists(conn: Any, table_name: str) -> bool:
+    if _is_sqlite_connection(conn):
+        row = fetch_one(
+            conn,
+            "SELECT name FROM sqlite_master WHERE type='table' AND name = ?",
+            (table_name,),
+        )
+        return bool(row)
+    row = fetch_one(
+        conn,
+        """
+        SELECT 1
+        FROM information_schema.tables
+        WHERE table_schema = 'public' AND table_name = %s
+        LIMIT 1
+        """,
+        (table_name,),
+    )
+    return bool(row)
+
+
+def _table_has_column(conn: Any, table_name: str, column_name: str) -> bool:
+    if not _table_exists(conn, table_name):
+        return False
+    if _is_sqlite_connection(conn):
+        rows = fetch_all(conn, f'PRAGMA table_info("{table_name}")')
+        for row in rows:
+            col = row["name"] if isinstance(row, dict) else row[1]
+            if str(col) == column_name:
+                return True
+        return False
+    row = fetch_one(
+        conn,
+        """
+        SELECT 1
+        FROM information_schema.columns
+        WHERE table_schema = 'public' AND table_name = %s AND column_name = %s
+        LIMIT 1
+        """,
+        (table_name, column_name),
+    )
+    return bool(row)
+
+
+def _drop_table_if_exists(conn: Any, table_name: str) -> None:
+    execute_query(conn, f'DROP TABLE IF EXISTS "{table_name}" CASCADE')
+
+
+def _ensure_postgres_tenant_schema_compat(conn: Any) -> None:
+    """
+    Detect legacy hosted schemas that predate tenant-aware columns.
+
+    In dev/test we allow a destructive reset because the user has explicitly
+    accepted data loss. For safety, reset is opt-in via
+    TOPOS_POSTGRES_RESET_INCOMPATIBLE_SCHEMA.
+    """
+    if _is_sqlite_connection(conn):
+        return
+
+    requires_reset = any(
+        _table_exists(conn, table) and not _table_has_column(conn, table, "tenant_id")
+        for table in ("messages", "oplog", "engine_config")
+    )
+    if not requires_reset:
+        return
+
+    if not settings.topos_postgres_reset_incompatible_schema:
+        raise PostgresConfigurationError(
+            "Detected legacy Postgres schema without tenant_id on messages/oplog/engine_config. "
+            "Set TOPOS_POSTGRES_RESET_INCOMPATIBLE_SCHEMA=true to reset these dev tables, or run a manual migration."
+        )
+
+    logger.warning(
+        "Resetting legacy Postgres tables (messages, oplog, engine_config) to tenant-aware schema."
+    )
+    _drop_table_if_exists(conn, "messages")
+    _drop_table_if_exists(conn, "oplog")
+    _drop_table_if_exists(conn, "engine_config")
+
+
+def execute_query(conn: Any, query: str, params: Sequence[Any] = ()) -> None:
+    normalized = _normalize_query_for_connection(conn, query)
+    with _cursor(conn) as cursor:
+        cursor.execute(normalized, tuple(params))
+
+
+def fetch_all(conn: Any, query: str, params: Sequence[Any] = ()) -> list[Tuple[Any, ...]]:
+    normalized = _normalize_query_for_connection(conn, query)
+    with _cursor(conn) as cursor:
+        cursor.execute(normalized, tuple(params))
+        rows = cursor.fetchall()
+    return list(rows or [])
+
+
+def fetch_one(conn: Any, query: str, params: Sequence[Any] = ()) -> Tuple[Any, ...] | None:
+    normalized = _normalize_query_for_connection(conn, query)
+    with _cursor(conn) as cursor:
+        cursor.execute(normalized, tuple(params))
+        row = cursor.fetchone()
+    return row
+
+
+@contextmanager
+def _cursor(conn: Any):
+    cursor = conn.cursor()
+    try:
+        yield cursor
+    finally:
+        try:
+            cursor.close()
+        except Exception:
+            pass
+
+
+def ensure_postgres_schema(conn: Any) -> None:
+    _ensure_postgres_tenant_schema_compat(conn)
+    statements: Iterable[str] = (
+        """
+        CREATE TABLE IF NOT EXISTS messages (
+            tenant_id TEXT NOT NULL,
+            dataset_id TEXT NOT NULL,
+            message_id TEXT PRIMARY KEY,
+            sender_type TEXT NOT NULL,
+            content TEXT NOT NULL,
+            ts TEXT NOT NULL,
+            user_id TEXT
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_messages_tenant_dataset_ts ON messages (tenant_id, dataset_id, ts DESC)",
+        """
+        CREATE TABLE IF NOT EXISTS oplog (
+            tenant_id TEXT NOT NULL,
+            dataset_id TEXT NOT NULL,
+            op_id TEXT PRIMARY KEY,
+            op_type TEXT NOT NULL,
+            payload_json TEXT NOT NULL,
+            hlc_ts TEXT NOT NULL
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_oplog_tenant_dataset_ts ON oplog (tenant_id, dataset_id, hlc_ts DESC)",
+        """
+        CREATE TABLE IF NOT EXISTS engine_config (
+            tenant_id TEXT NOT NULL,
+            key TEXT NOT NULL,
+            value TEXT NOT NULL,
+            PRIMARY KEY (tenant_id, key)
+        )
+        """,
+    )
+    for statement in statements:
+        execute_query(conn, statement)
+
+
+@contextmanager
+def connect_postgres():
+    dsn = _build_postgres_dsn()
+    if dsn.startswith("sqlite://"):
+        import sqlite3
+
+        sqlite_path = dsn.replace("sqlite://", "", 1)
+        conn = sqlite3.connect(sqlite_path)
+        conn.row_factory = sqlite3.Row
+        try:
+            ensure_postgres_schema(conn)
+            yield conn
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+        finally:
+            conn.close()
+        return
+
+    try:
+        import psycopg
+    except Exception as exc:  # pragma: no cover - depends on environment extras
+        raise PostgresConfigurationError(
+            "psycopg is required for postgres database_mode. Install optional dependency: psycopg[binary]."
+        ) from exc
+
+    conn = psycopg.connect(dsn)
+    try:
+        ensure_postgres_schema(conn)
+        yield conn
+        conn.commit()
+    except Exception:
+        conn.rollback()
+        raise
+    finally:
+        conn.close()

topos/storage/db/schema.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+
+def ensure_schema(conn) -> None:
+    """Ensure DB schema exists (stub)."""
+    _ = conn

topos/storage/enrichment/__init__.py

@@ -0,0 +1 @@
+"""Enrichment storage abstractions."""

topos/storage/enrichment/canonical_enrichment_store.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from .raw_enrichment_store import EnrichmentStore
+
+
+class CanonicalEnrichmentStore(EnrichmentStore):
+    """Store for canonical enrichment results."""

topos/storage/enrichment/raw_enrichment_store.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict
+
+
+@dataclass(frozen=True)
+class EnrichmentRef:
+    record_id: str
+
+
+class EnrichmentStore:
+    def write(self, result: Dict[str, str]) -> EnrichmentRef:
+        raise NotImplementedError
+
+
+class RawEnrichmentStore(EnrichmentStore):
+    """Store for raw enrichment results."""

topos/storage/normalized/__init__.py

@@ -0,0 +1 @@
+"""Normalized storage abstractions."""

topos/storage/normalized/normalized_store.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict
+
+
+@dataclass(frozen=True)
+class NormalizedRef:
+    record_id: str
+
+
+class NormalizedStore:
+    def write(self, record: Dict[str, str]) -> NormalizedRef:
+        raise NotImplementedError
+
+
+class InMemoryNormalizedStore(NormalizedStore):
+    def __init__(self):
+        self._records: Dict[str, Dict[str, str]] = {}
+
+    def write(self, record: Dict[str, str]) -> NormalizedRef:
+        record_id = record.get("record_id") or record.get("message_id") or ""
+        self._records[record_id] = record
+        return NormalizedRef(record_id=record_id)

topos/storage/oplog/__init__.py

@@ -0,0 +1 @@
+"""Oplog storage abstractions."""

topos/storage/oplog/decision.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+
+def should_write_to_oplog(op_type: str, payload: dict, enable_sync: bool) -> bool:
+    _ = (op_type, payload, enable_sync)
+    return True

topos/storage/oplog/oplog_store.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict
+
+
+@dataclass(frozen=True)
+class OplogEntry:
+    op_id: str
+    dataset_id: str
+    op_type: str
+    payload: Dict[str, str]
+
+
+class OplogStore:
+    def append(self, entry: OplogEntry) -> None:
+        raise NotImplementedError

topos/storage/oplog/postgres.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+
+class PostgresOplogStore:
+    def __init__(self, conn):
+        self.conn = conn
+
+    def append(self, entry):
+        _ = entry
+        raise NotImplementedError("PostgresOplogStore not implemented yet")

topos/storage/projections/__init__.py

@@ -0,0 +1 @@
+"""Projection storage abstractions."""

topos/storage/projections/index_ops_store.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+
+class IndexOpsStore:
+    def record_metric(self, metric: dict) -> None:
+        _ = metric

topos/storage/projections/vector_index_store.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+
+class VectorIndexStore:
+    def record_run(self, metadata: dict) -> None:
+        _ = metadata

topos/storage/raw/__init__.py

@@ -0,0 +1 @@
+"""Raw storage abstractions."""