topos-node 0.1.0__py3-none-any.whl

topos/storage/raw/raw_tables_manager.py

@@ -0,0 +1,295 @@
+"""Raw tables manager for storing original payloads before canonicalization."""
+
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger("topos.storage.raw.raw_tables_manager")
+
+
+class RawTablesManager:
+    """Manages raw retention tables for storing original payloads.
+    
+    According to architecture, raw tables are per-connector:
+    - `raw_chat_messages_{source}` for chat sources
+    - `raw_{source}_events` for event sources
+    """
+    
+    def __init__(self, conn: sqlite3.Connection):
+        """Initialize with database connection."""
+        self.conn = conn
+    
+    def get_raw_table_name(self, source_id: str, source_type: str = "chat_messages") -> str:
+        """Get raw table name for a source.
+        
+        Args:
+            source_id: Source identifier (e.g., "chatgpt", "chatgpt_ui_conversation")
+            source_type: Type of data ("chat_messages", "events", etc.)
+            
+        Returns:
+            Table name like "raw_chat_messages_chatgpt"
+        """
+        # Extract base source name (remove prefixes like "dev_test_")
+        if source_id in ("browser_visits", "browser_events", "starred_websites"):
+            base_source = source_id.replace("_", "")
+        else:
+            base_source = source_id
+            if "_" in source_id:
+                # For "chatgpt_ui_conversation", extract "chatgpt"
+                parts = source_id.split("_")
+                # Find the actual source name (usually after prefixes)
+                for part in parts:
+                    if part in ["chatgpt", "grok", "claude", "gemini"]:
+                        base_source = part
+                        break
+                # If no known source found, use the last meaningful part
+                if base_source == source_id:
+                    # For "chatgpt_ui_conversation", use "chatgpt_ui_conversation"
+                    # but normalize to just the source type
+                    if "chatgpt" in source_id.lower():
+                        base_source = "chatgpt"
+                    elif "grok" in source_id.lower():
+                        base_source = "grok"
+                    else:
+                        # Fallback: use a sanitized version
+                        base_source = source_id.replace("dev_test_", "").replace("_", "")
+        
+        if source_type == "chat_messages":
+            return f"raw_chat_messages_{base_source}"
+        else:
+            return f"raw_{base_source}_{source_type}"
+    
+    def ensure_raw_table(self, table_name: str) -> None:
+        """Ensure raw table exists with proper schema.
+        
+        Raw tables store original payloads verbatim with:
+        - source_system: Source identifier
+        - source_record_id: Unique record ID within source
+        - payload_json: Original payload as JSON string
+        - created_at: Timestamp when record was stored
+        - Uniqueness: (source_system, source_record_id)
+        """
+        try:
+            if table_name == "raw_chat_messages_browservisits":
+                self._ensure_browser_visits_raw_table(table_name)
+                return
+            self.conn.execute(f"""
+                CREATE TABLE IF NOT EXISTS {table_name} (
+                    source_system TEXT NOT NULL,
+                    source_record_id TEXT NOT NULL,
+                    payload_json TEXT NOT NULL,
+                    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+                    PRIMARY KEY (source_system, source_record_id)
+                )
+            """)
+            
+            # Create indexes
+            self.conn.execute(f"""
+                CREATE INDEX IF NOT EXISTS idx_{table_name}_source_system 
+                ON {table_name}(source_system)
+            """)
+            
+            self.conn.execute(f"""
+                CREATE INDEX IF NOT EXISTS idx_{table_name}_created_at 
+                ON {table_name}(created_at)
+            """)
+            
+            self.conn.commit()
+        except Exception as e:
+            self.conn.rollback()
+            logger.error("Failed to ensure raw table %s: %s", table_name, e)
+            raise
+
+    def _ensure_browser_visits_raw_table(self, table_name: str) -> None:
+        """Ensure browser visits raw table uses normalized columns (no payload_json)."""
+        cursor = self.conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
+            (table_name,),
+        )
+        table_exists = cursor.fetchone() is not None
+
+        def _create_schema(target_name: str) -> None:
+            self.conn.execute(f"""
+                CREATE TABLE IF NOT EXISTS {target_name} (
+                    source_system TEXT NOT NULL,
+                    source_record_id TEXT NOT NULL,
+                    record_id TEXT,
+                    dataset_id TEXT,
+                    url TEXT,
+                    visited_at TEXT,
+                    title TEXT,
+                    favicon_url TEXT,
+                    hostname TEXT,
+                    device_name TEXT,
+                    tab_id INTEGER,
+                    window_id INTEGER,
+                    incognito INTEGER,
+                    transition_type TEXT,
+                    pinned INTEGER,
+                    audible INTEGER,
+                    muted INTEGER,
+                    opener_tab_id INTEGER,
+                    referred_by TEXT,
+                    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+                    PRIMARY KEY (source_system, source_record_id)
+                )
+            """)
+
+        if not table_exists:
+            _create_schema(table_name)
+        else:
+            existing_cols_cursor = self.conn.execute(f"PRAGMA table_info({table_name})")
+            existing_cols = {row[1] for row in existing_cols_cursor.fetchall()}
+            needs_migration = "payload_json" in existing_cols or "url" not in existing_cols
+            if needs_migration:
+                pre_count = self.conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0]
+                tmp_table = f"{table_name}__migrated"
+                self.conn.execute(f"DROP TABLE IF EXISTS {tmp_table}")
+                _create_schema(tmp_table)
+                self.conn.execute(f"""
+                    INSERT OR REPLACE INTO {tmp_table} (
+                        source_system, source_record_id, record_id, dataset_id, url, visited_at, title,
+                        favicon_url, hostname, device_name, tab_id, window_id, incognito, transition_type,
+                        pinned, audible, muted, opener_tab_id, referred_by, created_at
+                    )
+                    SELECT
+                        source_system,
+                        source_record_id,
+                        COALESCE(json_extract(payload_json, '$.record_id'), source_record_id),
+                        json_extract(payload_json, '$.dataset_id'),
+                        json_extract(payload_json, '$.url'),
+                        json_extract(payload_json, '$.visited_at'),
+                        json_extract(payload_json, '$.title'),
+                        json_extract(payload_json, '$.favicon_url'),
+                        json_extract(payload_json, '$.hostname'),
+                        json_extract(payload_json, '$.device_name'),
+                        CAST(json_extract(payload_json, '$.tab_id') AS INTEGER),
+                        CAST(json_extract(payload_json, '$.window_id') AS INTEGER),
+                        CASE
+                            WHEN json_extract(payload_json, '$.incognito') IN (1, '1', 'true', 'TRUE') THEN 1
+                            WHEN json_extract(payload_json, '$.incognito') IN (0, '0', 'false', 'FALSE') THEN 0
+                            ELSE NULL
+                        END,
+                        json_extract(payload_json, '$.transition_type'),
+                        CASE
+                            WHEN json_extract(payload_json, '$.pinned') IN (1, '1', 'true', 'TRUE') THEN 1
+                            WHEN json_extract(payload_json, '$.pinned') IN (0, '0', 'false', 'FALSE') THEN 0
+                            ELSE NULL
+                        END,
+                        CASE
+                            WHEN json_extract(payload_json, '$.audible') IN (1, '1', 'true', 'TRUE') THEN 1
+                            WHEN json_extract(payload_json, '$.audible') IN (0, '0', 'false', 'FALSE') THEN 0
+                            ELSE NULL
+                        END,
+                        CASE
+                            WHEN json_extract(payload_json, '$.muted') IN (1, '1', 'true', 'TRUE') THEN 1
+                            WHEN json_extract(payload_json, '$.muted') IN (0, '0', 'false', 'FALSE') THEN 0
+                            ELSE NULL
+                        END,
+                        CAST(json_extract(payload_json, '$.opener_tab_id') AS INTEGER),
+                        json_extract(payload_json, '$.referred_by'),
+                        COALESCE(created_at, datetime('now'))
+                    FROM {table_name}
+                """)
+                self.conn.execute(f"DROP TABLE {table_name}")
+                self.conn.execute(f"ALTER TABLE {tmp_table} RENAME TO {table_name}")
+                post_count = self.conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0]
+                logger.info(
+                    "[PIPELINE:RAW] Migrated %s to normalized schema: rows_before=%d rows_after=%d",
+                    table_name,
+                    pre_count,
+                    post_count,
+                )
+
+        self.conn.execute(f"""
+            CREATE INDEX IF NOT EXISTS idx_{table_name}_source_system
+            ON {table_name}(source_system)
+        """)
+        self.conn.execute(f"""
+            CREATE INDEX IF NOT EXISTS idx_{table_name}_created_at
+            ON {table_name}(created_at)
+        """)
+        self.conn.execute(f"""
+            CREATE INDEX IF NOT EXISTS idx_{table_name}_visited_at
+            ON {table_name}(visited_at)
+        """)
+        self.conn.execute(f"""
+            CREATE INDEX IF NOT EXISTS idx_{table_name}_url
+            ON {table_name}(url)
+        """)
+        self.conn.commit()
+    
+    def write_raw_record(
+        self,
+        source_id: str,
+        source_record_id: str,
+        payload: Dict[str, Any],
+        source_type: str = "chat_messages",
+    ) -> None:
+        """Write raw record to raw table.
+        
+        Args:
+            source_id: Source identifier
+            source_record_id: Unique record ID within source
+            payload: Original payload dictionary
+            source_type: Type of data ("chat_messages", "events", etc.)
+        """
+        table_name = self.get_raw_table_name(source_id, source_type)
+        self.ensure_raw_table(table_name)
+        
+        try:
+            if table_name == "raw_chat_messages_browservisits":
+                self.conn.execute(f"""
+                    INSERT OR REPLACE INTO {table_name}
+                    (
+                        source_system, source_record_id, record_id, dataset_id, url, visited_at, title,
+                        favicon_url, hostname, device_name, tab_id, window_id, incognito, transition_type,
+                        pinned, audible, muted, opener_tab_id, referred_by, created_at
+                    )
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
+                """, (
+                    source_id,
+                    source_record_id,
+                    payload.get("record_id") or source_record_id,
+                    payload.get("dataset_id"),
+                    payload.get("url"),
+                    payload.get("visited_at"),
+                    payload.get("title"),
+                    payload.get("favicon_url"),
+                    payload.get("hostname"),
+                    payload.get("device_name"),
+                    payload.get("tab_id") if isinstance(payload.get("tab_id"), int) else None,
+                    payload.get("window_id") if isinstance(payload.get("window_id"), int) else None,
+                    1 if payload.get("incognito") is True else (0 if payload.get("incognito") is False else None),
+                    payload.get("transition_type"),
+                    1 if payload.get("pinned") is True else (0 if payload.get("pinned") is False else None),
+                    1 if payload.get("audible") is True else (0 if payload.get("audible") is False else None),
+                    1 if payload.get("muted") is True else (0 if payload.get("muted") is False else None),
+                    payload.get("opener_tab_id") if isinstance(payload.get("opener_tab_id"), int) else None,
+                    payload.get("referred_by"),
+                ))
+                self.conn.commit()
+                return
+
+            # Store payload as JSON string
+            payload_json = json.dumps(payload, ensure_ascii=False)
+            
+            self.conn.execute(f"""
+                INSERT OR REPLACE INTO {table_name} 
+                (source_system, source_record_id, payload_json, created_at)
+                VALUES (?, ?, ?, datetime('now'))
+            """, (source_id, source_record_id, payload_json))
+            
+            self.conn.commit()
+        except Exception as e:
+            self.conn.rollback()
+            logger.error(
+                "[PIPELINE:RAW] Failed to store raw record: source=%s, record_id=%s, error=%s",
+                source_id,
+                source_record_id,
+                e,
+            )
+            raise

topos/storage/raw/sqlite_raw_store.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from typing import Dict
+
+from .raw_store import RawRecordRef, RawStore
+
+
+class SQLiteRawStore(RawStore):
+    def __init__(self, db_conn):
+        self.db_conn = db_conn
+
+    def write_file(self, file):  # pragma: no cover - not used for sqlite raw store
+        raise NotImplementedError
+
+    def write_record(self, record: Dict[str, str]) -> RawRecordRef:
+        _ = record
+        raise NotImplementedError("SQLiteRawStore not implemented yet")

topos/storage/security/encryption.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+
+class EncryptionManager:
+    """No-op encryption manager placeholder."""
+
+    def __init__(self, dataset_id: str, user_id: str):
+        self.dataset_id = dataset_id
+        self.user_id = user_id
+        self._crypto_version = 1
+
+    def get_crypto_version(self) -> int:
+        return self._crypto_version
+
+    def encrypt_str(self, payload: str, crypto_version: int | None = None) -> str:
+        _ = crypto_version
+        return payload
+
+    def decrypt_str(self, ciphertext: str, crypto_version: int | None = None) -> str:
+        _ = crypto_version
+        return ciphertext

topos/storage/signal_identity.py

@@ -0,0 +1,71 @@
+"""Signal identity storage: my_phone_number (and optional my_signal_id) per dataset.
+
+Used to set sender_type (self vs contact) and owner when ingesting Signal messages.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+logger = logging.getLogger("topos.storage.signal_identity")
+
+TABLE = "signal_identity"
+
+
+def ensure_table(conn) -> None:
+    """Create signal_identity table if not exists."""
+    conn.execute(f"""
+        CREATE TABLE IF NOT EXISTS {TABLE} (
+            dataset_id TEXT NOT NULL PRIMARY KEY,
+            my_phone_number TEXT,
+            my_signal_id TEXT,
+            updated_at TEXT DEFAULT (datetime('now'))
+        )
+    """)
+    conn.commit()
+
+
+def get_signal_identity(conn, dataset_id: str) -> Optional[dict]:
+    """Return { my_phone_number, my_signal_id } for dataset_id, or None."""
+    if not conn or not dataset_id:
+        return None
+    try:
+        ensure_table(conn)
+        row = conn.execute(
+            f"SELECT my_phone_number, my_signal_id FROM {TABLE} WHERE dataset_id = ?",
+            (dataset_id,),
+        ).fetchone()
+        if not row:
+            return None
+        return {"my_phone_number": row[0], "my_signal_id": row[1]}
+    except Exception as e:
+        logger.warning("get_signal_identity failed: %s", e)
+        return None
+
+
+def put_signal_identity(
+    conn,
+    dataset_id: str,
+    *,
+    my_phone_number: Optional[str] = None,
+    my_signal_id: Optional[str] = None,
+) -> None:
+    """Set Signal identity for dataset_id. Pass None to leave a field unchanged."""
+    if not conn or not dataset_id:
+        return
+    try:
+        ensure_table(conn)
+        existing = get_signal_identity(conn, dataset_id)
+        phone = my_phone_number if my_phone_number is not None else (existing.get("my_phone_number") if existing else None)
+        sid = my_signal_id if my_signal_id is not None else (existing.get("my_signal_id") if existing else None)
+        conn.execute(
+            f"""
+            INSERT OR REPLACE INTO {TABLE} (dataset_id, my_phone_number, my_signal_id, updated_at)
+            VALUES (?, ?, ?, datetime('now'))
+            """,
+            (dataset_id, phone, sid),
+        )
+        conn.commit()
+    except Exception as e:
+        logger.warning("put_signal_identity failed: %s", e)

topos/storage/source_settings.py

@@ -0,0 +1,116 @@
+"""Per-source settings: enabled, last_sync_at, last_error (for local_sync sources)."""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+logger = logging.getLogger("topos.storage.source_settings")
+
+TABLE = "user_ingestion_sources"
+
+
+def ensure_table(conn) -> None:
+    conn.execute(f"""
+        CREATE TABLE IF NOT EXISTS {TABLE} (
+            dataset_id TEXT NOT NULL,
+            source_id TEXT NOT NULL,
+            enabled INTEGER NOT NULL DEFAULT 1,
+            last_sync_at TEXT,
+            last_error TEXT,
+            updated_at TEXT DEFAULT (datetime('now')),
+            PRIMARY KEY (dataset_id, source_id)
+        )
+    """)
+    conn.commit()
+
+
+def get_source_settings(conn, dataset_id: str, source_id: str) -> Optional[dict]:
+    """Return { enabled, last_sync_at, last_error } or None (then defaults: enabled true, no last_*)."""
+    if not conn or not dataset_id or not source_id:
+        return None
+    try:
+        ensure_table(conn)
+        row = conn.execute(
+            f"SELECT enabled, last_sync_at, last_error FROM {TABLE} WHERE dataset_id = ? AND source_id = ?",
+            (dataset_id, source_id),
+        ).fetchone()
+        if not row:
+            return {"enabled": True, "last_sync_at": None, "last_error": None}
+        return {"enabled": bool(row[0]), "last_sync_at": row[1], "last_error": row[2]}
+    except Exception as e:
+        logger.warning("get_source_settings failed: %s", e)
+        return {"enabled": True, "last_sync_at": None, "last_error": None}
+
+
+def put_source_settings(
+    conn,
+    dataset_id: str,
+    source_id: str,
+    *,
+    enabled: Optional[bool] = None,
+) -> None:
+    """Update enabled; leave last_sync_at/last_error unchanged."""
+    if not conn or not dataset_id or not source_id or enabled is None:
+        return
+    try:
+        ensure_table(conn)
+        cur = conn.execute(
+            f"SELECT 1 FROM {TABLE} WHERE dataset_id = ? AND source_id = ?",
+            (dataset_id, source_id),
+        ).fetchone()
+        if cur:
+            conn.execute(
+                f"UPDATE {TABLE} SET enabled = ?, updated_at = datetime('now') WHERE dataset_id = ? AND source_id = ?",
+                (1 if enabled else 0, dataset_id, source_id),
+            )
+        else:
+            conn.execute(
+                f"INSERT INTO {TABLE} (dataset_id, source_id, enabled, updated_at) VALUES (?, ?, ?, datetime('now'))",
+                (dataset_id, source_id, 1 if enabled else 0),
+            )
+        conn.commit()
+    except Exception as e:
+        logger.warning("put_source_settings failed: %s", e)
+
+
+def update_sync_result(
+    conn,
+    dataset_id: str,
+    source_id: str,
+    *,
+    success: bool,
+    last_sync_at: Optional[str] = None,
+    last_error: Optional[str] = None,
+) -> None:
+    """After sync: set last_sync_at (and clear last_error) on success, or set last_error on failure."""
+    if not conn or not dataset_id or not source_id:
+        return
+    try:
+        ensure_table(conn)
+        if success:
+            conn.execute(
+                f"""
+                INSERT INTO {TABLE} (dataset_id, source_id, enabled, last_sync_at, last_error, updated_at)
+                VALUES (?, ?, 1, ?, NULL, datetime('now'))
+                ON CONFLICT(dataset_id, source_id) DO UPDATE SET
+                    last_sync_at = ?,
+                    last_error = NULL,
+                    updated_at = datetime('now')
+                """,
+                (dataset_id, source_id, last_sync_at or "", last_sync_at or ""),
+            )
+        else:
+            conn.execute(
+                f"""
+                INSERT INTO {TABLE} (dataset_id, source_id, enabled, last_sync_at, last_error, updated_at)
+                VALUES (?, ?, 1, NULL, ?, datetime('now'))
+                ON CONFLICT(dataset_id, source_id) DO UPDATE SET
+                    last_error = ?,
+                    updated_at = datetime('now')
+                """,
+                (dataset_id, source_id, last_error or "", last_error or ""),
+            )
+        conn.commit()
+    except Exception as e:
+        logger.warning("update_sync_result failed: %s", e)

topos/storage/user_identity.py

@@ -0,0 +1,69 @@
+"""Dataset-scoped owner identity storage for self-name resolution."""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+logger = logging.getLogger("topos.storage.user_identity")
+
+TABLE = "user_identity"
+
+
+def ensure_table(conn) -> None:
+    """Create ``user_identity`` table if it does not exist."""
+    conn.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS {TABLE} (
+            dataset_id TEXT NOT NULL PRIMARY KEY,
+            display_name TEXT,
+            updated_at TEXT DEFAULT (datetime('now'))
+        )
+        """
+    )
+    conn.commit()
+
+
+def get_user_identity(conn, dataset_id: str) -> Optional[dict]:
+    """Return ``{display_name}`` for a dataset, or ``None`` when absent."""
+    if not conn or not dataset_id:
+        return None
+    try:
+        ensure_table(conn)
+        row = conn.execute(
+            f"SELECT display_name FROM {TABLE} WHERE dataset_id = ?",
+            (dataset_id,),
+        ).fetchone()
+        if not row:
+            return None
+        return {"display_name": row[0]}
+    except Exception as e:
+        logger.warning("get_user_identity failed: %s", e)
+        return None
+
+
+def put_user_identity(
+    conn,
+    dataset_id: str,
+    *,
+    display_name: Optional[str] = None,
+) -> None:
+    """Set dataset-scoped owner identity. ``None`` leaves the field unchanged."""
+    if not conn or not dataset_id:
+        return
+    try:
+        ensure_table(conn)
+        existing = get_user_identity(conn, dataset_id)
+        next_display_name = (
+            display_name if display_name is not None else (existing.get("display_name") if existing else None)
+        )
+        conn.execute(
+            f"""
+            INSERT OR REPLACE INTO {TABLE} (dataset_id, display_name, updated_at)
+            VALUES (?, ?, datetime('now'))
+            """,
+            (dataset_id, next_display_name),
+        )
+        conn.commit()
+    except Exception as e:
+        logger.warning("put_user_identity failed: %s", e)

topos/sync/__init__.py

@@ -0,0 +1,5 @@
+"""Sync layer for Topos."""
+
+from .client import SyncClient
+
+__all__ = ["SyncClient"]