topos-node 0.1.0__py3-none-any.whl

topos/storage/raw/browser_flat_tables.py

@@ -0,0 +1,303 @@
+"""Flat tables for browser plugin data: one row per event, one column per field.
+
+Stored in SQLite with explicit columns so DuckDB (or any SQL engine) can query
+without parsing JSON. Raw format = one row per event; good rows = flat columns
+for analytics (e.g. SELECT url, visited_at, hostname FROM browser_visits).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger("topos.storage.raw.browser_flat_tables")
+
+BROWSER_VISITS_TABLE = "browser_visits"
+BROWSER_EVENTS_TABLE = "browser_events"
+BROWSER_URL_CLASSIFICATION_TABLE = "browser_url_classification"
+# Normalized raw retention table for browser_visits (architecture layer).
+RAW_BROWSER_VISITS_TABLE = "raw_chat_messages_browservisits"
+
+
+def _to_sql_value(val: Any) -> Optional[str]:
+    """Convert Python value to SQL-friendly value (string or None)."""
+    if val is None:
+        return None
+    if isinstance(val, bool):
+        return "1" if val else "0"
+    if isinstance(val, (dict, list)):
+        return json.dumps(val, ensure_ascii=False)
+    return str(val)
+
+
+def ensure_browser_visits_table(conn) -> None:
+    """Create browser_visits table with flat columns if not exists."""
+    conn.execute(f"""
+        CREATE TABLE IF NOT EXISTS {BROWSER_VISITS_TABLE} (
+            record_id TEXT PRIMARY KEY,
+            dataset_id TEXT,
+            url TEXT NOT NULL,
+            visited_at TEXT NOT NULL,
+            title TEXT,
+            favicon_url TEXT,
+            hostname TEXT,
+            device_name TEXT,
+            tab_id INTEGER,
+            window_id INTEGER,
+            incognito INTEGER,
+            transition_type TEXT,
+            pinned INTEGER,
+            audible INTEGER,
+            muted INTEGER,
+            opener_tab_id INTEGER,
+            referred_by TEXT,
+            created_at TEXT DEFAULT (datetime('now'))
+        )
+    """)
+    conn.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_{BROWSER_VISITS_TABLE}_visited_at
+        ON {BROWSER_VISITS_TABLE}(visited_at)
+    """)
+    conn.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_{BROWSER_VISITS_TABLE}_hostname
+        ON {BROWSER_VISITS_TABLE}(hostname)
+    """)
+    conn.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_{BROWSER_VISITS_TABLE}_transition_type
+        ON {BROWSER_VISITS_TABLE}(transition_type)
+    """)
+    conn.commit()
+    logger.debug("Ensured table %s exists", BROWSER_VISITS_TABLE)
+
+
+def ensure_browser_events_table(conn) -> None:
+    """Create browser_events table with flat columns if not exists."""
+    conn.execute(f"""
+        CREATE TABLE IF NOT EXISTS {BROWSER_EVENTS_TABLE} (
+            record_id TEXT PRIMARY KEY,
+            dataset_id TEXT,
+            event_type TEXT NOT NULL,
+            url TEXT,
+            visited_at TEXT,
+            title TEXT,
+            favicon_url TEXT,
+            hostname TEXT,
+            device_name TEXT,
+            transition_type TEXT,
+            content TEXT,
+            tab_id INTEGER,
+            window_id INTEGER,
+            incognito INTEGER,
+            pinned INTEGER,
+            audible INTEGER,
+            muted INTEGER,
+            opener_tab_id INTEGER,
+            starred_at TEXT,
+            created_at TEXT DEFAULT (datetime('now'))
+        )
+    """)
+    conn.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_{BROWSER_EVENTS_TABLE}_event_type
+        ON {BROWSER_EVENTS_TABLE}(event_type)
+    """)
+    conn.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_{BROWSER_EVENTS_TABLE}_visited_at
+        ON {BROWSER_EVENTS_TABLE}(visited_at)
+    """)
+    conn.commit()
+    logger.debug("Ensured table %s exists", BROWSER_EVENTS_TABLE)
+
+
+def ensure_browser_url_classification_table(conn) -> None:
+    """Create browser URL classification table for enrichment output. Stage 9: enriched_from_table."""
+    conn.execute(f"""
+        CREATE TABLE IF NOT EXISTS {BROWSER_URL_CLASSIFICATION_TABLE} (
+            enriched_from_table TEXT NOT NULL,
+            record_id TEXT NOT NULL,
+            dataset_id TEXT,
+            url TEXT NOT NULL,
+            title TEXT,
+            url_category TEXT,
+            url_confidence REAL,
+            model_name TEXT,
+            created_at TEXT DEFAULT (datetime('now')),
+            updated_at TEXT DEFAULT (datetime('now')),
+            PRIMARY KEY (enriched_from_table, record_id)
+        )
+    """)
+    conn.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_{BROWSER_URL_CLASSIFICATION_TABLE}_category
+        ON {BROWSER_URL_CLASSIFICATION_TABLE}(url_category)
+    """)
+    conn.execute(f"""
+        CREATE INDEX IF NOT EXISTS idx_{BROWSER_URL_CLASSIFICATION_TABLE}_dataset
+        ON {BROWSER_URL_CLASSIFICATION_TABLE}(dataset_id)
+    """)
+    conn.commit()
+    logger.debug("Ensured table %s exists", BROWSER_URL_CLASSIFICATION_TABLE)
+
+
+def backfill_browser_visits_from_raw_retention(conn) -> int:
+    """Copy visits from raw retention into browser_visits when the flat table is empty.
+
+    Fresh Topos installs ingest into raw_chat_messages_browservisits first; this
+    one-time catch-up makes browser_visits visible in Data Explorer without manual SQL.
+    """
+    ensure_browser_visits_table(conn)
+    flat_count = conn.execute(f"SELECT COUNT(*) FROM {BROWSER_VISITS_TABLE}").fetchone()[0]
+    if flat_count:
+        return 0
+    raw_exists = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
+        (RAW_BROWSER_VISITS_TABLE,),
+    ).fetchone()
+    if not raw_exists:
+        return 0
+    raw_count = conn.execute(f"SELECT COUNT(*) FROM {RAW_BROWSER_VISITS_TABLE}").fetchone()[0]
+    if not raw_count:
+        return 0
+    conn.execute(f"""
+        INSERT OR REPLACE INTO {BROWSER_VISITS_TABLE}
+        (record_id, dataset_id, url, visited_at, title, favicon_url, hostname, device_name,
+         tab_id, window_id, incognito, transition_type, pinned, audible, muted, opener_tab_id, referred_by)
+        SELECT
+            COALESCE(NULLIF(TRIM(record_id), ''), source_record_id),
+            dataset_id,
+            url,
+            visited_at,
+            title,
+            favicon_url,
+            hostname,
+            device_name,
+            tab_id,
+            window_id,
+            incognito,
+            transition_type,
+            pinned,
+            audible,
+            muted,
+            opener_tab_id,
+            referred_by
+        FROM {RAW_BROWSER_VISITS_TABLE}
+        WHERE url IS NOT NULL AND TRIM(url) != ''
+    """)
+    conn.commit()
+    copied = conn.execute(f"SELECT COUNT(*) FROM {BROWSER_VISITS_TABLE}").fetchone()[0]
+    logger.info(
+        "[PIPELINE:RAW] Backfilled %d browser visit row(s) from %s into %s",
+        copied,
+        RAW_BROWSER_VISITS_TABLE,
+        BROWSER_VISITS_TABLE,
+    )
+    return int(copied)
+
+
+def write_browser_visit(conn, payload: Dict[str, Any]) -> None:
+    """Insert or replace one row in browser_visits (flat columns)."""
+    ensure_browser_visits_table(conn)
+    record_id = payload.get("record_id") or ""
+    url = _to_sql_value(payload.get("url")) or ""
+    visited_at = _to_sql_value(payload.get("visited_at")) or ""
+    conn.execute(f"""
+        INSERT OR REPLACE INTO {BROWSER_VISITS_TABLE}
+        (record_id, dataset_id, url, visited_at, title, favicon_url, hostname, device_name,
+         tab_id, window_id, incognito, transition_type, pinned, audible, muted, opener_tab_id, referred_by)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    """, (
+        record_id,
+        _to_sql_value(payload.get("dataset_id")),
+        url,
+        visited_at,
+        _to_sql_value(payload.get("title")),
+        _to_sql_value(payload.get("favicon_url")),
+        _to_sql_value(payload.get("hostname")),
+        _to_sql_value(payload.get("device_name")),
+        payload.get("tab_id") if isinstance(payload.get("tab_id"), (int, type(None))) else None,
+        payload.get("window_id") if isinstance(payload.get("window_id"), (int, type(None))) else None,
+        1 if payload.get("incognito") is True else (0 if payload.get("incognito") is False else None),
+        _to_sql_value(payload.get("transition_type")),
+        1 if payload.get("pinned") is True else (0 if payload.get("pinned") is False else None),
+        1 if payload.get("audible") is True else (0 if payload.get("audible") is False else None),
+        1 if payload.get("muted") is True else (0 if payload.get("muted") is False else None),
+        payload.get("opener_tab_id") if isinstance(payload.get("opener_tab_id"), (int, type(None))) else None,
+        _to_sql_value(payload.get("referred_by")),
+    ))
+    conn.commit()
+    logger.debug("[PIPELINE:RAW] Wrote flat row to %s: record_id=%s", BROWSER_VISITS_TABLE, record_id[:24] if record_id else None)
+
+
+def write_browser_event(conn, payload: Dict[str, Any]) -> None:
+    """Insert or replace one row in browser_events (flat columns)."""
+    ensure_browser_events_table(conn)
+    record_id = payload.get("record_id") or ""
+    event_type = _to_sql_value(payload.get("event_type")) or "unknown"
+    conn.execute(f"""
+        INSERT OR REPLACE INTO {BROWSER_EVENTS_TABLE}
+        (record_id, dataset_id, event_type, url, visited_at, title, favicon_url, hostname, device_name,
+         transition_type, content, tab_id, window_id, incognito, pinned, audible, muted, opener_tab_id, starred_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    """, (
+        record_id,
+        _to_sql_value(payload.get("dataset_id")),
+        event_type,
+        _to_sql_value(payload.get("url")),
+        _to_sql_value(payload.get("visited_at")),
+        _to_sql_value(payload.get("title")),
+        _to_sql_value(payload.get("favicon_url")),
+        _to_sql_value(payload.get("hostname")),
+        _to_sql_value(payload.get("device_name")),
+        _to_sql_value(payload.get("transition_type")),
+        _to_sql_value(payload.get("content")),
+        payload.get("tab_id") if isinstance(payload.get("tab_id"), (int, type(None))) else None,
+        payload.get("window_id") if isinstance(payload.get("window_id"), (int, type(None))) else None,
+        1 if payload.get("incognito") is True else (0 if payload.get("incognito") is False else None),
+        1 if payload.get("pinned") is True else (0 if payload.get("pinned") is False else None),
+        1 if payload.get("audible") is True else (0 if payload.get("audible") is False else None),
+        1 if payload.get("muted") is True else (0 if payload.get("muted") is False else None),
+        payload.get("opener_tab_id") if isinstance(payload.get("opener_tab_id"), (int, type(None))) else None,
+        _to_sql_value(payload.get("starred_at")),
+    ))
+    conn.commit()
+    logger.debug("[PIPELINE:RAW] Wrote flat row to %s: record_id=%s event_type=%s", BROWSER_EVENTS_TABLE, record_id[:24] if record_id else None, event_type)
+
+
+def write_browser_url_classification(
+    conn,
+    *,
+    source_table: str,
+    record_id: str,
+    dataset_id: Optional[str],
+    url: str,
+    title: Optional[str],
+    category: Optional[str],
+    confidence: Optional[float],
+    model_name: Optional[str],
+    ensure_table: bool = True,
+    log_write: bool = True,
+) -> None:
+    """Insert or replace one URL classification enrichment row."""
+    if ensure_table:
+        ensure_browser_url_classification_table(conn)
+    conn.execute(f"""
+        INSERT OR REPLACE INTO {BROWSER_URL_CLASSIFICATION_TABLE}
+        (enriched_from_table, record_id, dataset_id, url, title, url_category, url_confidence, model_name, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
+    """, (
+        _to_sql_value(source_table),
+        _to_sql_value(record_id),
+        _to_sql_value(dataset_id),
+        _to_sql_value(url),
+        _to_sql_value(title),
+        _to_sql_value(category),
+        confidence,
+        _to_sql_value(model_name),
+    ))
+    conn.commit()
+    if log_write:
+        logger.debug(
+            "[PIPELINE:RAW] Wrote URL classification row: source=%s record_id=%s category=%s",
+            source_table,
+            record_id[:24] if record_id else None,
+            category,
+        )

topos/storage/raw/file_store.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import logging
+import shutil
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from .raw_store import RawFile, RawFileRef
+
+logger = logging.getLogger("topos.storage.raw.file_store")
+
+
+@dataclass(frozen=True)
+class RawFileStore:
+    base_path: Path
+
+    def __init__(self, base_path: Optional[Path] = None):
+        env_override = os.getenv("TOPOS_INGESTION_BASE_PATH")
+        resolved_base = base_path or (Path(env_override) if env_override else Path.home() / ".topos" / "ingestion")
+        object.__setattr__(self, "base_path", resolved_base)
+        self.base_path.mkdir(parents=True, exist_ok=True)
+
+    def get_file_path(self, dataset_id: str, schema_id: str) -> Path:
+        safe_dataset_id = dataset_id.replace(":", "_").replace("/", "_")
+        safe_schema_id = schema_id.replace(".", "_").replace("/", "_")
+        dataset_dir = self.base_path / safe_dataset_id
+        dataset_dir.mkdir(parents=True, exist_ok=True)
+        return dataset_dir / f"{safe_schema_id}.jsonl"
+
+    def write_file(self, raw_file: RawFile) -> RawFileRef:
+        destination = self.get_file_path(
+            raw_file.metadata.get("dataset_id", "unknown"),
+            raw_file.metadata.get("schema_id", "unknown"),
+        )
+        source_path = Path(raw_file.file_path)
+        if source_path.resolve() == destination.resolve():
+            return RawFileRef(file_id=destination.stem, file_path=str(destination))
+        if destination.exists():
+            backup = destination.with_suffix(".jsonl.backup")
+            shutil.copy2(destination, backup)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(raw_file.file_path, destination)
+        logger.info("Saved raw file: %s", destination)
+        return RawFileRef(file_id=destination.stem, file_path=str(destination))
+
+    def write_bytes(self, dataset_id: str, schema_id: str, payload: bytes) -> RawFileRef:
+        destination = self.get_file_path(dataset_id, schema_id)
+        if destination.exists():
+            backup = destination.with_suffix(".jsonl.backup")
+            shutil.copy2(destination, backup)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_bytes(payload)
+        logger.info("Saved raw file bytes: %s", destination)
+        return RawFileRef(file_id=destination.stem, file_path=str(destination))
+
+    def append_record(self, dataset_id: str, schema_id: str, record: dict) -> RawFileRef:
+        destination = self.get_file_path(dataset_id, schema_id)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        with destination.open("a", encoding="utf-8") as handle:
+            handle.write(json.dumps(record))
+            handle.write("\n")
+        return RawFileRef(file_id=destination.stem, file_path=str(destination))
+
+    def list_datasets(self) -> list[dict]:
+        """List all datasets with their file stats."""
+        datasets = []
+        if not self.base_path.exists():
+            return datasets
+        for dataset_dir in self.base_path.iterdir():
+            if not dataset_dir.is_dir():
+                continue
+            dataset_id = dataset_dir.name.replace("_", ":")
+            total_size = 0
+            message_count = 0
+            schemas = []
+            for file_path in dataset_dir.glob("*.jsonl"):
+                if file_path.name.endswith(".backup"):
+                    continue
+                file_size = file_path.stat().st_size
+                total_size += file_size
+                schema_id = file_path.stem.replace("_", ".")
+                # Count messages in file
+                try:
+                    with file_path.open("r", encoding="utf-8") as f:
+                        for _ in f:
+                            message_count += 1
+                except Exception:
+                    pass
+                schemas.append({"schema_id": schema_id, "file_size": file_size})
+            if total_size > 0 or message_count > 0:
+                datasets.append({
+                    "dataset_id": dataset_id,
+                    "total_size": total_size,
+                    "message_count": message_count,
+                    "schemas": schemas,
+                })
+        return datasets

topos/storage/raw/raw_store.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict
+
+
+@dataclass(frozen=True)
+class RawFile:
+    file_path: str
+    metadata: Dict[str, str]
+
+
+@dataclass(frozen=True)
+class RawFileRef:
+    file_id: str
+    file_path: str
+
+
+@dataclass(frozen=True)
+class RawRecordRef:
+    record_id: str
+
+
+class RawStore:
+    def write_file(self, file: RawFile) -> RawFileRef:
+        raise NotImplementedError
+
+    def write_record(self, record: Dict[str, str]) -> RawRecordRef:
+        raise NotImplementedError