topos-node 0.1.0__py3-none-any.whl

topos/canonicalization/mappers/__init__.py

@@ -0,0 +1,22 @@
+"""Canonical mapper registry."""
+
+from .base import CanonicalMapper
+from .chatgpt_mapper import ChatGPTCanonicalMapper
+from .grok_mapper import GrokCanonicalMapper
+from .messenger_mapper import ImessageCanonicalMapper, SignalCanonicalMapper
+
+MAPPER_REGISTRY = {
+    "chatgpt": ChatGPTCanonicalMapper,
+    "grok": GrokCanonicalMapper,
+    "imessage": ImessageCanonicalMapper,  # Sprint 02: conversations canonical group
+    "signal": SignalCanonicalMapper,
+}
+
+__all__ = [
+    "CanonicalMapper",
+    "ChatGPTCanonicalMapper",
+    "GrokCanonicalMapper",
+    "ImessageCanonicalMapper",
+    "SignalCanonicalMapper",
+    "MAPPER_REGISTRY",
+]

topos/canonicalization/mappers/base.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict
+
+from ...ingestion.parsers.base import NormalizedRecord
+
+
+@dataclass(frozen=True)
+class CanonicalRecord:
+    record_id: str
+    payload: Dict[str, str]
+
+
+@dataclass(frozen=True)
+class MappingMetadata:
+    source_id: str
+    mapping_version: str
+
+
+class CanonicalMapper:
+    def map(self, normalized: NormalizedRecord) -> CanonicalRecord:
+        raise NotImplementedError
+
+    def mapping_metadata(self, normalized: NormalizedRecord) -> MappingMetadata:
+        raise NotImplementedError

topos/canonicalization/mappers/chatgpt_mapper.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+from ...ingestion.parsers.base import NormalizedRecord
+from ..models import CanonicalMessage
+from .base import CanonicalMapper, CanonicalRecord, MappingMetadata
+
+
+@dataclass
+class ChatGPTCanonicalMapper(CanonicalMapper):
+    version: str = "v1"
+
+    def map(self, normalized: NormalizedRecord) -> CanonicalRecord:
+        payload = normalized.payload
+        content = payload.get("content", "")
+        content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
+        message_id = payload.get("message_id", normalized.record_id)
+        
+        # Preserve _metadata for conversation tree reconstruction
+        metadata = {"mapper_version": self.version}
+        if "_metadata" in payload:
+            # Merge _metadata into metadata (preserves parent_id, node_id, etc.)
+            metadata.update(payload["_metadata"])
+        
+        canonical = CanonicalMessage(
+            message_id=message_id,
+            conversation_id=payload.get("thread_id", ""),
+            sender_type=payload.get("sender_type", ""),
+            content=content,
+            ts=payload.get("ts"),
+            source_id="chatgpt",
+            content_hash=content_hash,
+            metadata=metadata,
+        )
+        return CanonicalRecord(record_id=canonical.message_id, payload=canonical.__dict__)
+
+    def mapping_metadata(self, normalized: NormalizedRecord) -> MappingMetadata:
+        return MappingMetadata(source_id="chatgpt", mapping_version=self.version)

topos/canonicalization/mappers/grok_mapper.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ...ingestion.parsers.base import NormalizedRecord
+from .base import CanonicalMapper, CanonicalRecord, MappingMetadata
+
+
+@dataclass
+class GrokCanonicalMapper(CanonicalMapper):
+    version: str = "v1"
+
+    def map(self, normalized: NormalizedRecord) -> CanonicalRecord:
+        return CanonicalRecord(record_id=normalized.record_id, payload=normalized.payload)
+
+    def mapping_metadata(self, normalized: NormalizedRecord) -> MappingMetadata:
+        return MappingMetadata(source_id="grok", mapping_version=self.version)

topos/canonicalization/mappers/messenger_mapper.py

@@ -0,0 +1,58 @@
+"""Canonical mappers for messenger sources (imessage, signal).
+
+Produce the shape expected by conversation_messages. The ingestion pipeline
+routes canonical_group_id=conversations to ConversationsTablesManager and
+builds staging records from normalized payloads directly; these mappers
+are for registry completeness and any code that looks up imessage/signal by mapper_id.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ...ingestion.parsers.base import NormalizedRecord
+from .base import CanonicalMapper, CanonicalRecord, MappingMetadata
+
+
+def _normalized_to_canonical_payload(normalized: NormalizedRecord, source_id: str) -> dict:
+    """Convert normalized messenger record to canonical message shape for conversation_messages."""
+    p = normalized.payload
+    payload = {
+        "message_id": p.get("message_id", normalized.record_id),
+        "conversation_id": p.get("thread_id") or p.get("conversation_id") or "",
+        "sender_type": p.get("sender_type", "human"),
+        "sender_id": p.get("sender_id"),
+        "reply_to_message_id": p.get("reply_to_message_id"),
+        "message_type": p.get("message_type"),
+        "event_type": p.get("event_type"),
+        "content": p.get("content", ""),
+        "ts": p.get("ts", ""),
+        "source_id": source_id,
+    }
+    if "_metadata" in p:
+        payload["_metadata"] = p["_metadata"]
+    return payload
+
+
+@dataclass
+class ImessageCanonicalMapper(CanonicalMapper):
+    """Maps normalized iMessage record to canonical shape for conversation_messages."""
+
+    def map(self, normalized: NormalizedRecord) -> CanonicalRecord:
+        payload = _normalized_to_canonical_payload(normalized, "imessage")
+        return CanonicalRecord(record_id=payload["message_id"], payload=payload)
+
+    def mapping_metadata(self, normalized: NormalizedRecord) -> MappingMetadata:
+        return MappingMetadata(source_id="imessage", mapping_version="v1")
+
+
+@dataclass
+class SignalCanonicalMapper(CanonicalMapper):
+    """Maps normalized Signal record to canonical shape for conversation_messages."""
+
+    def map(self, normalized: NormalizedRecord) -> CanonicalRecord:
+        payload = _normalized_to_canonical_payload(normalized, "signal")
+        return CanonicalRecord(record_id=payload["message_id"], payload=payload)
+
+    def mapping_metadata(self, normalized: NormalizedRecord) -> MappingMetadata:
+        return MappingMetadata(source_id="signal", mapping_version="v1")

topos/canonicalization/models.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Optional
+
+
+@dataclass(frozen=True)
+class CanonicalConversation:
+    conversation_id: str
+    source_id: str
+    metadata: Dict[str, str]
+
+
+@dataclass(frozen=True)
+class CanonicalParticipant:
+    participant_id: str
+    conversation_id: str
+    role: str
+    metadata: Dict[str, str]
+
+
+@dataclass(frozen=True)
+class CanonicalMessage:
+    message_id: str
+    conversation_id: str
+    sender_type: str
+    content: str
+    ts: Optional[str]
+    source_id: str
+    content_hash: str
+    metadata: Dict[str, str]

topos/canonicalization/resolver.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from typing import Dict
+
+from .mappers.base import CanonicalRecord
+
+
+def deterministic_id(namespace: str, value: str) -> str:
+    seed = f"{namespace}:{value}".encode("utf-8")
+    return hashlib.sha256(seed).hexdigest()
+
+
+@dataclass
+class CanonicalResolver:
+    """Resolves canonical records and handles collisions (stub)."""
+
+    def resolve_message(self, payload: Dict[str, str]) -> CanonicalRecord:
+        content = payload.get("content", "")
+        source_id = payload.get("source_id", "unknown")
+        message_id = deterministic_id(source_id, content)
+        return CanonicalRecord(record_id=message_id, payload=payload)

topos/cli/__init__.py

@@ -0,0 +1 @@
+"""CLI package for Topos."""

topos/cli/__main__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from .commands import main
+
+if __name__ == "__main__":
+    main()

topos/cli/commands.py

@@ -0,0 +1,132 @@
+"""CLI entry points for Topos."""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+import click
+import uvicorn
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from topos.storage.db.paths import discover_databases
+
+USER_ENV_PATH = Path.home() / ".topos" / ".env"
+LEGACY_ENV_PATH = Path(__file__).resolve().parent.parent / ".env"
+
+
+def _load_env_file(env_path: Path) -> None:
+    if not env_path.exists():
+        return
+    for line in env_path.read_text(encoding="utf-8").splitlines():
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#") or "=" not in stripped:
+            continue
+        key, value = stripped.split("=", 1)
+        key = key.strip()
+        value = value.strip().strip('"').strip("'")
+        if key and key not in os.environ:
+            os.environ[key] = value
+
+
+def _save_topos_key(topos_key: str, env_path: Path = USER_ENV_PATH) -> Path:
+    env_path.parent.mkdir(parents=True, exist_ok=True)
+    lines: list[str] = []
+    if env_path.exists():
+        lines = env_path.read_text(encoding="utf-8").splitlines()
+
+    updated = False
+    new_lines: list[str] = []
+    for line in lines:
+        stripped = line.strip()
+        if stripped.startswith("TOPOS_KEY="):
+            new_lines.append(f"TOPOS_KEY={topos_key}")
+            updated = True
+        else:
+            new_lines.append(line)
+
+    if not updated:
+        if new_lines and new_lines[-1].strip():
+            new_lines.append("")
+        new_lines.append(f"TOPOS_KEY={topos_key}")
+
+    env_path.write_text("\n".join(new_lines) + "\n", encoding="utf-8")
+    try:
+        env_path.chmod(0o600)
+    except OSError:
+        # Best-effort permissions (e.g. may fail on some filesystems).
+        pass
+    return env_path
+
+
+@click.command()
+@click.option(
+    "--db-path",
+    help="Database file path (SQLite). If not specified, uses auto-discovery.",
+)
+@click.option(
+    "--topos-key",
+    help="Topos key for this run (overrides saved key).",
+)
+@click.option(
+    "--set-topos-key",
+    metavar="KEY",
+    help="Save TOPOS_KEY to ~/.topos/.env and exit.",
+)
+@click.option(
+    "--discover",
+    is_flag=True,
+    help="Show discovered databases and exit",
+)
+@click.option(
+    "--port",
+    default=9000,
+    help="Server port (default: 9000)",
+)
+@click.option(
+    "--host",
+    default="0.0.0.0",
+    help="Host to bind to (default: 0.0.0.0)",
+)
+def main(db_path, topos_key, set_topos_key, discover, port, host) -> None:
+    """Topos Control Plane API entry point."""
+    if set_topos_key:
+        env_path = _save_topos_key(set_topos_key)
+        click.echo(f"Saved TOPOS_KEY to {env_path}")
+        click.echo("You can now run: topos-node")
+        return
+
+    if discover:
+        databases = discover_databases()
+        if databases:
+            click.echo("Discovered databases:")
+            for db in databases:
+                click.echo(f"  - {db}")
+        else:
+            click.echo("No existing databases found")
+        return
+
+    _load_env_file(USER_ENV_PATH)
+    _load_env_file(LEGACY_ENV_PATH)
+
+    if topos_key:
+        os.environ["TOPOS_KEY"] = topos_key
+    elif not os.getenv("TOPOS_KEY"):
+        os.environ["TOPOS_KEY"] = "dev-key"
+        click.echo("TOPOS_KEY not set; using local dev key")
+
+    from topos.app import app
+
+    if db_path:
+        os.environ["TOPOS_DATABASE_PATH"] = db_path
+        click.echo(f"Database path: {db_path}")
+
+    click.echo(f"Starting topos API on {host}:{port}")
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    main()

topos/config/__init__.py

@@ -0,0 +1 @@
+"""Configuration package for Topos."""

topos/config/sanitization_ollama.py

@@ -0,0 +1,189 @@
+"""
+Sanitization LLM (Ollama) configuration: file/env defaults + device DB overrides.
+
+Device overrides are stored in SQLite `engine_config` under
+`ENGINE_CONFIG_KEY_SANITIZATION_OLLAMA_DEVICE` as JSON (see DeviceSanitizationOllamaOverrides).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+if TYPE_CHECKING:
+    from topos.config.settings import Settings
+
+logger = logging.getLogger("topos.config.sanitization_ollama")
+
+# Transform IDs that use Ollama in topos.sanitization.ollama_transforms
+SANITIZATION_OLLAMA_TRANSFORM_IDS: tuple[str, ...] = (
+    "pii_redaction",
+    "nsfw_sanitization",
+    "raw_to_summary",
+    "raw_to_sentiment",
+    "third_party_anonymization",
+    "name_removal",
+    "contact_removal",
+)
+
+ENGINE_CONFIG_KEY_SANITIZATION_OLLAMA_DEVICE = "sanitization_ollama_device"
+
+
+class DeviceSanitizationOllamaOverrides(BaseModel):
+    """Partial overrides stored on device (engine_config JSON). Omitted keys keep file/env defaults."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    version: int = Field(1, ge=1)
+    enabled: Optional[bool] = None
+    host: Optional[str] = None
+    default_model: Optional[str] = None
+    timeout_sec: Optional[float] = None
+    max_input_chars: Optional[int] = None
+    models: Optional[Dict[str, str]] = None
+
+
+class SanitizationOllamaEffective(BaseModel):
+    """Fully resolved config used at runtime (after merge)."""
+
+    enabled: bool
+    host: str
+    default_model: str
+    timeout_sec: float
+    auto_pull: bool
+    max_input_chars: int
+    models: Dict[str, str]
+
+
+def _settings_transform_model_map(settings: Any) -> Dict[str, Optional[str]]:
+    """Map transform_id -> optional per-transform model from Settings."""
+    return {
+        "pii_redaction": getattr(settings, "sanitization_ollama_model_pii_redaction", None),
+        "nsfw_sanitization": getattr(settings, "sanitization_ollama_model_nsfw_sanitization", None),
+        "raw_to_summary": getattr(settings, "sanitization_ollama_model_raw_to_summary", None),
+        "raw_to_sentiment": getattr(settings, "sanitization_ollama_model_raw_to_sentiment", None),
+        "third_party_anonymization": getattr(settings, "sanitization_ollama_model_third_party_anonymization", None),
+        "name_removal": getattr(settings, "sanitization_ollama_model_name_removal", None),
+        "contact_removal": getattr(settings, "sanitization_ollama_model_contact_removal", None),
+    }
+
+
+def _read_engine_config_value(conn: sqlite3.Connection, key: str) -> Optional[str]:
+    """Read engine_config without importing topos.core.state (avoids circular imports)."""
+    try:
+        row = conn.execute("SELECT value FROM engine_config WHERE key = ?", (key,)).fetchone()
+        if not row:
+            return None
+        return str(row[0] if not isinstance(row, sqlite3.Row) else row["value"])
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("engine_config read failed for %s: %s", key, exc)
+        return None
+
+
+def parse_device_overrides_json(raw: Optional[str]) -> DeviceSanitizationOllamaOverrides:
+    if not raw or not str(raw).strip():
+        return DeviceSanitizationOllamaOverrides()
+    try:
+        data = json.loads(raw)
+        if not isinstance(data, dict):
+            return DeviceSanitizationOllamaOverrides()
+        return DeviceSanitizationOllamaOverrides.model_validate(data)
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("Invalid sanitization_ollama device JSON, ignoring: %s", exc)
+        return DeviceSanitizationOllamaOverrides()
+
+
+def resolve_sanitization_ollama_effective(
+    settings: Settings,
+    conn: Optional[sqlite3.Connection],
+) -> SanitizationOllamaEffective:
+    device = DeviceSanitizationOllamaOverrides()
+    if conn is not None:
+        raw = _read_engine_config_value(conn, ENGINE_CONFIG_KEY_SANITIZATION_OLLAMA_DEVICE)
+        device = parse_device_overrides_json(raw)
+
+    enabled = settings.sanitization_ollama_enabled
+    if device.enabled is not None:
+        enabled = device.enabled
+
+    host = (device.host or settings.sanitization_ollama_host or settings.engine_ollama_base_url or "").strip()
+    if not host:
+        host = "http://127.0.0.1:11434"
+
+    default_model = (device.default_model or settings.sanitization_ollama_default_model or "llama3.2").strip()
+
+    timeout_sec = float(device.timeout_sec if device.timeout_sec is not None else settings.sanitization_ollama_timeout_sec)
+    auto_pull = bool(getattr(settings, "sanitization_ollama_auto_pull", True))
+    max_input_chars = int(
+        device.max_input_chars if device.max_input_chars is not None else settings.sanitization_ollama_max_input_chars
+    )
+
+    st_map = _settings_transform_model_map(settings)
+    models: Dict[str, str] = {}
+    for tid in SANITIZATION_OLLAMA_TRANSFORM_IDS:
+        m: Optional[str] = None
+        if device.models and tid in device.models and device.models[tid]:
+            m = str(device.models[tid]).strip()
+        if not m:
+            sm = st_map.get(tid)
+            if sm and str(sm).strip():
+                m = str(sm).strip()
+        if not m:
+            m = default_model
+        models[tid] = m
+
+    return SanitizationOllamaEffective(
+        enabled=enabled,
+        host=host,
+        default_model=default_model,
+        timeout_sec=timeout_sec,
+        auto_pull=auto_pull,
+        max_input_chars=max_input_chars,
+        models=models,
+    )
+
+
+def effective_config_for_api(settings: Settings, conn: Optional[sqlite3.Connection]) -> dict[str, Any]:
+    """Payload for GET /v1/sanitization-ollama-config (no secrets)."""
+    eff = resolve_sanitization_ollama_effective(settings, conn)
+    device_raw = None
+    device_obj: Optional[DeviceSanitizationOllamaOverrides] = None
+    if conn is not None:
+        device_raw = _read_engine_config_value(conn, ENGINE_CONFIG_KEY_SANITIZATION_OLLAMA_DEVICE)
+        device_obj = parse_device_overrides_json(device_raw)
+
+    defaults = {
+        "enabled": settings.sanitization_ollama_enabled,
+        "host": settings.sanitization_ollama_host or settings.engine_ollama_base_url,
+        "default_model": settings.sanitization_ollama_default_model,
+        "timeout_sec": settings.sanitization_ollama_timeout_sec,
+        "auto_pull": settings.sanitization_ollama_auto_pull,
+        "max_input_chars": settings.sanitization_ollama_max_input_chars,
+        "models": {k: v for k, v in _settings_transform_model_map(settings).items() if v},
+    }
+    return {
+        "transform_ids": list(SANITIZATION_OLLAMA_TRANSFORM_IDS),
+        "defaults_from_settings": defaults,
+        "device_overrides": device_obj.model_dump(exclude_none=True) if device_obj else {},
+        "effective": eff.model_dump(),
+    }
+
+
+def normalize_put_device_overrides(body: dict[str, Any]) -> str:
+    """Validate and return JSON string for engine_config."""
+    raw = body.get("device_overrides")
+    if raw is None:
+        raise ValueError("device_overrides is required")
+    if not isinstance(raw, dict):
+        raise ValueError("device_overrides must be an object")
+    parsed = DeviceSanitizationOllamaOverrides.model_validate(raw)
+    for tid, name in (parsed.models or {}).items():
+        if tid not in SANITIZATION_OLLAMA_TRANSFORM_IDS:
+            raise ValueError(f"Unknown transform_id in models: {tid!r}")
+        if not str(name).strip():
+            raise ValueError(f"Empty model for transform_id {tid!r}")
+    return json.dumps(parsed.model_dump(exclude_none=True))