topos-node 0.1.0__py3-none-any.whl

topos/sanitization/ollama_transforms.py

@@ -0,0 +1,276 @@
+"""
+v1 LLM-backed field transforms via Ollama (e.g. llama3.2).
+
+Configuration (precedence: device DB overrides → Settings / env → safe defaults):
+  - File/env: see `topos.config.settings` (SANITIZATION_OLLAMA_*).
+  - Device: `engine_config` key `sanitization_ollama_device` (JSON), via PUT /v1/sanitization-ollama-config.
+
+If /api/chat returns 404 with a model-not-found error and SANITIZATION_OLLAMA_AUTO_PULL is true (default),
+the engine pulls the model via Ollama /api/pull (see OllamaAdapter.ensure_model) then retries chat once.
+
+Fail-open: on error, callers keep the original text.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+import threading
+from contextlib import contextmanager
+from typing import Any, Dict, Final, Iterator, List, Optional, Tuple
+
+from topos.config.sanitization_ollama import (
+    SANITIZATION_OLLAMA_TRANSFORM_IDS,
+    SanitizationOllamaEffective,
+    resolve_sanitization_ollama_effective,
+)
+
+logger = logging.getLogger("topos.sanitization.ollama")
+
+# Serialize Ollama pulls when many UMA rows hit a missing model at once.
+_pull_locks: Dict[Tuple[str, str], threading.Lock] = {}
+_pull_registry_lock = threading.Lock()
+
+
+@contextmanager
+def _serialized_ollama_pull(host: str, model: str) -> Iterator[None]:
+    key = (host.rstrip("/"), model)
+    with _pull_registry_lock:
+        if key not in _pull_locks:
+            _pull_locks[key] = threading.Lock()
+        lock = _pull_locks[key]
+    with lock:
+        yield
+
+
+def _response_suggests_missing_ollama_model(response: Any) -> bool:
+    """Heuristic: Ollama /api/chat 404 JSON error for an unknown local model tag."""
+    if getattr(response, "status_code", None) != 404:
+        return False
+    err = ""
+    try:
+        data = response.json()
+        if isinstance(data, dict):
+            err = str(data.get("error") or "")
+    except Exception:
+        err = str(getattr(response, "text", None) or "")
+    el = err.lower()
+    return "not found" in el and "model" in el
+
+# Backward-compatible alias
+OLLAMA_TRANSFORM_IDS: Final[tuple[str, ...]] = SANITIZATION_OLLAMA_TRANSFORM_IDS
+
+_SYSTEM_STRICT: Final[str] = (
+    "You are a precise text processor for a privacy-preserving data export pipeline. "
+    "Follow instructions exactly. Output ONLY the requested result with no markdown fences, "
+    "no preamble, and no explanation unless the format explicitly asks for structured fields."
+)
+
+
+def _user_pii_redaction(text: str) -> str:
+    return f"""Redact personally identifiable information in the following text.
+Replace each span of PII with a bracketed token: [NAME], [EMAIL], [PHONE], [ADDRESS], [ID], [URL], [DATE_OF_BIRTH], [ACCOUNT], [OTHER_PII].
+Preserve structure (paragraphs, newlines) and non-PII wording. Do not invent content.
+
+TEXT:
+---
+{text}
+---
+OUTPUT (redacted text only):"""
+
+
+def _user_nsfw_sanitization(text: str) -> str:
+    return f"""Sanitize the following text for a general audience: remove or replace sexually explicit, graphically violent, or illegal-content descriptions with [REMOVED].
+Keep the rest of the message meaning where possible. Do not add commentary.
+
+TEXT:
+---
+{text}
+---
+OUTPUT (sanitized text only):"""
+
+
+def _user_raw_to_summary(text: str, params: Dict[str, Any]) -> str:
+    style = str(params.get("style") or "neutral").strip() or "neutral"
+    max_len = params.get("max_length")
+    cap = ""
+    if isinstance(max_len, int) and max_len > 0:
+        cap = f" Maximum length: about {max_len} words."
+    return f"""Summarize the following text in a {style} tone.{cap}
+Output a single concise paragraph.
+
+TEXT:
+---
+{text}
+---
+OUTPUT (summary only):"""
+
+
+def _user_raw_to_sentiment(text: str, params: Dict[str, Any]) -> str:
+    scale = str(params.get("scale") or "ternary").strip() or "ternary"
+    return f"""Classify the overall sentiment of the text on scale "{scale}".
+Respond with EXACTLY one line of JSON (no markdown) with keys: "label" (string) and "confidence" (number 0-1).
+Example: {{"label":"positive","confidence":0.71}}
+
+TEXT:
+---
+{text}
+---
+OUTPUT (one JSON line only):"""
+
+
+def _user_third_party_anonymization(text: str, params: Dict[str, Any]) -> str:
+    mode = str(params.get("mode") or "replace").strip() or "replace"
+    return f"""Anonymize third-party identities in the text (other people, companies, or clients mentioned by name).
+Mode: {mode}. Replace identifiable third-party names with [PERSON_1], [ORG_1], etc. Keep the narrator's own voice and first-person references unchanged if clearly "I/me/my".
+Do not remove non-identifying roles ("my therapist", "a colleague") unless they include a name.
+
+TEXT:
+---
+{text}
+---
+OUTPUT (anonymized text only):"""
+
+
+def _user_name_removal(text: str) -> str:
+    return f"""Remove personal names (people and well-known public figures) from the text. Replace each removed name with [NAME].
+Keep dates, numbers, and non-name entities. Preserve readability.
+
+TEXT:
+---
+{text}
+---
+OUTPUT (text only):"""
+
+
+def _user_contact_removal(text: str) -> str:
+    return f"""Remove contact details: email addresses, phone numbers, street addresses, social handles (@user), and messaging IDs.
+Replace each removed span with [CONTACT]. Keep the rest of the message.
+
+TEXT:
+---
+{text}
+---
+OUTPUT (text only):"""
+
+
+def _build_messages(transform_id: str, text: str, params: Dict[str, Any]) -> List[Dict[str, str]]:
+    params = params or {}
+    builders = {
+        "pii_redaction": lambda: _user_pii_redaction(text),
+        "nsfw_sanitization": lambda: _user_nsfw_sanitization(text),
+        "raw_to_summary": lambda: _user_raw_to_summary(text, params),
+        "raw_to_sentiment": lambda: _user_raw_to_sentiment(text, params),
+        "third_party_anonymization": lambda: _user_third_party_anonymization(text, params),
+        "name_removal": lambda: _user_name_removal(text),
+        "contact_removal": lambda: _user_contact_removal(text),
+    }
+    fn = builders.get(transform_id)
+    if fn is None:
+        raise ValueError(f"No Ollama prompt builder for transform_id={transform_id!r}")
+    user_content = fn()
+    return [
+        {"role": "system", "content": _SYSTEM_STRICT},
+        {"role": "user", "content": user_content},
+    ]
+
+
+def ollama_sanitization_enabled() -> bool:
+    """Whether sanitization is enabled (Settings + device overrides)."""
+    from topos.config.settings import settings
+    from topos.core.state import get_db_connection
+
+    return resolve_sanitization_ollama_effective(settings, get_db_connection()).enabled
+
+
+def _ollama_chat(
+    messages: List[Dict[str, str]],
+    *,
+    host: str,
+    model: str,
+    timeout_sec: float,
+    auto_pull: bool = True,
+) -> str:
+    import httpx
+
+    url = f"{host.rstrip('/')}/api/chat"
+    body = {"model": model, "messages": messages, "stream": False}
+    with httpx.Client(timeout=timeout_sec) as client:
+        resp = client.post(url, json=body)
+        if (
+            auto_pull
+            and resp.status_code == 404
+            and _response_suggests_missing_ollama_model(resp)
+        ):
+            logger.info(
+                "Ollama model %r not available at %s; pulling via /api/pull then retrying /api/chat",
+                model,
+                host.rstrip("/"),
+            )
+            with _serialized_ollama_pull(host, model):
+                from topos.engine.backends.ollama import OllamaAdapter
+
+                pulled = OllamaAdapter(base_url=host).ensure_model(model)
+                if pulled:
+                    logger.info("Sanitization: finished pulling Ollama model %r", model)
+            resp = client.post(url, json=body)
+        resp.raise_for_status()
+        data = resp.json()
+    msg = data.get("message") or {}
+    content = msg.get("content")
+    if not isinstance(content, str):
+        raise ValueError("Ollama response missing message.content")
+    return content.strip()
+
+
+def _strip_fences(s: str) -> str:
+    s = s.strip()
+    m = re.match(r"^```(?:\w+)?\s*\n?(.*?)\n?```\s*$", s, re.DOTALL)
+    return m.group(1).strip() if m else s
+
+
+def apply_text_transform_with_ollama(
+    text: str,
+    transform_id: str,
+    params: Optional[Dict[str, Any]] = None,
+    *,
+    effective: SanitizationOllamaEffective,
+    model_override: Optional[str] = None,
+) -> str:
+    """
+    Run a single catalog transform on string `text` via Ollama.
+    `effective` should be from `resolve_sanitization_ollama_effective` (caller resolves once per batch).
+    `model_override`: when set (e.g. Filter Lab compare), use this Ollama tag instead of per-transform model.
+    """
+    if transform_id not in SANITIZATION_OLLAMA_TRANSFORM_IDS:
+        raise ValueError(f"transform_id {transform_id!r} is not handled by Ollama sanitization")
+
+    model = (model_override or "").strip() or effective.models.get(transform_id) or effective.default_model
+    max_in = effective.max_input_chars
+    if max_in > 0 and len(text) > max_in:
+        text = text[:max_in] + "\n[TRUNCATED_FOR_LLM]"
+
+    messages = _build_messages(transform_id, text, dict(params or {}))
+    raw = _ollama_chat(
+        messages,
+        host=effective.host,
+        model=model,
+        timeout_sec=effective.timeout_sec,
+        auto_pull=effective.auto_pull,
+    )
+    out = _strip_fences(raw)
+
+    if transform_id == "raw_to_sentiment":
+        try:
+            line = out.splitlines()[0].strip()
+            parsed = json.loads(line)
+            label = str(parsed.get("label", "unknown"))
+            conf = parsed.get("confidence")
+            if isinstance(conf, (int, float)):
+                return f"{label} ({float(conf):.2f})"
+            return label
+        except Exception:
+            return out[:500]
+
+    return out

topos/scope_resolution.py

@@ -0,0 +1,89 @@
+# Scope → canonical table resolution (Sprint 05 Stage 1)
+# Maps MVP scope IDs from RPT to canonical table names. Reference: roles_scopes/MVP_TAXONOMY.md §4
+
+from __future__ import annotations
+
+from typing import List, Optional, Set
+
+# Scope ID -> canonical table(s). One scope can map to one or more tables.
+# Table names match MVP_TAXONOMY §4 (messages, ai_messages, events, ai_chat, activity, journal, Profile).
+SCOPE_TO_TABLES: dict[str, list[str]] = {
+    "messages:read": ["messages"],
+    "messages:write": ["messages"],
+    "aiMessages:read": ["ai_messages"],
+    "events:read": ["events"],
+    "events:write": ["events"],
+    "aiChat:read": ["ai_chat"],
+    "activity:read": ["activity"],
+    "activity:write": ["activity"],
+    "journal:read": ["journal"],
+    "journal:write": ["journal"],
+    "activitySummary:read": ["activity_summary"],
+    "wellnessSummary:read": ["wellness_summary"],
+    "publicBio:read": ["public_bio"],
+    "contacts:resolve": ["contacts", "contact_identifiers"],
+    "all:read": ["*"],
+    "all:write": ["*"],
+}
+
+# All canonical table names (for "all" resolution). Order doesn't matter.
+ALL_CANONICAL_TABLES: set[str] = {
+    "messages",
+    "ai_messages",
+    "events",
+    "ai_chat",
+    "activity",
+    "journal",
+    "activity_summary",
+    "wellness_summary",
+    "public_bio",
+    "contacts",
+    "contact_identifiers",
+}
+
+
+def resolve_scopes_to_tables(scope_ids: Optional[List[str]]) -> Set[str]:
+    """
+    Map MVP scope IDs to canonical table names.
+    If all:read or all:write is in scope_ids, returns ALL_CANONICAL_TABLES.
+    Otherwise returns the union of tables for each scope in scope_ids.
+    """
+    if not scope_ids:
+        return set()
+    tables: Set[str] = set()
+    for s in scope_ids:
+        s = (s or "").strip()
+        if not s:
+            continue
+        if s in ("all:read", "all:write"):
+            return set(ALL_CANONICAL_TABLES)
+        if s in SCOPE_TO_TABLES:
+            mapped = SCOPE_TO_TABLES[s]
+            if "*" in mapped:
+                return set(ALL_CANONICAL_TABLES)
+            tables.update(mapped)
+    return tables
+
+
+def may_access_table(allowed_tables: Set[str], table_or_alias: str) -> bool:
+    """
+    True if the allowed set (from resolve_scopes_to_tables) permits access to the given table.
+    table_or_alias can be canonical name (e.g. messages) or implementation table (e.g. ai_chat_messages).
+    """
+    if not allowed_tables:
+        return False
+    if "*" in allowed_tables or "all" in allowed_tables:
+        return True
+    # Exact match
+    if table_or_alias in allowed_tables:
+        return True
+    if table_or_alias == "conversation_messages" and "messages" in allowed_tables:
+        return True
+    # Implementation detail: ai_chat_messages table holds ai_chat and ai_messages data
+    if table_or_alias == "ai_chat_messages" and ("ai_chat" in allowed_tables or "ai_messages" in allowed_tables):
+        return True
+    if table_or_alias == "messages" and "messages" in allowed_tables:
+        return True
+    if table_or_alias in ("contacts", "contact_identifiers"):
+        return "contacts" in allowed_tables and "contact_identifiers" in allowed_tables
+    return False

topos/services/__init__.py

@@ -0,0 +1 @@
+"""Service interfaces and adapters for Topos."""

topos/services/container.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..config.settings import settings
+from .embeddings.base import EmbeddingsService
+from .embeddings.local import LocalEmbeddingsService
+from .embeddings.remote import RemoteEmbeddingsService
+from .interfaces import DbService, DeviceService, LLMService, SyncService
+from .local import LocalDbService, LocalDeviceService, LocalSyncService
+from .llm.openai import OpenAILLMService
+from .postgres import HostedDeviceService, HostedSyncService, PostgresDbService
+
+
+@dataclass(frozen=True)
+class Services:
+    db: DbService
+    sync: SyncService
+    device: DeviceService
+    llm: LLMService
+    embeddings: EmbeddingsService
+
+
+_services: Services | None = None
+
+
+def get_services() -> Services:
+    global _services
+    if _services is None:
+        if settings.topos_database_mode == "postgres":
+            _services = Services(
+                db=PostgresDbService(),
+                sync=HostedSyncService(),
+                device=HostedDeviceService(),
+                llm=OpenAILLMService(),
+                embeddings=RemoteEmbeddingsService(),
+            )
+        else:
+            _services = Services(
+                db=LocalDbService(),
+                sync=LocalSyncService(),
+                device=LocalDeviceService(),
+                llm=OpenAILLMService(),
+                embeddings=LocalEmbeddingsService(),
+            )
+    return _services

topos/services/embeddings/__init__.py

@@ -0,0 +1 @@
+"""Embeddings service adapters."""

topos/services/embeddings/base.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from typing import List, Protocol
+
+
+class EmbeddingsService(Protocol):
+    async def embed(self, texts: List[str]) -> List[List[float]]: ...

topos/services/embeddings/local.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from typing import List
+
+
+class LocalEmbeddingsService:
+    async def embed(self, texts: List[str]) -> List[List[float]]:
+        _ = texts
+        raise NotImplementedError("Local embeddings not implemented yet")

topos/services/embeddings/remote.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from typing import List
+
+
+class RemoteEmbeddingsService:
+    async def embed(self, texts: List[str]) -> List[List[float]]:
+        _ = texts
+        raise NotImplementedError("Remote embeddings not implemented yet")

topos/services/interfaces.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Protocol
+
+from ..core.api_models import (
+    DeviceInfoResponse,
+    DeviceNameResponse,
+    PairDeviceResponse,
+    PairingCodeResponse,
+    StoreMessageResponse,
+    SyncDatabaseResponse,
+    SyncResponse,
+)
+
+
+class DbService(Protocol):
+    async def store_message(self, payload: Dict[str, Any]) -> StoreMessageResponse: ...
+    async def get_oplog(self, dataset_id: Optional[str], limit: int, offset: int) -> Dict[str, Any]: ...
+    async def get_messages(self, dataset_id: Optional[str], limit: int, offset: int) -> Dict[str, Any]: ...
+    async def replay_projection(self, dataset_id: Optional[str]) -> Dict[str, Any]: ...
+    async def reset_database(self) -> Dict[str, Any]: ...
+    async def sync_database(self) -> SyncDatabaseResponse: ...
+    async def backup_database(self, encrypted: bool) -> Any: ...
+    async def restore_database(self, file, authenticated_user_id: str, encrypted: bool) -> Dict[str, Any]: ...
+
+
+class SyncService(Protocol):
+    async def trigger_sync(self) -> SyncResponse: ...
+
+
+class DeviceService(Protocol):
+    async def get_pairing_code(self) -> PairingCodeResponse: ...
+    async def pair_device(self, pairing_code: str, keep_existing_data: bool) -> PairDeviceResponse: ...
+    async def get_device_info(self) -> DeviceInfoResponse: ...
+    async def set_device_name(self, device_name: str) -> DeviceNameResponse: ...
+
+
+class LLMService(Protocol):
+    async def generate(self, payload: Dict[str, Any]) -> Dict[str, Any]: ...
+    async def list_ollama_models(self) -> Dict[str, Any]: ...

topos/services/llm/__init__.py

@@ -0,0 +1 @@
+"""LLM service adapters."""

topos/services/llm/base.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from typing import Dict, Protocol
+
+
+class LLMService(Protocol):
+    async def generate(self, payload: Dict[str, str]) -> Dict[str, str]: ...

topos/services/llm/openai.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict
+
+import httpx
+from fastapi import HTTPException, status
+
+from ...core import state
+from ...openai_client import OpenAIError
+from ...config.settings import settings
+
+logger = logging.getLogger("topos.services.llm")
+
+
+def _normalize_provider(raw: Any) -> str:
+    if raw is None:
+        return "openai"
+    if isinstance(raw, str):
+        v = raw.lower().strip()
+        if v in ("openai", "ollama"):
+            return v
+    return "openai"
+
+
+async def _ollama_generate(payload: Dict[str, Any]) -> Dict[str, Any]:
+    prompt = payload.get("prompt") or ""
+    model_raw = payload.get("model")
+    model = (
+        (model_raw.strip() if isinstance(model_raw, str) else "")
+        or settings.sanitization_ollama_default_model
+    )
+    max_tokens = payload.get("max_tokens")
+    temperature = payload.get("temperature")
+    base = settings.engine_ollama_base_url.rstrip("/")
+    body: Dict[str, Any] = {"model": model, "prompt": prompt, "stream": False}
+    opts: Dict[str, Any] = {}
+    if max_tokens is not None:
+        opts["num_predict"] = max_tokens
+    if temperature is not None:
+        opts["temperature"] = temperature
+    if opts:
+        body["options"] = opts
+    timeout = httpx.Timeout(settings.sanitization_ollama_timeout_sec, connect=10.0)
+    logger.info(
+        "Ollama generate: model=%r base=%s prompt_chars=%d max_tokens=%s temperature=%s",
+        model,
+        base,
+        len(prompt),
+        max_tokens,
+        temperature,
+    )
+    try:
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            r = await client.post(f"{base}/api/generate", json=body)
+    except httpx.RequestError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=f"Ollama unreachable at {base}: {exc}",
+        ) from exc
+    if r.status_code >= 400:
+        detail = (r.text or str(r.status_code))[:800]
+        raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Ollama error: {detail}")
+    data = r.json()
+    text = (data.get("response") or "").strip()
+    resp_model = str(data.get("model") or model)
+    logger.info(
+        "Ollama generate complete: model=%r response_chars=%d",
+        resp_model,
+        len(text),
+    )
+    return {"output": text, "model": resp_model, "usage": {}}
+
+
+async def _ollama_list_model_names() -> list[str]:
+    base = settings.engine_ollama_base_url.rstrip("/")
+    timeout = httpx.Timeout(settings.sanitization_ollama_timeout_sec, connect=10.0)
+    logger.info("Ollama list models: base=%s", base)
+    try:
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            r = await client.get(f"{base}/api/tags")
+    except httpx.RequestError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=f"Ollama unreachable at {base}: {exc}",
+        ) from exc
+    if r.status_code >= 400:
+        detail = (r.text or str(r.status_code))[:800]
+        raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Ollama error: {detail}")
+    data = r.json()
+    names = [str(m.get("name", "")).strip() for m in data.get("models", []) if m.get("name")]
+    unique = sorted(set(names))
+    logger.info("Ollama list models complete: count=%d", len(unique))
+    return unique
+
+
+class OpenAILLMService:
+    async def list_ollama_models(self) -> Dict[str, Any]:
+        if not settings.enable_llm or state.get_engine_mode() != "full":
+            raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="LLM is disabled")
+        models = await _ollama_list_model_names()
+        return {"models": models}
+
+    async def generate(self, payload: Dict[str, Any]) -> Dict[str, Any]:
+        if not settings.enable_llm or state.get_engine_mode() != "full":
+            raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="LLM is disabled")
+
+        provider = _normalize_provider(payload.get("provider"))
+        if provider == "ollama":
+            logger.info("LLM generate routed to Ollama (model=%r)", payload.get("model"))
+            return await _ollama_generate(payload)
+
+        if not settings.openai_api_key:
+            raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="OPENAI_API_KEY is not set")
+        try:
+            result = await state.openai_client.generate(
+                prompt=payload.get("prompt", ""),
+                max_tokens=payload.get("max_tokens"),
+                temperature=payload.get("temperature"),
+            )
+            return {"output": result["output"], "model": settings.openai_model, "usage": result["usage"]}
+        except OpenAIError as exc:
+            detail = str(exc)
+            if "rate_limited" in detail:
+                raise HTTPException(status_code=status.HTTP_429_TOO_MANY_REQUESTS, detail="LLM rate limited") from exc
+            raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail="LLM upstream error") from exc