topos-node 0.1.0__py3-none-any.whl

topos/ingestion/validation/schema_registry.py

@@ -0,0 +1,111 @@
+"""Minimal schema registry for ingestion validation."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger("topos.ingestion.schema_registry")
+
+SCHEMAS: Dict[str, Dict[str, Any]] = {
+    "chatgpt.conversation.v1": {
+        "name": "ChatGPT Conversation Export",
+        "version": "1",
+        "required_fields": ["id", "thread_id", "role", "content", "created_at"],
+        "field_types": {
+            "id": str,
+            "thread_id": str,
+            "role": str,
+            "content": str,
+            "created_at": (int, float, str),
+        },
+        "description": "ChatGPT conversation export format (JSONL - flat records)",
+        "file_format": "jsonl",
+    },
+    "chatgpt.conversation.v2": {
+        "name": "ChatGPT Real Export Format",
+        "version": "2",
+        "required_fields": ["id", "thread_id", "role", "content", "created_at"],
+        "field_types": {
+            "id": str,
+            "thread_id": str,
+            "role": str,
+            "content": str,
+            "created_at": (int, float, str),
+        },
+        "description": "ChatGPT real export format (JSON array of conversation objects, flattened to v1 format)",
+        "file_format": "json",
+        "source_structure": "conversation_array",
+        "note": "Flattened records match v1 format, but source is nested conversation objects",
+    },
+    # Sprint 02: Messenger ingestion (same logical shape as chat for conversation_messages)
+    "imessage.messages.v1": {
+        "name": "iMessage Messages",
+        "version": "1",
+        "required_fields": ["id", "thread_id", "role", "content", "created_at"],
+        "field_types": {
+            "id": str,
+            "thread_id": str,
+            "role": str,
+            "content": str,
+            "created_at": (int, float, str),
+        },
+        "description": "iMessage message format (normalized from chat.db or sync); id may be imessage:ROWID",
+        "file_format": "jsonl",
+    },
+    "signal.messages.v1": {
+        "name": "Signal Messages",
+        "version": "1",
+        "required_fields": ["id", "thread_id", "role", "content", "created_at"],
+        "field_types": {
+            "id": str,
+            "thread_id": str,
+            "role": str,
+            "content": str,
+            "created_at": (int, float, str),
+        },
+        "description": "Signal Desktop message format (normalized from SQLCipher or export); id may be signal:uuid",
+        "file_format": "jsonl",
+    },
+}
+
+
+def get_schema_definition(schema_id: str) -> Optional[Dict[str, Any]]:
+    return SCHEMAS.get(schema_id)
+
+
+def validate_schema(record: Dict[str, Any], schema_id: str) -> tuple[bool, Optional[str]]:
+    schema = get_schema_definition(schema_id)
+    if not schema:
+        return False, f"Unknown schema: {schema_id}"
+
+    required_fields = schema.get("required_fields", [])
+    for field in required_fields:
+        if field not in record:
+            return False, f"Missing required field: {field}"
+
+    field_types = schema.get("field_types", {})
+    for field, expected_type in field_types.items():
+        if field not in record:
+            continue
+        value = record[field]
+        if expected_type is None:
+            continue
+        if isinstance(expected_type, tuple):
+            if not any(isinstance(value, t) for t in expected_type):
+                return False, (
+                    f"Field '{field}' has invalid type. "
+                    f"Expected one of {expected_type}, got {type(value).__name__}"
+                )
+        elif not isinstance(value, expected_type):
+            return False, (
+                f"Field '{field}' has invalid type. "
+                f"Expected {expected_type.__name__}, got {type(value).__name__}"
+            )
+
+    return True, None
+
+
+def register_schema(schema_id: str, schema_def: Dict[str, Any]) -> None:
+    SCHEMAS[schema_id] = schema_def
+    logger.info("Registered schema: %s", schema_id)

topos/ingestion/validation/schema_validator.py

@@ -0,0 +1,13 @@
+"""Default schema validator (no-op placeholder)."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+
+from .base import SchemaDefinition, SchemaValidator, ValidationResult
+
+
+class NoOpSchemaValidator(SchemaValidator):
+    def validate(self, record: Dict[str, Any], schema: Optional[SchemaDefinition] = None) -> ValidationResult:
+        _ = (record, schema)
+        return ValidationResult(is_valid=True, errors=[], metadata={})

topos/lineage/__init__.py

@@ -0,0 +1 @@
+"""Lineage and provenance tracking for Topos."""

topos/lineage/provenance.py

@@ -0,0 +1,9 @@
+"""Provenance stubs for Topos."""
+
+from __future__ import annotations
+
+from typing import Dict
+
+
+def record_provenance(payload: Dict[str, str]) -> None:
+    _ = payload

topos/lineage/tracker.py

@@ -0,0 +1,9 @@
+"""Lineage tracking stubs."""
+
+from __future__ import annotations
+
+from typing import Dict
+
+
+def record_lineage(payload: Dict[str, str]) -> None:
+    _ = payload

topos/mcp_stdio_proxy.py

@@ -0,0 +1,83 @@
+"""
+Stdio MCP proxy: Claude Desktop → local Topos engine (no Control Plane).
+
+Run this so Claude talks MCP over stdio to the proxy; the proxy forwards tool calls
+to the engine's /api/local/* HTTP endpoints. Use when the engine and Claude run on
+the same machine.
+
+Usage:
+  ENGINE_URL=http://localhost:9000 BEARER_TOKEN=your_key python -m topos.mcp_stdio_proxy
+  # or with args:
+  python -m topos.mcp_stdio_proxy --url http://localhost:9000
+
+Claude Desktop config (direct to local engine): use scripts/run_local_mcp_proxy.sh
+with full path, args ["--url", "http://localhost:9000"], and env BEARER_TOKEN.
+
+Only list_database_tables and get_table_schema are exposed (engine's /api/local/*).
+For full tools (get_analytics, get_messages, get_oplog) use the Control Plane.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+import httpx
+from mcp.server.fastmcp import FastMCP
+
+# Engine URL and token; set in main() before FastMCP runs.
+_engine_url: str = ""
+_bearer_token: str = ""
+
+
+def _headers() -> dict[str, str]:
+    return {"Authorization": f"Bearer {_bearer_token}", "Content-Type": "application/json"}
+
+
+async def _call_engine(path: str, json_body: dict | None = None) -> dict:
+    url = f"{_engine_url.rstrip('/')}{path}"
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        r = await client.post(url, headers=_headers(), json=json_body or {})
+    r.raise_for_status()
+    data = r.json()
+    if isinstance(data, dict) and data.get("status") == "error":
+        raise ValueError(data.get("error", "engine error"))
+    return data
+
+
+def main() -> int:
+    global _engine_url, _bearer_token
+    parser = argparse.ArgumentParser(
+        description="MCP stdio proxy to local Topos engine (/api/local/*). No Control Plane."
+    )
+    parser.add_argument(
+        "--url",
+        default=os.environ.get("ENGINE_URL", "http://localhost:9000"),
+        help="Engine base URL (default: ENGINE_URL or http://localhost:9000)",
+    )
+    args = parser.parse_args()
+    _engine_url = args.url.rstrip("/")
+    _bearer_token = (os.environ.get("BEARER_TOKEN") or "").strip()
+    if not _bearer_token:
+        print("Error: BEARER_TOKEN env var required.", file=sys.stderr)
+        return 1
+
+    mcp = FastMCP("Topos (local engine)", port=0)
+
+    @mcp.tool()
+    async def list_database_tables() -> dict:
+        """List all database tables grouped by layer with row counts."""
+        return await _call_engine("/api/local/list_database_tables")
+
+    @mcp.tool()
+    async def get_table_schema(table_name: str) -> dict:
+        """Get column info (schema) for a table. Use list_database_tables to see available tables."""
+        return await _call_engine("/api/local/get_table_schema", {"table_name": table_name})
+
+    mcp.run(transport="stdio")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

topos/observability/__init__.py

@@ -0,0 +1 @@
+"""Observability utilities for Topos."""

topos/observability/alerts.py

@@ -0,0 +1,7 @@
+"""Alerting stubs for Topos."""
+
+from __future__ import annotations
+
+
+def send_alert(message: str) -> None:
+    _ = message

topos/observability/metrics.py

@@ -0,0 +1,25 @@
+"""Observability metrics (Sprint 07). In-memory counters; extend for Prometheus later."""
+
+from __future__ import annotations
+
+import threading
+from typing import Dict
+
+
+_counts: Dict[str, float] = {}
+_lock = threading.Lock()
+
+
+def record_metric(name: str, value: float) -> None:
+    with _lock:
+        _counts[name] = _counts.get(name, 0) + value
+
+
+def get_metric(name: str) -> float:
+    with _lock:
+        return _counts.get(name, 0.0)
+
+
+def reset_metrics() -> None:
+    with _lock:
+        _counts.clear()

topos/observability/tracing.py

@@ -0,0 +1,18 @@
+"""Tracing stubs for Topos."""
+
+from __future__ import annotations
+
+
+class Span:
+    def __init__(self, name: str):
+        self.name = name
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        _ = (exc_type, exc, tb)
+
+
+def start_span(name: str) -> Span:
+    return Span(name)

topos/openai_client.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Optional
+
+import httpx
+
+from .config.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAIError(Exception):
+    """Wrapper for upstream OpenAI errors."""
+
+
+class OpenAIClient:
+    """Minimal OpenAI chat completions client."""
+
+    def __init__(self, api_key: str | None = None, base_url: str | None = None) -> None:
+        self.api_key = api_key or settings.openai_api_key
+        self.base_url = base_url or settings.openai_base_url
+        self.timeout = settings.openai_timeout_seconds
+
+    async def generate(
+        self,
+        prompt: str,
+        max_tokens: Optional[int],
+        temperature: Optional[float],
+    ) -> Dict[str, Any]:
+        url = f"{self.base_url.rstrip('/')}/chat/completions"
+        headers = {"Authorization": f"Bearer {self.api_key}"}
+        payload: Dict[str, Any] = {
+            "model": settings.openai_model,
+            "messages": [
+                {"role": "system", "content": "You are a concise assistant."},
+                {"role": "user", "content": prompt},
+            ],
+        }
+        if max_tokens is not None:
+            payload["max_tokens"] = max_tokens
+        if temperature is not None:
+            payload["temperature"] = temperature
+
+        try:
+            async with httpx.AsyncClient(timeout=self.timeout) as client:
+                resp = await client.post(url, headers=headers, json=payload)
+        except Exception as exc:  # noqa: BLE001
+            logger.error("OpenAI request failed: %s", exc)
+            raise OpenAIError(f"request failed: {exc}") from exc
+
+        if resp.status_code == 429:
+            raise OpenAIError("rate_limited")
+
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json()
+            except Exception:
+                detail = resp.text
+            raise OpenAIError(f"upstream_error: {resp.status_code}: {detail}")
+
+        data = resp.json()
+        try:
+            message = data["choices"][0]["message"]["content"]
+            usage = data.get("usage", {})
+        except Exception as exc:  # noqa: BLE001
+            raise OpenAIError("invalid_response") from exc
+
+        return {"output": message, "usage": usage}

topos/projections/__init__.py

@@ -0,0 +1 @@
+"""Projection builders for Topos."""

topos/projections/vector_index/__init__.py

@@ -0,0 +1 @@
+"""Vector index projection abstractions."""

topos/projections/vector_index/base.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterable, List
+
+
+@dataclass(frozen=True)
+class EmbeddingRow:
+    record_id: str
+    vector: List[float]
+
+
+@dataclass(frozen=True)
+class ProjectionStatus:
+    status: str
+    count: int
+
+
+class ProjectionBuilder:
+    def build(self, embeddings: Iterable[EmbeddingRow]) -> ProjectionStatus:
+        raise NotImplementedError

topos/projections/vector_index/builders.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from typing import Iterable
+
+from .base import EmbeddingRow, ProjectionBuilder, ProjectionStatus
+
+
+class VectorIndexBuilder(ProjectionBuilder):
+    def build(self, embeddings: Iterable[EmbeddingRow]) -> ProjectionStatus:
+        count = sum(1 for _ in embeddings)
+        return ProjectionStatus(status="stub", count=count)

topos/projections/vector_index/health_checks.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+
+def check_index_health() -> dict:
+    return {"status": "stub"}

topos/rate_limit.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import time
+from typing import Dict
+
+from fastapi import HTTPException, Request, status
+
+from .config.settings import settings
+
+
+class TokenBucket:
+    def __init__(self, rate_per_minute: int) -> None:
+        self.capacity = rate_per_minute
+        self.tokens = rate_per_minute
+        self.refill_time = time.time()
+        self.rate_per_second = rate_per_minute / 60.0
+
+    def consume(self, tokens: int = 1) -> bool:
+        now = time.time()
+        elapsed = now - self.refill_time
+        refill = elapsed * self.rate_per_second
+        if refill > 0:
+            self.tokens = min(self.capacity, self.tokens + refill)
+            self.refill_time = now
+        if self.tokens >= tokens:
+            self.tokens -= tokens
+            return True
+        return False
+
+
+_buckets: Dict[str, TokenBucket] = {}
+
+
+def rate_limit(request: Request) -> None:
+    """Simple in-memory rate limit per client IP."""
+    ip = request.client.host if request.client else "unknown"
+    bucket = _buckets.get(ip)
+    if bucket is None:
+        bucket = TokenBucket(settings.rate_limit_per_minute)
+        _buckets[ip] = bucket
+
+    if not bucket.consume():
+        raise HTTPException(status_code=status.HTTP_429_TOO_MANY_REQUESTS, detail="Too many requests")

topos/sanitization/__init__.py

@@ -0,0 +1,16 @@
+"""Runtime text sanitization helpers (optional Ollama-backed field transforms)."""
+
+from topos.config.sanitization_ollama import SANITIZATION_OLLAMA_TRANSFORM_IDS
+
+from .ollama_transforms import (
+    OLLAMA_TRANSFORM_IDS,
+    apply_text_transform_with_ollama,
+    ollama_sanitization_enabled,
+)
+
+__all__ = [
+    "SANITIZATION_OLLAMA_TRANSFORM_IDS",
+    "OLLAMA_TRANSFORM_IDS",
+    "apply_text_transform_with_ollama",
+    "ollama_sanitization_enabled",
+]