topos-node 0.1.0__py3-none-any.whl

topos/engine/tasks.py

@@ -0,0 +1,154 @@
+"""Task contract for the Topos Engine (PRD §6)."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+# --- Nested structures (PRD §6.1) ---
+
+
+class ModelRequest(BaseModel):
+    """Model selection for a task."""
+
+    provider: str = Field(..., description="e.g. ollama, huggingface")
+    model: Optional[str] = Field(None, description="Model name or id; default from registry if omitted")
+
+
+class ExecutionSpec(BaseModel):
+    """Execution mode and scheduling hints."""
+
+    mode: str = Field(default="sync", description="sync, async, batch, etc.")
+    priority: int = Field(default=100, description="Lower = higher priority")
+    batch_key: Optional[str] = Field(None, description="Group tasks for model-aware batching")
+
+
+class TaskOptions(BaseModel):
+    """Per-task options."""
+
+    store_result: bool = Field(default=True, description="Whether to persist result")
+    apply_fisher_filter: bool = Field(default=False, description="Apply Fisher filter at output")
+
+
+class RequestedBy(BaseModel):
+    """Who requested the task and from where."""
+
+    user_id: Optional[str] = None
+    origin: Optional[str] = Field(None, description="e.g. write_event, batch, manual")
+
+
+# --- ProcessingTask (PRD §6.1) ---
+
+
+class ProcessingTask(BaseModel):
+    """Input contract for the Engine. All work is represented as a task."""
+
+    id: str = Field(..., description="Unique task id")
+    type: str = Field(..., description="enrichment, transformation, derivation, query, etc.")
+    subtype: Optional[str] = Field(None, description="e.g. emotion_classification, url_classification")
+    source_id: Optional[str] = None
+    record_ids: List[str] = Field(default_factory=list)
+    input: Dict[str, Any] = Field(default_factory=dict)
+    model_request: ModelRequest = Field(...)
+    execution: ExecutionSpec = Field(default_factory=ExecutionSpec)
+    options: TaskOptions = Field(default_factory=TaskOptions)
+    requested_by: Optional[RequestedBy] = None
+    created_at: Optional[str] = Field(
+        default_factory=lambda: datetime.now(tz=timezone.utc).isoformat()
+    )
+
+    model_config = {"extra": "forbid"}
+
+    def model_dump_json_roundtrip(self) -> Dict[str, Any]:
+        """Serialize to JSON-compatible dict (for transport and tests)."""
+        return self.model_dump(mode="json")
+
+
+# --- ProcessingResult (PRD §6.2) ---
+
+
+class Provenance(BaseModel):
+    """Provenance of the result."""
+
+    source_id: Optional[str] = None
+    record_ids: List[str] = Field(default_factory=list)
+
+
+class ExecutionMeta(BaseModel):
+    """Execution metadata."""
+
+    provider: Optional[str] = None
+    model: Optional[str] = None
+    duration_ms: Optional[int] = None
+    cache_hit: bool = False
+
+
+class ProcessingResult(BaseModel):
+    """Output contract for the Engine."""
+
+    task_id: str = Field(...)
+    status: str = Field(..., description="completed, failed, error, etc.")
+    output: Dict[str, Any] = Field(default_factory=dict)
+    output_type: str = Field(default="json")
+    confidence: Optional[float] = None
+    provenance: Optional[Provenance] = None
+    execution_meta: Optional[ExecutionMeta] = None
+    error: Optional[str] = None
+
+    model_config = {"extra": "forbid"}
+
+    def model_dump_json_roundtrip(self) -> Dict[str, Any]:
+        """Serialize to JSON-compatible dict."""
+        return self.model_dump(mode="json")
+
+
+# --- Helper ---
+
+
+def build_task(
+    task_id: str,
+    task_type: str,
+    model_request: ModelRequest,
+    *,
+    subtype: Optional[str] = None,
+    source_id: Optional[str] = None,
+    record_ids: Optional[List[str]] = None,
+    input_data: Optional[Dict[str, Any]] = None,
+    execution: Optional[ExecutionSpec] = None,
+    requested_by: Optional[RequestedBy] = None,
+) -> ProcessingTask:
+    """Build a ProcessingTask with required fields and optional overrides."""
+    return ProcessingTask(
+        id=task_id,
+        type=task_type,
+        subtype=subtype,
+        source_id=source_id,
+        record_ids=record_ids or [],
+        input=input_data or {},
+        model_request=model_request,
+        execution=execution or ExecutionSpec(),
+        requested_by=requested_by,
+    )
+
+
+def build_url_classification_task(
+    task_id: str,
+    url: str,
+    title: Optional[str] = None,
+    *,
+    source_id: Optional[str] = None,
+    record_ids: Optional[List[str]] = None,
+) -> ProcessingTask:
+    """Build a ProcessingTask for URL classification (enrichment, url_classification)."""
+    return build_task(
+        task_id=task_id,
+        task_type="enrichment",
+        model_request=ModelRequest(provider="huggingface"),
+        subtype="url_classification",
+        source_id=source_id,
+        record_ids=record_ids or [],
+        input_data={"url": url, "title": title or ""},
+    )

topos/engine/transport.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable, Dict
+
+MessageHandler = Callable[[Dict[str, Any]], Awaitable[Dict[str, Any] | None]]
+
+
+@dataclass(frozen=True)
+class TransportConfig:
+    mode: str = "ws"
+
+
+def normalize_transport_mode(value: str | None) -> str:
+    mode = (value or "").strip().lower()
+    if mode in {"ws", "endpoint"}:
+        return mode
+    return "ws"
+
+
+async def dispatch_compute_message(
+    *,
+    mode: str,
+    payload: Dict[str, Any],
+    handler: MessageHandler,
+) -> Dict[str, Any] | None:
+    normalized = normalize_transport_mode(mode)
+    if normalized == "ws":
+        return await handler(payload)
+    # endpoint transport currently shares the same business handler.
+    return await handler(payload)
+
+
+async def handle_ws_raw_message(raw: str, handler: MessageHandler) -> str | None:
+    message = json.loads(raw)
+    response = await dispatch_compute_message(mode="ws", payload=message, handler=handler)
+    if response is None:
+        return None
+    return json.dumps(response)
+
+
+async def handle_endpoint_request(payload: Dict[str, Any], handler: MessageHandler) -> Dict[str, Any] | None:
+    return await dispatch_compute_message(mode="endpoint", payload=payload, handler=handler)

topos/engine/usage_guard.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict, Optional
+
+import httpx
+
+from ..config.settings import settings
+
+logger = logging.getLogger("topos.engine.usage_guard")
+
+_KNOWN_GUARD_REASON_CODES = {
+    "LIMIT_EXCEEDED",
+    "MISSING_POLICY_RULE",
+    "UNTRUSTED_USAGE_SOURCE",
+    "METRIC_NOT_REGISTERED",
+    "PERIOD_RESOLUTION_FALLBACK",
+}
+
+
+def parse_guard_denial_payload(detail: Any) -> Optional[Dict[str, Any]]:
+    if not isinstance(detail, dict):
+        return None
+    reason_code = str(detail.get("reason_code") or "").strip()
+    metric_key = str(detail.get("metric_key") or "").strip()
+    period_start = str(detail.get("period_start") or "").strip()
+    period_end = str(detail.get("period_end") or "").strip()
+    if not reason_code or reason_code not in _KNOWN_GUARD_REASON_CODES:
+        return None
+    return {
+        "reason_code": reason_code,
+        "metric_key": metric_key or None,
+        "period_start": period_start or None,
+        "period_end": period_end or None,
+        "used": detail.get("used"),
+        "limit": detail.get("limit"),
+        "unlimited": detail.get("unlimited"),
+        "policy_version": detail.get("policy_version"),
+        "message": str(detail.get("message") or "Usage request denied by guard."),
+    }
+
+
+def classify_guard_submission_error(status_code: int, detail: Any) -> str:
+    denial = parse_guard_denial_payload(detail)
+    if status_code == 402 and denial is not None:
+        return "guard_denial"
+    return "transport_or_system_error"
+
+
+async def submit_usage_guard_check(
+    *,
+    usage_kind: str,
+    units: int,
+    timeout_seconds: float = 10.0,
+) -> Dict[str, Any]:
+    if os.getenv("TOPOS_USAGE_GUARD_ENFORCEMENT_ENABLED", "").strip().lower() not in {"1", "true", "yes", "on"}:
+        return {"allowed": True, "source": "guard_not_enabled"}
+
+    control_plane_http = str(settings.topos_control_plane_url or "").strip()
+    api_key = str(settings.topos_key or "").strip()
+    if control_plane_http.startswith("wss://"):
+        control_plane_http = control_plane_http.replace("wss://", "https://")
+    elif control_plane_http.startswith("ws://"):
+        control_plane_http = control_plane_http.replace("ws://", "http://")
+    control_plane_http = control_plane_http.split("/ws/")[0].rstrip("/")
+
+    # Allow dev/test operation when control-plane connectivity is absent.
+    if not control_plane_http or not api_key:
+        return {"allowed": True, "source": "guard_not_configured"}
+
+    url = f"{control_plane_http}/v1/billing/usage/charge"
+    headers = {"Authorization": f"Bearer {api_key}"}
+    payload = {"usage_kind": usage_kind, "units": int(max(0, units))}
+    async with httpx.AsyncClient(timeout=timeout_seconds) as client:
+        response = await client.post(url, headers=headers, json=payload)
+    if response.status_code == 200:
+        body = response.json() if response.content else {}
+        return {"allowed": True, "source": "control_plane", "payload": body}
+
+    detail = None
+    try:
+        body = response.json() if response.content else {}
+        detail = body.get("detail", body)
+    except Exception:
+        detail = response.text
+    classification = classify_guard_submission_error(response.status_code, detail)
+    if classification == "guard_denial":
+        denial = parse_guard_denial_payload(detail) or {}
+        logger.info(
+            "usage guard denied metric=%s reason=%s period_start=%s period_end=%s",
+            denial.get("metric_key"),
+            denial.get("reason_code"),
+            denial.get("period_start"),
+            denial.get("period_end"),
+        )
+        return {"allowed": False, "source": "control_plane", "denial": denial}
+    logger.warning("usage guard transport/system failure status=%s", response.status_code)
+    response.raise_for_status()
+    return {"allowed": True, "source": "control_plane_unreachable"}

topos/engine/usage_observation.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+
+from ..config.settings import settings
+
+
+ACTION_TO_METRIC_KEY: Dict[str, str] = {
+    "llm.generate": "llm_tokens",
+    "ingestion.file_processed": "file_transfer_mb",
+    "uma.permission_ticket.validated": "permission_tickets",
+    "source.install.completed": "source_installs",
+    "contacts.google.connect.started": "third_party_connections",
+}
+
+
+def map_action_to_metric_key(action: str) -> Optional[str]:
+    return ACTION_TO_METRIC_KEY.get(str(action or "").strip())
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _stable_identity_hash(identity: Dict[str, Any]) -> str:
+    canonical = json.dumps(identity, sort_keys=True, separators=(",", ":"), default=str)
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def derive_usage_idempotency_key(
+    *,
+    producer: str,
+    metric_key: str,
+    action: str,
+    canonical_action_identity: Dict[str, Any],
+) -> str:
+    return ":".join(
+        [
+            str(producer or "engine"),
+            str(metric_key or "unknown"),
+            str(action or "unknown"),
+            _stable_identity_hash(canonical_action_identity),
+        ]
+    )
+
+
+def build_usage_observation_envelope(
+    *,
+    action: str,
+    quantity: int,
+    producer: str,
+    canonical_action_identity: Dict[str, Any],
+    topos_id: Optional[str] = None,
+    source: Optional[str] = None,
+    observed_by: str = "engine",
+    trust_class: str = "observe_only",
+    metadata: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    metric_key = map_action_to_metric_key(action)
+    if not metric_key:
+        raise ValueError(f"Unknown usage action: {action}")
+    idempotency_key = derive_usage_idempotency_key(
+        producer=str(producer or "engine"),
+        metric_key=metric_key,
+        action=action,
+        canonical_action_identity=canonical_action_identity,
+    )
+    resolved_topos_id = str(topos_id or "").strip() or f"engine:{str(settings.topos_key or '')[:12]}"
+    return {
+        "event_id": f"eng_usage_{uuid.uuid4().hex}",
+        "topos_id": resolved_topos_id,
+        "metric_key": metric_key,
+        "quantity": int(max(0, quantity)),
+        "event_at": _now_iso(),
+        "source": str(source or f"engine.{producer}"),
+        "observed_by": observed_by,
+        "idempotency_key": idempotency_key,
+        "producer": producer,
+        "action": action,
+        "trust_class": trust_class,
+        "metadata": metadata or {},
+    }
+
+
+async def emit_usage_observation(
+    *,
+    action: str,
+    quantity: int,
+    producer: str,
+    canonical_action_identity: Dict[str, Any],
+    topos_id: Optional[str] = None,
+    source: Optional[str] = None,
+    observed_by: str = "engine",
+    trust_class: str = "observe_only",
+    metadata: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    envelope = build_usage_observation_envelope(
+        action=action,
+        quantity=quantity,
+        producer=producer,
+        canonical_action_identity=canonical_action_identity,
+        topos_id=topos_id,
+        source=source,
+        observed_by=observed_by,
+        trust_class=trust_class,
+        metadata=metadata,
+    )
+
+    # Use existing control-plane transport abstraction (WS unsolicited message).
+    try:
+        from ..core import state as engine_state
+
+        cp_client = getattr(engine_state, "control_plane_client", None)
+        if cp_client is not None:
+            await cp_client.send_message(
+                {
+                    "id": f"usage_obs_{uuid.uuid4().hex}",
+                    "type": "usage_observation",
+                    "payload": envelope,
+                }
+            )
+    except Exception:
+        # Observation emission must be non-blocking for product flows.
+        pass
+    return envelope

topos/engine/validator.py

@@ -0,0 +1,23 @@
+"""Task validator: reject invalid tasks."""
+
+from __future__ import annotations
+
+from typing import Optional, Tuple
+
+from .tasks import ProcessingTask
+
+
+def validate_task(task: ProcessingTask) -> Tuple[bool, Optional[str]]:
+    """
+    Validate required fields and obvious invariants.
+    Returns (is_valid, error_message).
+    """
+    if not task.id or not str(task.id).strip():
+        return False, "task id is required and must be non-empty"
+    if not task.type or not str(task.type).strip():
+        return False, "task type is required and must be non-empty"
+    if not task.model_request:
+        return False, "model_request is required"
+    if not task.model_request.provider or not str(task.model_request.provider).strip():
+        return False, "model_request.provider is required"
+    return True, None

topos/enrichment/__init__.py

@@ -0,0 +1 @@
+"""Enrichment layer for Topos."""

topos/enrichment/derived_tables.py

@@ -0,0 +1,214 @@
+"""Derived tables manager for enrichment data storage."""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from ..utils.base_object import BaseObject
+
+logger = logging.getLogger("topos.enrichment.derived_tables")
+
+
+class DerivedTablesManager(BaseObject):
+    """Manages derived tables for enrichment data."""
+
+    def __init__(self, conn: Optional[sqlite3.Connection] = None, *, name: Optional[str] = None):
+        """Initialize with optional database connection.
+        
+        Args:
+            conn: SQLite connection. If None, will try to get from state or create new.
+            name: Optional custom name. Defaults to `ClassName#N`
+        """
+        super().__init__(name=name)
+        self.conn = conn
+        if self.conn is None:
+            # Try to get connection from state
+            try:
+                from ..core.state import db_conn
+                self.conn = db_conn
+            except Exception:
+                pass
+        
+        # If still no connection, try to create one
+        if self.conn is None:
+            try:
+                from ..storage.db.paths import get_database_path
+                from ..config.settings import settings
+                
+                db_path = get_database_path(settings.topos_database_path)
+                if db_path.exists() or db_path.parent.exists():
+                    self.conn = sqlite3.connect(str(db_path))
+                    logger.debug("%s: Created database connection: %s", self, db_path)
+            except Exception as e:
+                logger.warning("%s: Could not create database connection: %s", self, e)
+                self.conn = None
+        
+        # Ensure tables exist
+        if self.conn:
+            self._ensure_tables()
+
+    def _ensure_tables(self) -> None:
+        """Ensure enrichment tables exist."""
+        if not self.conn:
+            return
+        
+        try:
+            # Create message_emotions table (Stage 9: model_name, all_emotions_json)
+            self.conn.execute("""
+                CREATE TABLE IF NOT EXISTS message_emotions (
+                    message_id TEXT NOT NULL,
+                    source_id TEXT,
+                    emotion_label TEXT,
+                    confidence REAL,
+                    model_name TEXT,
+                    all_emotions_json TEXT,
+                    created_at TEXT NOT NULL,
+                    PRIMARY KEY (message_id, model_name)
+                )
+            """)
+            
+            # Add source_id column if it doesn't exist (migration for existing tables)
+            try:
+                self.conn.execute("ALTER TABLE message_emotions ADD COLUMN source_id TEXT")
+            except sqlite3.OperationalError:
+                # Column already exists, ignore
+                pass
+            
+            self.conn.execute("""
+                CREATE INDEX IF NOT EXISTS idx_message_emotions_message 
+                ON message_emotions(message_id)
+            """)
+            
+            self.conn.execute("""
+                CREATE INDEX IF NOT EXISTS idx_message_emotions_label 
+                ON message_emotions(emotion_label)
+            """)
+            
+            self.conn.execute("""
+                CREATE INDEX IF NOT EXISTS idx_message_emotions_source 
+                ON message_emotions(source_id)
+            """)
+            
+            self.conn.commit()
+            logger.debug("%s: Ensured message_emotions table exists", self)
+        except Exception as e:
+            logger.error("%s: Failed to ensure enrichment tables: %s", self, e)
+            if self.conn:
+                self.conn.rollback()
+
+    def write_enrichment_batch(
+        self,
+        enrichment_records: List[Dict[str, Any]],
+        table_name: str,
+        batch_size: int = 1000,
+    ) -> int:
+        """Write enrichment records to derived table in batches.
+        
+        Args:
+            enrichment_records: List of enrichment record dicts
+            table_name: Derived table name (e.g., 'message_emotions', 'message_sentiment')
+            batch_size: Number of records per batch
+            
+        Returns:
+            Number of records written
+        """
+        if not enrichment_records:
+            return 0
+        
+        if not self.conn:
+            logger.warning("%s: No database connection available, skipping storage of %d records", self, len(enrichment_records))
+            return 0
+        
+        # Determine table schema based on table_name
+        if table_name == "message_emotions":
+            return self._write_emotions_batch(enrichment_records, batch_size)
+        elif table_name == "message_topics":
+            return self._write_topics_batch(enrichment_records, batch_size)
+        elif table_name == "message_sentiment":
+            return self._write_sentiment_batch(enrichment_records, batch_size)
+        elif table_name == "message_embeddings":
+            return self._write_embeddings_batch(enrichment_records, batch_size)
+        else:
+            logger.warning("%s: Unknown derived table: %s", self, table_name)
+            return 0
+
+    def _write_emotions_batch(
+        self,
+        records: List[Dict[str, Any]],
+        batch_size: int,
+    ) -> int:
+        """Write emotion records to message_emotions table."""
+        written = 0
+        try:
+            self._ensure_tables()
+            extracted_at = datetime.now(timezone.utc).isoformat()
+            
+            for i in range(0, len(records), batch_size):
+                batch = records[i:i + batch_size]
+                
+                values = []
+                for record in batch:
+                    import json
+                    all_emotions_val = record.get("all_emotions_json") or record.get("all_emotions") or []
+                    all_emotions_str = json.dumps(all_emotions_val) if isinstance(all_emotions_val, list) else (all_emotions_val if isinstance(all_emotions_val, str) else "[]")
+                    values.append((
+                        record.get("message_id"),
+                        record.get("source_id"),
+                        record.get("emotion_label"),
+                        record.get("confidence"),
+                        record.get("model_name") or record.get("model"),
+                        all_emotions_str,
+                        extracted_at,
+                    ))
+                
+                self.conn.executemany("""
+                    INSERT OR REPLACE INTO message_emotions (
+                        message_id, source_id, emotion_label, confidence, model_name, all_emotions_json, created_at
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?)
+                """, values)
+                
+                self.conn.commit()
+                written += len(batch)
+                logger.debug(
+                    "[PIPELINE:ENRICHMENT] %s: Wrote batch of %d emotion records (total: %d)",
+                    self,
+                    len(batch),
+                    written,
+                )
+        except Exception as e:
+            if self.conn:
+                self.conn.rollback()
+            logger.error("[PIPELINE:ENRICHMENT] %s: Failed to write emotions batch: %s", self, e)
+            raise
+        
+        return written
+
+    def _write_topics_batch(
+        self,
+        records: List[Dict[str, Any]],
+        batch_size: int,
+    ) -> int:
+        """Write topics records (stub)."""
+        logger.debug("%s: Topics batch write not yet implemented", self)
+        return 0
+
+    def _write_sentiment_batch(
+        self,
+        records: List[Dict[str, Any]],
+        batch_size: int,
+    ) -> int:
+        """Write sentiment records (stub)."""
+        logger.debug("%s: Sentiment batch write not yet implemented", self)
+        return 0
+
+    def _write_embeddings_batch(
+        self,
+        records: List[Dict[str, Any]],
+        batch_size: int,
+    ) -> int:
+        """Write embeddings records (stub)."""
+        logger.debug("%s: Embeddings batch write not yet implemented", self)
+        return 0