topos-node 1.3.55__tar.gz → 1.3.57__tar.gz

{topos_node-1.3.55/topos_node.egg-info → topos_node-1.3.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.55
+Version: 1.3.57
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai

{topos_node-1.3.55 → topos_node-1.3.57}/pyproject.toml

@@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.55"
+version = "1.3.57"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"

{topos_node-1.3.55 → topos_node-1.3.57}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.55"
-__version_info__ = (1, 3, 55)
+__version__ = "1.3.57"
+__version_info__ = (1, 3, 57)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.3.55 → topos_node-1.3.57}/topos/api/uma_data.py

@@ -6,17 +6,17 @@ from __future__ import annotations
 
 import hashlib
 
-from typing import Any, Dict, List, Optional, Set, Tuple
+from typing import Any, Dict, List, Optional, Set
 
 from fastapi import APIRouter, HTTPException, Query, Request, status
 
 from ..core.state import get_db_connection
 from ..uma_rpt import RPTValidationError, get_control_plane_http_base, introspect_for_resource
-from ..uma_filters import UMAFilterError, apply_filter_manifest, apply_filters, extract_field_transforms, extract_filter_manifest, get_limit_cap
-from ..scope_resolution import resolve_scopes_to_tables, may_access_table
+from ..uma_filters import UMAFilterError, apply_filter_manifest, extract_field_transforms, extract_filter_manifest, get_limit_cap
 from ..uma_contact_enrichment import apply_message_contact_pipeline, strip_contact_runtime_filters
 from ..uma_resource_id import parse_dataset_id_from_uma_dataset_resource_id
 from ..engine.usage_observation import emit_usage_observation
+from ..uma_authority import bound_uma_scope, dataset_scope_predicate, message_stream_granted, local_node_resource_scope, require_local_resource_binding
 
 router = APIRouter(prefix="/v1/uma/resources", tags=["uma-data"])
 
@@ -56,111 +56,42 @@ def _get_messages_from_db(
     limit: int,
     offset: int,
     allowed_tables: Optional[Set[str]] = None,
+    owner_user_id: Optional[str] = None,
+    *, whole_engine_scope: bool = False,
 ) -> List[Dict[str, Any]]:
-    """Query only the message-like tables permitted by the granted scopes."""
-
-    def _fetch_rows(query: str, params: Tuple[int, int]) -> List[Dict[str, Any]]:
-        cursor = conn.execute(query, params)
-        columns = [d[0] for d in cursor.description]
-        return [dict(zip(columns, row)) for row in cursor.fetchall()]
-
-    def _allowed_message_sources() -> List[str]:
-        if not allowed_tables:
-            candidates = []
-            if _table_exists(conn, "conversation_messages"):
-                candidates.append("conversation_messages")
-            elif _table_exists(conn, "messages"):
-                candidates.append("messages")
-            if _table_exists(conn, "ai_chat_messages"):
-                candidates.append("ai_chat_messages")
-            return candidates
-
-        candidates: List[str] = []
-        if "messages" in allowed_tables:
-            if _table_exists(conn, "conversation_messages"):
-                candidates.append("conversation_messages")
-            elif _table_exists(conn, "messages"):
-                candidates.append("messages")
-        if "conversation_messages" in allowed_tables and _table_exists(conn, "conversation_messages"):
-            if "conversation_messages" not in candidates:
-                candidates.append("conversation_messages")
-        if (
-            {"ai_chat", "ai_messages", "ai_chat_messages"} & set(allowed_tables)
-            and _table_exists(conn, "ai_chat_messages")
-        ):
-            candidates.append("ai_chat_messages")
-        return candidates
-
-    sources = _allowed_message_sources()
-    if not sources:
+    """Read only explicitly granted message tables, scoped before pagination."""
+    if not allowed_tables:
         return []
-
-    # Keep single-table queries paginated in SQL; when multiple tables are allowed,
-    # fetch and merge in Python so limit/offset apply to the combined result set.
-    if sources == ["conversation_messages"]:
-        oc = _message_time_order_column(conn, "conversation_messages")
-        return _fetch_rows(
-            f"SELECT * FROM conversation_messages ORDER BY {oc} DESC LIMIT ? OFFSET ?",
-            (limit, offset),
-        )
-    if sources == ["messages"]:
-        return _fetch_rows(
-            "SELECT * FROM messages ORDER BY ts DESC LIMIT ? OFFSET ?",
-            (limit, offset),
-        )
-    if sources == ["ai_chat_messages"]:
-        oc = _message_time_order_column(conn, "ai_chat_messages")
-        return _fetch_rows(
-            f"SELECT * FROM ai_chat_messages ORDER BY {oc} DESC LIMIT ? OFFSET ?",
-            (limit, offset),
-        )
-
+    sources: List[str] = []
+    if {"messages", "conversation_messages"} & allowed_tables:
+        if _table_exists(conn, "conversation_messages"):
+            sources.append("conversation_messages")
+        elif _table_exists(conn, "messages"):
+            sources.append("messages")
+    if {"ai_chat", "ai_messages", "ai_chat_messages"} & allowed_tables and _table_exists(conn, "ai_chat_messages"):
+        sources.append("ai_chat_messages")
     merged: List[Dict[str, Any]] = []
-    if "conversation_messages" in sources:
-        oc_cm = _message_time_order_column(conn, "conversation_messages")
-        merged.extend(
-            _fetch_rows(
-                f"SELECT * FROM conversation_messages ORDER BY {oc_cm} DESC LIMIT ? OFFSET ?",
-                (limit + offset, 0),
-            )
-        )
-    if "messages" in sources:
-        merged.extend(
-            _fetch_rows(
-                "SELECT * FROM messages ORDER BY ts DESC LIMIT ? OFFSET ?",
-                (limit + offset, 0),
-            )
+    from ..disclosure.tier import apply_disclosure_tier_to_rows
+    for table in sources:
+        scope, scope_params = dataset_scope_predicate(
+            _sqlite_table_columns(conn, table), dataset_id, owner_user_id,
+            whole_engine_scope=whole_engine_scope,
         )
-    if "ai_chat_messages" in sources:
-        oc_ac = _message_time_order_column(conn, "ai_chat_messages")
-        merged.extend(
-            _fetch_rows(
-                f"SELECT * FROM ai_chat_messages ORDER BY {oc_ac} DESC LIMIT ? OFFSET ?",
-                (limit + offset, 0),
-            )
+        order_column = _message_time_order_column(conn, table)
+        # Fetch at most the requested prefix from each eligible table; no
+        # out-of-dataset row can consume the limit or alter the returned count.
+        cursor = conn.execute(
+            f'SELECT * FROM "{table}" WHERE {scope} ORDER BY "{order_column}" DESC, message_id DESC LIMIT ?',
+            scope_params + (limit + offset,),
         )
+        columns = [column[0] for column in cursor.description]
+        rows = [dict(zip(columns, row)) for row in cursor.fetchall()]
+        disclosure_table = "conversation_messages" if table == "messages" else table
+        merged.extend(apply_disclosure_tier_to_rows(rows, table=disclosure_table, tier="default_disclosure"))
     merged.sort(key=lambda row: (row.get("event_at") or row.get("ts") or "", row.get("message_id") or ""), reverse=True)
     return merged[offset:offset + limit]
 
 
-def _get_oplog_from_db(
-    conn,
-    dataset_id: Optional[str],
-    limit: int,
-    offset: int,
-) -> List[Dict[str, Any]]:
-    """Query oplog table if it exists; otherwise return empty list."""
-    if not _table_exists(conn, "oplog"):
-        return []
-    query = "SELECT * FROM oplog ORDER BY hlc_ts LIMIT ? OFFSET ?"
-    try:
-        cursor = conn.execute(query, (limit, offset))
-        columns = [d[0] for d in cursor.description]
-        return [dict(zip(columns, row)) for row in cursor.fetchall()]
-    except Exception:
-        return []
-
-
 def _extract_bearer(request: Request) -> Optional[str]:
     auth = request.headers.get("authorization")
     if not auth or not auth.startswith("Bearer "):
@@ -216,28 +147,37 @@ async def get_uma_messages(
 ):
     """
     Return messages for the UMA resource, filtered by the permission's filters.
-    Sprint 05: returns data only if RPT allows messages, ai_messages, or ai_chat scope.
+    Exact message-family scopes select the tables; resource authority selects rows.
     """
     resource_id = resource_id.strip()
     payload = await require_uma_rpt(request, resource_id)
+    try:
+        # Validate both introspected hints and any narrower query hint.
+        bound_uma_scope({**payload, "resource_id": resource_id})
+        effective_dataset_id, bound_owner = bound_uma_scope({
+            **payload, "resource_id": resource_id, "dataset_id": dataset_id,
+        })
+    except ValueError as exc:
+        raise HTTPException(status_code=403, detail=str(exc)) from exc
     allowed_scopes = payload.get("allowed_scopes") or []
-    allowed_tables = resolve_scopes_to_tables(allowed_scopes)
+    # This endpoint can combine the two streams only when both are granted.
+    allowed_tables: Set[str] = set()
+    if message_stream_granted(allowed_scopes, "conversation"):
+        allowed_tables.update({"conversation_messages", "messages"})
+    if message_stream_granted(allowed_scopes, "ai_chat"):
+        allowed_tables.add("ai_chat_messages")
     if not allowed_tables:
-        return {"messages": [], "count": 0}
-    if not (
-        may_access_table(allowed_tables, "messages")
-        or may_access_table(allowed_tables, "conversation_messages")
-        or may_access_table(allowed_tables, "ai_chat_messages")
-        or may_access_table(allowed_tables, "ai_chat")
-        or may_access_table(allowed_tables, "ai_messages")
-    ):
-        return {"messages": [], "count": 0}
+        raise HTTPException(status_code=403, detail="message_scope_required")
     conn = get_db_connection()
     if not conn:
         raise HTTPException(
             status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
             detail="Database not initialized",
         )
+    try:
+        require_local_resource_binding(conn, resource_id, bound_owner)
+    except ValueError as exc:
+        raise HTTPException(status_code=403, detail=str(exc)) from exc
     filters = (request.state.uma_introspection or {}).get("filters")
     manifest = extract_filter_manifest(filters if isinstance(filters, dict) else None)
     ai_only = bool(
@@ -248,10 +188,14 @@ async def get_uma_messages(
     )
     logical_table = "ai_chat_messages" if ai_only else "conversation_messages" if conv_only else None
     limited = get_limit_cap(limit, manifest, logical_table)
-    effective_dataset_id = (dataset_id or "").strip() or parse_dataset_id_from_uma_dataset_resource_id(
-        resource_id
-    )
-    items = _get_messages_from_db(conn, effective_dataset_id, limited, offset, allowed_tables=allowed_tables)
+    try:
+        items = _get_messages_from_db(
+            conn, effective_dataset_id, limited, offset,
+            allowed_tables=allowed_tables, owner_user_id=bound_owner,
+            whole_engine_scope=local_node_resource_scope(conn, resource_id, rpt_validated=True),
+        )
+    except ValueError as exc:
+        raise HTTPException(status_code=403, detail=str(exc)) from exc
     try:
         # UMA data proxy = the grantee HTTP lane; filter unconditionally.
         from ..features.lifecycle.blackhole_guard import BlackholeGuard, CallerClass
@@ -290,26 +234,6 @@ async def get_uma_oplog(
     offset: int = Query(0, ge=0),
     dataset_id: Optional[str] = Query(None),
 ):
-    """
-    Return oplog entries for the UMA resource, filtered by the permission's filters.
-    Sprint 05: returns data only if RPT has at least one allowed scope (any read access).
-    """
-    resource_id = resource_id.strip()
-    payload = await require_uma_rpt(request, resource_id)
-    allowed_scopes = payload.get("allowed_scopes") or []
-    allowed_tables = resolve_scopes_to_tables(allowed_scopes)
-    if not allowed_tables:
-        return {"ops": [], "count": 0}
-    conn = get_db_connection()
-    if not conn:
-        raise HTTPException(
-            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-            detail="Database not initialized",
-        )
-    items = _get_oplog_from_db(conn, dataset_id, limit, offset)
-    filters = (request.state.uma_introspection or {}).get("filters")
-    try:
-        filtered = apply_filters(items, filters)
-    except UMAFilterError as exc:
-        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) from exc
-    return {"ops": filtered, "count": len(filtered)}
+    """No safe grantable projection exists for raw operation logs."""
+    await require_uma_rpt(request, resource_id.strip())
+    raise HTTPException(status_code=403, detail="shared_oplog_unavailable")

{topos_node-1.3.55 → topos_node-1.3.57}/topos/app.py

@@ -229,6 +229,26 @@ async def _reap_upgrade_runner(timeout_s: float = _UPGRADE_JOIN_TIMEOUT_S) -> No
         )
 
 
+async def _reap_rebuild_children(grace_s: float = 2.0) -> None:
+    """Stop any entity-graph rebuild child this process started.
+
+    The child is a separate process writing the same database, and nothing
+    else stops it: the supervisor signals the node's pid only. Left alone it
+    ran on after the node exited, holding the rebuild lock (eight node starts
+    on 2026-09-08 found one), and a rebuild awaited through asyncio.to_thread
+    held this process's exit in asyncio.run until the child finished.
+    """
+    from .features.entities.rebuild_subprocess import stop_rebuild_children
+
+    try:
+        stopped = await asyncio.to_thread(stop_rebuild_children, grace_s)
+    except Exception as exc:  # noqa: BLE001 — teardown never raises
+        logger.warning("could not stop graph rebuild subprocess at shutdown: %s", exc)
+        return
+    if stopped:
+        logger.info("stopped %d graph rebuild subprocess(es) at shutdown", stopped)
+
+
 @app.on_event("startup")
 async def startup_event() -> None:
     from .runtime_shutdown import begin_runtime, install_shutdown_signal_hooks
@@ -239,6 +259,18 @@ async def startup_event() -> None:
     # leaves theirs retired and gives this run its own.
     begin_runtime("app_startup")
     install_shutdown_signal_hooks()
+    # The last run's shutdown closed the graph-rebuild spawn gate; this run may
+    # rebuild again. And a stop request ends rebuild children at the signal,
+    # not only in shutdown_event, which an in-flight rebuild request can keep
+    # uvicorn from ever reaching (see rebuild_subprocess.signal_rebuild_children).
+    from .features.entities.rebuild_subprocess import (
+        allow_rebuild_children,
+        signal_rebuild_children,
+    )
+    from .runtime_shutdown import add_shutdown_listener
+
+    allow_rebuild_children()
+    add_shutdown_listener(signal_rebuild_children)
     # This process holds the database, so its backups are the ones the model
     # manager may spend before it evicts a model. The disk floor never resolves
     # that directory for itself — on a split node it runs beside the models, on
@@ -624,6 +656,10 @@ async def shutdown_event() -> None:
     # Reaped before the network teardown below, because these are the writers.
     if state.upgrade_runner_stop is not None:
         state.upgrade_runner_stop.set()
+    # First, because it unblocks the rest: a background task awaiting a rebuild
+    # through asyncio.to_thread is stuck until the child dies, however it is
+    # cancelled.
+    await _reap_rebuild_children()
     await _reap_background_tasks()
     await stop_pipeline_worker()
     await _reap_upgrade_runner()
@@ -649,3 +685,6 @@ async def shutdown_event() -> None:
         state.hosted_pool_lease_client = None
     if state.sync_client:
         await state.sync_client.stop()
+    # Again, last: a worker drained above could have started a rebuild after the
+    # first pass, and one awaited from an executor thread would hold exit open.
+    await _reap_rebuild_children()

{topos_node-1.3.55 → topos_node-1.3.57}/topos/cli/tray.py

@@ -617,6 +617,12 @@ def _reexec_topos_node() -> None:
     PyInstaller bundle on Windows all differ), while the interpreter running
     us always is.
     """
+    # execv keeps this pid, so a graph rebuild child's parent watch never fires,
+    # and the new image starts with an empty child registry: stop it here or it
+    # runs on unowned, holding the rebuild lock against the node we become.
+    from topos.features.entities.rebuild_subprocess import stop_rebuild_children
+
+    stop_rebuild_children()
     os.execv(sys.executable, [sys.executable, "-m", "topos.cli.commands", *sys.argv[1:]])
 
 
@@ -697,7 +703,19 @@ def serve_with_tray(
     server = uvicorn.Server(config)
 
     def stop_server() -> None:
-        server.should_exit = True
+        # Every tray exit comes through here (Quit, Select Topos, Ctrl+C). None
+        # is a signal the node's shutdown hooks see — uvicorn runs off the main
+        # thread here, so they never install — and uvicorn drains an in-flight
+        # graph rebuild request before shutdown_event can stop the child. So
+        # stop it now; the request then ends and the drain can finish.
+        try:
+            from topos.features.entities.rebuild_subprocess import signal_rebuild_children
+
+            signal_rebuild_children()
+        except Exception as exc:  # noqa: BLE001 — never let this cost the exit itself
+            print(f"Could not stop graph rebuild children ({exc}); quitting anyway.")
+        finally:
+            server.should_exit = True
 
     server_thread = threading.Thread(target=server.run, daemon=True)
     server_thread.start()

{topos_node-1.3.55 → topos_node-1.3.57}/topos/config/settings.py

@@ -163,7 +163,7 @@ class Settings(BaseSettings):
     sanitization_ollama_timeout_sec: float = Field(120.0)
     # llm_generation / routine hops — must cover local thinking models. Do NOT
     # reuse sanitization_ollama_timeout_sec (120s): that cuts digest synthesis
-    # mid-flight and surfaces as "Ollama unreachable".
+    # mid-flight and surfaces as "Ollama did not answer within 120s".
     engine_ollama_generate_timeout_sec: float = Field(300.0)
     ollama_list_timeout_sec: float = Field(10.0)
     sanitization_ollama_auto_pull: bool = Field(True)

{topos_node-1.3.55 → topos_node-1.3.57}/topos/control_plane_client.py

@@ -517,7 +517,16 @@ class ControlPlaneClient:
             await ws.send(json.dumps(payload))
             return True
         except Exception as exc:  # noqa: BLE001
-            logger.error("Failed to send message to control plane: %s", exc)
+            # Routing fields only, never the payload: this is the one line that
+            # says which request's answer died on a dead socket.
+            logger.error(
+                "Failed to send message to control plane: id=%s type=%s status=%s exc=%s: %s",
+                payload.get("id"),
+                payload.get("type"),
+                payload.get("status"),
+                type(exc).__name__,
+                exc,
+            )
             return False
 
     def set_device_info_snapshot(self, payload: dict[str, Any] | None) -> None:

{topos_node-1.3.55 → topos_node-1.3.57}/topos/core/handlers/device.py

@@ -280,13 +280,34 @@ async def handle_llm_generation(message: Dict[str, Any]) -> Optional[Dict[str, A
     unusable = _unusable_num_ctx(payload)
     if unusable is not None:
         return {"id": req_id, "status": "error", "error": unusable, "error_code": 422}
+    started, wall_started = time.monotonic(), time.time()
+    # Every outcome line carries the id: two routines overlap on the node, and
+    # the service-layer lines have none, so pairing by order is ambiguous.
     logger.info(
-        "llm_generation request: provider=%r model=%r prompt_chars=%d stream=%s",
+        "llm_generation request: id=%s provider=%r model=%r prompt_chars=%d stream=%s",
+        req_id,
         payload.get("provider"),
         payload.get("model"),
         len(str(payload.get("prompt") or "")),
         bool(payload.get("stream")),
     )
+
+    def _log_failed(exc: BaseException, code: Any) -> None:
+        # Classes and codes only: an error's text can quote an Ollama body.
+        # openai.py raises `from exc`, so the cause names the transport error.
+        logger.warning(
+            "llm_generation failed: id=%s provider=%r model=%r code=%s cause=%s"
+            " elapsed_s=%.1f wall_s=%.1f stream=%s",
+            req_id,
+            payload.get("provider"),
+            payload.get("model"),
+            code,
+            type(exc.__cause__ or exc).__name__,
+            time.monotonic() - started,
+            time.time() - wall_started,
+            bool(payload.get("stream")),
+        )
+
     current = asyncio.current_task()
     if current is not None:
         _LLM_GENERATION_TASKS[str(req_id)] = current
@@ -335,6 +356,14 @@ async def handle_llm_generation(message: Dict[str, Any]) -> Optional[Dict[str, A
         result = await get_services().llm.generate(
             payload, on_delta=on_delta, on_protocol=on_protocol
         )
+        logger.info(
+            "llm_generation done: id=%s model=%r output_chars=%d elapsed_s=%.1f wall_s=%.1f",
+            req_id,
+            payload.get("model"),
+            len(str((result or {}).get("output") or "")),
+            time.monotonic() - started,
+            time.time() - wall_started,
+        )
         return {"id": req_id, "status": "ok", "payload": result}
     except asyncio.CancelledError:
         # llm_cancel: answer the original request with a typed 499 so the
@@ -354,10 +383,13 @@ async def handle_llm_generation(message: Dict[str, Any]) -> Optional[Dict[str, A
                         }
                     )
                 )
-            except Exception:  # noqa: BLE001 — reporting must not mask the cancel
-                pass
+            except Exception as e:  # noqa: BLE001 — reporting must not mask the cancel
+                logger.warning(
+                    "llm_generation cancel reply not sent: id=%s exc=%s", req_id, type(e).__name__
+                )
         raise
     except LlmTypedError as exc:
+        _log_failed(exc, exc.code)
         return {
             "id": req_id,
             "status": "error",
@@ -365,6 +397,7 @@ async def handle_llm_generation(message: Dict[str, Any]) -> Optional[Dict[str, A
             "error_code": exc.code,
         }
     except HTTPException as exc:
+        _log_failed(exc, exc.status_code)
         detail = exc.detail
         err_msg = detail if isinstance(detail, str) else str(detail)
         return {
@@ -374,7 +407,10 @@ async def handle_llm_generation(message: Dict[str, Any]) -> Optional[Dict[str, A
             "error_code": exc.status_code,
         }
     except Exception as exc:  # noqa: BLE001
-        return {"id": req_id, "status": "error", "error": str(exc)}
+        _log_failed(exc, None)
+        # A bare `ValueError()` has empty text; the control plane would record
+        # a failure with no reason at all.
+        return {"id": req_id, "status": "error", "error": str(exc) or type(exc).__name__}
     finally:
         _LLM_GENERATION_TASKS.pop(str(req_id), None)