topos-node 1.5.3__tar.gz → 1.5.4__tar.gz

{topos_node-1.5.3/topos_node.egg-info → topos_node-1.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.5.3
+Version: 1.5.4
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai

{topos_node-1.5.3 → topos_node-1.5.4}/pyproject.toml

@@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.5.3"
+version = "1.5.4"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"

{topos_node-1.5.3 → topos_node-1.5.4}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.5.3"
-__version_info__ = (1, 5, 3)
+__version__ = "1.5.4"
+__version_info__ = (1, 5, 4)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.5.3 → topos_node-1.5.4}/topos/app.py

@@ -287,6 +287,13 @@ async def startup_event() -> None:
 
     allow_rebuild_children()
     add_shutdown_listener(signal_rebuild_children)
+    # BL-168: the stop's own steps (control-plane client first, the ingestion queue parked, the permissions
+    # runtime's background work stopped) start at the signal, not when uvicorn's drain reaches shutdown_event.
+    from . import node_stop
+
+    node_stop.bind_loop(asyncio.get_running_loop())
+    node_stop.set_background_reaper(_reap_background_tasks)
+    add_shutdown_listener(node_stop.begin_stop)
     # This process holds the database, so its backups are the ones the model
     # manager may spend before it evicts a model. The disk floor never resolves
     # that directory for itself — on a split node it runs beside the models, on
@@ -706,11 +713,19 @@ async def startup_event() -> None:
 
 @app.on_event("shutdown")
 async def shutdown_event() -> None:
+    from . import node_stop
     from .pipeline.job_runner import stop_pipeline_worker
-    from .runtime_shutdown import end_runtime
+    from .runtime_shutdown import current_generation, end_runtime
 
     # Wake cooperative workers (fact_llm / Ollama threads) before awaiting
     # network teardown so Ctrl+C does not wait out a full enrichment batch.
+    run = current_generation()
+    run.request_stop("app_shutdown")
+    # BL-168: the control-plane client closes first, then the ingestion queue
+    # is parked (the held job requeued) and the permissions runtime's background
+    # work stopped. The same task when a signal or the tray already began it.
+    # Before end_runtime, so the queue is parked under THIS run's generation.
+    await node_stop.stop_now(run)
     # end_runtime retires THIS run's generation — its workers stop for good —
     # and installs a fresh one, so anything that runs later in this process
     # (another app run, a CLI command, the next test) does not inherit a
@@ -723,10 +738,10 @@ async def shutdown_event() -> None:
     # Reaped before the network teardown below, because these are the writers.
     if state.upgrade_runner_stop is not None:
         state.upgrade_runner_stop.set()
-    # First, because it unblocks the rest: a background task awaiting a rebuild
-    # through asyncio.to_thread is stuck until the child dies, however it is
-    # cancelled.
-    await _reap_rebuild_children()
+    # Rebuild children were stopped first inside stop_now (after the control-
+    # plane client), because it unblocks the rest: a background task awaiting a
+    # rebuild through asyncio.to_thread is stuck until the child dies, however
+    # it is cancelled.
     await _reap_background_tasks()
     await stop_pipeline_worker()
     await _reap_upgrade_runner()
@@ -738,8 +753,7 @@ async def shutdown_event() -> None:
         except asyncio.CancelledError:
             pass
         state.engine_presence_task = None
-    if state.control_plane_client:
-        await state.control_plane_client.stop()
+    # The control-plane client was stopped first, by node_stop.stop_now above.
     # The first-pin thread waits between tries; tell it the app is gone (never blocks).
     from .relay_stamp import stop_first_pin
 

{topos_node-1.5.3 → topos_node-1.5.4}/topos/cli/commands.py

@@ -419,7 +419,10 @@ def main(
             log_path=get_log_file_path(),
         )
     else:
-        uvicorn.run(app, host=host, port=port, log_config=get_uvicorn_log_config())
+        from topos import node_stop
+
+        node_stop.enable_exit_deadline()
+        uvicorn.run(app, host=host, port=port, log_config=get_uvicorn_log_config(), **node_stop.uvicorn_options())
 
 
 from topos.cli.reprocess_cmd import reprocess_command  # noqa: E402

{topos_node-1.5.3 → topos_node-1.5.4}/topos/cli/tray.py

@@ -823,8 +823,11 @@ def serve_with_tray(
     """
     import uvicorn
 
-    config = uvicorn.Config(app, host=host, port=port, log_config=log_config)
+    from topos import node_stop
+
+    config = uvicorn.Config(app, host=host, port=port, log_config=log_config, **node_stop.uvicorn_options())
     server = uvicorn.Server(config)
+    node_stop.enable_exit_deadline()
 
     def stop_server() -> None:
         # Every tray exit comes through here (Quit, Select Topos, Ctrl+C). None
@@ -838,6 +841,12 @@ def serve_with_tray(
             signal_rebuild_children()
         except Exception as exc:  # noqa: BLE001 — never let this cost the exit itself
             print(f"Could not stop graph rebuild children ({exc}); quitting anyway.")
+        try:
+            # BL-168: the control-plane client, the ingestion queue and the
+            # permissions runtime's background work stop now, not after the drain.
+            node_stop.begin_stop("tray_quit")
+        except Exception as exc:  # noqa: BLE001
+            print(f"Could not begin the node's stop ({exc}); quitting anyway.")
         finally:
             server.should_exit = True
 

{topos_node-1.5.3 → topos_node-1.5.4}/topos/control_plane_client.py

@@ -46,6 +46,11 @@ logger = logging.getLogger("topos.control_plane_client")
 #: reconnects itself rather than being evicted.
 PING_INTERVAL_SECONDS = 30.0
 PING_TIMEOUT_SECONDS = 90.0
+#: R-N154 M2: at a stop, how long the asks already being handled get to send their replies before the socket closes.
+#: Inside node_stop's 12 s bound for the client's whole stop.
+INBOUND_DRAIN_SECONDS = 5.0
+#: What an ask that arrives while the node is stopping is answered with, at once.
+STOPPING_ERROR = "Engine is stopping. Retry shortly."
 
 _FAST_INBOUND_MESSAGE_TYPES = frozenset(
     {
@@ -145,6 +150,9 @@ class ControlPlaneClient:
         self._inbound_semaphore = asyncio.Semaphore(self._inbound_concurrency_limit)
         self._inbound_tasks: set[asyncio.Task] = set()
         self._inbound_lock = asyncio.Lock()
+        # R-N154 M2: set at the start of a stop; new asks are refused while the ones in hand finish.
+        self._draining = False
+        self._inbound_drain_seconds = INBOUND_DRAIN_SECONDS
 
         self._presence_outbox_size = max(1, int(settings.control_plane_presence_outbox_size))
         self._presence_outbox: deque[dict[str, Any]] = deque(maxlen=self._presence_outbox_size)
@@ -188,6 +196,7 @@ class ControlPlaneClient:
         if self._task and not self._task.done():
             return
         self._stop_requested = False
+        self._draining = False
         try:
             # Engine handlers run here — the loop the app lives on.
             self._handler_loop = asyncio.get_running_loop()
@@ -277,6 +286,8 @@ class ControlPlaneClient:
             await asyncio.to_thread(self._thread.join, 10.0)
             self._thread = None
         else:
+            self._draining = True
+            await self._drain_inbound_tasks(self._inbound_drain_seconds)
             self._stop.set()
             if self._ws:
                 try:
@@ -302,7 +313,14 @@ class ControlPlaneClient:
         self._set_state("idle")
 
     async def _shutdown_on_client_loop(self) -> None:
-        """Runs on the client thread's loop; unblocks _run so the thread exits."""
+        """Runs on the client thread's loop; unblocks _run so the thread exits.
+
+        Two phases (R-N154 M2). First the asks already being handled get up to ``_inbound_drain_seconds`` to send
+        their replies while every new ask is refused at once with ``STOPPING_ERROR``: a search past its checkpoint
+        has written its receipt and spent its share's allowance, and an admitted answer has its id, so closing under
+        them lost the reply the node had already paid for. Then the socket closes and what is left is cancelled."""
+        self._draining = True
+        await self._drain_inbound_tasks(self._inbound_drain_seconds)
         self._stop.set()
         if self._ws:
             try:
@@ -311,6 +329,12 @@ class ControlPlaneClient:
                 pass
         await self._cancel_inbound_tasks()
 
+    async def _drain_inbound_tasks(self, timeout_s: float) -> None:
+        async with self._inbound_lock:
+            pending = [task for task in self._inbound_tasks if not task.done()]
+        if pending and timeout_s > 0:
+            await asyncio.wait(pending, timeout=timeout_s)
+
     async def _cancel_inbound_tasks(self) -> None:
         async with self._inbound_lock:
             pending = list(self._inbound_tasks)
@@ -420,6 +444,12 @@ class ControlPlaneClient:
 
     async def _schedule_inbound_message(self, ws, data: Dict[str, Any]) -> None:
         msg_type = str(data.get("type") or "")
+        if self._draining and msg_type not in _FAST_INBOUND_MESSAGE_TYPES and msg_type != "ping":
+            # R-N154 M2: the node is stopping. Refuse at once, so the asker hears now rather than at a timeout.
+            request_id = data.get("id")
+            if request_id:
+                await self._send_ws_json(ws, {"id": request_id, "status": "error", "error": STOPPING_ERROR})
+            return
         if msg_type in _FAST_INBOUND_MESSAGE_TYPES:
             task = asyncio.create_task(self._handle_message(ws, data))
             self._inbound_tasks.add(task)

{topos_node-1.5.3 → topos_node-1.5.4}/topos/engine/registration.py

@@ -70,11 +70,17 @@ def _search_batch_version() -> int:
         return 0
 
 
+# The answer protocol this node serves, 0 when answers are off: 1 wrote answers (1.5.0); 2 also carries the writer
+# a policy's `writer_disclosure` releases (A2A-4 amendment 9). The control plane offers the sharer that choice only
+# to a node advertising 2, because an older node refuses a policy naming the field.
+ANSWER_VERSION = 2
+
+
 def _answer_version() -> int:
     try:
         from ..permissions_v2 import switches
-        return int(switches.is_bound() and switches.on(switches.MESSAGE_SEARCH)
-                   and switches.on(switches.ANSWERS))
+        return ANSWER_VERSION if (switches.is_bound() and switches.on(switches.MESSAGE_SEARCH)
+                                  and switches.on(switches.ANSWERS)) else 0
     except Exception:
         return 0
 

topos_node-1.5.4/topos/node_stop.py

@@ -0,0 +1,272 @@
+"""What a stop does the moment it is asked for (BL-168).
+
+A node told to stop used to do almost nothing until uvicorn reached the app's
+lifespan shutdown, and uvicorn reaches it only after every in-flight request
+has finished: with no ``timeout_graceful_shutdown`` that wait has no bound. On
+10 Oct 2026 (1.5.3) the port closed at the SIGTERM, then the node ran on for 14
+minutes: its pipeline worker kept running the job it held, nothing stopped the
+permissions runtime's background threads (the catch-up assessment pass, the
+share refresh loop, the index rebuild queue), and the control-plane client,
+which ``shutdown_event`` stops last of all, stayed connected and kept taking
+relayed asks it could no longer answer ("Dropping inbound request due to
+saturation", ~150). It exited when that work happened to end.
+
+Now a stop has three parts:
+
+1. **At the signal** (``begin_stop``, a ``runtime_shutdown`` listener; the
+   tray's Quit calls it too). It only schedules work and starts a timer, as a
+   signal handler must. On the app loop, in this order (``stop_now``):
+   close the control-plane client, so the control plane stops relaying asks
+   to a node that is going away and answers the asker at once; stop any graph
+   rebuild child (what a held job may be waiting on); park the
+   ingestion queue (``job_runner.park_pipeline_worker``: the job a worker
+   holds goes back to the queue, and no worker claims or starts again in
+   this run); stop the permissions runtime's background work
+   (``runtime.stop_background_work``: catch-up passes, refresh loop, rebuild
+   queue, answer jobs; assessments are durable checkpoints and a pass a stop
+   cuts short is planned again at the next start). Then the app's own
+   background tasks.
+2. **The drain is bounded.** uvicorn waits ``drain_seconds()`` for in-flight
+   requests and then cancels them (``uvicorn_options``), so lifespan shutdown
+   always runs; ``shutdown_event`` runs ``stop_now`` again first (it is the
+   same task when the signal already started it).
+3. **The process is bounded.** Cancelling a coroutine never stops the thread it
+   awaits, and on Python 3.10 ``asyncio.run`` and the interpreter join every
+   executor thread with no timeout. ``exit_deadline_seconds()`` after the stop
+   began, a process still alive logs the threads it was waiting on (names only)
+   and exits. SQLite rolls an interrupted transaction back at the next open; a
+   claimed job whose requeue never landed is recovered at the next start
+   (``recover_pipeline_jobs``).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import threading
+import time
+from typing import Any, Callable, Optional
+
+logger = logging.getLogger("topos.node_stop")
+
+#: How long uvicorn lets in-flight requests finish before it cancels them.
+DEFAULT_DRAIN_SECONDS = 10.0
+#: From the stop request to a forced exit, if the process is still alive then.
+DEFAULT_EXIT_DEADLINE_SECONDS = 60.0
+#: The control-plane client's own stop waits up to 10 s for its close handshake
+#: and 10 s for its thread; one bound over both.
+CONTROL_PLANE_STOP_SECONDS = 12.0
+#: Parking hands the held job back under the write gate (bounded to 2 s inside).
+PARK_SECONDS = 5.0
+#: The permissions runtime's stops only set events; the bound is for the gate.
+PERMISSIONS_STOP_SECONDS = 5.0
+BACKGROUND_REAP_SECONDS = 5.0
+
+
+def _env_seconds(name: str, default: float) -> float:
+    raw = os.environ.get(name, "").strip()
+    if not raw:
+        return default
+    try:
+        value = float(raw)
+    except ValueError:
+        return default
+    return max(0.0, value)
+
+
+def drain_seconds() -> float:
+    """uvicorn's ``timeout_graceful_shutdown``: ``TOPOS_SHUTDOWN_DRAIN_SECONDS``, default 10."""
+    return _env_seconds("TOPOS_SHUTDOWN_DRAIN_SECONDS", DEFAULT_DRAIN_SECONDS)
+
+
+def exit_deadline_seconds() -> float:
+    """``TOPOS_SHUTDOWN_EXIT_DEADLINE_SECONDS``, default 60; 0 turns the forced exit off."""
+    return _env_seconds("TOPOS_SHUTDOWN_EXIT_DEADLINE_SECONDS", DEFAULT_EXIT_DEADLINE_SECONDS)
+
+
+def uvicorn_options() -> dict:
+    """Keyword arguments every node uvicorn server is built with (CLI and tray)."""
+    return {"timeout_graceful_shutdown": max(1, int(round(drain_seconds())))}
+
+
+_lock = threading.Lock()
+_loop: Optional[asyncio.AbstractEventLoop] = None
+_task: Optional[asyncio.Task] = None
+_begun = False
+
+
+def bind_loop(loop: asyncio.AbstractEventLoop) -> None:
+    """App startup: the loop ``begin_stop`` schedules on. A new run starts with no stop begun."""
+    global _loop, _task, _begun
+    with _lock:
+        _loop = loop
+        _task = None
+        _begun = False
+
+
+def begin_stop(reason: str = "signal") -> bool:
+    """Start the stop now. Safe inside a signal handler and from any thread.
+
+    Blocks on no lock the interrupted main thread could be holding: this
+    module's ``_lock`` is only tried (a busy lock means a stop is being begun or
+    joined already); ``current_generation`` reads without a lock; the
+    generation's own lock is reentrant; ``Event.set`` and
+    ``call_soon_threadsafe`` take only locks no main-thread code here holds
+    across a call out. Idempotent per run. Returns True when this call began it."""
+    global _begun
+    from .runtime_shutdown import current_generation
+
+    if not _lock.acquire(blocking=False):
+        return False
+    try:
+        if _begun:
+            return False
+        _begun = True
+        loop = _loop
+    finally:
+        _lock.release()
+    generation = current_generation()
+    if not generation.is_stop_requested():
+        # The tray's Quit is no signal; the run is stopping all the same.
+        generation.request_stop(reason)
+    _arm_exit_deadline()
+    if loop is not None and not loop.is_closed():
+        try:
+            loop.call_soon_threadsafe(_spawn_on_loop, generation)
+        except RuntimeError:  # the loop closed between the check and the call
+            pass
+    return True
+
+
+def _spawn_on_loop(generation: Any) -> None:
+    global _task
+    with _lock:
+        if _task is None:
+            _task = asyncio.get_running_loop().create_task(_stop_steps(generation), name="node-stop")
+
+
+async def stop_now(generation: Any = None) -> None:
+    """Run the stop's steps once per run (``shutdown_event``; the signal's task when it already began)."""
+    global _task, _begun
+    from .runtime_shutdown import current_generation
+
+    with _lock:
+        _begun = True
+        task = _task
+        if task is None or task.get_loop() is not asyncio.get_running_loop():
+            task = asyncio.get_running_loop().create_task(
+                _stop_steps(generation if generation is not None else current_generation()), name="node-stop"
+            )
+            _task = task
+    await asyncio.shield(task)
+
+
+async def _bounded(name: str, step: Callable[[], Any], seconds: float) -> None:
+    started = time.monotonic()
+    try:
+        await asyncio.wait_for(step(), timeout=seconds)
+    except asyncio.TimeoutError:
+        logger.warning("node stop: %s did not finish within %.0fs; going on", name, seconds)
+    except Exception as exc:  # noqa: BLE001 — a stop step never stops the stop
+        logger.warning("node stop: %s failed (%s)", name, type(exc).__name__)
+    else:
+        logger.info("node stop: %s done in %.1fs", name, time.monotonic() - started)
+
+
+async def _stop_steps(generation: Any) -> None:
+    from .core import state
+    from .pipeline.job_runner import park_pipeline_worker
+
+    client = getattr(state, "control_plane_client", None)
+    if client is not None:
+        await _bounded("control-plane client", client.stop, CONTROL_PLANE_STOP_SECONDS)
+    # Before the tasks and workers it unblocks: cancelling a task never stops the thread it awaits, and a job
+    # waiting on a rebuild child waits until the child dies (the signal listener has usually stopped it already).
+    await _bounded("graph rebuild children", _stop_rebuild_children, REBUILD_STOP_SECONDS)
+    await _bounded("ingestion queue", lambda: park_pipeline_worker(generation), PARK_SECONDS)
+
+    def _permissions() -> Any:
+        from .permissions_v2.runtime import stop_background_work
+
+        return asyncio.to_thread(stop_background_work)
+
+    await _bounded("permissions background work", _permissions, PERMISSIONS_STOP_SECONDS)
+    reap = _background_reaper
+    if reap is not None:
+        await _bounded("background tasks", reap, BACKGROUND_REAP_SECONDS)
+
+
+REBUILD_STOP_SECONDS = 5.0
+
+
+def _stop_rebuild_children() -> Any:
+    from .features.entities import rebuild_subprocess
+
+    return asyncio.to_thread(rebuild_subprocess.stop_rebuild_children, 2.0)
+
+
+#: ``app._reap_background_tasks``, set at startup (this module never imports the app).
+_background_reaper: Optional[Callable[[], Any]] = None
+
+
+def set_background_reaper(reap: Optional[Callable[[], Any]]) -> None:
+    global _background_reaper
+    _background_reaper = reap
+
+
+_stop_begun = threading.Event()
+_deadline_thread: Optional[threading.Thread] = None
+
+
+def enable_exit_deadline() -> bool:
+    """Serving entry points only (``topos-node`` and its tray), before uvicorn starts: arm the forced exit.
+
+    The timer thread starts here, not at the stop: starting a thread inside a signal handler can deadlock on
+    ``threading``'s own lock when the signal lands while the main thread is starting one (asyncio's executor
+    does, on the loop thread). A process that never calls this (tests, CLI commands) never force-exits."""
+    global _deadline_thread
+    seconds = exit_deadline_seconds()
+    if seconds <= 0:
+        return False
+    with _lock:
+        if _deadline_thread is not None and _deadline_thread.is_alive():
+            return True
+        _deadline_thread = threading.Thread(
+            target=_exit_after_stop, args=(seconds,), name="topos-exit-deadline", daemon=True
+        )
+        _deadline_thread.start()
+    return True
+
+
+def _arm_exit_deadline() -> None:
+    _stop_begun.set()
+
+
+def _exit_after_stop(seconds: float) -> None:
+    _stop_begun.wait()
+    _exit_after(seconds)
+
+
+def _still_running() -> list[str]:
+    me = threading.current_thread()
+    return sorted(
+        t.name for t in threading.enumerate() if t is not me and t.is_alive() and t is not threading.main_thread()
+    )
+
+
+def _exit_after(seconds: float, *, exit: Callable[[int], Any] = os._exit, sleep: Callable[[float], Any] = time.sleep) -> None:
+    sleep(seconds)
+    names = _still_running()
+    logger.warning(
+        "node stop: still running %.0fs after the stop was asked for; exiting now (%d thread(s): %s)",
+        seconds,
+        len(names),
+        ", ".join(names[:16]) or "-",
+    )
+    for handler in logging.getLogger().handlers + logger.handlers:
+        try:
+            handler.flush()
+        except Exception:  # noqa: BLE001
+            pass
+    exit(0)

{topos_node-1.5.3 → topos_node-1.5.4}/topos/permissions_v2/answer_generation.py

@@ -325,7 +325,7 @@ DOMAIN_FOLDS = frozenset(_stem(word) for domain, words in CATEGORY_TOPICS.items(
 # the walk are unchanged. A family word in a question is a label of the share, never a person's name (`_name_terms`).
 SOURCE_FAMILIES = ("imessage", "chatgpt", "journal", "whatsapp", "signal", "telegram", "gmail", "slack", "discord")
 # AR152 class Q, per-item carriage: on when shipped; a pinned constant so a measurement may run both ways.
-PER_ITEM_CARRIAGE = True   # 1.5.3 ships it on (review R-RULES); the owner may turn it off in 1.5.4 once lending is measured on live receipts
+PER_ITEM_CARRIAGE = False  # off from 1.5.4 (owner, 11 Oct, BL-163): on the rig it cost 6 of 84 answerable questions with must-abstain 0/60 either way; R-RULES F2's steer fix makes off a precision cost only
 _COORDINATORS = frozenset({"and", "or"})
 _SUBJECT_TOKEN = re.compile(r"[^\W_]+|[,&/]", re.UNICODE)