topos-node 1.3.0__tar.gz → 1.3.1__tar.gz

{topos_node-1.3.0/topos_node.egg-info → topos_node-1.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.0
+Version: 1.3.1
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai

{topos_node-1.3.0 → topos_node-1.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.0"
+version = "1.3.1"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"

{topos_node-1.3.0 → topos_node-1.3.1}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.0"
-__version_info__ = (1, 3, 0)
+__version__ = "1.3.1"
+__version_info__ = (1, 3, 1)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.3.0 → topos_node-1.3.1}/topos/app.py

@@ -5,6 +5,7 @@ import importlib.util
 import logging
 import os
 import sys
+import threading
 
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
@@ -171,19 +172,9 @@ async def startup_event() -> None:
                 logger.info("Stage 9 migrations applied at startup: %d renames", len(result["applied"]))
     except Exception as e:
         logger.debug("Stage 9 migrations at startup (non-fatal): %s", e)
-    # Upgrade runner: compare the stamped derivation baseline against the
-    # shipped version and re-derive what this release invalidated (manifest in
-    # topos/upgrades). Heavy steps run in a daemon thread; the node serves
-    # stale derived data meanwhile. Fresh installs stamp-and-skip.
-    try:
-        from .core.state import get_db_connection as _get_conn_for_upgrades
-        from .upgrades.runner import start_background as _start_upgrades
-
-        _upgrade_conn = state.db_conn if state.db_conn is not None else _get_conn_for_upgrades()
-        if _upgrade_conn is not None:
-            _start_upgrades(_upgrade_conn)
-    except Exception as e:
-        logger.warning("Upgrade runner at startup failed (non-fatal): %s", e)
+    # Upgrade runner is armed later — after control-plane / local readiness —
+    # so the React UI can fetch bootstrap data before enrichment reprocess
+    # contends for SQLite / Ollama / MPS. See _arm_upgrade_runner below.
     try:
         from .core.state import get_db_connection as _get_conn_for_pipeline
         from .pipeline.job_runner import recover_pipeline_jobs, start_pipeline_worker
@@ -295,6 +286,41 @@ async def startup_event() -> None:
     start_runtime_update_monitor()
     start_update_hotkey_listener()
 
+    # Upgrade runner: plan/stamp cheaply, but defer heavy re-derivation until the
+    # UI can bootstrap (CP connected + grace). Fresh installs stamp-and-skip.
+    try:
+        from .core.state import get_db_connection as _get_conn_for_upgrades
+        from .upgrades.runner import (
+            ready_timeout_seconds as _upgrade_ready_timeout,
+            start_background as _start_upgrades,
+            ui_grace_seconds as _upgrade_ui_grace,
+        )
+
+        _upgrade_conn = state.db_conn if state.db_conn is not None else _get_conn_for_upgrades()
+        if _upgrade_conn is not None:
+            _upgrade_ready = threading.Event()
+            _start_upgrades(
+                _upgrade_conn,
+                ready_event=_upgrade_ready,
+                ui_grace_s=_upgrade_ui_grace(),
+                ready_timeout_s=_upgrade_ready_timeout(),
+            )
+
+            async def _signal_upgrade_ui_ready() -> None:
+                try:
+                    if state.control_plane_client is not None:
+                        await state.control_plane_client.wait_until_connected(
+                            timeout_s=_upgrade_ready_timeout()
+                        )
+                except Exception as ready_exc:  # noqa: BLE001
+                    logger.debug("Upgrade UI-ready wait ended: %s", ready_exc)
+                finally:
+                    _upgrade_ready.set()
+
+            asyncio.create_task(_signal_upgrade_ui_ready())
+    except Exception as e:
+        logger.warning("Upgrade runner at startup failed (non-fatal): %s", e)
+
     if settings.enable_sync and settings.topos_user_id:
         state.sync_client = SyncClient(
             sync_url=settings.get_sync_url(),

{topos_node-1.3.0 → topos_node-1.3.1}/topos/control_plane_client.py

@@ -49,6 +49,10 @@ _FAST_INBOUND_MESSAGE_TYPES = frozenset(
         "signal_entity_graph",
         "signal_list_entities",
         "signal_get_entity",
+        # Shell bootstrap / upgrade banner — must not sit behind inbound saturation
+        # while a version upgrade reprocess is walking sources.
+        "get_runtime_bootstrap",
+        "get_upgrade_status",
     }
 )
 

{topos_node-1.3.0 → topos_node-1.3.1}/topos/engine/usage_observation.py

@@ -56,7 +56,12 @@ def resolve_llm_usage_purpose(
         return origin_key
     source_key = str(source_id or "").strip().lower()
     # Signal derivation helpers reuse query_inference plumbing but are ingestion work.
-    if source_key in {"cluster_labeler", "cluster_labels", "topic_clusters"}:
+    if source_key in {
+        "cluster_labeler",
+        "cluster_labels",
+        "topic_clusters",
+        "conversation_context",
+    }:
         return PURPOSE_INGESTION_PIPELINE
     subtype_key = str(subtype or "").strip().lower()
     type_key = str(task_type or "").strip().lower()

{topos_node-1.3.0 → topos_node-1.3.1}/topos/features/signal/conversation_context.py

@@ -70,7 +70,7 @@ def _parse_label(answer: Any) -> Optional[str]:
 def _classify_with_model(model: str, excerpts: List[str]) -> Optional[str]:
     from ...config.settings import settings  # noqa: F401 — engine client env
     from ...engine.client import get_engine_client_or_local
-    from ...engine.tasks import ModelRequest, ProcessingTask
+    from ...engine.tasks import ModelRequest, ProcessingTask, RequestedBy
 
     task = ProcessingTask(
         id="conversation_context",
@@ -83,6 +83,8 @@ def _classify_with_model(model: str, excerpts: List[str]) -> Optional[str]:
             "context": "\n---\n".join(excerpts),
         },
         model_request=ModelRequest(provider="ollama", model=model),
+        # Piggybacked on dimension_summary / enrichment — not an interactive UI ask.
+        requested_by=RequestedBy(origin="ingestion_pipeline"),
     )
     try:
         result = get_engine_client_or_local(None).run(task)

{topos_node-1.3.0 → topos_node-1.3.1}/topos/upgrades/manifests.json

@@ -18,12 +18,13 @@
           "params": {
             "job_names": ["entities"],
             "force_reprocess": true,
+            "include_signal": false,
             "per_source": true,
             "source_filter": "sources with canonical message/journal/activity records"
           },
-          "progress": "GET /v1/enrichment/progress/{job_id}",
+          "progress": "TUI progress bar on the node + GET /v1/upgrade/status",
           "idempotent": "record_mention is INSERT OR IGNORE; re-running converges",
-          "estimated_duration": "~10–20 min per 1.5k records (CPU-bound NER); node stays usable, serves stale graph meanwhile",
+          "estimated_duration": "~10–20 min per 1.5k records (CPU-bound NER); node stays usable, serves stale graph meanwhile; signal/LLM lane stays off (graph rebuild is the next step)",
           "order": 1
         },
         {
@@ -76,12 +77,13 @@
           "params": {
             "job_names": ["attention_triage"],
             "force_reprocess": true,
+            "include_signal": false,
             "per_source": true,
             "source_filter": "sources with canonical message/journal/activity records"
           },
-          "progress": "GET /v1/enrichment/progress/{job_id}",
+          "progress": "TUI progress bar on the node + GET /v1/upgrade/status",
           "idempotent": "verdicts are (day, record_id) upserts; signal objects no-op when unchanged",
-          "estimated_duration": "seconds–minutes depending on day span; CPU-bound instruments, no LLM",
+          "estimated_duration": "seconds–minutes depending on day span; CPU-bound instruments, no LLM (signal lane off)",
           "order": 1
         }
       ],

{topos_node-1.3.0 → topos_node-1.3.1}/topos/upgrades/runner.py

@@ -15,6 +15,11 @@ re-derivation steps all completed) against the shipped version and executes
   * kill-switch: TOPOS_UPGRADE_RUNNER=off disables execution (nothing stamps,
     so re-enabling picks up where it left off).
 
+Heavy steps are deferred until the UI can bootstrap (control-plane ready +
+grace window), then run in a daemon thread with a TUI progress bar. Enrichment
+reprocess steps run only the declared ``job_names`` — they do NOT fan out into
+the signal/LLM lane (that was starving startup UI traffic).
+
 Executors are injectable for tests; the real ones dispatch step kinds to
 engine internals (no HTTP self-calls).
 """
@@ -26,6 +31,7 @@ import logging
 import os
 import sqlite3
 import threading
+import time
 from datetime import datetime, timezone
 from typing import Any, Callable, Dict, List, Optional
 
@@ -35,12 +41,20 @@ logger = logging.getLogger("topos.upgrades.runner")
 
 _BASELINE_KEY = "engine.upgrade.baseline"
 _BOOTSTRAP_BASELINE = "1.1.0"
+_DEFAULT_UI_GRACE_SECONDS = 20.0
+_DEFAULT_READY_TIMEOUT_SECONDS = 60.0
 
 ExecutorFn = Callable[[Dict[str, Any], sqlite3.Connection], Dict[str, Any]]
 
 # Status snapshot for the /upgrade/status surface (module-level, single node).
 _state_lock = threading.Lock()
-_runner_state: Dict[str, Any] = {"running": False, "current_step": None, "last_result": None}
+_runner_state: Dict[str, Any] = {
+    "running": False,
+    "waiting_for_ui": False,
+    "current_step": None,
+    "progress": None,
+    "last_result": None,
+}
 
 
 def _now() -> str:
@@ -53,12 +67,66 @@ def _enabled() -> bool:
     )
 
 
+def _env_float(name: str, default: float) -> float:
+    raw = os.environ.get(name)
+    if raw is None or not str(raw).strip():
+        return default
+    try:
+        return max(0.0, float(raw))
+    except (TypeError, ValueError):
+        return default
+
+
+def ui_grace_seconds() -> float:
+    """Seconds to wait after readiness so the React UI can fetch bootstrap data."""
+    return _env_float("TOPOS_UPGRADE_UI_GRACE_SECONDS", _DEFAULT_UI_GRACE_SECONDS)
+
+
+def ready_timeout_seconds() -> float:
+    return _env_float("TOPOS_UPGRADE_READY_TIMEOUT_SECONDS", _DEFAULT_READY_TIMEOUT_SECONDS)
+
+
 def _shipped_version() -> str:
     from topos.__version__ import __version__
 
     return __version__
 
 
+def _set_runner_state(**updates: Any) -> None:
+    with _state_lock:
+        _runner_state.update(updates)
+
+
+def _progress_snapshot(
+    *,
+    step_id: str,
+    step_index: int,
+    steps_total: int,
+    unit_label: str,
+    completed: int,
+    total: int,
+    detail: Optional[str] = None,
+) -> Dict[str, Any]:
+    total = max(0, int(total))
+    completed = max(0, min(int(completed), total if total else int(completed)))
+    fraction = (completed / total) if total else 0.0
+    # Blend step index with in-step fraction so multi-step upgrades read smoothly.
+    if steps_total > 0:
+        overall = (max(0, step_index) + fraction) / steps_total
+    else:
+        overall = fraction
+    return {
+        "step_id": step_id,
+        "step_index": step_index,
+        "steps_total": steps_total,
+        "unit_label": unit_label,
+        "completed": completed,
+        "total": total,
+        "percent": round(100.0 * overall, 1),
+        "detail": detail,
+    }
+
+
 def read_baseline(conn: sqlite3.Connection) -> Optional[str]:
     try:
         row = conn.execute(
@@ -186,28 +254,74 @@ def _real_source_ids(conn: sqlite3.Connection) -> List[str]:
 
 
 def _exec_enrichment_reprocess(step: Dict[str, Any], conn: sqlite3.Connection) -> Dict[str, Any]:
-    """Walk real sources through the enrichment core with force_reprocess."""
+    """Walk real sources through the enrichment core with force_reprocess.
+
+    Upgrade manifests declare specific ``job_names`` (e.g. attention_triage).
+    Unless a step explicitly sets ``include_signal: true``, we keep the signal /
+    LLM derivation lane OFF — otherwise every source fans out into embeddings,
+    cluster labeling, dimension briefs, and conversation-context LLM calls and
+    starves the UI during first boot after an upgrade.
+    """
     import asyncio
 
     from ..api.enrichment import _process_enrichment_core
+    from ..enrichment.progress_bar import ProgressBar
 
     params = step.get("params") or {}
     job_names = params.get("job_names") or None
-    detail: Dict[str, Any] = {"sources": {}}
-    for source_id in _real_source_ids(conn):
-        try:
-            out = asyncio.run(
-                _process_enrichment_core(
-                    source_id=source_id,
-                    job_names=list(job_names) if job_names else None,
-                    force_reprocess=bool(params.get("force_reprocess", True)),
+    include_signal = bool(params.get("include_signal", False))
+    source_ids = _real_source_ids(conn)
+    step_id = str(step.get("id") or "enrichment_reprocess")
+    step_index = int(step.get("_runner_step_index") or 0)
+    steps_total = int(step.get("_runner_steps_total") or 1)
+    detail: Dict[str, Any] = {"sources": {}, "include_signal": include_signal}
+    def _run_sources(pbar: Optional[ProgressBar] = None) -> None:
+        for idx, source_id in enumerate(source_ids):
+            _set_runner_state(
+                progress=_progress_snapshot(
+                    step_id=step_id,
+                    step_index=step_index,
+                    steps_total=steps_total,
+                    unit_label="sources",
+                    completed=idx,
+                    total=len(source_ids),
+                    detail=source_id,
                 )
             )
-            detail["sources"][source_id] = str(out.get("status") or "ok")
-        except ValueError:
-            detail["sources"][source_id] = "unknown_source_skipped"
-        except Exception as exc:  # noqa: BLE001 — one source must not kill the walk
-            detail["sources"][source_id] = f"error: {exc}"
+            if pbar is not None:
+                pbar.set_description(f"Upgrade {step_id} · {source_id}")
+            try:
+                out = asyncio.run(
+                    _process_enrichment_core(
+                        source_id=source_id,
+                        job_names=list(job_names) if job_names else None,
+                        force_reprocess=bool(params.get("force_reprocess", True)),
+                        include_signal=include_signal,
+                    )
+                )
+                detail["sources"][source_id] = str(out.get("status") or "ok")
+            except ValueError:
+                detail["sources"][source_id] = "unknown_source_skipped"
+            except Exception as exc:  # noqa: BLE001 — one source must not kill the walk
+                detail["sources"][source_id] = f"error: {exc}"
+            if pbar is not None:
+                pbar.update(1)
+
+    if source_ids:
+        with ProgressBar(total=len(source_ids), desc=f"Upgrade {step_id}", width=40) as pbar:
+            _run_sources(pbar)
+    else:
+        _run_sources(None)
+    _set_runner_state(
+        progress=_progress_snapshot(
+            step_id=step_id,
+            step_index=step_index,
+            steps_total=steps_total,
+            unit_label="sources",
+            completed=len(source_ids),
+            total=len(source_ids),
+        )
+    )
     return detail
 
 
@@ -256,7 +370,9 @@ def run_pending_upgrades(
 
     ran = failed = 0
     failed_ids: set = set()
-    for step in plan["steps"]:
+    steps = list(plan["steps"])
+    steps_total = len(steps)
+    for step_index, step in enumerate(steps):
         step_id = str(step["id"])
         status = _ledger_status(conn, shipped_v, step_id)
         if status == "done":
@@ -274,12 +390,26 @@ def run_pending_upgrades(
             failed_ids.add(step_id)
             failed += 1
             continue
-        with _state_lock:
-            _runner_state.update(running=True, current_step=step_id)
+        annotated = dict(step)
+        annotated["_runner_step_index"] = step_index
+        annotated["_runner_steps_total"] = steps_total
+        _set_runner_state(
+            running=True,
+            waiting_for_ui=False,
+            current_step=step_id,
+            progress=_progress_snapshot(
+                step_id=step_id,
+                step_index=step_index,
+                steps_total=steps_total,
+                unit_label="steps",
+                completed=0,
+                total=1,
+            ),
+        )
         _ledger_set(conn, shipped_v, step_id, "running", started=True)
         logger.info("upgrade step %s (%s) starting", step_id, step["kind"])
         try:
-            detail = executor(step, conn)
+            detail = executor(annotated, conn)
             _ledger_set(conn, shipped_v, step_id, "done", detail if isinstance(detail, dict) else None)
             ran += 1
         except Exception as exc:  # noqa: BLE001 — ledger the failure, keep the node up
@@ -287,8 +417,7 @@ def run_pending_upgrades(
             _ledger_set(conn, shipped_v, step_id, "failed", {"error": str(exc)})
             failed_ids.add(step_id)
             failed += 1
-    with _state_lock:
-        _runner_state.update(running=False, current_step=None)
+    _set_runner_state(running=False, waiting_for_ui=False, current_step=None, progress=None)
 
     all_done = all(
         _ledger_status(conn, shipped_v, str(s["id"])) == "done" for s in plan["steps"]
@@ -297,27 +426,74 @@ def run_pending_upgrades(
         _stamp_baseline(conn, shipped_v)
         logger.info("upgrade to %s complete (%d steps)", shipped_v, ran)
     result = {"steps_run": ran, "steps_failed": failed, "baseline_advanced": all_done}
-    with _state_lock:
-        _runner_state["last_result"] = result
+    _set_runner_state(last_result=result)
     return result
 
 
-def start_background(conn: sqlite3.Connection) -> Optional[threading.Thread]:
-    """Boot entrypoint: plan quickly; heavy steps run off the event loop."""
+def start_background(
+    conn: sqlite3.Connection,
+    *,
+    ready_event: Optional[threading.Event] = None,
+    ui_grace_s: Optional[float] = None,
+    ready_timeout_s: Optional[float] = None,
+) -> Optional[threading.Thread]:
+    """Boot entrypoint: plan quickly; heavy steps wait for UI, then run off-loop.
+
+    ``ready_event`` should be set once the control plane (or local HTTP) is able
+    to serve the React app's initial bootstrap. A grace window then lets those
+    requests finish before enrichment reprocess contends for SQLite / Ollama /
+    MPS. Fresh installs stamp-and-skip inline (no thread).
+    """
     if not _enabled():
         return None
     plan = plan_upgrade(conn)
     if plan["fresh_install"] or not plan["steps"]:
         run_pending_upgrades(conn)  # cheap: stamp/no-op inline
         return None
+    grace = _DEFAULT_UI_GRACE_SECONDS if ui_grace_s is None else max(0.0, float(ui_grace_s))
+    ready_timeout = (
+        _DEFAULT_READY_TIMEOUT_SECONDS if ready_timeout_s is None else max(0.0, float(ready_timeout_s))
+    )
     logger.info(
-        "upgrade %s → %s: %d step(s) queued (%s)",
+        "upgrade %s → %s: %d step(s) queued (%s); waiting for UI readiness "
+        "(timeout=%.0fs, grace=%.0fs) before starting",
         plan["baseline"], plan["shipped"], len(plan["steps"]),
         ", ".join(s["id"] for s in plan["steps"]),
+        ready_timeout,
+        grace,
     )
-    thread = threading.Thread(
-        target=run_pending_upgrades, args=(conn,), name="topos-upgrade-runner", daemon=True
+    _set_runner_state(
+        waiting_for_ui=True,
+        running=False,
+        current_step=None,
+        progress={
+            "step_id": None,
+            "percent": 0.0,
+            "detail": "waiting_for_ui",
+            "grace_seconds": grace,
+        },
     )
+
+    def _target() -> None:
+        try:
+            if ready_event is not None:
+                signaled = ready_event.wait(timeout=ready_timeout if ready_timeout > 0 else None)
+                if not signaled:
+                    logger.warning(
+                        "upgrade ready-event timed out after %.0fs; starting anyway",
+                        ready_timeout,
+                    )
+            if grace > 0:
+                logger.info(
+                    "UI grace: deferring upgrade work for %.0fs so bootstrap can finish",
+                    grace,
+                )
+                time.sleep(grace)
+            run_pending_upgrades(conn)
+        finally:
+            _set_runner_state(waiting_for_ui=False)
+
+    thread = threading.Thread(target=_target, name="topos-upgrade-runner", daemon=True)
     thread.start()
     return thread
 
@@ -333,7 +509,9 @@ def runner_status(conn: sqlite3.Connection) -> Dict[str, Any]:
         "fresh_install": plan["fresh_install"],
         "pending_steps": [s["id"] for s in plan["steps"]],
         "running": state["running"],
+        "waiting_for_ui": state.get("waiting_for_ui", False),
         "current_step": state["current_step"],
+        "progress": state.get("progress"),
         "last_result": state["last_result"],
         "ledger": ledger_rows(conn),
     }

{topos_node-1.3.0 → topos_node-1.3.1/topos_node.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.0
+Version: 1.3.1
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai