topos-node 1.3.55__tar.gz → 1.3.56__tar.gz

{topos_node-1.3.55/topos_node.egg-info → topos_node-1.3.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.55
+Version: 1.3.56
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai

{topos_node-1.3.55 → topos_node-1.3.56}/pyproject.toml

@@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.55"
+version = "1.3.56"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"

{topos_node-1.3.55 → topos_node-1.3.56}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.55"
-__version_info__ = (1, 3, 55)
+__version__ = "1.3.56"
+__version_info__ = (1, 3, 56)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.3.55 → topos_node-1.3.56}/topos/config/settings.py

@@ -163,7 +163,7 @@ class Settings(BaseSettings):
     sanitization_ollama_timeout_sec: float = Field(120.0)
     # llm_generation / routine hops — must cover local thinking models. Do NOT
     # reuse sanitization_ollama_timeout_sec (120s): that cuts digest synthesis
-    # mid-flight and surfaces as "Ollama unreachable".
+    # mid-flight and surfaces as "Ollama did not answer within 120s".
     engine_ollama_generate_timeout_sec: float = Field(300.0)
     ollama_list_timeout_sec: float = Field(10.0)
     sanitization_ollama_auto_pull: bool = Field(True)

{topos_node-1.3.55 → topos_node-1.3.56}/topos/control_plane_client.py

@@ -517,7 +517,16 @@ class ControlPlaneClient:
             await ws.send(json.dumps(payload))
             return True
         except Exception as exc:  # noqa: BLE001
-            logger.error("Failed to send message to control plane: %s", exc)
+            # Routing fields only, never the payload: this is the one line that
+            # says which request's answer died on a dead socket.
+            logger.error(
+                "Failed to send message to control plane: id=%s type=%s status=%s exc=%s: %s",
+                payload.get("id"),
+                payload.get("type"),
+                payload.get("status"),
+                type(exc).__name__,
+                exc,
+            )
             return False
 
     def set_device_info_snapshot(self, payload: dict[str, Any] | None) -> None:

{topos_node-1.3.55 → topos_node-1.3.56}/topos/core/handlers/device.py

@@ -280,13 +280,34 @@ async def handle_llm_generation(message: Dict[str, Any]) -> Optional[Dict[str, A
     unusable = _unusable_num_ctx(payload)
     if unusable is not None:
         return {"id": req_id, "status": "error", "error": unusable, "error_code": 422}
+    started, wall_started = time.monotonic(), time.time()
+    # Every outcome line carries the id: two routines overlap on the node, and
+    # the service-layer lines have none, so pairing by order is ambiguous.
     logger.info(
-        "llm_generation request: provider=%r model=%r prompt_chars=%d stream=%s",
+        "llm_generation request: id=%s provider=%r model=%r prompt_chars=%d stream=%s",
+        req_id,
         payload.get("provider"),
         payload.get("model"),
         len(str(payload.get("prompt") or "")),
         bool(payload.get("stream")),
     )
+
+    def _log_failed(exc: BaseException, code: Any) -> None:
+        # Classes and codes only: an error's text can quote an Ollama body.
+        # openai.py raises `from exc`, so the cause names the transport error.
+        logger.warning(
+            "llm_generation failed: id=%s provider=%r model=%r code=%s cause=%s"
+            " elapsed_s=%.1f wall_s=%.1f stream=%s",
+            req_id,
+            payload.get("provider"),
+            payload.get("model"),
+            code,
+            type(exc.__cause__ or exc).__name__,
+            time.monotonic() - started,
+            time.time() - wall_started,
+            bool(payload.get("stream")),
+        )
+
     current = asyncio.current_task()
     if current is not None:
         _LLM_GENERATION_TASKS[str(req_id)] = current
@@ -335,6 +356,14 @@ async def handle_llm_generation(message: Dict[str, Any]) -> Optional[Dict[str, A
         result = await get_services().llm.generate(
             payload, on_delta=on_delta, on_protocol=on_protocol
         )
+        logger.info(
+            "llm_generation done: id=%s model=%r output_chars=%d elapsed_s=%.1f wall_s=%.1f",
+            req_id,
+            payload.get("model"),
+            len(str((result or {}).get("output") or "")),
+            time.monotonic() - started,
+            time.time() - wall_started,
+        )
         return {"id": req_id, "status": "ok", "payload": result}
     except asyncio.CancelledError:
         # llm_cancel: answer the original request with a typed 499 so the
@@ -354,10 +383,13 @@ async def handle_llm_generation(message: Dict[str, Any]) -> Optional[Dict[str, A
                         }
                     )
                 )
-            except Exception:  # noqa: BLE001 — reporting must not mask the cancel
-                pass
+            except Exception as e:  # noqa: BLE001 — reporting must not mask the cancel
+                logger.warning(
+                    "llm_generation cancel reply not sent: id=%s exc=%s", req_id, type(e).__name__
+                )
         raise
     except LlmTypedError as exc:
+        _log_failed(exc, exc.code)
         return {
             "id": req_id,
             "status": "error",
@@ -365,6 +397,7 @@ async def handle_llm_generation(message: Dict[str, Any]) -> Optional[Dict[str, A
             "error_code": exc.code,
         }
     except HTTPException as exc:
+        _log_failed(exc, exc.status_code)
         detail = exc.detail
         err_msg = detail if isinstance(detail, str) else str(detail)
         return {
@@ -374,7 +407,10 @@ async def handle_llm_generation(message: Dict[str, Any]) -> Optional[Dict[str, A
             "error_code": exc.status_code,
         }
     except Exception as exc:  # noqa: BLE001
-        return {"id": req_id, "status": "error", "error": str(exc)}
+        _log_failed(exc, None)
+        # A bare `ValueError()` has empty text; the control plane would record
+        # a failure with no reason at all.
+        return {"id": req_id, "status": "error", "error": str(exc) or type(exc).__name__}
     finally:
         _LLM_GENERATION_TASKS.pop(str(req_id), None)
 

{topos_node-1.3.55 → topos_node-1.3.56}/topos/services/llm/openai.py

@@ -74,6 +74,37 @@ def _ollama_generate_timeout() -> httpx.Timeout:
     )
 
 
+def _log_ollama_failure(
+    outcome: str,
+    *,
+    model: str,
+    stream: bool,
+    started: float,
+    wall_started: float,
+    budget_s: float,
+    exc: Optional[BaseException] = None,
+    status_code: Optional[int] = None,
+) -> None:
+    """One line per failed generation. Classes and numbers only: the prompt,
+    the response and Ollama's body never go into it.
+
+    ``elapsed_s`` is monotonic and ``wall_s`` wall-clock. On macOS the
+    monotonic clock stops while the Mac sleeps, so a gap between the two is
+    time asleep mid-generation — which a node log could not show before.
+    """
+    logger.warning(
+        "Ollama generate %s: model=%r stream=%s exc=%s status=%s elapsed_s=%.1f wall_s=%.1f timeout_s=%.0f",
+        outcome,
+        model,
+        stream,
+        type(exc).__name__ if exc is not None else None,
+        status_code,
+        time.monotonic() - started,
+        time.time() - wall_started,
+        budget_s,
+    )
+
+
 def _ensure_ollama_running(base_url: str) -> None:
     """Start a local Ollama if this request needs one and :11434 is down.
 
@@ -113,6 +144,9 @@ async def _ollama_generate(payload: Dict[str, Any]) -> Dict[str, Any]:
     num_ctx = payload.get("num_ctx")
     base = settings.engine_ollama_base_url.rstrip("/")
     await asyncio.to_thread(_ensure_ollama_running, base)
+    # After the cold start, so elapsed_s is comparable to timeout_s: the httpx
+    # budget does not cover ensure_running either.
+    started, wall_started = time.monotonic(), time.time()
     # Routines / non-stream API: suppress CoT so the visible answer is not
     # starved. Models that reject think=false still get a request without it
     # (Ollama returns 400); we retry once without the flag below.
@@ -131,14 +165,16 @@ async def _ollama_generate(payload: Dict[str, Any]) -> Dict[str, Any]:
         body["options"] = opts
     timeout = _ollama_generate_timeout()
     logger.info(
-        "Ollama generate: model=%r base=%s prompt_chars=%d max_tokens=%s temperature=%s think=%s",
+        "Ollama generate: model=%r base=%s prompt_chars=%d max_tokens=%s temperature=%s think=%s timeout_s=%.0f",
         model,
         base,
         len(prompt),
         max_tokens,
         temperature,
         think,
+        timeout.read,
     )
+    clocks = {"model": model, "stream": False, "started": started, "wall_started": wall_started}
     try:
         async with httpx.AsyncClient(timeout=timeout) as client:
             r = await client.post(f"{base}/api/generate", json=body)
@@ -153,12 +189,33 @@ async def _ollama_generate(payload: Dict[str, Any]) -> Dict[str, Any]:
                 )
                 body.pop("think", None)
                 r = await client.post(f"{base}/api/generate", json=body)
+    except httpx.ConnectTimeout as exc:
+        # Ahead of TimeoutException, which it also is: no connection within the
+        # connect budget is a host that is off or asleep, not a slow model. The
+        # detail avoids the word "timeout" -- the web app's failure copy reads
+        # that as a stalled local model.
+        _log_ollama_failure("failed", budget_s=timeout.connect, exc=exc, **clocks)
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=f"Ollama unreachable at {base}: no connection within {timeout.connect:.0f}s",
+        ) from exc
+    except httpx.TimeoutException as exc:
+        # Ahead of RequestError, which every httpx timeout also is. A timeout's
+        # text is empty, so it used to read "Ollama unreachable at …: " while
+        # Ollama was up and simply had not answered yet.
+        _log_ollama_failure("timed out", budget_s=timeout.read, exc=exc, **clocks)
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=f"Ollama did not answer within {timeout.read:.0f}s at {base} ({type(exc).__name__})",
+        ) from exc
     except httpx.RequestError as exc:
+        _log_ollama_failure("failed", budget_s=timeout.read, exc=exc, **clocks)
         raise HTTPException(
             status_code=status.HTTP_502_BAD_GATEWAY,
-            detail=f"Ollama unreachable at {base}: {exc}",
+            detail=f"Ollama unreachable at {base}: {str(exc) or type(exc).__name__}",
         ) from exc
     if r.status_code >= 400:
+        _log_ollama_failure("error", budget_s=timeout.read, status_code=r.status_code, **clocks)
         detail = (r.text or str(r.status_code))[:800]
         raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Ollama error: {detail}")
     data = r.json()
@@ -168,11 +225,14 @@ async def _ollama_generate(payload: Dict[str, Any]) -> Dict[str, Any]:
     completion_tokens = int(data.get("eval_count") or 0)
     total_tokens = prompt_tokens + completion_tokens
     logger.info(
-        "Ollama generate complete: model=%r response_chars=%d prompt_tokens=%d completion_tokens=%d",
+        "Ollama generate complete: model=%r response_chars=%d prompt_tokens=%d completion_tokens=%d"
+        " elapsed_s=%.1f wall_s=%.1f",
         resp_model,
         len(text),
         prompt_tokens,
         completion_tokens,
+        time.monotonic() - started,
+        time.time() - wall_started,
     )
     usage: Dict[str, Any] = {}
     if total_tokens > 0:
@@ -353,11 +413,16 @@ async def _ollama_stream_generate(
             if time.monotonic() - last_frame_at >= interval:
                 await _emit_protocol("heartbeat", {"phase": phase})
 
+    started, wall_started = time.monotonic(), time.time()
+    clocks = {"model": model, "stream": True, "started": started, "wall_started": wall_started}
     hb_task = asyncio.create_task(_heartbeats()) if on_protocol is not None else None
     try:
         async with httpx.AsyncClient(timeout=timeout) as client:
             async with client.stream("POST", f"{base}/api/generate", json=body) as response:
                 if response.status_code >= 400:
+                    _log_ollama_failure(
+                        "error", budget_s=timeout.read, status_code=response.status_code, **clocks
+                    )
                     detail = (await response.aread()).decode("utf-8", errors="replace")[:800]
                     raise HTTPException(
                         status_code=status.HTTP_502_BAD_GATEWAY,
@@ -393,10 +458,25 @@ async def _ollama_stream_generate(
                         completion_tokens = int(data.get("eval_count") or completion_tokens)
                         done_reason = str(data.get("done_reason") or "")
                         timings = _ollama_timings(data)
+    except httpx.ConnectTimeout as exc:
+        # Same as the non-stream path: a host that never answered the connect.
+        _log_ollama_failure("failed", budget_s=timeout.connect, exc=exc, **clocks)
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=f"Ollama unreachable at {base}: no connection within {timeout.connect:.0f}s",
+        ) from exc
+    except httpx.TimeoutException as exc:
+        # Same split as the non-stream path: a timeout is not "unreachable".
+        _log_ollama_failure("timed out", budget_s=timeout.read, exc=exc, **clocks)
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=f"Ollama did not answer within {timeout.read:.0f}s at {base} ({type(exc).__name__})",
+        ) from exc
     except httpx.RequestError as exc:
+        _log_ollama_failure("failed", budget_s=timeout.read, exc=exc, **clocks)
         raise HTTPException(
             status_code=status.HTTP_502_BAD_GATEWAY,
-            detail=f"Ollama unreachable at {base}: {exc}",
+            detail=f"Ollama unreachable at {base}: {str(exc) or type(exc).__name__}",
         ) from exc
     finally:
         if hb_task is not None:

{topos_node-1.3.55 → topos_node-1.3.56}/topos/upgrades/manifests.json

@@ -857,6 +857,12 @@
       "steps": [],
       "notes": []
     },
+    {
+      "version": "1.3.56",
+      "summary": "Release 1.3.56.",
+      "steps": [],
+      "notes": []
+    },
     {
       "version": "unreleased",
       "summary": "Staging entry for in-flight PRs. cut_release.py stamps this with the next version at release cut. Never executed by steps_between.",

{topos_node-1.3.55 → topos_node-1.3.56/topos_node.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.55
+Version: 1.3.56
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai