topos-node 1.3.36__tar.gz → 1.3.38__tar.gz

{topos_node-1.3.36/topos_node.egg-info → topos_node-1.3.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.36
+Version: 1.3.38
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai

{topos_node-1.3.36 → topos_node-1.3.38}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.36"
+version = "1.3.38"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"

{topos_node-1.3.36 → topos_node-1.3.38}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.36"
-__version_info__ = (1, 3, 36)
+__version__ = "1.3.38"
+__version_info__ = (1, 3, 38)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.3.36 → topos_node-1.3.38}/topos/api/ingestion_sources.py

@@ -158,7 +158,7 @@ async def put_source_settings_endpoint(
     dataset_id: Optional[str] = Query(default=None),
     body: Optional[dict] = Body(default=None),
 ):
-    """Set source settings (e.g. enabled). Valid for local_sync sources (imessage, signal)."""
+    """Set source settings (e.g. enabled, exclude_spam). Valid for local_sync sources (imessage, signal)."""
     source = REGISTRY.get(source_id)
     if not source:
         return {"status": "error", "error": "unknown source_id"}
@@ -167,14 +167,26 @@ async def put_source_settings_endpoint(
     dataset_id = dataset_id or (body.get("dataset_id") if body else None)
     if not dataset_id:
         return {"status": "error", "error": "dataset_id required"}
-    enabled = body.get("enabled") if body else None
-    if enabled is None:
-        return {"status": "error", "error": "enabled required in body"}
+    body = body or {}
+    enabled = body.get("enabled")
+    posture_provided = "posture" in body
+    exclude_spam_provided = "exclude_spam" in body
+    if enabled is None and not posture_provided and not exclude_spam_provided:
+        return {"status": "error", "error": "enabled, posture, or exclude_spam required in body"}
     conn = get_db_connection()
     if not conn:
         return {"status": "error", "error": "Database not available"}
-    put_source_settings(conn, dataset_id, source_id, enabled=enabled)
-    return {"status": "ok", "dataset_id": dataset_id, "source_id": source_id, "enabled": bool(enabled)}
+    put_kwargs: dict = {"enabled": enabled}
+    if posture_provided:
+        put_kwargs["posture"] = body.get("posture")
+    if exclude_spam_provided:
+        put_kwargs["exclude_spam"] = bool(body.get("exclude_spam"))
+    try:
+        put_source_settings(conn, dataset_id, source_id, **put_kwargs)
+    except ValueError as exc:
+        return {"status": "error", "error": str(exc)}
+    settings = get_source_settings(conn, dataset_id, source_id) or {}
+    return {"status": "ok", "dataset_id": dataset_id, "source_id": source_id, **settings}
 
 
 @router.post("/sources/{source_id}/sync", dependencies=[Depends(require_api_key)])

{topos_node-1.3.36 → topos_node-1.3.38}/topos/api/signal.py

@@ -682,28 +682,37 @@ async def blackhole_entity(
     """
     import asyncio
 
+    from ..core.state import close_thread_db_connection
     from ..features.lifecycle.blackhole import BlackholeStore
     from ..features.lifecycle.blackhole_rebuild import rebuild_for_blackhole
 
-    conn = _entities_conn()
+    _entities_conn()  # fail fast with 503 before spawning the worker
+    processing_tier = body.processing_tier
+    note = body.note
+
+    def _blackhole():
+        try:
+            conn = _entities_conn()
+            result = BlackholeStore(conn).blackhole_entity(
+                entity_ref=entity_id,
+                processing_tier=processing_tier,
+                note=note,
+            )
+            # D4: the notification above was raised first, so the owner already
+            # knows the hide is incomplete; now do the rebuild that makes it
+            # true. Same worker / same connection — do not hop back to the loop
+            # with a write-gate hold still possible.
+            if not result.get("already_blackholed"):
+                result["rebuild"] = rebuild_for_blackhole(conn, entity_id).as_dict()
+            return result
+        finally:
+            close_thread_db_connection()
+
     try:
-        result = BlackholeStore(conn).blackhole_entity(
-            entity_ref=entity_id,
-            processing_tier=body.processing_tier,
-            note=body.note,
-        )
+        return await asyncio.to_thread(_blackhole)
     except ValueError as exc:
         raise HTTPException(status_code=400, detail=str(exc))
 
-    # D4: the notification above was raised first, so the owner already knows the
-    # hide is incomplete; now do the rebuild that makes it true. Non-owners are
-    # withheld the prose artifacts for the duration, so this running late is safe
-    # while it running silently would not be.
-    if not result.get("already_blackholed"):
-        result["rebuild"] = await asyncio.to_thread(rebuild_for_blackhole, conn, entity_id)
-        result["rebuild"] = result["rebuild"].as_dict()
-    return result
-
 
 @router.delete("/entities/{entity_id}/blackhole")
 async def unblackhole_entity(
@@ -711,9 +720,20 @@ async def unblackhole_entity(
     _api_key: str = Depends(require_api_key),
 ):
     """Lift a black hole. Existing grants are not restored — normal permissions resume."""
+    import asyncio
+
+    from ..core.state import close_thread_db_connection
     from ..features.lifecycle.blackhole import BlackholeStore
 
-    return BlackholeStore(_entities_conn()).unblackhole_entity(entity_ref=entity_id)
+    _entities_conn()  # fail fast with 503 before spawning the worker
+
+    def _unblackhole():
+        try:
+            return BlackholeStore(_entities_conn()).unblackhole_entity(entity_ref=entity_id)
+        finally:
+            close_thread_db_connection()
+
+    return await asyncio.to_thread(_unblackhole)
 
 
 @router.get("/blackholes")

{topos_node-1.3.36 → topos_node-1.3.38}/topos/cli/tray.py

@@ -2,8 +2,9 @@
 
 Port of the original open-source topos-cli ``menu_bar_app`` (pystray + PIL):
 the Topos glyph with a status dot composited in the corner — yellow while
-starting, green while the node answers ``/healthcheck``, red while it does
-not, orange when a newer topos-node is published on PyPI. The menu offers
+starting, green while the node answers ``/healthcheck``, red only after
+repeated misses (a single late probe is a busy node, not a dead one),
+orange when a newer topos-node is published on PyPI. The menu offers
 API docs, the hosted Topos app, a one-click update when one is available,
 and Quit.
 
@@ -33,10 +34,36 @@ STATUS_COLORS = {
 }
 
 HEALTH_POLL_SECONDS = 5.0
+#: Must sit above ``probe_db_health``'s 2s budget. A 3s client deadline raced
+#: the probe under ingest load and painted the tray red for a live node.
+HEALTH_TIMEOUT_SECONDS = 10.0
+#: One missed probe is a stalled event loop, not a down node. Two in a row
+#: is enough to go red; a single success clears the count (same shape as
+#: the web app's ``HEALTH_FAILURE_THRESHOLD``).
+HEALTH_FAILURE_THRESHOLD = 2
 TOPOS_APP_URL = "https://topos.dialogues.ai"
 TOPOS_DOCS_URL = "https://topos.dialogues.ai/docs/welcome"
 
 
+def resolve_tray_health_status(
+    *,
+    probe_ok: bool,
+    consecutive_failures: int,
+    current_status: str,
+    failure_threshold: int = HEALTH_FAILURE_THRESHOLD,
+) -> tuple[str, int]:
+    """Map one ``/healthcheck`` result to (next_status, next_failure_count).
+
+    A single timeout must not flip a green (or still-starting) icon to red.
+    """
+    if probe_ok:
+        return "healthy", 0
+    failures = consecutive_failures + 1
+    if failures >= failure_threshold:
+        return "down", failures
+    return current_status, failures
+
+
 def tray_available() -> bool:
     try:
         import pystray  # noqa: F401
@@ -214,16 +241,24 @@ class ToposTray:
     def _poll_health(self) -> None:
         import httpx
 
+        consecutive_failures = 0
         while self._icon is not None and self._icon.visible:
             try:
                 # 401/403 = health auth enabled; the node answered, so it's up.
-                healthy = httpx.get(self.health_url, timeout=3.0).status_code in (200, 401, 403)
+                healthy = httpx.get(
+                    self.health_url, timeout=HEALTH_TIMEOUT_SECONDS
+                ).status_code in (200, 401, 403)
             except Exception:
                 healthy = False
             if healthy:
                 self._fetch_shell_status()
                 self._maybe_fetch_topos_name()
-            self._set_status("healthy" if healthy else "down")
+            next_status, consecutive_failures = resolve_tray_health_status(
+                probe_ok=healthy,
+                consecutive_failures=consecutive_failures,
+                current_status=self.status,
+            )
+            self._set_status(next_status)
             time.sleep(HEALTH_POLL_SECONDS)
 
     def _fetch_shell_status(self) -> None:

{topos_node-1.3.36 → topos_node-1.3.38}/topos/core/handlers/signal_features.py

@@ -1031,7 +1031,10 @@ async def handle_signal_unblackhole_entity(message: Dict[str, Any]) -> Optional[
     try:
         from ...features.lifecycle.blackhole import BlackholeStore
 
-        result = BlackholeStore(hub.get_db_connection()).unblackhole_entity(entity_ref=entity_id)
+        def _unblackhole(conn):
+            return BlackholeStore(conn).unblackhole_entity(entity_ref=entity_id)
+
+        result = await run_db_write(_unblackhole)
         return {"id": req_id, "status": "ok", "payload": result}
     except Exception as exc:  # noqa: BLE001
         return {"id": req_id, "status": "error", "error": str(exc)}

{topos_node-1.3.36 → topos_node-1.3.38}/topos/core/handlers/sources.py

@@ -198,6 +198,8 @@ async def handle_put_source_settings(message: Dict[str, Any]) -> Optional[Dict[s
     # present in the body — absent means "leave unchanged".
     posture_provided = "posture" in payload
     posture = payload.get("posture")
+    exclude_spam_provided = "exclude_spam" in payload
+    exclude_spam = payload.get("exclude_spam")
     if not source_id or not dataset_id:
         return {"id": req_id, "status": "error", "error": "source_id and dataset_id required"}
     source = REGISTRY.get(source_id)
@@ -208,16 +210,20 @@ async def handle_put_source_settings(message: Dict[str, Any]) -> Optional[Dict[s
     # enabled toggle is being set.
     if enabled is not None and getattr(source, "source_type", None) != "local_sync":
         return {"id": req_id, "status": "error", "error": "enabled only applies to local_sync sources"}
-    if enabled is None and not posture_provided:
-        return {"id": req_id, "status": "error", "error": "enabled or posture required in body"}
+    if exclude_spam_provided and getattr(source, "source_type", None) != "local_sync":
+        return {"id": req_id, "status": "error", "error": "exclude_spam only applies to local_sync sources"}
+    if enabled is None and not posture_provided and not exclude_spam_provided:
+        return {"id": req_id, "status": "error", "error": "enabled, posture, or exclude_spam required in body"}
     conn = hub.get_db_connection()
     if not conn:
         return {"id": req_id, "status": "error", "error": "Database not available"}
     try:
+        put_kwargs: Dict[str, Any] = {"enabled": enabled}
         if posture_provided:
-            put_source_settings(conn, dataset_id, source_id, enabled=enabled, posture=posture)
-        else:
-            put_source_settings(conn, dataset_id, source_id, enabled=enabled)
+            put_kwargs["posture"] = posture
+        if exclude_spam_provided:
+            put_kwargs["exclude_spam"] = bool(exclude_spam)
+        put_source_settings(conn, dataset_id, source_id, **put_kwargs)
     except ValueError as exc:
         return {"id": req_id, "status": "error", "error": str(exc)}
     # Echo the effective settings back (posture reflects what was persisted /
@@ -229,6 +235,7 @@ async def handle_put_source_settings(message: Dict[str, Any]) -> Optional[Dict[s
     else:
         result_payload["enabled"] = bool(settings_data.get("enabled", True))
     result_payload["posture"] = settings_data.get("posture")
+    result_payload["exclude_spam"] = bool(settings_data.get("exclude_spam", True))
     return {"id": req_id, "status": "ok", "payload": result_payload}
 
 @handles("get_source_contacts")

{topos_node-1.3.36 → topos_node-1.3.38}/topos/engine/backends/ollama.py

@@ -110,6 +110,12 @@ class OllamaAdapter:
         except Exception:
             return False
 
+    def _ensure_server(self) -> None:
+        """Start a local Ollama if this request needs one and :11434 is down."""
+        from ..ollama_runtime import ensure_running
+
+        ensure_running(base_url=self._base_url)
+
     def model_supports_thinking(self, model: str) -> bool:
         """True when /api/show lists the 'thinking' capability for ``model``.
 
@@ -172,6 +178,7 @@ class OllamaAdapter:
         curated table — Ollama has no size for a tag that is not on disk
         (PLAN_LOCAL_MODEL_QUICKSTART D4).
         """
+        self._ensure_server()
         req = urllib.request.Request(f"{self._base_url}/api/tags", method="GET")
         try:
             with urllib.request.urlopen(req, timeout=10) as resp:
@@ -211,6 +218,7 @@ class OllamaAdapter:
         show the download to a human. A callback that raises must not abort a
         multi-gigabyte transfer that is otherwise healthy.
         """
+        self._ensure_server()
         body = {"model": model_name, "stream": stream}
         req = urllib.request.Request(
             f"{self._base_url}/api/pull",
@@ -361,6 +369,7 @@ class OllamaAdapter:
         # rejected think=false before keep thinking but get a wider output
         # budget so the CoT cannot starve the answer (Ollama returns thinking
         # in a separate field, so 'response' stays clean either way).
+        self._ensure_server()
         requested_think = think
         think = self.resolve_think(model, think)
         if (

topos_node-1.3.38/topos/engine/ollama_runtime.py

@@ -0,0 +1,142 @@
+"""Start a local Ollama server when a request needs it and :11434 is down.
+
+Reachability is already probed everywhere; nothing on the product path used
+to spawn the server. Install opens Ollama.app after a one-click install —
+this module does the same (or ``ollama serve``) at request time, only for a
+localhost base URL, and only once under a process lock.
+"""
+
+from __future__ import annotations
+
+import logging
+import shutil
+import subprocess
+import threading
+import time
+from typing import Callable, Optional
+from urllib.parse import urlparse
+
+logger = logging.getLogger("topos.engine.ollama_runtime")
+
+_LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
+_WAIT_SEC = 20.0
+_POLL_INTERVAL = 0.5
+
+_LOCK = threading.Lock()
+
+
+def _default_base_url() -> str:
+    try:
+        from ..config.settings import settings
+
+        return str(getattr(settings, "engine_ollama_base_url", None) or "http://localhost:11434").rstrip("/")
+    except Exception:
+        return "http://localhost:11434"
+
+
+def is_local_base_url(url: str) -> bool:
+    raw = str(url or "").strip()
+    if not raw:
+        return False
+    parsed = urlparse(raw if "://" in raw else f"http://{raw}")
+    host = (parsed.hostname or "").lower()
+    return host in _LOCAL_HOSTS
+
+
+def default_is_reachable(base_url: str) -> bool:
+    from .backends.ollama import OllamaAdapter
+
+    return bool(OllamaAdapter(base_url=base_url).is_reachable())
+
+
+def default_app_present() -> bool:
+    from .ollama_install import default_app_present as _app_present
+
+    return bool(_app_present())
+
+
+def default_open_app() -> None:
+    from .ollama_install import default_open_app as _open_app
+
+    _open_app()
+
+
+def default_cli_present() -> bool:
+    return shutil.which("ollama") is not None
+
+
+def default_spawn_serve() -> None:
+    subprocess.Popen(
+        ["ollama", "serve"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        start_new_session=True,
+    )
+
+
+def ensure_running(
+    *,
+    base_url: Optional[str] = None,
+    is_reachable: Optional[Callable[[], bool]] = None,
+    is_local: Optional[Callable[[str], bool]] = None,
+    app_present: Optional[Callable[[], bool]] = None,
+    open_app: Optional[Callable[[], None]] = None,
+    cli_present: Optional[Callable[[], bool]] = None,
+    spawn_serve: Optional[Callable[[], None]] = None,
+    sleep: Optional[Callable[[float], None]] = None,
+    wait_sec: float = _WAIT_SEC,
+    poll_interval: float = _POLL_INTERVAL,
+) -> bool:
+    """Bring a local Ollama up if a request needs it.
+
+    Returns True when the server answers ``/api/tags`` (already up, or up
+    after we launched). Returns False when the URL is remote, nothing is
+    installed, or the wait timed out — callers then hit the existing 502 /
+    deferred path.
+    """
+    url = (base_url if base_url is not None else _default_base_url()).rstrip("/")
+    local_check = is_local if is_local is not None else is_local_base_url
+    if not local_check(url):
+        return False
+
+    reachable = is_reachable if is_reachable is not None else (lambda: default_is_reachable(url))
+    if reachable():
+        return True
+
+    with _LOCK:
+        if reachable():
+            return True
+
+        launched = False
+        present = app_present if app_present is not None else default_app_present
+        if present():
+            try:
+                (open_app if open_app is not None else default_open_app)()
+                launched = True
+                logger.info("opened Ollama.app; waiting for %s", url)
+            except Exception as exc:  # noqa: BLE001 — launch is best-effort
+                logger.warning("failed to open Ollama.app: %s", exc)
+
+        if not launched:
+            has_cli = cli_present if cli_present is not None else default_cli_present
+            if has_cli():
+                try:
+                    (spawn_serve if spawn_serve is not None else default_spawn_serve)()
+                    launched = True
+                    logger.info("spawned `ollama serve`; waiting for %s", url)
+                except Exception as exc:  # noqa: BLE001
+                    logger.warning("failed to spawn ollama serve: %s", exc)
+
+        if not launched:
+            logger.info("ollama is down and nothing on this machine can start it")
+            return False
+
+        sleeper = sleep if sleep is not None else time.sleep
+        deadline = time.monotonic() + float(wait_sec)
+        interval = max(0.0, float(poll_interval))
+        while time.monotonic() < deadline:
+            if reachable():
+                return True
+            sleeper(interval)
+        logger.warning("ollama did not become reachable at %s within %.1fs", url, wait_sec)
+        return False

{topos_node-1.3.36 → topos_node-1.3.38}/topos/features/entities/reads.py

@@ -106,6 +106,123 @@ def list_entities(
     }
 
 
+def entity_card_aggregates(
+    conn: sqlite3.Connection,
+    entity_id: str,
+    *,
+    guard: BlackholeGuard,
+) -> Dict[str, Any]:
+    """Full-population counts a type-specific entity card reads.
+
+    Separate from ``recent_mentions`` because they answer a different question.
+    ``recent_mentions`` is a SAMPLE — twenty rows, newest first — and a card that
+    derived "83% of this came from browsing" by counting a sample would be
+    stating a measurement it had not made. Every count here is over the whole
+    mention set.
+
+    Two aggregates, because the card designs turn on two distinctions the spine
+    already records and nothing has ever read:
+
+    ``mention_sources``
+        Mentions grouped by canonical table, with the owner-authored share. This
+        is what separates an organization the owner *writes about* from one whose
+        name appeared in a page title, a place they have *been* (``location_events``)
+        from one they only ever talked about, and a project they *work on*
+        (``activity_events``) from one they have only planned. The whole judgement
+        line of six of the ten cards rests on it.
+
+    ``neighbor_counts``
+        Edges grouped by ``(edge_type, direction, neighbour type)``. The card needs
+        "6 people around this" and "14 goals point at it", and ``connections`` is
+        capped at twelve rows ordered by weight — a cap is the right answer for a
+        list and the wrong one for a count.
+
+    Direction is kept because it is load-bearing for at least one type: a
+    conversation's ``participates_in`` (inbound, someone was there) and its
+    ``mentions`` (outbound, someone was named) are different claims about a
+    person, and a card that merged them would say a person who was discussed was
+    in the room.
+    """
+    eid = str(entity_id)
+
+    # A record that names a protected entity is withheld whole, so its mentions
+    # of THIS entity cannot be counted either — otherwise the total silently
+    # disagrees with the records the reader is allowed to open, and the size of
+    # the disagreement is a signal about what was hidden. Inert for the owner UI,
+    # where `sees_everything` short-circuits `blocked_record_ids` to an empty set.
+    blocked_records = guard.blocked_record_ids()
+
+    mention_rows = conn.execute(
+        """
+        SELECT record_id, COALESCE(canonical_table, ''), COALESCE(authored_by_owner, 0)
+        FROM entity_mentions WHERE entity_id = ?
+        """,
+        (eid,),
+    ).fetchall()
+
+    by_table: Dict[str, Dict[str, int]] = {}
+    for record_id, table, authored in mention_rows:
+        if blocked_records and str(record_id) in blocked_records:
+            continue
+        # An unattributed mention is its own fact — bucketed under "" rather than
+        # dropped, so the card's percentages are over the same denominator as its
+        # total and cannot quietly exceed 100%.
+        slot = by_table.setdefault(str(table), {"count": 0, "owner_authored": 0})
+        slot["count"] += 1
+        if int(authored or 0) == 1:
+            slot["owner_authored"] += 1
+
+    mention_sources = sorted(
+        (
+            {"table": table, "count": v["count"], "owner_authored": v["owner_authored"]}
+            for table, v in by_table.items()
+        ),
+        key=lambda r: (-int(r["count"]), str(r["table"])),
+    )
+
+    # Counted in Python rather than pushed into a GROUP BY so the guard applies
+    # per neighbour: a protected entity must not be counted, and a SQL count
+    # would include it while the neighbour LIST beside it does not.
+    edge_rows = conn.execute(
+        """
+        SELECT e.src_entity_id, e.dst_entity_id, e.edge_type,
+               COALESCE(s.entity_type, ''), COALESCE(t.entity_type, ''),
+               s.canonical_name, t.canonical_name
+        FROM entity_edges e
+        LEFT JOIN entities s ON s.entity_id = e.src_entity_id
+        LEFT JOIN entities t ON t.entity_id = e.dst_entity_id
+        WHERE (e.src_entity_id = ? OR e.dst_entity_id = ?) AND e.valid_to IS NULL
+        """,
+        (eid, eid),
+    ).fetchall()
+
+    tally: Dict[tuple, int] = {}
+    for src, dst, edge_type, src_type, dst_type, src_name, dst_name in edge_rows:
+        outbound = str(src) == eid
+        other_id = dst if outbound else src
+        other_type = dst_type if outbound else src_type
+        other_name = dst_name if outbound else src_name
+        if guard.blocks_entity_id(other_id) or guard.blocks_name(other_name):
+            continue
+        key = (str(edge_type), "out" if outbound else "in", str(other_type))
+        tally[key] = tally.get(key, 0) + 1
+
+    neighbor_counts = sorted(
+        (
+            {
+                "edge_type": edge_type,
+                "direction": direction,
+                "entity_type": other_type,
+                "count": count,
+            }
+            for (edge_type, direction, other_type), count in tally.items()
+        ),
+        key=lambda r: (-int(r["count"]), str(r["edge_type"]), str(r["entity_type"])),
+    )
+
+    return {"mention_sources": mention_sources, "neighbor_counts": neighbor_counts}
+
+
 def get_entity_detail(
     conn: sqlite3.Connection,
     entity_id: str,
@@ -175,6 +292,12 @@ def get_entity_detail(
     entity["affinity_connections"] = guard.filter_rows(
         affinity, id_keys=id_keys, name_keys=name_keys
     )
+    # Full-population counts for the type-specific cards. Attached to every
+    # entity rather than branching on type here: the card layer decides what a
+    # place does with `location_events` and what an org does with the
+    # owner-authored share, and a read that guessed for it would have to be
+    # changed every time a card learns a new question.
+    entity.update(entity_card_aggregates(conn, str(entity_id), guard=guard))
     dossier = load_dossier_for_entity(conn, str(entity_id))
     if isinstance(dossier, dict) and not guard.sees_everything:
         dossier = dict(dossier)