topos-node 1.3.36__tar.gz → 1.3.37__tar.gz

{topos_node-1.3.36/topos_node.egg-info → topos_node-1.3.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.36
+Version: 1.3.37
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai

{topos_node-1.3.36 → topos_node-1.3.37}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.36"
+version = "1.3.37"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"

{topos_node-1.3.36 → topos_node-1.3.37}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.36"
-__version_info__ = (1, 3, 36)
+__version__ = "1.3.37"
+__version_info__ = (1, 3, 37)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.3.36 → topos_node-1.3.37}/topos/api/ingestion_sources.py

@@ -158,7 +158,7 @@ async def put_source_settings_endpoint(
     dataset_id: Optional[str] = Query(default=None),
     body: Optional[dict] = Body(default=None),
 ):
-    """Set source settings (e.g. enabled). Valid for local_sync sources (imessage, signal)."""
+    """Set source settings (e.g. enabled, exclude_spam). Valid for local_sync sources (imessage, signal)."""
     source = REGISTRY.get(source_id)
     if not source:
         return {"status": "error", "error": "unknown source_id"}
@@ -167,14 +167,26 @@ async def put_source_settings_endpoint(
     dataset_id = dataset_id or (body.get("dataset_id") if body else None)
     if not dataset_id:
         return {"status": "error", "error": "dataset_id required"}
-    enabled = body.get("enabled") if body else None
-    if enabled is None:
-        return {"status": "error", "error": "enabled required in body"}
+    body = body or {}
+    enabled = body.get("enabled")
+    posture_provided = "posture" in body
+    exclude_spam_provided = "exclude_spam" in body
+    if enabled is None and not posture_provided and not exclude_spam_provided:
+        return {"status": "error", "error": "enabled, posture, or exclude_spam required in body"}
     conn = get_db_connection()
     if not conn:
         return {"status": "error", "error": "Database not available"}
-    put_source_settings(conn, dataset_id, source_id, enabled=enabled)
-    return {"status": "ok", "dataset_id": dataset_id, "source_id": source_id, "enabled": bool(enabled)}
+    put_kwargs: dict = {"enabled": enabled}
+    if posture_provided:
+        put_kwargs["posture"] = body.get("posture")
+    if exclude_spam_provided:
+        put_kwargs["exclude_spam"] = bool(body.get("exclude_spam"))
+    try:
+        put_source_settings(conn, dataset_id, source_id, **put_kwargs)
+    except ValueError as exc:
+        return {"status": "error", "error": str(exc)}
+    settings = get_source_settings(conn, dataset_id, source_id) or {}
+    return {"status": "ok", "dataset_id": dataset_id, "source_id": source_id, **settings}
 
 
 @router.post("/sources/{source_id}/sync", dependencies=[Depends(require_api_key)])

{topos_node-1.3.36 → topos_node-1.3.37}/topos/core/handlers/sources.py

@@ -198,6 +198,8 @@ async def handle_put_source_settings(message: Dict[str, Any]) -> Optional[Dict[s
     # present in the body — absent means "leave unchanged".
     posture_provided = "posture" in payload
     posture = payload.get("posture")
+    exclude_spam_provided = "exclude_spam" in payload
+    exclude_spam = payload.get("exclude_spam")
     if not source_id or not dataset_id:
         return {"id": req_id, "status": "error", "error": "source_id and dataset_id required"}
     source = REGISTRY.get(source_id)
@@ -208,16 +210,20 @@ async def handle_put_source_settings(message: Dict[str, Any]) -> Optional[Dict[s
     # enabled toggle is being set.
     if enabled is not None and getattr(source, "source_type", None) != "local_sync":
         return {"id": req_id, "status": "error", "error": "enabled only applies to local_sync sources"}
-    if enabled is None and not posture_provided:
-        return {"id": req_id, "status": "error", "error": "enabled or posture required in body"}
+    if exclude_spam_provided and getattr(source, "source_type", None) != "local_sync":
+        return {"id": req_id, "status": "error", "error": "exclude_spam only applies to local_sync sources"}
+    if enabled is None and not posture_provided and not exclude_spam_provided:
+        return {"id": req_id, "status": "error", "error": "enabled, posture, or exclude_spam required in body"}
     conn = hub.get_db_connection()
     if not conn:
         return {"id": req_id, "status": "error", "error": "Database not available"}
     try:
+        put_kwargs: Dict[str, Any] = {"enabled": enabled}
         if posture_provided:
-            put_source_settings(conn, dataset_id, source_id, enabled=enabled, posture=posture)
-        else:
-            put_source_settings(conn, dataset_id, source_id, enabled=enabled)
+            put_kwargs["posture"] = posture
+        if exclude_spam_provided:
+            put_kwargs["exclude_spam"] = bool(exclude_spam)
+        put_source_settings(conn, dataset_id, source_id, **put_kwargs)
     except ValueError as exc:
         return {"id": req_id, "status": "error", "error": str(exc)}
     # Echo the effective settings back (posture reflects what was persisted /
@@ -229,6 +235,7 @@ async def handle_put_source_settings(message: Dict[str, Any]) -> Optional[Dict[s
     else:
         result_payload["enabled"] = bool(settings_data.get("enabled", True))
     result_payload["posture"] = settings_data.get("posture")
+    result_payload["exclude_spam"] = bool(settings_data.get("exclude_spam", True))
     return {"id": req_id, "status": "ok", "payload": result_payload}
 
 @handles("get_source_contacts")

{topos_node-1.3.36 → topos_node-1.3.37}/topos/engine/backends/ollama.py

@@ -110,6 +110,12 @@ class OllamaAdapter:
         except Exception:
             return False
 
+    def _ensure_server(self) -> None:
+        """Start a local Ollama if this request needs one and :11434 is down."""
+        from ..ollama_runtime import ensure_running
+
+        ensure_running(base_url=self._base_url)
+
     def model_supports_thinking(self, model: str) -> bool:
         """True when /api/show lists the 'thinking' capability for ``model``.
 
@@ -172,6 +178,7 @@ class OllamaAdapter:
         curated table — Ollama has no size for a tag that is not on disk
         (PLAN_LOCAL_MODEL_QUICKSTART D4).
         """
+        self._ensure_server()
         req = urllib.request.Request(f"{self._base_url}/api/tags", method="GET")
         try:
             with urllib.request.urlopen(req, timeout=10) as resp:
@@ -211,6 +218,7 @@ class OllamaAdapter:
         show the download to a human. A callback that raises must not abort a
         multi-gigabyte transfer that is otherwise healthy.
         """
+        self._ensure_server()
         body = {"model": model_name, "stream": stream}
         req = urllib.request.Request(
             f"{self._base_url}/api/pull",
@@ -361,6 +369,7 @@ class OllamaAdapter:
         # rejected think=false before keep thinking but get a wider output
         # budget so the CoT cannot starve the answer (Ollama returns thinking
         # in a separate field, so 'response' stays clean either way).
+        self._ensure_server()
         requested_think = think
         think = self.resolve_think(model, think)
         if (

topos_node-1.3.37/topos/engine/ollama_runtime.py

@@ -0,0 +1,142 @@
+"""Start a local Ollama server when a request needs it and :11434 is down.
+
+Reachability is already probed everywhere; nothing on the product path used
+to spawn the server. Install opens Ollama.app after a one-click install —
+this module does the same (or ``ollama serve``) at request time, only for a
+localhost base URL, and only once under a process lock.
+"""
+
+from __future__ import annotations
+
+import logging
+import shutil
+import subprocess
+import threading
+import time
+from typing import Callable, Optional
+from urllib.parse import urlparse
+
+logger = logging.getLogger("topos.engine.ollama_runtime")
+
+_LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
+_WAIT_SEC = 20.0
+_POLL_INTERVAL = 0.5
+
+_LOCK = threading.Lock()
+
+
+def _default_base_url() -> str:
+    try:
+        from ..config.settings import settings
+
+        return str(getattr(settings, "engine_ollama_base_url", None) or "http://localhost:11434").rstrip("/")
+    except Exception:
+        return "http://localhost:11434"
+
+
+def is_local_base_url(url: str) -> bool:
+    raw = str(url or "").strip()
+    if not raw:
+        return False
+    parsed = urlparse(raw if "://" in raw else f"http://{raw}")
+    host = (parsed.hostname or "").lower()
+    return host in _LOCAL_HOSTS
+
+
+def default_is_reachable(base_url: str) -> bool:
+    from .backends.ollama import OllamaAdapter
+
+    return bool(OllamaAdapter(base_url=base_url).is_reachable())
+
+
+def default_app_present() -> bool:
+    from .ollama_install import default_app_present as _app_present
+
+    return bool(_app_present())
+
+
+def default_open_app() -> None:
+    from .ollama_install import default_open_app as _open_app
+
+    _open_app()
+
+
+def default_cli_present() -> bool:
+    return shutil.which("ollama") is not None
+
+
+def default_spawn_serve() -> None:
+    subprocess.Popen(
+        ["ollama", "serve"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        start_new_session=True,
+    )
+
+
+def ensure_running(
+    *,
+    base_url: Optional[str] = None,
+    is_reachable: Optional[Callable[[], bool]] = None,
+    is_local: Optional[Callable[[str], bool]] = None,
+    app_present: Optional[Callable[[], bool]] = None,
+    open_app: Optional[Callable[[], None]] = None,
+    cli_present: Optional[Callable[[], bool]] = None,
+    spawn_serve: Optional[Callable[[], None]] = None,
+    sleep: Optional[Callable[[float], None]] = None,
+    wait_sec: float = _WAIT_SEC,
+    poll_interval: float = _POLL_INTERVAL,
+) -> bool:
+    """Bring a local Ollama up if a request needs it.
+
+    Returns True when the server answers ``/api/tags`` (already up, or up
+    after we launched). Returns False when the URL is remote, nothing is
+    installed, or the wait timed out — callers then hit the existing 502 /
+    deferred path.
+    """
+    url = (base_url if base_url is not None else _default_base_url()).rstrip("/")
+    local_check = is_local if is_local is not None else is_local_base_url
+    if not local_check(url):
+        return False
+
+    reachable = is_reachable if is_reachable is not None else (lambda: default_is_reachable(url))
+    if reachable():
+        return True
+
+    with _LOCK:
+        if reachable():
+            return True
+
+        launched = False
+        present = app_present if app_present is not None else default_app_present
+        if present():
+            try:
+                (open_app if open_app is not None else default_open_app)()
+                launched = True
+                logger.info("opened Ollama.app; waiting for %s", url)
+            except Exception as exc:  # noqa: BLE001 — launch is best-effort
+                logger.warning("failed to open Ollama.app: %s", exc)
+
+        if not launched:
+            has_cli = cli_present if cli_present is not None else default_cli_present
+            if has_cli():
+                try:
+                    (spawn_serve if spawn_serve is not None else default_spawn_serve)()
+                    launched = True
+                    logger.info("spawned `ollama serve`; waiting for %s", url)
+                except Exception as exc:  # noqa: BLE001
+                    logger.warning("failed to spawn ollama serve: %s", exc)
+
+        if not launched:
+            logger.info("ollama is down and nothing on this machine can start it")
+            return False
+
+        sleeper = sleep if sleep is not None else time.sleep
+        deadline = time.monotonic() + float(wait_sec)
+        interval = max(0.0, float(poll_interval))
+        while time.monotonic() < deadline:
+            if reachable():
+                return True
+            sleeper(interval)
+        logger.warning("ollama did not become reachable at %s within %.1fs", url, wait_sec)
+        return False

{topos_node-1.3.36 → topos_node-1.3.37}/topos/ingestion/local_sync.py

@@ -64,6 +64,41 @@ def _run_local_sync_enrichment_if_enabled(
         )
 
 
+def _as_bool(value: Any, *, default: bool = True) -> bool:
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, (int, float)):
+        return bool(value)
+    text = str(value).strip().lower()
+    if text in {"1", "true", "yes", "on"}:
+        return True
+    if text in {"0", "false", "no", "off"}:
+        return False
+    return default
+
+
+def _resolve_exclude_spam(
+    options: Optional[Dict[str, Any]],
+    *,
+    db_conn: Any,
+    dataset_id: str,
+) -> bool:
+    """Default on. sync_options.exclude_spam wins over the stored source setting."""
+    if isinstance(options, dict) and "exclude_spam" in options:
+        return _as_bool(options.get("exclude_spam"), default=True)
+    try:
+        from ..storage.source_settings import get_source_settings
+
+        settings = get_source_settings(db_conn, dataset_id, SOURCE_ID_IMESSAGE) or {}
+        if "exclude_spam" in settings:
+            return _as_bool(settings.get("exclude_spam"), default=True)
+    except Exception:
+        logger.debug("exclude_spam setting lookup failed; defaulting to skip spam", exc_info=True)
+    return True
+
+
 def _resolve_sync_start_unix(options: Optional[Dict[str, Any]]) -> tuple[Optional[float], Optional[str]]:
     """Resolve sync start timestamp from sync options."""
     if not options:
@@ -318,16 +353,16 @@ def run_imessage_sync(
 ) -> Dict[str, Any]:
     """
     Run iMessage sync: load checkpoint → read from chat.db → parse → write to conversation_messages → save checkpoint.
-    Returns dict with status, records_processed, last_record_id, error (if any).
+    Returns dict with status, records_processed, records_skipped, last_record_id, error (if any).
     """
     if not dataset_id:
-        return {"status": "error", "error": "dataset_id required", "records_processed": 0}
+        return {"status": "error", "error": "dataset_id required", "records_processed": 0, "records_skipped": 0}
 
     if db_conn is None:
         from ..core.state import get_db_connection
         db_conn = get_db_connection()
     if db_conn is None:
-        return {"status": "error", "error": "Database connection not available", "records_processed": 0}
+        return {"status": "error", "error": "Database connection not available", "records_processed": 0, "records_skipped": 0}
 
     store = checkpoint_store if checkpoint_store is not None else SqliteCheckpointStore(db_conn)
     checkpoint = store.get_checkpoint(dataset_id, IMESSAGE_SCHEMA_ID)
@@ -355,7 +390,7 @@ def run_imessage_sync(
             e,
             exc_info=True,
         )
-        return {"status": "error", "error": str(e), "records_processed": 0}
+        return {"status": "error", "error": str(e), "records_processed": 0, "records_skipped": 0}
 
 
 def _run_imessage_sync_impl(
@@ -371,36 +406,39 @@ def _run_imessage_sync_impl(
     """Implementation of run_imessage_sync (called inside try so we never raise)."""
     start_unix, start_error = _resolve_sync_start_unix(sync_options)
     if start_error:
-        return {"status": "error", "error": start_error, "records_processed": 0}
+        return {"status": "error", "error": start_error, "records_processed": 0, "records_skipped": 0}
 
     parser_cls = PARSER_REGISTRY.get(IMESSAGE_SCHEMA_ID)
     if not parser_cls:
-        return {"status": "error", "error": "No parser for imessage.messages.v1", "records_processed": 0}
+        return {"status": "error", "error": "No parser for imessage.messages.v1", "records_processed": 0, "records_skipped": 0}
     parser = parser_cls(dataset_id=dataset_id, _schema_id=IMESSAGE_SCHEMA_ID)
     from ..storage.canonical import ConversationsTablesManager
     manager = ConversationsTablesManager(db_conn)
-    from .sources.imessage_reader import read_imessage_rows_list, get_chat_db_path
+    from .sources.imessage_reader import read_imessage_batch, get_chat_db_path
     path = chat_db_path or get_chat_db_path()
+    exclude_spam = _resolve_exclude_spam(sync_options, db_conn=db_conn, dataset_id=dataset_id)
 
     # For bounded history sync, restart from row 0 and apply time filter.
     current_last_record_id = "0" if start_unix is not None else last_record_id
     final_last_record_id = last_record_id
     total_processed = 0
+    total_skipped = 0
     batch_num = 0
 
     while True:
         batch_num += 1
         try:
-            rows = read_imessage_rows_list(
+            batch = read_imessage_batch(
                 last_rowid=current_last_record_id if current_last_record_id != "0" else None,
                 chat_db_path=path,
                 batch_size=batch_size,
                 start_unix=start_unix,
+                exclude_spam=exclude_spam,
             )
         except FileNotFoundError as e:
-            return {"status": "error", "error": str(e), "records_processed": total_processed}
+            return {"status": "error", "error": str(e), "records_processed": total_processed, "records_skipped": total_skipped}
         except PermissionError as e:
-            return {"status": "error", "error": str(e), "records_processed": total_processed}
+            return {"status": "error", "error": str(e), "records_processed": total_processed, "records_skipped": total_skipped}
         except OSError as e:
             logger.warning(
                 "imessage read failed (OSError errno=%s) on batch %d: %s",
@@ -409,7 +447,7 @@ def _run_imessage_sync_impl(
                 e,
                 exc_info=True,
             )
-            return {"status": "error", "error": str(e), "records_processed": total_processed}
+            return {"status": "error", "error": str(e), "records_processed": total_processed, "records_skipped": total_skipped}
         except sqlite3.Error as e:
             logger.warning(
                 "imessage read failed (sqlite3.Error) on batch %d: %s",
@@ -417,12 +455,14 @@ def _run_imessage_sync_impl(
                 e,
                 exc_info=True,
             )
-            return {"status": "error", "error": str(e), "records_processed": total_processed}
+            return {"status": "error", "error": str(e), "records_processed": total_processed, "records_skipped": total_skipped}
         except Exception as e:
             logger.warning("imessage read failed on batch %d: %s", batch_num, e, exc_info=True)
-            return {"status": "error", "error": str(e), "records_processed": total_processed}
+            return {"status": "error", "error": str(e), "records_processed": total_processed, "records_skipped": total_skipped}
 
-        if not rows:
+        rows = batch.rows
+        total_skipped += batch.records_skipped
+        if batch.scanned_count == 0:
             break
 
         # Persist raw iMessage payloads for traceability and debugging (non-fatal on failure).
@@ -440,7 +480,6 @@ def _run_imessage_sync_impl(
             logger.warning("[PIPELINE:RAW] iMessage raw write failed (non-fatal): %s", e)
 
         normalized_records: List[Any] = []
-        max_rowid: Optional[int] = None
         for row in rows:
             raw = RawRecord(record_id=row["id"], payload=row)
             validation = parser.validate(raw)
@@ -449,9 +488,6 @@ def _run_imessage_sync_impl(
                 continue
             norm = parser.parse(raw)
             normalized_records.append(norm)
-            rid = row.get("ROWID")
-            if rid is not None and (max_rowid is None or rid > max_rowid):
-                max_rowid = rid
 
         if normalized_records:
             mapped_records = _map_normalized_records_with_canonical_mapper(
@@ -484,7 +520,12 @@ def _run_imessage_sync_impl(
                 manager.upsert_message_batch(staging_records, dataset_id, SOURCE_ID_IMESSAGE)
             except Exception as e:
                 logger.exception("ConversationsTablesManager.upsert_message_batch failed")
-                return {"status": "error", "error": str(e), "records_processed": total_processed}
+                return {
+                    "status": "error",
+                    "error": str(e),
+                    "records_processed": total_processed,
+                    "records_skipped": total_skipped,
+                }
 
             canonical_messages = [
                 {
@@ -509,25 +550,27 @@ def _run_imessage_sync_impl(
 
             total_processed += len(normalized_records)
 
-        if max_rowid is None:
+        if batch.max_scanned_rowid is None:
             # Defensive: avoid infinite loops if no valid rowid in batch.
             break
 
-        final_last_record_id = f"imessage:{max_rowid}"
+        final_last_record_id = f"imessage:{batch.max_scanned_rowid}"
         store.save_checkpoint(IngestionCheckpoint(
             dataset_id=dataset_id,
             schema_id=IMESSAGE_SCHEMA_ID,
             last_record_id=final_last_record_id,
-            metadata={},
+            metadata={"exclude_spam": exclude_spam},
         ))
         current_last_record_id = final_last_record_id
 
-        if len(rows) < batch_size:
+        if batch.scanned_count < batch_size:
             break
 
     return {
         "status": "ok",
         "records_processed": total_processed,
+        "records_skipped": total_skipped,
+        "exclude_spam": exclude_spam,
         "last_record_id": final_last_record_id,
     }