topos-node 1.3.8__tar.gz → 1.3.9__tar.gz

{topos_node-1.3.8/topos_node.egg-info → topos_node-1.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.8
+Version: 1.3.9
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai

{topos_node-1.3.8 → topos_node-1.3.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.8"
+version = "1.3.9"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"

{topos_node-1.3.8 → topos_node-1.3.9}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.8"
-__version_info__ = (1, 3, 8)
+__version__ = "1.3.9"
+__version_info__ = (1, 3, 9)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.3.8 → topos_node-1.3.9}/topos/enrichment/derivation_recovery.py

@@ -240,6 +240,70 @@ def list_pending_derivation_retries(
     return out
 
 
+async def retry_single_derivation(
+    conn: sqlite3.Connection,
+    *,
+    source_id: str,
+    sync_batch_id: str,
+    job_name: str,
+    record_ids: Optional[Sequence[str]] = None,
+) -> Dict[str, Any]:
+    """Re-run exactly one recorded derivation debt.
+
+    Reloads the debt's canonical records and re-runs ONLY the job that failed,
+    so a retry cannot disturb jobs that already succeeded. Jobs are idempotent
+    per (batch, job) at the ledger level; a retry that fails again simply leaves
+    the debt in place for the next attempt.
+
+    Returns ``{"outcome": "recovered" | "still_failing" | "skipped", ...}``.
+    "skipped" means the debt cannot be re-run mechanically (unknown source, no
+    surviving records) — the caller decides whether that discharges it.
+    """
+    if not job_name or not source_id:
+        return {"outcome": "skipped", "reason": "incomplete record"}
+    try:
+        from ..ingestion.canonical_pipeline import load_canonical_records_for_signal
+        # Reuses reprocess's resolver so a runtime-installed source rehydrates
+        # the same way it does for a manual reprocess.
+        from ..ingestion.reprocess import _resolve_source_def
+
+        try:
+            source_def = _resolve_source_def(source_id)
+        except ValueError:
+            return {"outcome": "skipped", "reason": "unknown source"}
+        records = load_canonical_records_for_signal(conn, source_def)
+        wanted = {str(r) for r in (record_ids or []) if r}
+        if wanted:
+            scoped = [
+                r
+                for r in records
+                if str(r.get("message_id") or r.get("record_id") or "") in wanted
+            ]
+            records = scoped or records
+        if not records:
+            return {"outcome": "skipped", "reason": "no records"}
+
+        from .orchestrator import SignalDerivationOrchestrator
+
+        result = await SignalDerivationOrchestrator().run_signal_derivation(
+            records, source_id, job_names=[job_name], sync_batch_id=sync_batch_id
+        )
+        if result.get("errors"):
+            return {
+                "outcome": "still_failing",
+                "error": result["errors"][0].get("error"),
+            }
+        # run_signal_derivation clears the debt itself on success.
+        return {
+            "outcome": "recovered",
+            "records": len(records),
+            "created": result.get("records_created", {}).get(job_name, 0),
+        }
+    except Exception as exc:  # noqa: BLE001 — one bad debt must not stop the rest
+        logger.warning("[DERIVE:RETRY] retry failed job=%s source=%s: %s", job_name, source_id, exc)
+        return {"outcome": "still_failing", "error": str(exc)}
+
+
 async def retry_pending_derivations(
     conn: Optional[sqlite3.Connection] = None,
     *,
@@ -249,11 +313,6 @@ async def retry_pending_derivations(
 ) -> Dict[str, Any]:
     """Re-run derivation jobs that previously failed, clearing settled debts.
 
-    Reloads each debt's canonical records and re-runs ONLY the job that failed,
-    so a retry cannot disturb jobs that already succeeded. Jobs are idempotent
-    per (batch, job) at the ledger level; a retry that fails again simply leaves
-    the debt in place for the next attempt.
-
     ``dry_run`` reports what would run without touching anything.
     """
     from ..core.state import get_db_connection
@@ -278,61 +337,27 @@ async def retry_pending_derivations(
         ]
         return outcome
 
-    from ..ingestion.canonical_pipeline import load_canonical_records_for_signal
-    # Reuses reprocess's resolver so a runtime-installed source rehydrates the
-    # same way it does for a manual reprocess.
-    from ..ingestion.reprocess import _resolve_source_def
-
     for debt in pending:
         job_name = debt["job_name"]
         src = debt["source_id"]
         batch = debt["sync_batch_id"]
-        if not job_name or not src:
-            outcome["skipped"].append({**debt, "reason": "incomplete record"})
-            continue
-        try:
-            try:
-                source_def = _resolve_source_def(src)
-            except ValueError:
-                outcome["skipped"].append({"job_name": job_name, "source_id": src, "reason": "unknown source"})
-                continue
-            records = load_canonical_records_for_signal(conn, source_def)
-            wanted = set(debt.get("record_ids") or [])
-            if wanted:
-                scoped = [
-                    r
-                    for r in records
-                    if str(r.get("message_id") or r.get("record_id") or "") in wanted
-                ]
-                records = scoped or records
-            if not records:
-                outcome["skipped"].append({"job_name": job_name, "source_id": src, "reason": "no records"})
-                continue
-
-            from .orchestrator import SignalDerivationOrchestrator
-
+        result = await retry_single_derivation(
+            conn,
+            source_id=src,
+            sync_batch_id=batch,
+            job_name=job_name,
+            record_ids=debt.get("record_ids") or [],
+        )
+        kind = result.pop("outcome")
+        entry = {"job_name": job_name, "source_id": src, **result}
+        if kind == "recovered":
             outcome["attempted"] += 1
-            result = await SignalDerivationOrchestrator().run_signal_derivation(
-                records, src, job_names=[job_name], sync_batch_id=batch
-            )
-            if result.get("errors"):
-                outcome["still_failing"].append(
-                    {"job_name": job_name, "source_id": src, "error": result["errors"][0].get("error")}
-                )
-            else:
-                # run_signal_derivation clears the debt itself on success.
-                outcome["recovered"].append(
-                    {
-                        "job_name": job_name,
-                        "source_id": src,
-                        "batch": batch,
-                        "records": len(records),
-                        "created": result.get("records_created", {}).get(job_name, 0),
-                    }
-                )
-        except Exception as exc:  # noqa: BLE001 — one bad debt must not stop the rest
-            logger.warning("[DERIVE:RETRY] retry failed job=%s source=%s: %s", job_name, src, exc)
-            outcome["still_failing"].append({"job_name": job_name, "source_id": src, "error": str(exc)})
+            outcome["recovered"].append({**entry, "batch": batch})
+        elif kind == "still_failing":
+            outcome["attempted"] += 1
+            outcome["still_failing"].append(entry)
+        else:
+            outcome["skipped"].append(entry)
 
     outcome["pending_after"] = len(list_pending_derivation_retries(conn, source_id=source_id, limit=1000))
     logger.info(
@@ -344,6 +369,67 @@ async def retry_pending_derivations(
     return outcome
 
 
+#: How long a worker-claimed retry waits for an active derivation batch to end
+#: before handing the job back to the queue. Debts are recorded MID-batch, and
+#: the worker claims within its poll interval — re-running immediately would
+#: re-contend for the same write gate that produced the original "database is
+#: locked". The wait doubles as the requeue backoff: while a long batch runs,
+#: the worker cycles claim → wait → requeue at this period instead of spinning.
+_IN_FLIGHT_WAIT_SECONDS = 30.0
+_IN_FLIGHT_POLL_SECONDS = 0.5
+
+
+async def run_derivation_retry_job(payload: Dict[str, Any]) -> Dict[str, Any]:
+    """Worker executor for ``SIGNAL_DERIVE_RETRY_KIND`` pipeline jobs.
+
+    This is the "by the worker" half of the module contract: the debt record IS
+    the queue row, so the worker claims it and re-runs the one derivation it
+    describes. Success marks the row done (the orchestrator clears the debt,
+    then the worker completes the job). A retry that fails again returns an
+    error status, parking the row 'failed' — still visible in every debt
+    listing, re-queued by the next organic failure of the same (batch, job), and
+    re-runnable via ``POST /signal/derivation-debt/retry``. One claim is one
+    attempt; nothing here loops.
+    """
+    import asyncio
+
+    from .pipeline_activity import is_derivation_in_flight
+
+    waited = 0.0
+    while is_derivation_in_flight():
+        if waited >= _IN_FLIGHT_WAIT_SECONDS:
+            return {"status": "requeue", "reason": "derivation in flight"}
+        await asyncio.sleep(_IN_FLIGHT_POLL_SECONDS)
+        waited += _IN_FLIGHT_POLL_SECONDS
+
+    from ..core.state import get_db_connection
+
+    conn = get_db_connection()
+    if conn is None:
+        return {"status": "error", "error": "no database connection"}
+
+    job_name = str(payload.get("job_name") or "")
+    result = await retry_single_derivation(
+        conn,
+        source_id=str(payload.get("source_id") or ""),
+        sync_batch_id=str(payload.get("sync_batch_id") or "unknown"),
+        job_name=job_name,
+        record_ids=payload.get("record_ids") or [],
+    )
+    outcome = result.pop("outcome")
+    if outcome == "recovered":
+        return {
+            "status": "ok",
+            "records_created": {job_name: result.get("created", 0)},
+            **result,
+        }
+    if outcome == "skipped":
+        # A debt that cannot be re-run is still recorded data loss. Failing the
+        # job keeps it visible for a human instead of silently discharging it.
+        return {"status": "error", "error": f"cannot retry: {result.get('reason')}"}
+    return {"status": "error", "error": str(result.get("error") or "retry failed")}
+
+
 def pending_derivation_summary(conn: Optional[sqlite3.Connection]) -> Dict[str, Any]:
     """What derived data this node KNOWS it is missing, and since when it knew.
 

{topos_node-1.3.8 → topos_node-1.3.9}/topos/pipeline/job_runner.py

@@ -11,6 +11,8 @@ import uuid
 from typing import Any, Awaitable, Callable, Dict, Optional
 
 from ..storage.db.write_gate import is_busy_error
+# Safe at import time: derivation_recovery has no module-level heavy imports.
+from ..enrichment.derivation_recovery import SIGNAL_DERIVE_RETRY_KIND
 from .job_store import (
     claim_matching_queued_jobs,
     claim_next_job,
@@ -18,6 +20,7 @@ from .job_store import (
     fail_job,
     recover_stale_jobs,
     record_derivation_completion,
+    requeue_job,
     update_job_progress,
 )
 
@@ -167,14 +170,29 @@ async def _execute_topic_consolidation(payload: Dict[str, Any]) -> Dict[str, Any
     return {"status": "ok", "result": result}
 
 
+async def _execute_signal_derive_retry(payload: Dict[str, Any]) -> Dict[str, Any]:
+    from ..enrichment.derivation_recovery import run_derivation_retry_job
+
+    return await run_derivation_retry_job(payload)
+
+
 EXECUTORS: Dict[str, ExecutorFn] = {
     "inbox_deferred_enrichment": _execute_inbox_deferred_enrichment,
     "file_ingestion": _execute_file_ingestion,
     "enrichment_process_source": _execute_enrichment_process_source,
     "topic_consolidation": _execute_topic_consolidation,
+    SIGNAL_DERIVE_RETRY_KIND: _execute_signal_derive_retry,
 }
 
 
+def _executable_kinds() -> list[str]:
+    """Kinds this worker may claim. Computed per claim so tests that patch
+    EXECUTORS are honored. A queued row of any other kind (e.g. written by a
+    newer node version) stays queued and visible instead of being claimed and
+    immediately failed as unknown."""
+    return sorted(EXECUTORS)
+
+
 #: Upper bound on inbox jobs merged into one derive batch. A CP backlog after
 #: downtime arrives as one job per delivery (usually one record each); merging
 #: them runs the per-batch jobs (dimension briefs, topic work) once instead of
@@ -277,6 +295,18 @@ async def process_job(conn_factory: Callable[[], Any], job: Dict[str, Any]) -> N
         await _run_db(conn_factory, _mark_crashed)
         return
 
+    if str(result.get("status") or "ok") == "requeue":
+        # The executor declined to run right now (e.g. a derivation batch holds
+        # the write gate). Hand the claim back untouched — this is a deferral,
+        # not an attempt, so the row must not read as failed and must not burn
+        # a retry.
+        def _requeue(own: Any) -> None:
+            for entry in jobs:
+                requeue_job(own, str(entry["job_id"]))
+
+        await _run_db(conn_factory, _requeue)
+        return
+
     if str(result.get("status") or "ok") == "error":
         error = str(result.get("error") or result.get("message") or "failed")
 
@@ -345,7 +375,7 @@ async def _worker_loop(conn_factory: Callable[[], Any]) -> None:
                 own = conn_factory()
                 if own is None:
                     return None
-                return claim_next_job(own, lease_owner=_lease_owner)
+                return claim_next_job(own, lease_owner=_lease_owner, kinds=_executable_kinds())
 
             job = await asyncio.to_thread(_claim)
             if job is None:
@@ -409,7 +439,7 @@ async def process_pending_jobs_once(conn_factory: Callable[[], Any], *, limit: i
         own = conn_factory()
         if own is None:
             return None
-        return claim_next_job(own, lease_owner=_lease_owner)
+        return claim_next_job(own, lease_owner=_lease_owner, kinds=_executable_kinds())
 
     for _ in range(max(1, int(limit))):
         job = await asyncio.to_thread(_claim)

{topos_node-1.3.8 → topos_node-1.3.9}/topos/pipeline/job_store.py

@@ -273,6 +273,29 @@ def fail_job(
         commit_connection(conn)
 
 
+def requeue_job(conn: sqlite3.Connection, job_id: str) -> None:
+    """Return a claimed job to the queue without recording an attempt.
+
+    For executors that decline to run right now (e.g. a derivation batch is in
+    flight). Unlike ``fail_job`` the row goes back to 'queued', so nothing
+    downstream reads a deferral as a failure. Guarded on status='running' so a
+    job that was settled out-of-band while claimed (``clear_derivation_retry``
+    marks the row 'done' when a later batch succeeds) is not resurrected.
+    """
+    ensure_pipeline_jobs_schema(conn)
+    with with_db_write():
+        conn.execute(
+            """
+            UPDATE pipeline_jobs
+            SET status='queued', lease_owner=NULL, lease_expires_at=NULL,
+                updated_at=datetime('now')
+            WHERE job_id=? AND status='running'
+            """,
+            (job_id,),
+        )
+        commit_connection(conn)
+
+
 def recover_stale_jobs(conn: sqlite3.Connection) -> int:
     """Reset expired or orphaned running jobs to queued."""
     ensure_pipeline_jobs_schema(conn)

{topos_node-1.3.8 → topos_node-1.3.9}/topos/upgrades/manifests.json

@@ -212,6 +212,30 @@
         "Worker-thread write offload + statistics txn hygiene close the database-is-locked / event-loop-stall class that hit repeat ingests after 1.3.7."
       ]
     },
+    {
+      "version": "1.3.9",
+      "summary": "Derivation-retry repair: the signal_derive_retry executor was never registered, so every recorded derivation debt was claimed and failed as an unknown job kind \u2014 the retry meant to repair a lost derivation destroyed the record of it instead. The executor is wired, workers claim only kinds they can execute, and an executor may requeue rather than read as failed. One consent:auto step sweeps debt already parked 'failed', which nothing else would re-queue. Also: public_bio:read / work_context:read gain test cover for the restored profile_records lane, and just harness-gate scores answer quality against a ratcheting floor instead of a 70/70 bar it cannot reach on a signal-less seed DB. No schema change.",
+      "steps": [
+        {
+          "id": "retry-recorded-derivation-debt",
+          "kind": "engine_endpoint",
+          "title": "Re-run derivation debt that was failed as an unknown job kind",
+          "why": "record_failed_derivation() has enqueued pipeline rows of kind 'signal_derive_retry' since 1.3.x, but no executor was ever registered for that kind, so the worker claimed each row and immediately failed it with 'Unknown job kind: signal_derive_retry'. The debt record IS the queue row, so the retry meant to repair a lost derivation destroyed the record of it instead \u2014 nodes carry silent derivation gaps with a 'failed' row and nothing pointing at them. The executor is registered as of this release, but that does not revive the existing rows: a failed row is only re-queued by the next organic failure of the same (batch, job), and recover_stale_jobs resets 'running' rows, never 'failed' ones. This step sweeps them once on upgrade so the repair happens without an operator remembering to POST /v1/signal/derivation-debt/retry.",
+          "params": {
+            "method": "POST",
+            "path": "/v1/signal/derivation-debt/retry",
+            "limit": 200
+          },
+          "consent": "auto",
+          "cost": "no LLM, no network; re-runs only the derivations already recorded as lost",
+          "progress": "synchronous; response is the attempted/recovered/still_failing report",
+          "idempotent": "re-running is a no-op once pending is empty; a debt that fails again is left 'failed' and still listed",
+          "estimated_duration": "seconds on a node with no debt (the common case); proportional to recorded debt otherwise",
+          "order": 1
+        }
+      ],
+      "notes": []
+    },
     {
       "version": "unreleased",
       "summary": "Staging entry for in-flight PRs. cut_release.py stamps this with the next version at release cut. Never executed by steps_between.",

{topos_node-1.3.8 → topos_node-1.3.9}/topos/upgrades/runner.py

@@ -376,6 +376,25 @@ def _exec_engine_endpoint(step: Dict[str, Any], conn: sqlite3.Connection) -> Dic
         from ..features.entities.rebuild_subprocess import run_graph_rebuild
 
         return dict(run_graph_rebuild(conn))
+    if path == "/v1/signal/derivation-debt/retry":
+        # Recorded debt is not self-healing: a failed row is only re-queued by
+        # the next organic failure of the same (batch, job), and recover_stale_jobs
+        # resets 'running' rows, never 'failed' ones. So a node that recorded debt
+        # while its executor was missing keeps the gap until something sweeps it.
+        import asyncio
+
+        from ..enrichment.derivation_recovery import retry_pending_derivations
+
+        params = step.get("params") or {}
+        return dict(
+            asyncio.run(
+                retry_pending_derivations(
+                    conn,
+                    source_id=params.get("source_id"),
+                    limit=int(params.get("limit") or 200),
+                )
+            )
+        )
     raise ValueError(f"no internal dispatch for endpoint step: {path!r}")
 
 

{topos_node-1.3.8 → topos_node-1.3.9/topos_node.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.8
+Version: 1.3.9
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai