topos-node 1.3.3__tar.gz → 1.3.5__tar.gz

{topos_node-1.3.3/topos_node.egg-info → topos_node-1.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.3
+Version: 1.3.5
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai
@@ -25,7 +25,7 @@ Requires-Dist: google-cloud-run~=0.16.0
 Requires-Dist: asgi-lifespan~=2.1.0
 Requires-Dist: psycopg[binary]~=3.3.3
 Requires-Dist: duckdb~=1.5.2
-Requires-Dist: torch<3,>=2.8.0
+Requires-Dist: torch<2.13,>=2.8.0
 Requires-Dist: transformers~=5.10.2
 Requires-Dist: huggingface-hub~=1.18.0
 Requires-Dist: tqdm~=4.67.3
@@ -33,6 +33,7 @@ Requires-Dist: hf-xet~=1.5.1
 Requires-Dist: sentence-transformers~=5.6.0
 Requires-Dist: networkx<3.7,>=3.2.1
 Requires-Dist: python-louvain<0.17,>=0.16
+Requires-Dist: grand-cypher<0.13,>=0.12
 Requires-Dist: numpy<3,>=1.26
 Requires-Dist: scipy<2,>=1.11
 Requires-Dist: scikit-learn<2,>=1.4
@@ -43,7 +44,7 @@ Provides-Extra: database
 Requires-Dist: psycopg[binary]~=3.3.3; extra == "database"
 Provides-Extra: engine
 Requires-Dist: duckdb~=1.5.2; extra == "engine"
-Requires-Dist: torch<3,>=2.8.0; extra == "engine"
+Requires-Dist: torch<2.13,>=2.8.0; extra == "engine"
 Requires-Dist: transformers~=5.10.2; extra == "engine"
 Requires-Dist: huggingface-hub~=1.18.0; extra == "engine"
 Requires-Dist: tqdm~=4.67.3; extra == "engine"
@@ -51,6 +52,7 @@ Requires-Dist: hf-xet~=1.5.1; extra == "engine"
 Requires-Dist: sentence-transformers~=5.6.0; extra == "engine"
 Requires-Dist: networkx<3.7,>=3.2.1; extra == "engine"
 Requires-Dist: python-louvain<0.17,>=0.16; extra == "engine"
+Requires-Dist: grand-cypher<0.13,>=0.12; extra == "engine"
 Requires-Dist: numpy<3,>=1.26; extra == "engine"
 Requires-Dist: scipy<2,>=1.11; extra == "engine"
 Requires-Dist: scikit-learn<2,>=1.4; extra == "engine"
@@ -126,6 +128,14 @@ Topos Engine uses both local and model-hub paths so users can choose flexibility
 
 The `shared/` package in this repo is part of the node runtime contract. It contains common schema and filtering definitions used by both API and engine paths.
 
+### Architecture notes (where truth lives)
+
+This README is orientation, not the architecture ledger. Prefer:
+
+- **As-built / why:** sibling `topos-research-wiki` → `20_ARCHITECTURE/` (`As_Built_Node`, firewall, query, MCP, graph, …)
+- **As-is invariants / checks:** sibling `topos-ops-wiki` → `25_SYSTEMS/` (`SYS-node`, `SYS-cognitive-firewall`, …)
+- **Target design:** `topos-data-wiki` (`status: target`) — do not implement from it without checking as-built
+
 ## Quick Start
 
 ### 1) Install

{topos_node-1.3.3 → topos_node-1.3.5}/README.md

@@ -61,6 +61,14 @@ Topos Engine uses both local and model-hub paths so users can choose flexibility
 
 The `shared/` package in this repo is part of the node runtime contract. It contains common schema and filtering definitions used by both API and engine paths.
 
+### Architecture notes (where truth lives)
+
+This README is orientation, not the architecture ledger. Prefer:
+
+- **As-built / why:** sibling `topos-research-wiki` → `20_ARCHITECTURE/` (`As_Built_Node`, firewall, query, MCP, graph, …)
+- **As-is invariants / checks:** sibling `topos-ops-wiki` → `25_SYSTEMS/` (`SYS-node`, `SYS-cognitive-firewall`, …)
+- **Target design:** `topos-data-wiki` (`status: target`) — do not implement from it without checking as-built
+
 ## Quick Start
 
 ### 1) Install

{topos_node-1.3.3 → topos_node-1.3.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.3"
+version = "1.3.5"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"
@@ -27,7 +27,7 @@ dependencies = [
   "asgi-lifespan~=2.1.0",
   "psycopg[binary]~=3.3.3",
   "duckdb~=1.5.2",
-  "torch>=2.8.0,<3",
+  "torch>=2.8.0,<2.13",
   "transformers~=5.10.2",
   "huggingface-hub~=1.18.0",
   "tqdm~=4.67.3",
@@ -35,6 +35,10 @@ dependencies = [
   "sentence-transformers~=5.6.0",
   "networkx>=3.2.1,<3.7",
   "python-louvain>=0.16,<0.17",
+  # openCypher over the materialised entity graph, owner-only
+  # (topos.features.entities.cypher; PLAN_GRAPH_QUERY_AND_LATENT_EDGES §4).
+  # Pure-Python parser (lark) + matcher over networkx — no graph database.
+  "grand-cypher>=0.12,<0.13",
   # Complexity readings (topos.features.complexity): entropy/MI + graph math.
   "numpy>=1.26,<3",
   "scipy>=1.11,<2",
@@ -58,7 +62,7 @@ database = [
 # ML + enrichment runtime (hosted topos-engine Cloud Run image).
 engine = [
   "duckdb~=1.5.2",
-  "torch>=2.8.0,<3",
+  "torch>=2.8.0,<2.13",
   "transformers~=5.10.2",
   "huggingface-hub~=1.18.0",
   "tqdm~=4.67.3",
@@ -66,6 +70,7 @@ engine = [
   "sentence-transformers~=5.6.0",
   "networkx>=3.2.1,<3.7",
   "python-louvain>=0.16,<0.17",
+  "grand-cypher>=0.12,<0.13",
   "numpy>=1.26,<3",
   "scipy>=1.11,<2",
   "scikit-learn>=1.4,<2",
@@ -106,6 +111,8 @@ filterwarnings = [
   "ignore:on_event is deprecated.*:DeprecationWarning:fastapi.*",
 ]
 markers = [
+  "journey: binds this test to a J-* journey note in topos-ops-wiki (the id is the marker arg)",
+  "check: binds this test to a C-* check note in topos-ops-wiki (the id is the marker arg)",
   "gap: gap verification test (sprint pack); must pass after sprint lands",
   "qq_eval: live query quality/latency eval against owner database",
   "acceptance: Wiki MVP acceptance test (AT-1 through AT-8)",

topos_node-1.3.5/tests/test_protocol_handled_types.py

@@ -0,0 +1,293 @@
+"""Guards for the handled-message-types protocol snapshot.
+
+The snapshot (topos/protocol/handled_message_types.json) is the machine-readable
+list of control-plane message types this engine dispatches. Two invariants:
+
+1. Freshness — the committed snapshot equals the runtime HANDLERS registry.
+   Adding or removing an @handles registration requires regenerating it:
+   uv run python scripts/dump_handled_types.py > topos/protocol/handled_message_types.json
+
+2. Static parity — the set of string literals in @handles(...) decorators equals
+   the runtime registry, so no handler is registered dynamically in a way that
+   static consumers of the snapshot (or this repo's source) would miss.
+
+The snapshot's second key, ``owner_only_message_types``, carries the subset
+registered with ``@handles(..., owner_only=True)``. Those are types the engine
+answers for the node's owner but which must never be reachable from an agent or
+cross-user surface; the tests below are the leak gates that enforce it.
+
+What "not reachable" is measured against
+----------------------------------------
+The control plane's ``/mcp`` gateway (``control_plane/mcp_gateway.py``) is the
+only place the CP forwards an engine message on behalf of an agent or a UMA
+grantee, so it is the forwarding set these gates assert against: both the tool
+names it publishes and the message types those tools forward.
+
+``control_plane/engine_protocol.py::SENT_MESSAGE_TYPES`` is deliberately *not*
+the gate. It declares what the owner's own authenticated dashboard routes send
+to the owner's own node — ``delete_database_table`` is in it precisely because
+the owner's data explorer exists. Owner-only means owner-only, not unreachable.
+"""
+from __future__ import annotations
+
+import ast
+import json
+import os
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+SNAPSHOT = REPO_ROOT / "topos" / "protocol" / "handled_message_types.json"
+
+MCP_GATEWAY_RELPATH = Path("control_plane") / "mcp_gateway.py"
+
+
+def _runtime_handled() -> set[str]:
+    from topos.core.handlers import HANDLERS
+
+    return set(HANDLERS)
+
+
+def _runtime_owner_only() -> set[str]:
+    from topos.core.handlers import OWNER_ONLY_MESSAGE_TYPES
+
+    return set(OWNER_ONLY_MESSAGE_TYPES)
+
+
+def _handles_calls() -> list[ast.Call]:
+    calls: list[ast.Call] = []
+    for py in sorted((REPO_ROOT / "topos").rglob("*.py")):
+        if "__pycache__" in py.parts:
+            continue
+        try:
+            tree = ast.parse(py.read_text(encoding="utf-8"))
+        except SyntaxError:
+            continue
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "handles":
+                calls.append(node)
+    return calls
+
+
+def _literal_str_args(call: ast.Call) -> set[str]:
+    return {
+        arg.value
+        for arg in call.args
+        if isinstance(arg, ast.Constant) and isinstance(arg.value, str)
+    }
+
+
+def _static_handled() -> set[str]:
+    found: set[str] = set()
+    for call in _handles_calls():
+        found |= _literal_str_args(call)
+    return found
+
+
+def _static_owner_only() -> set[str]:
+    found: set[str] = set()
+    for call in _handles_calls():
+        marked = any(
+            kw.arg == "owner_only"
+            and isinstance(kw.value, ast.Constant)
+            and kw.value.value is True
+            for kw in call.keywords
+        )
+        if marked:
+            found |= _literal_str_args(call)
+    return found
+
+
+def _snapshot() -> dict:
+    return json.loads(SNAPSHOT.read_text(encoding="utf-8"))
+
+
+def _mcp_gateway_tree() -> ast.AST:
+    """The CP's /mcp gateway module, or skip if the CP repo isn't checked out.
+
+    Mirrors the CP's own engine lookup in its tests/contract/test_engine_protocol.py.
+    """
+    env_path = os.environ.get("CONTROL_PLANE_REPO_PATH", "").strip()
+    roots = [Path(env_path)] if env_path else [REPO_ROOT.parent / "topos-control-plane"]
+    for root in roots:
+        gateway = root / MCP_GATEWAY_RELPATH
+        if gateway.is_file():
+            return ast.parse(gateway.read_text(encoding="utf-8"))
+    pytest.skip(
+        "control-plane repo not found; set CONTROL_PLANE_REPO_PATH to run the "
+        "owner-only /mcp leak gates (CI must provide it)"
+    )
+
+
+def _mcp_tool_names(tree: ast.AST) -> set[str]:
+    """Names published as /mcp tools: functions decorated with @fastmcp.tool()."""
+    names: set[str] = set()
+    for node in ast.walk(tree):
+        if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            continue
+        for dec in node.decorator_list:
+            func = dec.func if isinstance(dec, ast.Call) else dec
+            if isinstance(func, ast.Attribute) and func.attr == "tool":
+                names.add(node.name)
+    return names
+
+
+def _mcp_forwarded_message_types(tree: ast.AST) -> set[str]:
+    """Engine message types the /mcp gateway forwards (_forward/_forward_owner_scoped)."""
+    found: set[str] = set()
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call) or not node.args:
+            continue
+        name = getattr(node.func, "id", "") or getattr(node.func, "attr", "")
+        if not name.startswith("_forward"):
+            continue
+        first = node.args[0]
+        if isinstance(first, ast.Constant) and isinstance(first.value, str):
+            found.add(first.value)
+    return found
+
+
+def _mcp_string_literals(tree: ast.AST) -> set[str]:
+    return {
+        node.value
+        for node in ast.walk(tree)
+        if isinstance(node, ast.Constant) and isinstance(node.value, str)
+    }
+
+
+def test_snapshot_matches_runtime_registry():
+    snapshot = set(_snapshot()["handled_message_types"])
+    runtime = _runtime_handled()
+    missing = sorted(runtime - snapshot)
+    stale = sorted(snapshot - runtime)
+    assert not missing and not stale, (
+        "topos/protocol/handled_message_types.json is stale "
+        f"(missing={missing}, stale={stale}). Regenerate with: "
+        "uv run python scripts/dump_handled_types.py > topos/protocol/handled_message_types.json"
+    )
+
+
+def test_static_handles_literals_match_runtime_registry():
+    runtime = _runtime_handled()
+    static = _static_handled()
+    dynamic_only = sorted(runtime - static)
+    unregistered = sorted(static - runtime)
+    assert not dynamic_only, (
+        "Handlers registered without a literal @handles(...) decorator; static "
+        f"consumers of the registry cannot see them: {dynamic_only}"
+    )
+    assert not unregistered, (
+        f"@handles(...) literals that never reach the runtime registry: {unregistered}"
+    )
+
+
+def test_owner_only_snapshot_matches_runtime_registry():
+    snapshot = set(_snapshot()["owner_only_message_types"])
+    runtime = _runtime_owner_only()
+    missing = sorted(runtime - snapshot)
+    stale = sorted(snapshot - runtime)
+    assert not missing and not stale, (
+        "owner_only_message_types in topos/protocol/handled_message_types.json is "
+        f"stale (missing={missing}, stale={stale}). Regenerate with: "
+        "uv run python scripts/dump_handled_types.py > topos/protocol/handled_message_types.json"
+    )
+
+
+def test_owner_only_is_a_subset_of_handled():
+    snapshot = _snapshot()
+    orphans = sorted(
+        set(snapshot["owner_only_message_types"]) - set(snapshot["handled_message_types"])
+    )
+    assert not orphans, f"owner_only types with no registered handler: {orphans}"
+
+
+def test_static_owner_only_markers_match_runtime_registry():
+    runtime = _runtime_owner_only()
+    static = _static_owner_only()
+    dynamic_only = sorted(runtime - static)
+    unregistered = sorted(static - runtime)
+    assert not dynamic_only, (
+        "Types marked owner-only at runtime with no literal "
+        f"@handles(..., owner_only=True) decorator: {dynamic_only}"
+    )
+    assert not unregistered, (
+        "@handles(..., owner_only=True) literals that never reach the runtime "
+        f"registry: {unregistered}"
+    )
+
+
+def test_destructive_database_explorer_types_are_owner_only():
+    """These predate the marker and are why it exists; they may not be unmarked."""
+    runtime = _runtime_owner_only()
+    expected = {
+        "get_database_explorer_summary",
+        "delete_database_rows",
+        "delete_database_table",
+    }
+    assert expected <= runtime, f"no longer marked owner-only: {sorted(expected - runtime)}"
+
+
+def test_graph_cypher_query_is_owner_only_and_off_every_agent_surface():
+    """The §4.1 gate, hard 0.
+
+    An arbitrary traversal punches straight through row-level redaction:
+    ``MATCH (a)-[*1..3]->(b)`` does not respect a scope unless the subgraph was
+    scoped BEFORE the query ran, and nothing scopes it today. Until a
+    scoped-subgraph materialisation story exists, openCypher is owner-only.
+    """
+    assert "graph_cypher_query" in _runtime_owner_only(), (
+        "graph_cypher_query must be registered @handles(..., owner_only=True)"
+    )
+    assert "graph_cypher_query" in set(_snapshot()["owner_only_message_types"])
+
+    tree = _mcp_gateway_tree()
+    assert "graph_cypher_query" not in _mcp_forwarded_message_types(tree), (
+        "graph_cypher_query is forwarded by the control plane's /mcp gateway"
+    )
+    assert "graph_cypher_query" not in _mcp_tool_names(tree), (
+        "graph_cypher_query is published as a /mcp tool"
+    )
+    assert "graph_cypher_query" not in _mcp_string_literals(tree), (
+        "graph_cypher_query is named in control_plane/mcp_gateway.py"
+    )
+
+
+def test_owner_only_types_are_absent_from_the_local_mcp_tool_list():
+    """The engine's own stdio MCP proxy is an agent surface too.
+
+    ``_mcp_gateway_tree`` skips when the control-plane repo is not checked out;
+    this list ships in this repo, so this half of the gate can never be skipped.
+    """
+    owner_only = _runtime_owner_only()
+    assert owner_only, "owner-only marker registry is empty; the gate would be vacuous"
+
+    proxy = REPO_ROOT / "topos" / "mcp_stdio_proxy.py"
+    tree = ast.parse(proxy.read_text(encoding="utf-8"))
+
+    published = sorted(owner_only & _mcp_tool_names(tree))
+    assert not published, f"owner-only types published as local /mcp tools: {published}"
+
+    mentioned = sorted(owner_only & _mcp_string_literals(tree))
+    assert not mentioned, f"owner-only types named in {proxy.name}: {mentioned}"
+
+
+def test_owner_only_types_are_absent_from_the_mcp_surface():
+    """Hard 0: no owner-only type may be reachable through the /mcp gateway."""
+    owner_only = _runtime_owner_only()
+    assert owner_only, "owner-only marker registry is empty; the gate would be vacuous"
+
+    tree = _mcp_gateway_tree()
+
+    forwarded = sorted(owner_only & _mcp_forwarded_message_types(tree))
+    assert not forwarded, f"owner-only message types forwarded by the /mcp gateway: {forwarded}"
+
+    published = sorted(owner_only & _mcp_tool_names(tree))
+    assert not published, f"owner-only message types published as /mcp tool names: {published}"
+
+    # Belt and braces: the forward helpers can be renamed or wrapped, so also fail
+    # on an owner-only type name appearing anywhere in the gateway module at all.
+    mentioned = sorted(owner_only & _mcp_string_literals(tree))
+    assert not mentioned, (
+        f"owner-only message types named in control_plane/mcp_gateway.py: {mentioned}"
+    )

{topos_node-1.3.3 → topos_node-1.3.5}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.3"
-__version_info__ = (1, 3, 3)
+__version__ = "1.3.5"
+__version_info__ = (1, 3, 5)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.3.3 → topos_node-1.3.5}/topos/api/enrichment.py

@@ -415,9 +415,9 @@ def _delete_enrichment_data_core(source_id: str, job_name: str) -> Dict[str, Any
                 ]
                 cur = conn.execute(f"DELETE FROM signal_embeddings WHERE {where}", params)
                 try:
-                    from ..storage.adapters.sqlite.vector_search import delete_vec_rows
+                    from ..storage.adapters.sqlite.stores import SQLiteVectorIndex
 
-                    delete_vec_rows(conn, ids)
+                    SQLiteVectorIndex(conn).delete_embeddings(ids)
                 except Exception as exc:
                     logger.warning("Vector companion cleanup failed: %s", exc)
             else:
@@ -572,19 +572,33 @@ async def _generic_backfill_core(
                     "errors": [],
                 }
         else:
+            from ..enrichment.catalog import catalog_spec_version
+
             coverage_table = coverage_table_for_job(job_name)
             if coverage_table and _table_exists(db_conn, coverage_table):
                 cols = _table_columns(db_conn, coverage_table)
                 id_col = "message_id" if "message_id" in cols else "record_id"
                 if "source_id" in cols and id_col in cols:
-                    done = {
-                        str(row[0])
-                        for row in db_conn.execute(
-                            f"SELECT DISTINCT {id_col} FROM {coverage_table} WHERE source_id=?",
-                            (source_id,),
-                        ).fetchall()
-                        if row[0]
-                    }
+                    min_spec = catalog_spec_version(job_name)
+                    if "spec_version" in cols:
+                        done = {
+                            str(row[0])
+                            for row in db_conn.execute(
+                                f"SELECT DISTINCT {id_col} FROM {coverage_table} "
+                                f"WHERE source_id=? AND COALESCE(spec_version, 0) >= ?",
+                                (source_id, min_spec),
+                            ).fetchall()
+                            if row[0]
+                        }
+                    else:
+                        done = {
+                            str(row[0])
+                            for row in db_conn.execute(
+                                f"SELECT DISTINCT {id_col} FROM {coverage_table} WHERE source_id=?",
+                                (source_id,),
+                            ).fetchall()
+                            if row[0]
+                        }
                     records = [r for r in records if _record_identifier(r) not in done]
 
     if isinstance(limit, int) and limit > 0:
@@ -650,13 +664,45 @@ async def _generic_backfill_core(
     }
 
 
-def _get_enriched_message_ids(table_name: str, conn) -> set[str]:
-    """Get set of message_ids that have enrichment records in the given table."""
+def _get_enriched_message_ids(
+    table_name: str,
+    conn,
+    *,
+    min_spec_version: Optional[int] = None,
+) -> set[str]:
+    """Ids that already satisfy coverage for ``table_name``.
+
+    When ``min_spec_version`` is set and the table has ``spec_version``, only
+    rows with ``COALESCE(spec_version, 0) >= min_spec_version`` count as done
+    (PLAN M3 stale predicate). Otherwise presence-only.
+    """
     if not conn:
         return set()
     try:
-        cursor = conn.execute(f"SELECT DISTINCT message_id FROM {table_name}")
-        return {row[0] for row in cursor.fetchall()}
+        cols = _table_columns(conn, table_name)
+        if not cols:
+            return set()
+        if "message_id" in cols and "record_id" in cols:
+            id_expr = "COALESCE(message_id, record_id)"
+        elif "message_id" in cols:
+            id_expr = "message_id"
+        elif "record_id" in cols:
+            id_expr = "record_id"
+        else:
+            return set()
+        if (
+            min_spec_version is not None
+            and "spec_version" in cols
+            and isinstance(min_spec_version, int)
+        ):
+            cursor = conn.execute(
+                f"SELECT DISTINCT {id_expr} FROM {table_name} "
+                f"WHERE COALESCE(spec_version, 0) >= ?",
+                (int(min_spec_version),),
+            )
+        else:
+            cursor = conn.execute(f"SELECT DISTINCT {id_expr} FROM {table_name}")
+        return {str(row[0]) for row in cursor.fetchall() if row[0]}
     except Exception as e:
         logger.warning("Failed to query enriched message IDs from %s: %s", table_name, e)
         return set()
@@ -758,12 +804,18 @@ async def _find_unprocessed_messages(
         db_conn, source_def, dataset_id=dataset_id
     )
     if generic_messages:
+        from ..enrichment.catalog import catalog_spec_version
+
         enriched: Optional[set[str]] = None
         job_table_map = {job.get_job_name(): job.get_derived_table() for job in CANONICAL_JOBS}
         for job_name in jobs_to_check:
             table_name = job_table_map.get(job_name)
             if table_name:
-                job_ids = _get_enriched_message_ids(table_name, db_conn)
+                job_ids = _get_enriched_message_ids(
+                    table_name,
+                    db_conn,
+                    min_spec_version=catalog_spec_version(job_name),
+                )
                 enriched = job_ids if enriched is None else (enriched & job_ids)
             else:
                 logger.warning("Unknown enrichment job: %s (skipping check)", job_name)
@@ -947,6 +999,8 @@ async def _find_unprocessed_messages(
     # row for it (intersection). The previous union behavior skipped messages
     # that had been enriched by any single job, leaving partial enrichment
     # permanently un-backfillable.
+    from ..enrichment.catalog import catalog_spec_version
+
     enriched_ids: Optional[set[str]] = None
     # Create a mapping from job name to table name using the job registry
     job_to_table = {job.get_job_name(): job.get_derived_table() for job in CANONICAL_JOBS}
@@ -954,7 +1008,11 @@ async def _find_unprocessed_messages(
     for job_name in jobs_to_check:
         table_name = job_to_table.get(job_name)
         if table_name:
-            job_ids = _get_enriched_message_ids(table_name, db_conn)
+            job_ids = _get_enriched_message_ids(
+                table_name,
+                db_conn,
+                min_spec_version=catalog_spec_version(job_name),
+            )
             enriched_ids = job_ids if enriched_ids is None else (enriched_ids & job_ids)
         else:
             logger.warning("Unknown enrichment job: %s (skipping check)", job_name)

{topos_node-1.3.3 → topos_node-1.3.5}/topos/api/ingestion_api.py

@@ -21,13 +21,27 @@ async def report_progress() -> dict:
 
 @router.post("/ingestion/reprocess", dependencies=[Depends(require_api_key)])
 async def ingestion_reprocess(payload: dict = Body(default_factory=dict)) -> dict:
+    """Re-run raw→canonical for a source (optional newest-N via ``limit``).
+
+    Body:
+      - source_id (required)
+      - dataset_id (required)
+      - from_stage: ``raw`` (default) | ``canonical``
+      - limit: optional int — newest N raw rows only
+      - run_enrichment: optional bool (default true)
+      - force / sync_batch_id: optional
+    """
     try:
+        limit_raw = payload.get("limit")
+        limit = int(limit_raw) if limit_raw is not None and str(limit_raw).strip() != "" else None
         return await reprocess_source(
             source_id=str(payload["source_id"]),
             dataset_id=str(payload["dataset_id"]),
             from_stage=payload.get("from_stage") or "raw",
             sync_batch_id=payload.get("sync_batch_id"),
             force=bool(payload.get("force", False)),
+            limit=limit,
+            run_enrichment=bool(payload.get("run_enrichment", True)),
         )
     except KeyError as exc:
         raise HTTPException(status_code=400, detail=f"Missing field: {exc}") from exc