topos-node 1.3.5__tar.gz → 1.3.6__tar.gz

{topos_node-1.3.5/topos_node.egg-info → topos_node-1.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.5
+Version: 1.3.6
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai

{topos_node-1.3.5 → topos_node-1.3.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.5"
+version = "1.3.6"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"

topos_node-1.3.6/tests/test_prewarm_first_run.py

@@ -0,0 +1,61 @@
+"""first_run decides whether the UI says "downloading" or "preparing".
+
+Getting it wrong is a lie in the copy: a node that reports first_run=False shows
+"preparing its language models… should only take a moment" while it fetches 2.9GB.
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from topos.sanitization import prewarm
+
+
+@pytest.fixture
+def cache_root(tmp_path, monkeypatch):
+    monkeypatch.setattr(prewarm, "_hf_cache_root", lambda: str(tmp_path))
+    return tmp_path
+
+
+def _write_blobs(cache_root, repo_id: str, names: list[str]) -> None:
+    blobs = os.path.join(prewarm._repo_dir(str(cache_root), repo_id), "blobs")
+    os.makedirs(blobs, exist_ok=True)
+    for name in names:
+        open(os.path.join(blobs, name), "w").close()
+
+
+REPO = prewarm.PRIVACY_FILTER_REPO
+
+
+def test_absent_repo_is_not_present(cache_root):
+    assert prewarm._repo_present(REPO) is False
+
+
+def test_started_but_empty_is_not_present(cache_root):
+    """The first instant of a first download — the directory exists, nothing is in it."""
+    _write_blobs(cache_root, REPO, [])
+    assert prewarm._repo_present(REPO) is False
+
+
+def test_interrupted_download_is_not_present(cache_root):
+    """The case that made this a bug: a killed first run leaves .incomplete blobs
+    behind, and directory-existence alone would call that done."""
+    _write_blobs(cache_root, REPO, ["abc123.incomplete"])
+    assert prewarm._repo_present(REPO) is False
+
+
+def test_partially_resumed_download_is_not_present(cache_root):
+    _write_blobs(cache_root, REPO, ["abc123", "def456.incomplete"])
+    assert prewarm._repo_present(REPO) is False
+
+
+def test_complete_download_is_present(cache_root):
+    _write_blobs(cache_root, REPO, ["abc123", "def456"])
+    assert prewarm._repo_present(REPO) is True
+
+
+def test_no_cache_root_is_not_present(monkeypatch):
+    monkeypatch.setattr(prewarm, "_hf_cache_root", lambda: None)
+    assert prewarm._repo_present(REPO) is False

topos_node-1.3.6/tests/test_system_computer_name.py

@@ -0,0 +1,113 @@
+"""The machine name a Mac owner recognises, reported alongside the hostname.
+
+`platform.node()` returns the network hostname, which on macOS is either a
+mangled form of the computer name ("jonnys-macbook-pro.local") or something
+meaningless from DHCP ("mac.lan" on the machine this was written on). The
+frontend names a new self-hosted Topos after this, so a bad answer here shows up
+as a Topos called "mac.lan".
+"""
+
+from __future__ import annotations
+
+import subprocess
+
+import pytest
+
+from topos.core import state
+
+
+@pytest.fixture(autouse=True)
+def _clear_cache():
+    state._computer_name_cache = state._UNRESOLVED
+    yield
+    state._computer_name_cache = state._UNRESOLVED
+
+
+def _fake_run(stdout: str):
+    def run(_cmd, **_kwargs):
+        return subprocess.CompletedProcess(args=_cmd, returncode=0, stdout=stdout, stderr="")
+
+    return run
+
+
+def test_reports_the_computer_name_on_macos(monkeypatch):
+    monkeypatch.setattr(state.platform, "system", lambda: "Darwin")
+    monkeypatch.setattr(state.subprocess, "run", _fake_run("Jonny's MacBook Pro\n"))
+
+    assert state._macos_computer_name() == "Jonny's MacBook Pro"
+
+
+def test_absent_off_macos(monkeypatch):
+    monkeypatch.setattr(state.platform, "system", lambda: "Linux")
+
+    def explode(*_a, **_k):  # pragma: no cover - must never be reached
+        raise AssertionError("scutil is macOS-only and must not be spawned elsewhere")
+
+    monkeypatch.setattr(state.subprocess, "run", explode)
+    assert state._macos_computer_name() is None
+
+
+@pytest.mark.parametrize("stdout", ["", "   \n"])
+def test_blank_output_is_no_name(monkeypatch, stdout):
+    monkeypatch.setattr(state.platform, "system", lambda: "Darwin")
+    monkeypatch.setattr(state.subprocess, "run", _fake_run(stdout))
+
+    assert state._macos_computer_name() is None
+
+
+def test_failure_is_swallowed_and_never_retried(monkeypatch):
+    """A nicer label is never worth a startup failure — nor a stall on every request.
+
+    get_system_info runs inside async handlers, so an uncached spawn blocks the
+    event loop for the full timeout. On a Mac with no ComputerName that would be
+    a permanent per-request stall, so a miss is cached exactly like a hit and the
+    node falls back to the hostname for the rest of its life.
+    """
+    monkeypatch.setattr(state.platform, "system", lambda: "Darwin")
+    calls = {"n": 0}
+
+    def always_times_out(_cmd, **_kwargs):
+        calls["n"] += 1
+        raise subprocess.TimeoutExpired(cmd=_cmd, timeout=2.0)
+
+    monkeypatch.setattr(state.subprocess, "run", always_times_out)
+
+    assert state._macos_computer_name() is None
+    assert state._macos_computer_name() is None
+    assert state._macos_computer_name() is None
+    assert calls["n"] == 1
+
+
+def test_non_macos_never_spawns_twice(monkeypatch):
+    monkeypatch.setattr(state.platform, "system", lambda: "Linux")
+    assert state._macos_computer_name() is None
+    # Cached, so a Linux node does not re-check platform on every request either.
+    assert state._computer_name_cache is None
+
+
+def test_spawns_once_per_process(monkeypatch):
+    """get_device_info calls get_system_info twice, and the CP hits it on every connect."""
+    monkeypatch.setattr(state.platform, "system", lambda: "Darwin")
+    calls = {"n": 0}
+
+    def counting(_cmd, **_kwargs):
+        calls["n"] += 1
+        return subprocess.CompletedProcess(args=_cmd, returncode=0, stdout="q4\n", stderr="")
+
+    monkeypatch.setattr(state.subprocess, "run", counting)
+
+    for _ in range(5):
+        state._macos_computer_name()
+
+    assert calls["n"] == 1
+
+
+def test_system_info_carries_computer_name_beside_hostname(monkeypatch):
+    """The dict is free-form and the control plane forwards it verbatim, so this
+    key is the entire wire contract — it lands as system_info.computer_name."""
+    monkeypatch.setattr(state, "_macos_computer_name", lambda: "Jonny's MacBook Pro")
+
+    info = state.get_system_info()
+
+    assert info["computer_name"] == "Jonny's MacBook Pro"
+    assert "hostname" in info

{topos_node-1.3.5 → topos_node-1.3.6}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.5"
-__version_info__ = (1, 3, 5)
+__version__ = "1.3.6"
+__version_info__ = (1, 3, 6)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.3.5 → topos_node-1.3.6}/topos/api/shell.py

@@ -44,6 +44,8 @@ def _client_is_local(request: Request) -> bool:
 
 @router.get("/status")
 async def shell_status() -> dict:
+    from ..sanitization.prewarm import prewarm_status
+
     info = get_update_info()
     log_file = get_log_file_path()
     return {
@@ -59,6 +61,11 @@ async def shell_status() -> dict:
             "applying": _apply_state["applying"],
             "last_result": _apply_state["last_result"],
         },
+        # Additive: existing shells decode a fixed set of keys and ignore this.
+        # The tray cannot show it until the Swift ShellStatus struct adds the
+        # field (its CodingKeys are explicit), which needs its own signed build —
+        # shipping the data first means that release is a display change only.
+        "models": prewarm_status(),
     }
 
 

{topos_node-1.3.5 → topos_node-1.3.6}/topos/api/upgrades.py

@@ -19,11 +19,17 @@ router = APIRouter()
 def _status_payload() -> Dict[str, Any]:
     from ..core.state import get_db_connection
     from ..features.entities.graph_refresh import status as refresh_status
+    from ..sanitization.prewarm import prewarm_status
     from ..upgrades.runner import runner_status
 
     conn = get_db_connection()
     upgrade = runner_status(conn) if conn is not None else {"enabled": False, "error": "no database"}
-    return {"upgrade": upgrade, "graph_refresh": refresh_status()}
+    # `models` rides this existing read because the control plane already
+    # proxies it verbatim (engine_config.py returns `result.get("payload")`),
+    # `get_upgrade_status` is already allowlisted, and it is in the node's
+    # fast-inbound set — so it answers in milliseconds even while the prewarm
+    # thread is saturated downloading. No control-plane change, no CP deploy.
+    return {"upgrade": upgrade, "graph_refresh": refresh_status(), "models": prewarm_status()}
 
 
 @router.get("/v1/upgrade/status", dependencies=[Depends(require_api_key)])

{topos_node-1.3.5 → topos_node-1.3.6}/topos/core/state.py

@@ -5,6 +5,7 @@ import asyncio
 import os
 import platform
 import shutil
+import subprocess
 import sqlite3
 import sys
 import threading
@@ -719,8 +720,10 @@ def _migrate_legacy_browser_plugin_app_ids(conn: sqlite3.Connection) -> int:
             f"UPDATE {UMA_ACCESS_REQUESTS_TABLE} SET app_id = ? WHERE app_id = ?",
             (BROWSER_HISTORY_PLUGIN_APP_ID, LEGACY_BROWSER_PLUGIN_APP_ID),
         )
+        # Commit even for 0 rows: the UPDATE opened an implicit transaction
+        # either way, and this runs at startup on the owner connection.
+        conn.commit()
         if cursor.rowcount:
-            conn.commit()
             logger.info(
                 "Migrated %s uma_access_requests rows: %s → %s",
                 cursor.rowcount,
@@ -1082,6 +1085,53 @@ def get_total_memory_bytes() -> Optional[int]:
     return None
 
 
+# Resolved at most once per process.
+#
+# get_device_info calls get_system_info twice per request (services/local.py,
+# for engine_name and for the system dict), the control plane hits it on every
+# websocket connect, and these are async handlers — an uncached subprocess.run
+# blocks the event loop for its full timeout every time. Failures are cached
+# too, deliberately: a Mac with ComputerName unset, or a slow configd, would
+# otherwise stall EVERY request by up to the timeout, forever. One stall is a
+# blip; a permanent per-request stall is an outage. The cost of caching a miss
+# is a Topos named after the hostname instead, which is exactly the fallback
+# that already exists.
+_UNRESOLVED = object()
+_computer_name_cache: Any = _UNRESOLVED
+
+
+def _macos_computer_name() -> Optional[str]:
+    """The name a Mac owner actually recognises — "Jonny's MacBook Pro".
+
+    `platform.node()` returns the network hostname, which on macOS is a mangled
+    form of the same thing ("jonnys-macbook-pro.local") or, on many networks,
+    something meaningless from DHCP. A Topos named after it reads like a server.
+
+    Read via scutil rather than a new dependency: pyobjc/SystemConfiguration
+    would be a wheel added to every install for one string. Absolute path,
+    bounded timeout, and every failure swallowed — a nicer label is never worth
+    risking node startup. (Measured at 8ms, but a 1s bound proved tight enough to
+    trip under load, and the first boot is the one call that has to answer.)
+    """
+    global _computer_name_cache
+    if _computer_name_cache is not _UNRESOLVED:
+        return _computer_name_cache
+    if platform.system() != "Darwin":
+        _computer_name_cache = None
+        return None
+    try:
+        out = subprocess.run(
+            ["/usr/sbin/scutil", "--get", "ComputerName"],
+            capture_output=True,
+            text=True,
+            timeout=2.0,
+        )
+        _computer_name_cache = (out.stdout or "").strip() or None
+    except Exception:  # noqa: BLE001
+        _computer_name_cache = None
+    return _computer_name_cache
+
+
 def get_system_info() -> Dict[str, Any]:
     disk_total = None
     disk_free = None
@@ -1097,6 +1147,10 @@ def get_system_info() -> Dict[str, Any]:
         disk_free = None
     return {
         "hostname": platform.node() or None,
+        # Free-form dict (api_models.py types `system` as Dict[str, Any]) and the
+        # control plane forwards it verbatim, so this reaches fleet status as
+        # system_info.computer_name with no CP change at all.
+        "computer_name": _macos_computer_name(),
         "os": platform.system() or None,
         "os_version": platform.version() or None,
         "platform": platform.platform() or None,

{topos_node-1.3.5 → topos_node-1.3.6}/topos/disclosure/privacy_layer.py

@@ -349,8 +349,9 @@ async def run_privacy_disclosure_layer(
                 ):
                     nsfw_tagged += 1
 
-    if updated or nsfw_tagged:
-        conn.commit()
+    # Commit unconditionally: 0-row updates still opened an implicit
+    # transaction, which must not outlive this call.
+    conn.commit()
 
     duration_ms = int((time.perf_counter() - started) * 1000)
     logger.debug(

{topos_node-1.3.5 → topos_node-1.3.6}/topos/enrichment/derivation_recovery.py

@@ -172,8 +172,11 @@ def clear_derivation_retry(
                 (key,),
             )
             changed = int(cur.rowcount or 0)
-            if changed:
-                commit_connection(conn)
+            # Commit even when 0 rows matched: the UPDATE opened an implicit
+            # transaction regardless, and leaving it open makes the next
+            # BEGIN IMMEDIATE on this connection fail — which broke every
+            # topic_clusters batch (this runs after each successful job).
+            commit_connection(conn)
         if changed:
             logger.info(
                 "[DERIVE:RETRY] cleared derivation debt job=%s batch=%s",

{topos_node-1.3.5 → topos_node-1.3.6}/topos/enrichment/jobs/canonical/pii_redaction_job.py

@@ -123,6 +123,7 @@ class PiiRedactionJob(BaseEnrichmentJob):
             if progress_callback:
                 progress_callback(idx + 1, total)
 
-        if updated:
-            conn.commit()
+        # Commit unconditionally: a no-change pass may still have opened an
+        # implicit transaction, which must not outlive this job.
+        conn.commit()
         return [{"records_updated": updated}] if updated else []

{topos_node-1.3.5 → topos_node-1.3.6}/topos/enrichment/jobs/canonical/topic_clusters_job.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import logging
 import os
 from typing import Any, Callable, Dict, List, Optional
@@ -138,6 +139,21 @@ class TopicClusterJob(BaseEnrichmentJob):
         self,
         canonical_messages: List[Dict[str, Any]],
         progress_callback: Optional[Callable[[int, int], None]] = None,
+    ) -> List[Dict[str, Any]]:
+        # The clustering path blocks for seconds at a time (k-means, the
+        # label-pool wait, persist under the write gate); on the event loop
+        # that starves every coroutine including the control-plane keepalive.
+        # get_db_connection is thread-local, so the worker thread gets its own
+        # handle — passing the loop thread's connection across would reinstate
+        # the cross-thread sharing behind the 2026-07-30 corruption.
+        return await asyncio.to_thread(
+            self._enrich_sync, canonical_messages, progress_callback
+        )
+
+    def _enrich_sync(
+        self,
+        canonical_messages: List[Dict[str, Any]],
+        progress_callback: Optional[Callable[[int, int], None]] = None,
     ) -> List[Dict[str, Any]]:
         conn = get_db_connection()
         if conn is None:

{topos_node-1.3.5 → topos_node-1.3.6}/topos/features/signal/topic_clustering.py

@@ -1250,11 +1250,11 @@ def recompute_topic_clusters(
     )
 
     _strip_member_vectors(clusters)
-    from ...storage.db.write_gate import sqlite_retry_busy, with_db_write
+    from ...storage.db.write_gate import begin_immediate, sqlite_retry_busy, with_db_write
 
     with with_db_write():
         try:
-            sqlite_retry_busy(lambda: conn.execute("BEGIN IMMEDIATE"))
+            begin_immediate(conn)
             _clear_all_topic_clusters(conn)
             persist_result = persist_topic_clusters(
                 conn,

{topos_node-1.3.5 → topos_node-1.3.6}/topos/pipeline/job_runner.py

@@ -11,6 +11,7 @@ import uuid
 from typing import Any, Awaitable, Callable, Dict, Optional
 
 from .job_store import (
+    claim_matching_queued_jobs,
     claim_next_job,
     complete_job,
     fail_job,
@@ -144,17 +145,23 @@ async def _execute_topic_consolidation(payload: Dict[str, Any]) -> Dict[str, Any
         # what this executor exists to avoid.
         return {"status": "error", "error": "derivation in flight; will retry"}
 
-    conn = get_db_connection()
-    if conn is None:
-        return {"status": "error", "error": "no database connection"}
+    def _run() -> Dict[str, Any]:
+        # Fetched INSIDE the worker thread: get_db_connection is thread-local,
+        # and handing the loop thread's connection across threads is the
+        # sharing that caused the 2026-07-30 transaction corruption.
+        own = get_db_connection()
+        if own is None:
+            return {"status": "error", "error": "no database connection"}
+        return recompute_topic_clusters(
+            own,
+            source_ids=list(_resolved_topic_cluster_source_ids()),
+            sync_batch_id=None,
+            min_records=3,
+        )
 
-    result = await asyncio.to_thread(
-        recompute_topic_clusters,
-        conn,
-        source_ids=list(_resolved_topic_cluster_source_ids()),
-        sync_batch_id=None,
-        min_records=3,
-    )
+    result = await asyncio.to_thread(_run)
+    if str(result.get("status") or "") == "error":
+        return result
     logger.info("topic consolidation complete: %s", result)
     return {"status": "ok", "result": result}
 
@@ -167,13 +174,43 @@ EXECUTORS: Dict[str, ExecutorFn] = {
 }
 
 
+#: Upper bound on inbox jobs merged into one derive batch. A CP backlog after
+#: downtime arrives as one job per delivery (usually one record each); merging
+#: them runs the per-batch jobs (dimension briefs, topic work) once instead of
+#: once per delivery. Leftovers beyond the cap stay queued for the next cycle.
+_COALESCE_MAX_JOBS = 100
+
+
+def _coalesce_inbox_jobs(
+    conn, job: Dict[str, Any], payload: Dict[str, Any]
+) -> tuple[list[Dict[str, Any]], Dict[str, Any]]:
+    """Merge queued inbox jobs for the same source into this job's batch."""
+    siblings = claim_matching_queued_jobs(
+        conn,
+        lease_owner=_lease_owner,
+        kind=str(job["kind"]),
+        source_id=job.get("source_id"),
+        limit=_COALESCE_MAX_JOBS,
+    )
+    if not siblings:
+        return [job], payload
+    merged_records = list(payload.get("canonical_records") or [])
+    for sibling in siblings:
+        merged_records.extend((sibling.get("payload") or {}).get("canonical_records") or [])
+    merged_payload = {**payload, "canonical_records": merged_records}
+    logger.info(
+        "coalesced %d inbox jobs into one derive batch: source=%s records=%d",
+        len(siblings) + 1,
+        job.get("source_id"),
+        len(merged_records),
+    )
+    return [job, *siblings], merged_payload
+
+
 async def process_job(conn, job: Dict[str, Any]) -> None:
-    """Run one claimed job to completion."""
+    """Run one claimed job (plus any coalesced siblings) to completion."""
     job_id = str(job["job_id"])
     kind = str(job["kind"])
-    write_id = job.get("write_id")
-    source_id = job.get("source_id")
-    sync_batch_id = job.get("sync_batch_id")
 
     executor = EXECUTORS.get(kind)
     if executor is None:
@@ -184,38 +221,46 @@ async def process_job(conn, job: Dict[str, Any]) -> None:
         update_job_progress(conn, job_id, progress)
 
     payload = dict(job.get("payload") or {})
+    jobs = [job]
+    if kind == "inbox_deferred_enrichment":
+        jobs, payload = _coalesce_inbox_jobs(conn, job, payload)
     if kind == "enrichment_process_source":
         payload["_progress_updater"] = _progress_updater
 
     try:
         result = await executor(payload)
         if str(result.get("status") or "ok") == "error":
-            fail_job(conn, job_id, error=str(result.get("error") or result.get("message") or "failed"))
-            update_job_progress(conn, job_id, {"status": "failed", "result": result})
+            error = str(result.get("error") or result.get("message") or "failed")
+            for entry in jobs:
+                fail_job(conn, str(entry["job_id"]), error=error)
+                update_job_progress(conn, str(entry["job_id"]), {"status": "failed", "result": result})
             return
-        complete_job(conn, job_id, detail=result)
-        update_job_progress(
-            conn,
-            job_id,
-            {
-                "status": "completed",
-                "messages_processed": result.get("messages_processed", 0),
-                "records_created": result.get("records_created", {}),
-                "errors": result.get("errors", []),
-            },
-        )
-        if kind == "inbox_deferred_enrichment" and write_id:
-            record_derivation_completion(
+        for entry in jobs:
+            entry_id = str(entry["job_id"])
+            complete_job(conn, entry_id, detail=result)
+            update_job_progress(
                 conn,
-                write_id=str(write_id),
-                job_id=job_id,
-                source_id=source_id,
-                sync_batch_id=sync_batch_id,
+                entry_id,
+                {
+                    "status": "completed",
+                    "messages_processed": result.get("messages_processed", 0),
+                    "records_created": result.get("records_created", {}),
+                    "errors": result.get("errors", []),
+                },
             )
+            if kind == "inbox_deferred_enrichment" and entry.get("write_id"):
+                record_derivation_completion(
+                    conn,
+                    write_id=str(entry["write_id"]),
+                    job_id=entry_id,
+                    source_id=entry.get("source_id"),
+                    sync_batch_id=entry.get("sync_batch_id"),
+                )
     except Exception as exc:  # noqa: BLE001
         logger.warning("pipeline job failed job_id=%s kind=%s: %s", job_id, kind, exc, exc_info=exc)
-        fail_job(conn, job_id, error=str(exc))
-        update_job_progress(conn, job_id, {"status": "failed", "error": str(exc)})
+        for entry in jobs:
+            fail_job(conn, str(entry["job_id"]), error=str(exc))
+            update_job_progress(conn, str(entry["job_id"]), {"status": "failed", "error": str(exc)})
 
 
 #: Idle poll interval. Every tick claims against SQLite, so this is also how

{topos_node-1.3.5 → topos_node-1.3.6}/topos/pipeline/job_store.py

@@ -8,7 +8,7 @@ import uuid
 from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional
 
-from ..storage.db.write_gate import commit_connection, sqlite_retry_busy, with_db_write
+from ..storage.db.write_gate import begin_immediate, commit_connection, sqlite_retry_busy, with_db_write
 
 JOB_STATUSES = frozenset({"queued", "running", "done", "failed"})
 DEFAULT_LEASE_SECONDS = 300
@@ -145,7 +145,7 @@ def claim_next_job(
     expires = (datetime.now(timezone.utc) + timedelta(seconds=lease_seconds)).isoformat()
 
     with with_db_write():
-        sqlite_retry_busy(lambda: conn.execute("BEGIN IMMEDIATE"))
+        begin_immediate(conn)
         try:
             query = "SELECT job_id FROM pipeline_jobs WHERE status='queued'"
             params: list[Any] = []
@@ -179,6 +179,58 @@ def claim_next_job(
     return get_job(conn, job_id)
 
 
+def claim_matching_queued_jobs(
+    conn: sqlite3.Connection,
+    *,
+    lease_owner: str,
+    kind: str,
+    source_id: Optional[str],
+    lease_seconds: int = DEFAULT_LEASE_SECONDS,
+    limit: int = 100,
+) -> List[Dict[str, Any]]:
+    """Claim every queued job of ``kind`` for ``source_id``, oldest first.
+
+    Lets the worker coalesce a backlog — e.g. 40 one-record inbox deliveries —
+    into a single derive batch so the per-batch jobs (dimension briefs, topic
+    work) run once instead of once per delivery.
+    """
+    ensure_pipeline_jobs_schema(conn)
+    expires = (datetime.now(timezone.utc) + timedelta(seconds=lease_seconds)).isoformat()
+    claimed_ids: List[str] = []
+
+    with with_db_write():
+        begin_immediate(conn)
+        try:
+            query = "SELECT job_id FROM pipeline_jobs WHERE status='queued' AND kind=?"
+            params: list[Any] = [kind]
+            if source_id is None:
+                query += " AND source_id IS NULL"
+            else:
+                query += " AND source_id=?"
+                params.append(source_id)
+            query += " ORDER BY created_at ASC LIMIT ?"
+            params.append(int(limit))
+            rows = conn.execute(query, tuple(params)).fetchall()
+            for row in rows:
+                updated = conn.execute(
+                    """
+                    UPDATE pipeline_jobs
+                    SET status='running', lease_owner=?, lease_expires_at=?,
+                        started_at=COALESCE(started_at, datetime('now')),
+                        updated_at=datetime('now')
+                    WHERE job_id=? AND status='queued'
+                    """,
+                    (lease_owner, expires, str(row[0])),
+                )
+                if int(updated.rowcount or 0):
+                    claimed_ids.append(str(row[0]))
+            sqlite_retry_busy(conn.commit)
+        except Exception:
+            conn.execute("ROLLBACK")
+            raise
+    return [job for job in (get_job(conn, jid) for jid in claimed_ids) if job is not None]
+
+
 def complete_job(
     conn: sqlite3.Connection,
     job_id: str,