topos-node 1.3.38__tar.gz → 1.3.40__tar.gz

{topos_node-1.3.38/topos_node.egg-info → topos_node-1.3.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.38
+Version: 1.3.40
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai
@@ -163,13 +163,13 @@ Optional:
 ### 3) Run
 
 ```bash
-topos-node --host 0.0.0.0 --port 9000
+topos-node --host 0.0.0.0 --port 8676
 ```
 
 ### 4) Verify
 
 ```bash
-curl http://localhost:9000/health
+curl http://localhost:8676/health
 ```
 
 ## Common Commands

{topos_node-1.3.38 → topos_node-1.3.40}/README.md

@@ -96,13 +96,13 @@ Optional:
 ### 3) Run
 
 ```bash
-topos-node --host 0.0.0.0 --port 9000
+topos-node --host 0.0.0.0 --port 8676
 ```
 
 ### 4) Verify
 
 ```bash
-curl http://localhost:9000/health
+curl http://localhost:8676/health
 ```
 
 ## Common Commands

{topos_node-1.3.38 → topos_node-1.3.40}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.38"
+version = "1.3.40"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"
@@ -112,7 +112,7 @@ tmp_path_retention_count = 1
 tmp_path_retention_policy = "failed"
 # The DEFAULT lane is hermetic. Without this, a plain `pytest tests/` selects
 # every marker, and the three lanes below are the ones that read and WRITE the
-# developer's own ~/.topos/database.db or drive the node running on :9000 —
+# developer's own ~/.topos/database.db or drive the node running on :8676 —
 # `tests/conftest.py::_no_live_db_guard` waives its protection for exactly these
 # markers, which only makes sense if the ordinary gate never selects them.
 # On 2026-08-19 it did: one `pytest tests/ -q` run inserted 71 rows into the
@@ -137,7 +137,7 @@ markers = [
   "public: public-safe test lane for OSS CI",
   "private: internal architecture or fixtures; excluded from public CI",
   "e2e: end-to-end tests (live Keycloak + Control Plane; require config and running CP)",
-  "live: requires a running local engine (localhost:9000) and a real TOPOS_KEY; excluded from deterministic lanes",
+  "live: requires a running local engine (localhost:8676) and a real TOPOS_KEY; excluded from deterministic lanes",
   "release_pressure: iteration-4 live pressure smoke; run explicitly before releases",
   "p0: priority marker on release pressure tests",
   "retrieval_eval: dense-intelligence retrieval eval (loads local embedding model; run explicitly)",

{topos_node-1.3.38 → topos_node-1.3.40}/tests/test_active_database_binding.py

@@ -20,6 +20,7 @@ from pathlib import Path
 
 import pytest
 
+from topos.defaults import DEFAULT_NODE_PORT
 from topos.storage.db import paths
 
 pytestmark = [pytest.mark.public]
@@ -381,7 +382,7 @@ class TestSwitchPreflight:
     def _no_running_node(self, monkeypatch):
         from topos import profiles
 
-        monkeypatch.setattr(profiles, "node_is_running", lambda port=9000: False)
+        monkeypatch.setattr(profiles, "node_is_running", lambda port=DEFAULT_NODE_PORT: False)
 
     def test_refuses_a_topos_from_a_newer_engine(self, tmp_path):
         from topos import profiles

{topos_node-1.3.38 → topos_node-1.3.40}/tests/test_profile_hygiene.py

@@ -22,6 +22,7 @@ from pathlib import Path
 import pytest
 
 from topos import profiles
+from topos.defaults import DEFAULT_NODE_PORT
 from topos.storage.db.migrations import backup as backup_mod
 
 pytestmark = [pytest.mark.public]
@@ -70,7 +71,7 @@ def _seed_active(base: Path, *, key: str = "KEYAAA", real_db: bool = True) -> No
 
 @pytest.fixture(autouse=True)
 def _no_running_node(monkeypatch):
-    monkeypatch.setattr(profiles, "node_is_running", lambda port=9000: False)
+    monkeypatch.setattr(profiles, "node_is_running", lambda port=DEFAULT_NODE_PORT: False)
 
 
 class TestArchiveIsSelfContained:

{topos_node-1.3.38 → topos_node-1.3.40}/tests/test_profiles.py

@@ -17,6 +17,7 @@ from click.testing import CliRunner
 
 from topos import profiles
 from topos.cli.profile_cmd import profile_group
+from topos.defaults import DEFAULT_NODE_PORT
 
 pytestmark = [pytest.mark.public]
 
@@ -45,7 +46,7 @@ def _seed_profile(base: Path, slug: str, *, name: str | None = None, key: str =
 
 @pytest.fixture(autouse=True)
 def _no_running_node(monkeypatch):
-    monkeypatch.setattr(profiles, "node_is_running", lambda port=9000: False)
+    monkeypatch.setattr(profiles, "node_is_running", lambda port=DEFAULT_NODE_PORT: False)
 
 
 class TestNewProfile:
@@ -128,7 +129,7 @@ class TestSwitchProfile:
     def test_refuses_while_node_running(self, tmp_path, monkeypatch):
         _seed_active(tmp_path)
         _seed_profile(tmp_path, "work")
-        monkeypatch.setattr(profiles, "node_is_running", lambda port=9000: True)
+        monkeypatch.setattr(profiles, "node_is_running", lambda port=DEFAULT_NODE_PORT: True)
         with pytest.raises(profiles.ProfileError, match="running"):
             profiles.switch_profile("work", tmp_path)
         # Nothing moved.
@@ -328,7 +329,7 @@ class TestRemoveProfile:
     def test_does_not_require_the_node_to_be_stopped(self, tmp_path, monkeypatch):
         # An archived Topos is not the one a running engine has open, so unlike
         # new/switch this must work without asking anyone to quit Topos.
-        monkeypatch.setattr(profiles, "node_is_running", lambda port=9000: True)
+        monkeypatch.setattr(profiles, "node_is_running", lambda port=DEFAULT_NODE_PORT: True)
         _seed_profile(tmp_path, "work")
 
         profiles.remove_profile("work", tmp_path)

{topos_node-1.3.38 → topos_node-1.3.40}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.38"
-__version_info__ = (1, 3, 38)
+__version__ = "1.3.40"
+__version_info__ = (1, 3, 40)
 
 __all__ = ["__version__", "__version_info__"]

topos_node-1.3.40/topos/api/ingestion_api.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Body, Depends, HTTPException, Request
+
+from ..auth import require_api_key
+from ..ingestion.reprocess import reprocess_source
+from .ingestion_sources import ingest_source
+
+router = APIRouter()
+
+
+@router.post("/ingestion/start")
+async def start_ingestion() -> dict:
+    return {"status": "stub"}
+
+
+@router.post("/ingestion/progress")
+async def report_progress() -> dict:
+    return {"status": "stub"}
+
+
+@router.post("/ingestion/reprocess", dependencies=[Depends(require_api_key)])
+async def ingestion_reprocess(payload: dict = Body(default_factory=dict)) -> dict:
+    """Re-run raw→canonical for a source (optional newest-N via ``limit``).
+
+    Body:
+      - source_id (required)
+      - dataset_id (required)
+      - from_stage: ``raw`` (default) | ``canonical``
+      - limit: optional int — newest N raw rows only
+      - run_enrichment: optional bool (default true)
+      - force / sync_batch_id: optional
+    """
+    try:
+        limit_raw = payload.get("limit")
+        limit = int(limit_raw) if limit_raw is not None and str(limit_raw).strip() != "" else None
+        return await reprocess_source(
+            source_id=str(payload["source_id"]),
+            dataset_id=str(payload["dataset_id"]),
+            from_stage=payload.get("from_stage") or "raw",
+            sync_batch_id=payload.get("sync_batch_id"),
+            force=bool(payload.get("force", False)),
+            limit=limit,
+            run_enrichment=bool(payload.get("run_enrichment", True)),
+        )
+    except KeyError as exc:
+        raise HTTPException(status_code=400, detail=f"Missing field: {exc}") from exc
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+
+@router.get("/ingestion/audit", dependencies=[Depends(require_api_key)])
+async def get_ingestion_audit(sync_batch_id: str) -> dict:
+    from ..core.state import get_db_connection
+    from ..pipeline.audit import SQLiteIngestAuditStore
+
+    conn = get_db_connection()
+    if not conn:
+        raise HTTPException(status_code=503, detail="Database unavailable")
+    store = SQLiteIngestAuditStore(conn)
+    return {"sync_batch_id": sync_batch_id, "stages": store.query_by_batch(sync_batch_id)}
+
+
+@router.post("/ingestion/upload", dependencies=[Depends(require_api_key)])
+async def upload_ingestion_file(request: Request) -> dict:
+    return await ingest_source("chatgpt_file_ingestion", request)
+
+
+@router.get("/ingestion/local-exports", dependencies=[Depends(require_api_key)])
+async def list_local_exports() -> dict:
+    """Exports sitting on this machine, for the import screen to offer.
+
+    The node is the only party that can answer this: a browser cannot read a
+    filesystem, and cannot produce an absolute path even for a file the user
+    picks. So the node looks, and hands back opaque handles plus labels a person
+    can recognise -- never the paths themselves.
+
+    Bounded and stateless by construction (see ``local_exports``): a few known
+    folders, shallow, symlinks skipped, time-boxed, nothing written down.
+    """
+    import asyncio
+
+    from ..ingestion.local_exports import find_exports
+
+    result = await asyncio.to_thread(find_exports)
+    return result.as_payload()
+
+
+@router.get("/ingestion/local-exports/{handle}/describe", dependencies=[Depends(require_api_key)])
+async def describe_local_export(handle: str) -> dict:
+    """What is inside one discovered export: conversation count and date range.
+
+    Separate from discovery on purpose. Listing candidates reads only an
+    archive's central directory and stays in the low hundreds of milliseconds;
+    this opens the conversations file and parses it, which is about a second on
+    a 52 MB export. Paying that for every row of a scan to inform a choice about
+    one of them would be the wrong trade.
+
+    It exists because a window is measured against today while an export is
+    fixed in the past, and nothing in the UI could see the gap: a six-month
+    window against a fourteen-month-old export drops every conversation and
+    completes successfully with nothing imported.
+    """
+    import asyncio
+
+    from ..ingestion.local_exports import describe_export, resolve
+
+    path = await asyncio.to_thread(resolve, handle)
+    if path is None:
+        raise HTTPException(
+            status_code=404,
+            detail="That export is no longer where we found it. Scan again and pick it.",
+        )
+    try:
+        return await asyncio.to_thread(describe_export, path)
+    except Exception as exc:  # noqa: BLE001 — a describe failure must not block the import
+        raise HTTPException(status_code=422, detail=f"Could not read that export: {exc}") from exc
+
+
+@router.post("/ingestion/upload-local-path", dependencies=[Depends(require_api_key)])
+async def upload_ingestion_local_path(
+    request: Request, payload: dict = Body(default_factory=dict)
+) -> dict:
+    """Ingest a file that is already on this machine. No bytes cross a network.
+
+    Accepts either a ``handle`` from ``/ingestion/local-exports`` -- the path
+    never leaves the node -- or an explicit ``file_path``, which is still how
+    someone points at an export outside the searched folders.
+    """
+    from ..ingestion.local_exports import resolve
+
+    dataset_id = payload.get("dataset_id")
+    file_path = payload.get("file_path")
+    handle = str(payload.get("handle") or "").strip()
+
+    if handle and not file_path:
+        resolved = resolve(handle)
+        if resolved is None:
+            # Handles are re-found rather than remembered, so a stale one means
+            # the file moved or went away -- not that the user did anything odd.
+            raise HTTPException(
+                status_code=404,
+                detail="That export is no longer where we found it. Scan again and pick it.",
+            )
+        file_path = str(resolved)
+
+    if not file_path:
+        raise HTTPException(status_code=400, detail="Provide a handle or a file_path.")
+
+    return await ingest_source(
+        "chatgpt_file_ingestion",
+        request,
+        dataset_id=dataset_id,
+        file_path=file_path,
+    )

{topos_node-1.3.38 → topos_node-1.3.40}/topos/cli/commands.py

@@ -12,6 +12,7 @@ import uvicorn
 # Add parent directory to path for imports
 sys.path.insert(0, str(Path(__file__).parent.parent.parent))
 
+from topos.defaults import DEFAULT_NODE_PORT
 from topos.runtime_update import (
     DEFAULT_PACKAGE_NAME,
     apply_package_update,
@@ -232,8 +233,8 @@ def _maybe_offer_self_update(
 )
 @click.option(
     "--port",
-    default=9000,
-    help="Server port (default: 9000)",
+    default=DEFAULT_NODE_PORT,
+    help=f"Server port (default: {DEFAULT_NODE_PORT})",
 )
 @click.option(
     "--host",

{topos_node-1.3.38 → topos_node-1.3.40}/topos/cli/profile_cmd.py

@@ -17,6 +17,7 @@ from typing import Optional
 import click
 
 from topos import profiles
+from topos.defaults import DEFAULT_NODE_PORT
 
 
 def _echo(payload: dict, as_json: bool, human: str) -> None:
@@ -41,7 +42,7 @@ _BASE_OPTION = click.option(
 )
 _JSON_OPTION = click.option("--json", "as_json", is_flag=True, help="Machine-readable output.")
 _PORT_OPTION = click.option(
-    "--port", default=9000, show_default=True, help="Port probed for a running node."
+    "--port", default=DEFAULT_NODE_PORT, show_default=True, help="Port probed for a running node."
 )
 
 

{topos_node-1.3.38 → topos_node-1.3.40}/topos/core/handlers/__init__.py

@@ -170,6 +170,7 @@ from . import (  # noqa: F401  (imported for handler registration side effects)
     complexity,
     timeline_daily,
     graph_query,
+    graph_summary,
 )
 
 

topos_node-1.3.40/topos/core/handlers/graph_summary.py

@@ -0,0 +1,72 @@
+"""How much is in the graph — the smallest possible answer.
+
+Why this exists rather than a count taken from ``/v1/signal/graph``: that
+endpoint returns node and edge LISTS, capped at 200 and 500, and no totals. To
+learn "how many" from it you would fetch rows you throw away, and still not
+know the answer once the graph passed the cap.
+
+Why not carry the number on the ingestion job instead: the progress channel is
+a fixed set of columns on the control plane's ``ingestion_jobs`` table, with no
+JSON field, and ``stage_event`` is filtered down to four keys. Adding it there
+means a schema migration on the production database — a real cost to pay for a
+reassurance line, and a change that is hard to take back.
+
+So: three COUNT(*) queries, read on demand, owned by nothing.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, Optional
+
+from .registry import handles
+
+
+def _counts() -> Dict[str, Any]:
+    from ...core.state import get_db_connection
+
+    conn = get_db_connection()
+    if conn is None:
+        return {"available": False}
+
+    out: Dict[str, Any] = {"available": True}
+    # Each table is counted independently and a failure on one is not a failure
+    # of the reply: a missing table on an older node should cost that number,
+    # not the whole answer.
+    for key, table in (
+        ("nodes", "graph_nodes"),
+        ("edges", "graph_edges"),
+        ("entities", "entities"),
+    ):
+        try:
+            row = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()  # noqa: S608 — fixed names
+            out[key] = int(row[0]) if row else 0
+        except Exception:  # noqa: BLE001
+            out[key] = None
+    try:
+        row = conn.execute(
+            "SELECT MAX(materialized_at) FROM graph_materialization_state"
+        ).fetchone()
+        out["materialized_at"] = row[0] if row else None
+    except Exception:  # noqa: BLE001
+        out["materialized_at"] = None
+    return out
+
+
+@handles("graph_summary")
+async def handle_graph_summary(message: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """Node, edge and entity counts, for a screen that wants to show growth.
+
+    The reason it is worth showing during an import: entity data lands at the
+    third of twenty stages and the graph is materialised twice, so these numbers
+    move long before the import finishes. Watching them move is the difference
+    between a long wait and a long wait you can see the point of.
+    """
+    req_id = message.get("id")
+    if not req_id:
+        return None
+    try:
+        payload = await asyncio.to_thread(_counts)
+        return {"id": req_id, "status": "ok", "payload": payload}
+    except Exception as exc:  # noqa: BLE001
+        return {"id": req_id, "status": "error", "error": str(exc)}

{topos_node-1.3.38 → topos_node-1.3.40}/topos/core/handlers/ingest.py

@@ -365,6 +365,22 @@ async def handle_start_ingestion(message: Dict[str, Any]) -> Optional[Dict[str,
     file_url = payload.get("file_url")
     file_base64 = payload.get("file_base64")
     file_path = payload.get("file_path")
+    local_handle = str(payload.get("local_export_handle") or "").strip()
+    if local_handle and not file_path:
+        # A handle from `list_local_exports`. The path was never sent to the
+        # browser, so it is resolved here — and only ever to something still
+        # inside a searched folder, which is what makes a handle safe to accept
+        # back over the relay.
+        from ...ingestion.local_exports import resolve as _resolve_local_export
+
+        resolved = _resolve_local_export(local_handle)
+        if resolved is None:
+            return {
+                "id": req_id,
+                "status": "error",
+                "error": "That export is no longer where we found it. Scan again and pick it.",
+            }
+        file_path = str(resolved)
     job_id = payload.get("job_id")
     source_id = payload.get("source_id")  # Optional source_id from control plane
     source_definition = payload.get("source_definition")
@@ -408,10 +424,13 @@ async def handle_start_ingestion(message: Dict[str, Any]) -> Optional[Dict[str,
                 "payload": {"job_id": job_id, "status": "completed", "deduplicated": True},
             }
 
+        # Import-time policy chosen in the UI (date window, inclusion flags).
+        ingest_options = payload.get("ingest_options")
         job_payload: Dict[str, Any] = {
             "dataset_id": dataset_id or "",
             "schema_id": schema_id,
             "file_format": file_format,
+            "ingest_options": ingest_options if isinstance(ingest_options, dict) else None,
             "job_id": job_id,
             "source_id": source_id,
             "source_definition": source_definition,
@@ -478,6 +497,62 @@ async def handle_ingestion_reprocess(message: Dict[str, Any]) -> Optional[Dict[s
     except Exception as exc:  # noqa: BLE001
         return {"id": req_id, "status": "error", "error": str(exc)}
 
+@handles("list_local_exports")
+async def handle_list_local_exports(message: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """Exports sitting on this machine, for the import screen to offer.
+
+    Only the node can answer this. A browser cannot read a filesystem, and
+    cannot produce an absolute path even for a file the user picks — so without
+    this the only way to reach the free import path was to type one out by hand.
+
+    What comes back is opaque handles and labels a person can recognise, never
+    paths: the reply travels to the browser over the relay, and a home-directory
+    path carries the account name and whatever else lives beside it.
+    """
+    req_id = message.get("id")
+    if not req_id:
+        return None
+    try:
+        import asyncio
+
+        from ...ingestion.local_exports import find_exports
+
+        result = await asyncio.to_thread(find_exports)
+        return {"id": req_id, "status": "ok", "payload": result.as_payload()}
+    except Exception as exc:  # noqa: BLE001
+        return {"id": req_id, "status": "error", "error": str(exc)}
+
+
+@handles("describe_local_export")
+async def handle_describe_local_export(message: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """Conversation count and date range for one discovered export.
+
+    The window a person picks is measured from today; the export is fixed in
+    the past. Nothing in the UI could see that gap, so a six-month window
+    against a fourteen-month-old file imported nothing and said it succeeded.
+    This is the number that closes it, read where the dates are.
+    """
+    req_id = message.get("id")
+    if not req_id:
+        return None
+    payload = message.get("payload") or {}
+    handle = str(payload.get("handle") or "").strip()
+    if not handle:
+        return {"id": req_id, "status": "error", "error": "handle required"}
+    try:
+        import asyncio
+
+        from ...ingestion.local_exports import describe_export, resolve
+
+        path = await asyncio.to_thread(resolve, handle)
+        if path is None:
+            return {"id": req_id, "status": "error", "error": "export no longer found"}
+        described = await asyncio.to_thread(describe_export, path)
+        return {"id": req_id, "status": "ok", "payload": described}
+    except Exception as exc:  # noqa: BLE001
+        return {"id": req_id, "status": "error", "error": str(exc)}
+
+
 @handles("get_ingestion_audit")
 async def handle_get_ingestion_audit(message: Dict[str, Any]) -> Optional[Dict[str, Any]]:
     req_id = message.get("id")

topos_node-1.3.40/topos/defaults.py

@@ -0,0 +1,9 @@
+"""Stable defaults for a local Topos node.
+
+8676 is TOPO on a phone keypad. Keep this the only Python source of the
+listen port so CLI, probes, and the local MCP proxy cannot drift.
+"""
+
+DEFAULT_NODE_PORT = 8676
+DEFAULT_NODE_HTTP_URL = f"http://127.0.0.1:{DEFAULT_NODE_PORT}"
+DEFAULT_NODE_LOCALHOST_URL = f"http://localhost:{DEFAULT_NODE_PORT}"

{topos_node-1.3.38 → topos_node-1.3.40}/topos/disclosure/privacy_layer.py

@@ -169,13 +169,36 @@ class PrivacyLayerClient:
         return {"status": "failed", "error": str(last_exc), "items": []}
 
 
+# Names for the two model passes below. They are reported to the caller's
+# progress callback and surface verbatim in the UI, so they are written for a
+# person watching an import rather than for a log grep.
+#
+# "Filtering private information" rather than "checking for private details":
+# this pass does not merely look. It detects PII spans and REDACTS them into
+# the disclosure copy, which is the whole point of the layer — the node is
+# preparing what it can safely show. A name that says "checking" describes an
+# inspection and undersells the thing the product actually does for you.
+PRIVACY_STAGE_REDACT = "filtering private information"
+PRIVACY_STAGE_NSFW = "checking for sensitive content"
+
+# How many records to send per call. NOT a throughput choice: both engine
+# backends (redact_privacy_batch, classify_nsfw_batch) loop over `items` one at
+# a time, so grouping 32 records buys no batched inference -- it is purely
+# transport. The only thing the group size decides is how often progress can
+# move. Measured on a live import: ~3.75s per record on CPU, so a group of 32
+# meant the bar stepped once every two minutes and read as frozen; at 4 it
+# moves about every fifteen seconds. The per-call overhead is a dict and a
+# thread hop against ~15s of model work.
+PRIVACY_PROGRESS_CHUNK = 4
+
+
 async def run_privacy_disclosure_layer(
     conn,
     canonical_messages: List[Dict[str, Any]],
     *,
     source_group: Optional[str] = None,
     client: Optional[PrivacyLayerClient] = None,
-    progress_callback: Optional[Callable[[int, int], None]] = None,
+    progress_callback: Optional[Callable[[int, int, str], None]] = None,
     nsfw_only: bool = False,
 ) -> Dict[str, Any]:
     """Mandatory post-canonical Platform Privacy Layer: PII disclosure + NSFW tags.
@@ -242,7 +265,7 @@ async def run_privacy_disclosure_layer(
             table = canonical_table_for_message(msg, source_group=source_group)
             if not table:
                 if progress_callback:
-                    progress_callback(idx + 1, total)
+                    progress_callback(idx + 1, total, PRIVACY_STAGE_REDACT)
                 continue
             record_id = (
                 msg.get("message_id")
@@ -252,7 +275,7 @@ async def run_privacy_disclosure_layer(
             )
             if not record_id:
                 if progress_callback:
-                    progress_callback(idx + 1, total)
+                    progress_callback(idx + 1, total, PRIVACY_STAGE_REDACT)
                 continue
             record_id = str(record_id)
             for field in fields_for_table(table):
@@ -284,10 +307,24 @@ async def run_privacy_disclosure_layer(
 
         from ..sanitization.privacy_filter import PRIVACY_DISCLOSE_MAX_BATCH
 
-        for i in range(0, len(flat_pending), PRIVACY_DISCLOSE_MAX_BATCH):
-            batch = flat_pending[i : i + PRIVACY_DISCLOSE_MAX_BATCH]
+        # Report against the batched work, not the message list: this loop is
+        # where the model time goes, and the old callback fired only on the
+        # branches that SKIP a message -- so a run with nothing to skip
+        # reported nothing at all, for its entire duration.
+        _redact_total = len(flat_pending)
+        _redact_done = 0
+        _redact_chunk = min(PRIVACY_DISCLOSE_MAX_BATCH, PRIVACY_PROGRESS_CHUNK)
+        for i in range(0, len(flat_pending), _redact_chunk):
+            batch = flat_pending[i : i + _redact_chunk]
             items = [{"id": e["batch_key"], "text": e["raw"]} for e in batch]
+            if progress_callback:
+                progress_callback(_redact_done, _redact_total, PRIVACY_STAGE_REDACT)
             result = await privacy_client.redact_batch(items)
+            _redact_done += len(batch)
+            # Also after: without this the final group never reports, and the
+            # stage appears to stop one group short of its own total.
+            if progress_callback:
+                progress_callback(_redact_done, _redact_total, PRIVACY_STAGE_REDACT)
             if result.get("status") in ("unavailable", "failed"):
                 failed_batches += 1
                 logger.warning(
@@ -349,10 +386,18 @@ async def run_privacy_disclosure_layer(
         from ..sanitization.nsfw_classifier import NSFW_CLASSIFY_MAX_BATCH
 
         nsfw_items = list(nsfw_pending.items())
-        for i in range(0, len(nsfw_items), NSFW_CLASSIFY_MAX_BATCH):
-            batch = nsfw_items[i : i + NSFW_CLASSIFY_MAX_BATCH]
+        _nsfw_total = len(nsfw_items)
+        _nsfw_done = 0
+        _nsfw_chunk = min(NSFW_CLASSIFY_MAX_BATCH, PRIVACY_PROGRESS_CHUNK)
+        for i in range(0, len(nsfw_items), _nsfw_chunk):
+            batch = nsfw_items[i : i + _nsfw_chunk]
             items = [{"id": key, "text": entry["text"]} for key, entry in batch]
+            if progress_callback:
+                progress_callback(_nsfw_done, _nsfw_total, PRIVACY_STAGE_NSFW)
             nsfw_result = await privacy_client.classify_nsfw_batch(items)
+            _nsfw_done += len(batch)
+            if progress_callback:
+                progress_callback(_nsfw_done, _nsfw_total, PRIVACY_STAGE_NSFW)
             if nsfw_result.get("status") in ("failed",):
                 nsfw_failed_batches += 1
                 continue