topos-node 1.3.38__tar.gz → 1.3.39__tar.gz

{topos_node-1.3.38/topos_node.egg-info → topos_node-1.3.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: topos-node
-Version: 1.3.38
+Version: 1.3.39
 Summary: Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://topos.dialogues.ai
@@ -163,13 +163,13 @@ Optional:
 ### 3) Run
 
 ```bash
-topos-node --host 0.0.0.0 --port 9000
+topos-node --host 0.0.0.0 --port 8676
 ```
 
 ### 4) Verify
 
 ```bash
-curl http://localhost:9000/health
+curl http://localhost:8676/health
 ```
 
 ## Common Commands

{topos_node-1.3.38 → topos_node-1.3.39}/README.md

@@ -96,13 +96,13 @@ Optional:
 ### 3) Run
 
 ```bash
-topos-node --host 0.0.0.0 --port 9000
+topos-node --host 0.0.0.0 --port 8676
 ```
 
 ### 4) Verify
 
 ```bash
-curl http://localhost:9000/health
+curl http://localhost:8676/health
 ```
 
 ## Common Commands

{topos_node-1.3.38 → topos_node-1.3.39}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "topos-node"
-version = "1.3.38"
+version = "1.3.39"
 description = "Topos personal AI engine (FastAPI): local data, sync, control plane WebSocket client"
 readme = "README.md"
 license = "Apache-2.0"
@@ -112,7 +112,7 @@ tmp_path_retention_count = 1
 tmp_path_retention_policy = "failed"
 # The DEFAULT lane is hermetic. Without this, a plain `pytest tests/` selects
 # every marker, and the three lanes below are the ones that read and WRITE the
-# developer's own ~/.topos/database.db or drive the node running on :9000 —
+# developer's own ~/.topos/database.db or drive the node running on :8676 —
 # `tests/conftest.py::_no_live_db_guard` waives its protection for exactly these
 # markers, which only makes sense if the ordinary gate never selects them.
 # On 2026-08-19 it did: one `pytest tests/ -q` run inserted 71 rows into the
@@ -137,7 +137,7 @@ markers = [
   "public: public-safe test lane for OSS CI",
   "private: internal architecture or fixtures; excluded from public CI",
   "e2e: end-to-end tests (live Keycloak + Control Plane; require config and running CP)",
-  "live: requires a running local engine (localhost:9000) and a real TOPOS_KEY; excluded from deterministic lanes",
+  "live: requires a running local engine (localhost:8676) and a real TOPOS_KEY; excluded from deterministic lanes",
   "release_pressure: iteration-4 live pressure smoke; run explicitly before releases",
   "p0: priority marker on release pressure tests",
   "retrieval_eval: dense-intelligence retrieval eval (loads local embedding model; run explicitly)",

{topos_node-1.3.38 → topos_node-1.3.39}/tests/test_active_database_binding.py

@@ -20,6 +20,7 @@ from pathlib import Path
 
 import pytest
 
+from topos.defaults import DEFAULT_NODE_PORT
 from topos.storage.db import paths
 
 pytestmark = [pytest.mark.public]
@@ -381,7 +382,7 @@ class TestSwitchPreflight:
     def _no_running_node(self, monkeypatch):
         from topos import profiles
 
-        monkeypatch.setattr(profiles, "node_is_running", lambda port=9000: False)
+        monkeypatch.setattr(profiles, "node_is_running", lambda port=DEFAULT_NODE_PORT: False)
 
     def test_refuses_a_topos_from_a_newer_engine(self, tmp_path):
         from topos import profiles

{topos_node-1.3.38 → topos_node-1.3.39}/tests/test_profile_hygiene.py

@@ -22,6 +22,7 @@ from pathlib import Path
 import pytest
 
 from topos import profiles
+from topos.defaults import DEFAULT_NODE_PORT
 from topos.storage.db.migrations import backup as backup_mod
 
 pytestmark = [pytest.mark.public]
@@ -70,7 +71,7 @@ def _seed_active(base: Path, *, key: str = "KEYAAA", real_db: bool = True) -> No
 
 @pytest.fixture(autouse=True)
 def _no_running_node(monkeypatch):
-    monkeypatch.setattr(profiles, "node_is_running", lambda port=9000: False)
+    monkeypatch.setattr(profiles, "node_is_running", lambda port=DEFAULT_NODE_PORT: False)
 
 
 class TestArchiveIsSelfContained:

{topos_node-1.3.38 → topos_node-1.3.39}/tests/test_profiles.py

@@ -17,6 +17,7 @@ from click.testing import CliRunner
 
 from topos import profiles
 from topos.cli.profile_cmd import profile_group
+from topos.defaults import DEFAULT_NODE_PORT
 
 pytestmark = [pytest.mark.public]
 
@@ -45,7 +46,7 @@ def _seed_profile(base: Path, slug: str, *, name: str | None = None, key: str =
 
 @pytest.fixture(autouse=True)
 def _no_running_node(monkeypatch):
-    monkeypatch.setattr(profiles, "node_is_running", lambda port=9000: False)
+    monkeypatch.setattr(profiles, "node_is_running", lambda port=DEFAULT_NODE_PORT: False)
 
 
 class TestNewProfile:
@@ -128,7 +129,7 @@ class TestSwitchProfile:
     def test_refuses_while_node_running(self, tmp_path, monkeypatch):
         _seed_active(tmp_path)
         _seed_profile(tmp_path, "work")
-        monkeypatch.setattr(profiles, "node_is_running", lambda port=9000: True)
+        monkeypatch.setattr(profiles, "node_is_running", lambda port=DEFAULT_NODE_PORT: True)
         with pytest.raises(profiles.ProfileError, match="running"):
             profiles.switch_profile("work", tmp_path)
         # Nothing moved.
@@ -328,7 +329,7 @@ class TestRemoveProfile:
     def test_does_not_require_the_node_to_be_stopped(self, tmp_path, monkeypatch):
         # An archived Topos is not the one a running engine has open, so unlike
         # new/switch this must work without asking anyone to quit Topos.
-        monkeypatch.setattr(profiles, "node_is_running", lambda port=9000: True)
+        monkeypatch.setattr(profiles, "node_is_running", lambda port=DEFAULT_NODE_PORT: True)
         _seed_profile(tmp_path, "work")
 
         profiles.remove_profile("work", tmp_path)

{topos_node-1.3.38 → topos_node-1.3.39}/topos/__version__.py

@@ -1,6 +1,6 @@
 """Topos Control Plane Version Information."""
 
-__version__ = "1.3.38"
-__version_info__ = (1, 3, 38)
+__version__ = "1.3.39"
+__version_info__ = (1, 3, 39)
 
 __all__ = ["__version__", "__version_info__"]

{topos_node-1.3.38 → topos_node-1.3.39}/topos/cli/commands.py

@@ -12,6 +12,7 @@ import uvicorn
 # Add parent directory to path for imports
 sys.path.insert(0, str(Path(__file__).parent.parent.parent))
 
+from topos.defaults import DEFAULT_NODE_PORT
 from topos.runtime_update import (
     DEFAULT_PACKAGE_NAME,
     apply_package_update,
@@ -232,8 +233,8 @@ def _maybe_offer_self_update(
 )
 @click.option(
     "--port",
-    default=9000,
-    help="Server port (default: 9000)",
+    default=DEFAULT_NODE_PORT,
+    help=f"Server port (default: {DEFAULT_NODE_PORT})",
 )
 @click.option(
     "--host",

{topos_node-1.3.38 → topos_node-1.3.39}/topos/cli/profile_cmd.py

@@ -17,6 +17,7 @@ from typing import Optional
 import click
 
 from topos import profiles
+from topos.defaults import DEFAULT_NODE_PORT
 
 
 def _echo(payload: dict, as_json: bool, human: str) -> None:
@@ -41,7 +42,7 @@ _BASE_OPTION = click.option(
 )
 _JSON_OPTION = click.option("--json", "as_json", is_flag=True, help="Machine-readable output.")
 _PORT_OPTION = click.option(
-    "--port", default=9000, show_default=True, help="Port probed for a running node."
+    "--port", default=DEFAULT_NODE_PORT, show_default=True, help="Port probed for a running node."
 )
 
 

{topos_node-1.3.38 → topos_node-1.3.39}/topos/core/handlers/ingest.py

@@ -408,10 +408,13 @@ async def handle_start_ingestion(message: Dict[str, Any]) -> Optional[Dict[str,
                 "payload": {"job_id": job_id, "status": "completed", "deduplicated": True},
             }
 
+        # Import-time policy chosen in the UI (date window, inclusion flags).
+        ingest_options = payload.get("ingest_options")
         job_payload: Dict[str, Any] = {
             "dataset_id": dataset_id or "",
             "schema_id": schema_id,
             "file_format": file_format,
+            "ingest_options": ingest_options if isinstance(ingest_options, dict) else None,
             "job_id": job_id,
             "source_id": source_id,
             "source_definition": source_definition,

topos_node-1.3.39/topos/defaults.py

@@ -0,0 +1,9 @@
+"""Stable defaults for a local Topos node.
+
+8676 is TOPO on a phone keypad. Keep this the only Python source of the
+listen port so CLI, probes, and the local MCP proxy cannot drift.
+"""
+
+DEFAULT_NODE_PORT = 8676
+DEFAULT_NODE_HTTP_URL = f"http://127.0.0.1:{DEFAULT_NODE_PORT}"
+DEFAULT_NODE_LOCALHOST_URL = f"http://localhost:{DEFAULT_NODE_PORT}"

{topos_node-1.3.38 → topos_node-1.3.39}/topos/enrichment/jobs/canonical/entities_job.py

@@ -266,7 +266,9 @@ class EntitiesJob(BaseEnrichmentJob):
                 resolver.record_mention(
                     entity_id,
                     record_id=record_id,
-                    surface_text=surface,
+                    # A declared row may resolve on one string and be evidenced by
+                    # another: a cited host is the node, the full URL is the proof.
+                    surface_text=str(rec.get("surface_detail") or surface),
                     source_id=rec.get("source_id"),
                     canonical_table=rec.get("canonical_table"),
                     confidence=confidence,

topos_node-1.3.39/topos/features/entities/chatgpt_declared.py

@@ -0,0 +1,256 @@
+"""Declared entity minting for ChatGPT turns (PLAN_CHATGPT_IMPORT.md Sprint 4).
+
+A ChatGPT turn carries more than its text. The export declares, per message,
+which pages were cited, which searches were run, which canvas document was being
+written and which files were attached — and the reader already lifts all of it
+onto ``ai_chat_messages.metadata_json``. Until now none of it became a node, so
+the graph could not answer "what sources inform me" even though the answer was
+sitting in a column.
+
+This is the same class the 1.3.36 density pass ranked strongest: read from a
+structured column, no model, no confidence to discount.
+
+Three judgements are encoded here, and each one is a decision not a detail:
+
+**Hosts, not URLs.** 1,359 distinct URLs would mint 1,359 leaf nodes that
+connect to nothing and say nothing. The *host* is the source — ``docs.python.org``
+is a thing you read repeatedly — so hosts are minted and the full URL is kept as
+the mention's surface text. Subdomains stay distinct: reading Google Cloud's
+docs is not reading google.com.
+
+**Search queries are not minted.** They are declared, and they are read — they
+ride on the exposure edge and on the mention that the search produced — but a
+sentence is not an entity. Minting "Google Cloud roles/compute.instanceAdmin.v1
+permissions create start stop delete VM instances" as a node would add a row, not
+density. The entities *inside* a query reach the graph through the citations the
+query returned.
+
+**Attachments must be named by a human.** A ChatGPT export names half its
+attachments with a UUID. ``proposal-v3.pdf`` is a document; ``bb2b7a54-d935-4ac1
+-82d2-af0ff5df204c.png`` is a filename the system made up, and minting it would
+put a hash in the graph.
+
+Pure stdlib. The registry in ``declared_mappings`` dispatches to this.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Dict, Iterable, List, Optional
+from urllib.parse import urlsplit
+
+# Spine entity types minted here. Both are new to the vocabulary in
+# ``resolver._NER_TYPE_MAP``; declared rows keep their type verbatim, so no
+# mapping is needed, but a UI that switches on entity_type should learn them.
+TYPE_WEB_SOURCE = "web_source"
+TYPE_DOCUMENT = "document"
+
+# Owner edges. Exposure is not authorship: a page the assistant fetched and
+# showed is something the owner was exposed to, and the provenance layer must
+# never read it as something they said.
+EDGE_EXPOSED_TO = "exposed_to"
+EDGE_AUTHORED = "authored"
+
+# Hosts that identify no source: loopback, bare addresses, and the redirectors
+# a search tool leaves behind.
+_HOST_DENYLIST = frozenset({"localhost", "127.0.0.1", "0.0.0.0", "example.com"})
+_IPV4 = re.compile(r"^\d{1,3}(\.\d{1,3}){3}$")
+
+# A filename a person chose, versus one a system generated.
+_UUID_NAME = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE)
+_HEX_NAME = re.compile(r"^[0-9a-f]{16,}$", re.IGNORECASE)
+_MIN_DOCUMENT_NAME = 3
+
+
+def host_of(url: str) -> str:
+    """The source a URL names: hostname, lowercased, without ``www.``.
+
+    Returns "" for anything that does not identify a readable source.
+    """
+    text = str(url or "").strip()
+    if not text:
+        return ""
+    if "//" not in text:
+        text = "//" + text
+    try:
+        host = (urlsplit(text).hostname or "").lower()
+    except ValueError:
+        return ""
+    host = host.removeprefix("www.")
+    if not host or "." not in host or host in _HOST_DENYLIST or _IPV4.match(host):
+        return ""
+    return host
+
+
+def is_named_by_a_human(filename: str) -> bool:
+    """Did a person choose this filename, or did a machine?"""
+    name = str(filename or "").strip()
+    if len(name) < _MIN_DOCUMENT_NAME:
+        return False
+    stem = name.rsplit(".", 1)[0] if "." in name else name
+    if _UUID_NAME.match(stem) or _HEX_NAME.match(stem):
+        return False
+    # "file_00000000abc" and friends: no letters a person would have typed.
+    return bool(re.search(r"[A-Za-z]{3,}", stem))
+
+
+def _metadata(record: Dict[str, Any]) -> Dict[str, Any]:
+    """``metadata_json`` arrives as a dict at map time and a string from SQL."""
+    value = record.get("metadata_json")
+    if value is None:
+        value = record.get("metadata")
+    if isinstance(value, str):
+        try:
+            value = json.loads(value)
+        except (TypeError, ValueError):
+            return {}
+    return value if isinstance(value, dict) else {}
+
+
+def _row(
+    surface: str,
+    entity_type: str,
+    *,
+    record: Dict[str, Any],
+    record_id: str,
+    event_at: Optional[str],
+    self_edge: Optional[str] = None,
+    surface_detail: Optional[str] = None,
+) -> Dict[str, Any]:
+    row: Dict[str, Any] = {
+        "message_id": record_id,
+        "record_id": record_id,
+        "source_id": record.get("source_id"),
+        "event_at": event_at,
+        "canonical_table": record.get("_table") or record.get("canonical_table") or "ai_chat_messages",
+        "entity_text": surface,
+        "entity_type": entity_type,
+        "confidence": 1.0,
+        "provider": "declared",
+        "model": None,
+    }
+    if self_edge:
+        row["self_edge"] = self_edge
+    if surface_detail:
+        # What the graph shows is the host; what the evidence says is the URL.
+        row["surface_detail"] = surface_detail
+    return row
+
+
+def _citation_urls(metadata: Dict[str, Any]) -> List[str]:
+    urls: List[str] = []
+    for citation in metadata.get("citation_urls") or []:
+        if isinstance(citation, str) and citation.strip():
+            urls.append(citation.strip())
+    return urls
+
+
+def _canvas_title(metadata: Dict[str, Any]) -> str:
+    canvas = metadata.get("canvas")
+    if not isinstance(canvas, dict):
+        return ""
+    title = canvas.get("title")
+    if isinstance(title, str) and title.strip():
+        return title.strip()
+    return ""
+
+
+def _attachment_names(metadata: Dict[str, Any]) -> Iterable[str]:
+    for item in metadata.get("attachments") or []:
+        if isinstance(item, dict):
+            name = item.get("name")
+            if isinstance(name, str) and is_named_by_a_human(name):
+                yield name.strip()
+
+
+def declared_rows(
+    record: Dict[str, Any],
+    *,
+    record_id: str,
+    event_at: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+    """NER-result-shaped rows minted from one turn's declared metadata.
+
+    Shape matches ``declared_mappings.extract_declared_entities`` exactly, so the
+    entities job resolves these through the same declared path — at declared
+    confidence, with the type kept verbatim.
+    """
+    if not record_id:
+        return []
+    metadata = _metadata(record)
+    if not metadata:
+        return []
+
+    rows: List[Dict[str, Any]] = []
+    seen: set = set()
+
+    # Which search produced this message's citations, if any. Carried on the
+    # mention rather than minted: it is evidence, not an entity.
+    queries = [q for q in (metadata.get("search_queries") or []) if isinstance(q, str) and q.strip()]
+    query_note = queries[0].strip() if queries else ""
+
+    for url in _citation_urls(metadata):
+        host = host_of(url)
+        if not host:
+            continue
+        key = (host, TYPE_WEB_SOURCE)
+        if key in seen:
+            continue
+        seen.add(key)
+        detail = f"{url} — searched: {query_note}" if query_note else url
+        rows.append(_row(host, TYPE_WEB_SOURCE, record=record, record_id=record_id,
+                         event_at=event_at, self_edge=EDGE_EXPOSED_TO, surface_detail=detail))
+
+    canvas = _canvas_title(metadata)
+    if canvas:
+        key = (canvas.lower(), TYPE_DOCUMENT)
+        if key not in seen:
+            seen.add(key)
+            rows.append(_row(canvas, TYPE_DOCUMENT, record=record, record_id=record_id,
+                             event_at=event_at, self_edge=EDGE_AUTHORED))
+
+    for name in _attachment_names(metadata):
+        key = (name.lower(), TYPE_DOCUMENT)
+        if key in seen:
+            continue
+        seen.add(key)
+        rows.append(_row(name, TYPE_DOCUMENT, record=record, record_id=record_id,
+                         event_at=event_at, self_edge=EDGE_AUTHORED))
+
+    return rows
+
+
+def coverage(records: Iterable[Dict[str, Any]]) -> Dict[str, int]:
+    """What this lane would mint over a batch — the Sprint 4 measurement."""
+    hosts: set = set()
+    documents: set = set()
+    urls = 0
+    queries = 0
+    skipped_machine_names = 0
+    for record in records:
+        metadata = _metadata(record)
+        if not metadata:
+            continue
+        for url in _citation_urls(metadata):
+            urls += 1
+            host = host_of(url)
+            if host:
+                hosts.add(host)
+        queries += len([q for q in (metadata.get("search_queries") or []) if isinstance(q, str) and q.strip()])
+        title = _canvas_title(metadata)
+        if title:
+            documents.add(title.lower())
+        for item in metadata.get("attachments") or []:
+            if isinstance(item, dict) and isinstance(item.get("name"), str):
+                if is_named_by_a_human(item["name"]):
+                    documents.add(item["name"].strip().lower())
+                else:
+                    skipped_machine_names += 1
+    return {
+        "citation_urls": urls,
+        "web_sources": len(hosts),
+        "documents": len(documents),
+        "search_queries_read": queries,
+        "attachments_skipped_machine_named": skipped_machine_names,
+    }

{topos_node-1.3.38 → topos_node-1.3.39}/topos/features/entities/declared_mappings.py

@@ -183,6 +183,44 @@ def mapping_for(source_id: Optional[str]) -> Optional[Dict[str, Any]]:
     return DECLARED_ENTITY_MAPPINGS.get(str(source_id or "").strip())
 
 
+# Sources whose declared entities need code, not a path-and-transform rule: a
+# ChatGPT turn declares a *list* of cited URLs and a filename that has to be
+# judged before it is minted. The declaration DSL maps one path to one surface,
+# so these register a producer with the same output shape instead of bending it.
+def _chatgpt_declared(record, *, record_id, event_at):
+    from .chatgpt_declared import declared_rows
+
+    return declared_rows(record, record_id=record_id, event_at=event_at)
+
+
+DECLARED_ROW_PRODUCERS: Dict[str, Any] = {
+    "chatgpt_file_ingestion": _chatgpt_declared,
+    "chatgpt_ui_conversation": _chatgpt_declared,
+}
+
+
+def declared_producers_enabled() -> bool:
+    """``TOPOS_DECLARED_PRODUCERS=off`` turns the code-backed declared lanes off.
+
+    Exists so a before/after measurement is one variable on one build, instead of
+    two builds someone has to trust are otherwise identical.
+    """
+    import os
+
+    return os.environ.get("TOPOS_DECLARED_PRODUCERS", "on").strip().lower() not in (
+        "0",
+        "off",
+        "false",
+        "no",
+    )
+
+
+def producer_for(source_id: Optional[str]):
+    if not declared_producers_enabled():
+        return None
+    return DECLARED_ROW_PRODUCERS.get(str(source_id or "").strip())
+
+
 def ner_suppressed_source_ids() -> set:
     return {sid for sid, m in DECLARED_ENTITY_MAPPINGS.items() if m.get("suppress_ner")}
 
@@ -234,7 +272,8 @@ def extract_declared_entities(
     should pass record_id/event_at explicitly so the two never drift.
     """
     mapping = mapping_for(record.get("source_id"))
-    if not mapping:
+    producer = producer_for(record.get("source_id"))
+    if not mapping and not producer:
         return []
     if record_id is None:
         record_id = str(
@@ -255,6 +294,12 @@ def extract_declared_entities(
         )
     if not record_id:
         return []
+    if producer is not None:
+        rows = list(producer(record, record_id=record_id, event_at=event_at))
+        if not mapping:
+            return rows
+    else:
+        rows = []
     # Keyed on the TRANSFORMED surface, because that is what the entity rows below
     # carry. Without this the two halves agree only while every self-edge target
     # happens to be untransformed — true of github, and silently false the moment
@@ -266,8 +311,8 @@ def extract_declared_entities(
         for rule in mapping.get("self_edges", [])
     }
     edge_targets.pop("", None)
-    out: List[Dict[str, Any]] = []
-    seen: set = set()
+    out: List[Dict[str, Any]] = list(rows)
+    seen: set = {(str(r.get("entity_text") or "").lower(), str(r.get("entity_type") or "")) for r in out}
     for rule in mapping.get("entities", []):
         raw = _field(record, str(rule.get("path") or ""))
         surface = _transform(raw, rule.get("transform"))

{topos_node-1.3.38 → topos_node-1.3.39}/topos/ingestion/ingest_helpers.py

@@ -65,6 +65,7 @@ async def ingest_file_payload(
     source_definition: Optional[Dict[str, Any]] = None,
     progress_api_url: Optional[str] = None,
     progress_api_key: Optional[str] = None,
+    ingest_options: Optional[Dict[str, Any]] = None,
 ) -> dict:
     if not dataset_id:
         return {"status": "error", "error": "dataset_id required"}
@@ -112,10 +113,17 @@ async def ingest_file_payload(
         len(file_bytes) if file_bytes else None,
     )
     if file_path:
-        job = trigger.create_job(job_id, dataset_id, schema_id, file_path, file_format=file_format)
+        job = trigger.create_job(
+            job_id, dataset_id, schema_id, file_path, file_format=file_format, ingest_options=ingest_options
+        )
     else:
         job = trigger.create_job_from_bytes(
-            job_id, dataset_id, schema_id, file_bytes or b"", file_format=file_format
+            job_id,
+            dataset_id,
+            schema_id,
+            file_bytes or b"",
+            file_format=file_format,
+            ingest_options=ingest_options,
         )
     manager = IngestionManager(file_store=file_store)
     result = await manager.process_job(

{topos_node-1.3.38 → topos_node-1.3.39}/topos/ingestion/manager.py

@@ -797,7 +797,14 @@ class IngestionManager(BaseObject):
             pbar = ProgressBar(total=1000, desc=f"{self}: Parsing")  # Placeholder, will adjust
 
         try:
-            async for raw_payload in parse_file(_read_file_bytes(file_path), job.metadata.get("file_format", "jsonl")):
+            # Import-time policy (date window, inclusion flags) travels on the
+            # job so a re-run of the same job reproduces the same corpus.
+            ingest_options = job.metadata.get("ingest_options")
+            async for raw_payload in parse_file(
+                _read_file_bytes(file_path),
+                job.metadata.get("file_format", "jsonl"),
+                ingest_options if isinstance(ingest_options, dict) else None,
+            ):
                 expanded_payloads = _expand_file_records(raw_payload, source_def)
                 if not expanded_payloads:
                     expanded_payloads = [raw_payload] if isinstance(raw_payload, dict) else []