sql-code-graph 1.0.2__py3-none-any.whl → 1.1.0__py3-none-any.whl

sqlcg/indexer/indexer.py

@@ -4,10 +4,11 @@ import multiprocessing as mp
 import queue
 import time
 from collections.abc import Callable
+from dataclasses import dataclass, field
 from pathlib import Path
 
+from sqlcg.core.config import get_external_consumers, get_presentation_prefixes
 from sqlcg.core.graph_db import GraphBackend
-from sqlcg.core.queries import STALE_VIEWS_QUERY
 from sqlcg.core.schema import NodeLabel, RelType
 from sqlcg.indexer.error_classify import _classify_error, dominant_cause
 from sqlcg.indexer.pool import HardKillPool
@@ -22,6 +23,141 @@ from sqlcg.utils.logging import getLogger
 logger = getLogger(__name__)
 
 
+@dataclass
+class FileRowSet:
+    """Row dicts produced from a single ParsedFile (Phase A output).
+
+    Pure data container — no db access, no execute(). Used as the unit
+    passed from _build_file_rows to BatchRowBuffer.extend().
+    """
+
+    file_rows: list[dict] = field(default_factory=list)
+    table_rows: list[dict] = field(default_factory=list)
+    column_rows: list[dict] = field(default_factory=list)
+    query_rows: list[dict] = field(default_factory=list)
+    defined_in_edges: list[dict] = field(default_factory=list)
+    has_column_edges: list[dict] = field(default_factory=list)
+    query_defined_in_edges: list[dict] = field(default_factory=list)
+    selects_from_edges: list[dict] = field(default_factory=list)
+    column_lineage_edges: list[dict] = field(default_factory=list)
+    star_source_edges: list[dict] = field(default_factory=list)
+    counts: dict = field(
+        default_factory=lambda: {"tables": 0, "edges": 0, "columns_defined": 0, "star_sources": 0}
+    )
+    parse_quality_key: str = "full"
+
+
+@dataclass
+class BatchRowBuffer:
+    """Accumulated row dicts across all files in a batch.
+
+    Used by _flush_row_batch to dedup and flush once per batch instead of
+    once per file — the v1.1.1 perf improvement (~13,400 → ~270 execute()).
+    """
+
+    file_rows: list[dict] = field(default_factory=list)
+    table_rows: list[dict] = field(default_factory=list)
+    column_rows: list[dict] = field(default_factory=list)
+    query_rows: list[dict] = field(default_factory=list)
+    defined_in_edges: list[dict] = field(default_factory=list)
+    has_column_edges: list[dict] = field(default_factory=list)
+    query_defined_in_edges: list[dict] = field(default_factory=list)
+    selects_from_edges: list[dict] = field(default_factory=list)
+    column_lineage_edges: list[dict] = field(default_factory=list)
+    star_source_edges: list[dict] = field(default_factory=list)
+
+    def extend(self, rows: FileRowSet) -> None:
+        """Accumulate one file's row sets into this buffer."""
+        self.file_rows.extend(rows.file_rows)
+        self.table_rows.extend(rows.table_rows)
+        self.column_rows.extend(rows.column_rows)
+        self.query_rows.extend(rows.query_rows)
+        self.defined_in_edges.extend(rows.defined_in_edges)
+        self.has_column_edges.extend(rows.has_column_edges)
+        self.query_defined_in_edges.extend(rows.query_defined_in_edges)
+        self.selects_from_edges.extend(rows.selects_from_edges)
+        self.column_lineage_edges.extend(rows.column_lineage_edges)
+        self.star_source_edges.extend(rows.star_source_edges)
+
+    @classmethod
+    def from_single(cls, rows: FileRowSet) -> "BatchRowBuffer":
+        """Build a single-file batch buffer (used by the thin _upsert_parsed_file wrapper)."""
+        buf = cls()
+        buf.extend(rows)
+        return buf
+
+
+def _flush_row_batch(db: GraphBackend, buf: BatchRowBuffer) -> None:
+    """Dedup accumulated rows across the batch, then issue the 10 bulk calls ONCE.
+
+    This is the v1.1.1 batch-flush core: called once per batch (not once per file).
+    Dedup keys mirror the graph's MERGE cardinality:
+      - file_rows:             path (primary key)
+      - table_rows:            qualified (primary key); prefers row with non-empty
+                               defined_in_file so DEFINED_IN provenance is preserved.
+      - column_rows:           id (primary key)
+      - query_rows:            id (primary key, globally unique path:index)
+      - edge rows:             (src_key, dst_key) only — matches MERGE (src)-[r]->(dst)
+                               cardinality; KuzuDB cannot hold two edges of the same
+                               type between the same pair.
+
+    Performance invariant: calls each upsert_*_bulk at most ONCE per invocation;
+    never calls upsert_node / upsert_edge per row.
+    Guard: tests/unit/test_upsert_batch_invariant.py
+    """
+    # --- Phase B: batch-scoped dedup ---
+    # For table_rows, prefer defined rows (non-empty defined_in_file) so provenance
+    # is not lost when a shared table is referenced by multiple files.
+    table_dedup: dict[str, dict] = {}
+    for r in buf.table_rows:
+        key = r["qualified"]
+        existing = table_dedup.get(key)
+        if existing is None or (not existing.get("defined_in_file") and r.get("defined_in_file")):
+            table_dedup[key] = r
+    table_rows = list(table_dedup.values())
+
+    column_rows = list({r["id"]: r for r in buf.column_rows}.values())
+    query_rows = list({r["id"]: r for r in buf.query_rows}.values())
+    file_rows = list({r["path"]: r for r in buf.file_rows}.values())
+
+    # Edge dedup by (src_key, dst_key) — matches MERGE (src)-[r]->(dst) cardinality.
+    defined_in_edges = list(
+        {(r["src_key"], r["dst_key"]): r for r in buf.defined_in_edges}.values()
+    )
+    has_column_edges = list(
+        {(r["src_key"], r["dst_key"]): r for r in buf.has_column_edges}.values()
+    )
+    query_defined_in_edges = list(
+        {(r["src_key"], r["dst_key"]): r for r in buf.query_defined_in_edges}.values()
+    )
+    selects_from_edges = list(
+        {(r["src_key"], r["dst_key"]): r for r in buf.selects_from_edges}.values()
+    )
+    column_lineage_edges = list(
+        {(r["src_key"], r["dst_key"]): r for r in buf.column_lineage_edges}.values()
+    )
+    star_source_edges = list(
+        {(r["src_key"], r["dst_key"]): r for r in buf.star_source_edges}.values()
+    )
+
+    # --- Phase C: flush in dependency order (nodes before their edges) ---
+    db.upsert_nodes_bulk(NodeLabel.FILE, file_rows)
+    db.upsert_nodes_bulk(NodeLabel.TABLE, table_rows)
+    db.upsert_nodes_bulk(NodeLabel.COLUMN, column_rows)
+    db.upsert_nodes_bulk(NodeLabel.QUERY, query_rows)
+
+    db.upsert_edges_bulk(NodeLabel.TABLE, NodeLabel.FILE, RelType.DEFINED_IN, defined_in_edges)
+    db.upsert_edges_bulk(NodeLabel.TABLE, NodeLabel.COLUMN, RelType.HAS_COLUMN, has_column_edges)
+    db.upsert_edges_bulk(
+        NodeLabel.QUERY, NodeLabel.FILE, RelType.QUERY_DEFINED_IN, query_defined_in_edges
+    )
+    db.upsert_edges_bulk(NodeLabel.QUERY, NodeLabel.TABLE, RelType.SELECTS_FROM, selects_from_edges)
+    db.upsert_edges_bulk(
+        NodeLabel.COLUMN, NodeLabel.COLUMN, RelType.COLUMN_LINEAGE, column_lineage_edges
+    )
+    db.upsert_edges_bulk(NodeLabel.QUERY, NodeLabel.TABLE, RelType.STAR_SOURCE, star_source_edges)
+
+
 def _subprocess_parse_worker(parser_cls, dialect, path, sql, q):
     """Parse a single file in a subprocess; queue the ParsedFile (or exception).
 
@@ -55,7 +191,7 @@ class Indexer:
         dialect: str | None,
         db: GraphBackend,
         dbt_manifest: Path | None = None,
-        timeout_per_file: int = 5,
+        timeout_per_file: int = 10,
         use_git: bool = True,
         progress_callback: Callable[[int, int], None] | None = None,
         no_ddl: bool = False,
@@ -288,37 +424,15 @@ class Indexer:
         }
 
         def _flush_batch(batch: list[ParsedFile]) -> None:
-            """Upsert a batch of ParsedFile objects in a single transaction.
+            """Accumulate rows for the batch and flush once (v1.1.1 batch-upsert path).
 
-            On any per-file exception, the file is recorded as failed but the
-            transaction continues for the remaining files in the batch. This
-            matches the legacy per-file behaviour where one bad file did not
-            abort the whole index run.
+            Per-file row-building errors are isolated inside _upsert_file_batch; good
+            files always flush together in a single transaction.
 
             Note: sqlcg watch's reindex_file uses a separate code path with
             its own short per-file transaction. PERF-BATCH only affects index_repo.
             """
-            if not batch:
-                return
-            with db.transaction():
-                for parsed_in_batch in batch:
-                    try:
-                        counts = self._upsert_parsed_file(
-                            parsed_in_batch,
-                            db,
-                            defined_table_registry=defined_table_registry,
-                        )
-                        nonlocal_counts["tables"] += counts["tables"]
-                        nonlocal_counts["edges"] += counts["edges"]
-                        nonlocal_counts["star_sources"] += counts.get("star_sources", 0)
-                        nonlocal_counts["columns_defined"] += counts.get("columns_defined", 0)
-                        q_key = parsed_in_batch.parse_quality.value.lower()
-                        nonlocal_counts["quality"][q_key] += 1
-                    except Exception as exc:
-                        logger.warning(
-                            "Failed to upsert %s: %s — skipping", parsed_in_batch.path, exc
-                        )
-                        nonlocal_counts["quality"]["failed"] += 1
+            self._upsert_file_batch(batch, db, defined_table_registry, nonlocal_counts)
 
         if profile:
             _t_upsert_start = time.perf_counter()
@@ -345,6 +459,9 @@ class Indexer:
         if profile:
             _t_star_end = time.perf_counter()
 
+        # Post-ingestion: ingest declared external downstream consumers from .sqlcg.toml
+        ingest_result = self._ingest_external_consumers(db, path)
+
         # Persist the HEAD SHA so `sqlcg reindex` (no SHAs) can compute the delta.
         # Silent on failure — non-git repos still index cleanly.
         try:
@@ -402,6 +519,9 @@ class Indexer:
             "quality": nonlocal_counts["quality"],
             "error_summary": error_summary,
             "batch_size": batch_size,
+            "external_consumers": ingest_result["consumers"],
+            "external_consumer_edges": ingest_result["edges"],
+            "external_consumer_warnings": ingest_result["warnings"],
         }
 
         if profile:
@@ -434,7 +554,7 @@ class Indexer:
         dialect: str | None,
         *,
         batch_size: int = 50,
-        timeout_per_file: int = 5,
+        timeout_per_file: int = 10,
         max_closure_depth: int = 3,
     ) -> dict:
         """Incrementally resync the graph after a git branch change or pull.
@@ -722,41 +842,28 @@ class Indexer:
             for pf in closure_results:
                 db.delete_nodes_for_file(pf.path_str)
 
-        # Flush in batches using the same pattern as index_repo._flush_batch
+        # Flush in batches using the shared _upsert_file_batch path (v1.1.1 batch-flush).
+        # resync_changed does not surface tables/edges/quality counts in its summary, so
+        # a temporary counts dict is passed and discarded after each batch.
+        _resync_counts: dict = {
+            "tables": 0,
+            "edges": 0,
+            "star_sources": 0,
+            "columns_defined": 0,
+            "quality": {"full": 0, "table_only": 0, "scripting_fallback": 0, "failed": 0},
+        }
         batch: list[ParsedFile] = []
         for pf in all_results:
             batch.append(pf)
             if len(batch) >= batch_size:
-                with db.transaction():
-                    for pf_in_batch in batch:
-                        try:
-                            self._upsert_parsed_file(
-                                pf_in_batch,
-                                db,
-                                defined_table_registry=defined_table_registry,
-                            )
-                        except Exception as exc:
-                            logger.warning(
-                                "resync_changed: upsert failed for %s: %s — skipping",
-                                pf_in_batch.path,
-                                exc,
-                            )
+                self._upsert_file_batch(
+                    batch, db, defined_table_registry, _resync_counts, "resync_changed: "
+                )
                 batch = []
         if batch:
-            with db.transaction():
-                for pf_in_batch in batch:
-                    try:
-                        self._upsert_parsed_file(
-                            pf_in_batch,
-                            db,
-                            defined_table_registry=defined_table_registry,
-                        )
-                    except Exception as exc:
-                        logger.warning(
-                            "resync_changed: upsert failed for %s: %s — skipping",
-                            pf_in_batch.path,
-                            exc,
-                        )
+            self._upsert_file_batch(
+                batch, db, defined_table_registry, _resync_counts, "resync_changed: "
+            )
 
         # ---- Step 8: Single star expansion (same as index_repo) ------------------
         self._expand_star_sources(db)
@@ -767,15 +874,19 @@ class Indexer:
         return summary
 
     def reindex_file(self, file_path: str, db: GraphBackend, dialect: str | None) -> None:
-        """Re-index a single file and its dependent views.
+        """Re-index a single file.
+
+        Re-indexes only the changed file. The stale-view cascade that was here
+        previously was a no-op (the dead query matched kind='VIEW' which is never
+        written). It has been removed in PR-3 (#33). Full re-index
+        (db reset && sqlcg index) is the migration path for VIEW dependency
+        tracking, deferred to v1.2.
 
         Args:
             file_path: Path to the file to re-index
             db: GraphBackend instance
             dialect: SQL dialect (None for ANSI)
         """
-        stale_views = db.run_read(STALE_VIEWS_QUERY, {"path": file_path})
-
         with db.transaction():
             db.delete_nodes_for_file(file_path)
             schema_resolver = SchemaResolver(dialect=dialect)
@@ -785,11 +896,7 @@ class Indexer:
 
             self._upsert_parsed_file(parsed, db)
 
-        for row in stale_views:
-            self._reindex_view_definition(row["view_name"], db, dialect)
-
-        # Re-run star expansion after re-indexing and all stale views are processed
-        # Call this outside the transaction
+        # Re-run star expansion after re-indexing
         self._expand_star_sources(db)
 
     def _index_single_file(self, parser, path: Path, sql: str, timeout: int) -> ParsedFile:
@@ -864,40 +971,31 @@ class Indexer:
         """Canonical graph key for a table/column: lowercased full_id."""
         return full_id.lower()
 
-    def _upsert_parsed_file(
+    def _build_file_rows(
         self,
         parsed: ParsedFile,
-        db: GraphBackend,
         defined_table_registry: dict[str, str] | None = None,
-    ) -> dict:
-        """Map ParsedFile → graph nodes/edges using bulk upsert.
+    ) -> FileRowSet:
+        """Build all row dicts for one ParsedFile — Phase A (pure, no db access).
+
+        Extracts the four node-row lists and six edge-row lists from a ParsedFile.
+        No execute(), no db argument. A build-time exception here is caught by the
+        per-file try/except in _upsert_file_batch so the bad file is isolated from
+        the rest of the batch.
 
         Args:
-            parsed: ParsedFile to upsert
-            db: GraphBackend instance
+            parsed: ParsedFile to build rows for
+            defined_table_registry: Optional cross-file DDL dedup registry
 
         Returns:
-            Dict with keys: tables, edges, columns_defined, star_sources
+            FileRowSet with all row lists and per-file counts/quality key
         """
-        counts = {"tables": 0, "edges": 0, "columns_defined": 0, "star_sources": 0}
-
-        # --- Phase A: collect rows from parsed model ---
-
-        file_rows: list[dict] = []
-        table_rows: list[dict] = []  # NodeLabel.TABLE
-        column_rows: list[dict] = []  # NodeLabel.COLUMN
-        query_rows: list[dict] = []  # NodeLabel.QUERY
-
-        defined_in_edges: list[dict] = []  # TABLE -> FILE
-        has_column_edges: list[dict] = []  # TABLE -> COLUMN
-        query_defined_in_edges: list[dict] = []  # QUERY -> FILE
-        selects_from_edges: list[dict] = []  # QUERY -> TABLE
-        column_lineage_edges: list[dict] = []  # COLUMN -> COLUMN
-        star_source_edges: list[dict] = []  # QUERY -> TABLE
+        rows = FileRowSet()
+        rows.parse_quality_key = parsed.parse_quality.value.lower()
 
         # File node
         cause, failed = dominant_cause(parsed.errors)
-        file_rows.append(
+        rows.file_rows.append(
             {
                 "path": parsed.path_str,
                 "dialect": parsed.dialect or "",
@@ -924,18 +1022,18 @@ class Indexer:
                 if hasattr(parsed, "errors"):
                     parsed.errors.append(f"duplicate_ddl:{table.full_id}:already_in:{first_file}")
 
-            table_rows.append(
+            rows.table_rows.append(
                 {
                     "qualified": table.full_id,
                     "name": table.name,
                     "catalog": table.catalog or "",
                     "db": table.db or "",
-                    "kind": "TABLE",
+                    "kind": "table",
                     "defined_in_file": parsed.path_str,
                 }
             )
-            defined_in_edges.append({"src_key": table.full_id, "dst_key": parsed.path_str})
-            counts["tables"] += 1
+            rows.defined_in_edges.append({"src_key": table.full_id, "dst_key": parsed.path_str})
+            rows.counts["tables"] += 1
 
         # Build a map of target.full_id -> QueryNode for column lookup
         defined_by_query = {
@@ -955,7 +1053,7 @@ class Indexer:
 
             for col_name in qnode.defined_columns:
                 col_id = f"{table.full_id}.{col_name}"
-                column_rows.append(
+                rows.column_rows.append(
                     {
                         "id": col_id,
                         "col_name": col_name,
@@ -965,15 +1063,15 @@ class Indexer:
                         "table_name": table.name,
                     }
                 )
-                has_column_edges.append(
+                rows.has_column_edges.append(
                     {"src_key": table.full_id, "dst_key": col_id, "source": "ddl"}
                 )
-                counts["columns_defined"] += 1
+                rows.counts["columns_defined"] += 1
 
         # Query nodes and related edges
         for i, stmt in enumerate(parsed.statements):
             query_id = f"{parsed.path_str}:{i}"
-            query_rows.append(
+            rows.query_rows.append(
                 {
                     "id": query_id,
                     "file_path": parsed.path_str,
@@ -984,29 +1082,41 @@ class Indexer:
                     "parse_failed": stmt.parse_failed,
                     "confidence": stmt.confidence,
                     "parsing_mode": stmt.parsing_mode,
+                    "start_line": stmt.start_line,
                 }
             )
-            query_defined_in_edges.append({"src_key": query_id, "dst_key": parsed.path_str})
+            rows.query_defined_in_edges.append({"src_key": query_id, "dst_key": parsed.path_str})
 
-            # Source table edges
+            # Source table edges — apply structural role from TableRef.role.
+            # Real source tables have role="table"; CTE aliases have role="cte".
+            # Exclude <output> synthetic sinks (fallback dst when stmt.target is None).
             for src_table in stmt.sources:
-                table_rows.append(
+                if src_table.full_id == "<output>":
+                    continue
+                rows.table_rows.append(
                     {
                         "qualified": src_table.full_id,
                         "name": src_table.name,
                         "catalog": src_table.catalog or "",
                         "db": src_table.db or "",
-                        "kind": "TABLE",
+                        "kind": src_table.role,
                         "defined_in_file": "",
                     }
                 )
-                selects_from_edges.append({"src_key": query_id, "dst_key": src_table.full_id})
+                rows.selects_from_edges.append({"src_key": query_id, "dst_key": src_table.full_id})
 
-            # Column lineage edges
+            # Column lineage edges.
+            # CTE destination tables (role="cte") are not in stmt.sources so they are not
+            # emitted by the source-table loop above.  Collect them here from edge.dst so
+            # that each CTE alias gets a SqlTable node with kind="cte" in the graph.
+            # Also exclude <output> synthetic sinks from both table_rows and column_rows.
             for edge in stmt.column_lineage:
                 src_id = edge.src.full_id
                 dst_id = edge.dst.full_id
-                column_rows.append(
+                # Exclude <output> sink — it is a synthetic fallback, not a real table
+                if edge.dst.table.full_id == "<output>":
+                    continue
+                rows.column_rows.append(
                     {
                         "id": src_id,
                         "col_name": edge.src.name,
@@ -1016,7 +1126,7 @@ class Indexer:
                         "table_name": edge.src.table.name,
                     }
                 )
-                column_rows.append(
+                rows.column_rows.append(
                     {
                         "id": dst_id,
                         "col_name": edge.dst.name,
@@ -1026,7 +1136,19 @@ class Indexer:
                         "table_name": edge.dst.table.name,
                     }
                 )
-                column_lineage_edges.append(
+                # Emit CTE destination table nodes so they appear as SqlTable with kind="cte"
+                if edge.dst.table.role == "cte":
+                    rows.table_rows.append(
+                        {
+                            "qualified": edge.dst.table.full_id,
+                            "name": edge.dst.table.name,
+                            "catalog": edge.dst.table.catalog or "",
+                            "db": edge.dst.table.db or "",
+                            "kind": "cte",
+                            "defined_in_file": "",
+                        }
+                    )
+                rows.column_lineage_edges.append(
                     {
                         "src_key": src_id,
                         "dst_key": dst_id,
@@ -1035,21 +1157,21 @@ class Indexer:
                         "query_id": query_id,
                     }
                 )
-                counts["edges"] += 1
+                rows.counts["edges"] += 1
 
-            # STAR_SOURCE edges for graph-backend expansion
+            # STAR_SOURCE edges for graph-backend expansion — real source tables, keep kind="table"
             for star in stmt.star_sources:
-                table_rows.append(
+                rows.table_rows.append(
                     {
                         "qualified": star.source.full_id,
                         "name": star.source.name,
                         "catalog": star.source.catalog or "",
                         "db": star.source.db or "",
-                        "kind": "TABLE",
+                        "kind": "table",
                         "defined_in_file": "",
                     }
                 )
-                star_source_edges.append(
+                rows.star_source_edges.append(
                     {
                         "src_key": query_id,
                         "dst_key": star.source.full_id,
@@ -1058,53 +1180,95 @@ class Indexer:
                         "confidence": 0.8,
                     }
                 )
-                counts["star_sources"] += 1
+                rows.counts["star_sources"] += 1
 
             # Upsert target table node (if not already a defined_table)
             # so that star expansion can create destination columns
             if stmt.target and stmt.target.full_id not in defined_table_ids:
-                table_rows.append(
+                rows.table_rows.append(
                     {
                         "qualified": stmt.target.full_id,
                         "name": stmt.target.name,
                         "catalog": stmt.target.catalog or "",
                         "db": stmt.target.db or "",
-                        "kind": "TABLE",
+                        "kind": "table",
                         "defined_in_file": "",
                     }
                 )
 
-        # --- Phase B: deduplicate rows within each group ---
-        # Cypher MERGE is idempotent under deduplication, but we deduplicate in Python
-        # to reduce payload size
-        table_rows = list({r["qualified"]: r for r in table_rows}.values())
-        column_rows = list({r["id"]: r for r in column_rows}.values())
-        query_rows = list({r["id"]: r for r in query_rows}.values())
-
-        # --- Phase C: flush in dependency order (nodes before their edges) ---
-        db.upsert_nodes_bulk(NodeLabel.FILE, file_rows)
-        db.upsert_nodes_bulk(NodeLabel.TABLE, table_rows)
-        db.upsert_nodes_bulk(NodeLabel.COLUMN, column_rows)
-        db.upsert_nodes_bulk(NodeLabel.QUERY, query_rows)
-
-        db.upsert_edges_bulk(NodeLabel.TABLE, NodeLabel.FILE, RelType.DEFINED_IN, defined_in_edges)
-        db.upsert_edges_bulk(
-            NodeLabel.TABLE, NodeLabel.COLUMN, RelType.HAS_COLUMN, has_column_edges
-        )
-        db.upsert_edges_bulk(
-            NodeLabel.QUERY, NodeLabel.FILE, RelType.QUERY_DEFINED_IN, query_defined_in_edges
-        )
-        db.upsert_edges_bulk(
-            NodeLabel.QUERY, NodeLabel.TABLE, RelType.SELECTS_FROM, selects_from_edges
-        )
-        db.upsert_edges_bulk(
-            NodeLabel.COLUMN, NodeLabel.COLUMN, RelType.COLUMN_LINEAGE, column_lineage_edges
-        )
-        db.upsert_edges_bulk(
-            NodeLabel.QUERY, NodeLabel.TABLE, RelType.STAR_SOURCE, star_source_edges
-        )
+        return rows
+
+    def _upsert_file_batch(
+        self,
+        batch: list[ParsedFile],
+        db: GraphBackend,
+        defined_table_registry: dict[str, str],
+        nonlocal_counts: dict,
+        warning_prefix: str = "",
+    ) -> None:
+        """Accumulate rows for all files in batch, then flush once in one transaction.
+
+        Per-file failure isolation: _build_file_rows (pure) runs per file inside its
+        own try/except — a bad file is recorded as failed and excluded from the buffer;
+        the remaining good files flush together. Flush-time backend errors (rare) fail
+        the whole batch transaction, which is already the behaviour inside db.transaction().
+
+        Args:
+            batch: List of ParsedFile objects to upsert
+            db: GraphBackend instance
+            defined_table_registry: Cross-file DDL dedup registry
+            nonlocal_counts: Mutable summary dict updated in place (tables/edges/quality/…)
+            warning_prefix: Optional prefix for warning log messages (e.g. "resync_changed: ")
+        """
+        if not batch:
+            return
+        buf = BatchRowBuffer()
+        for parsed_in_batch in batch:
+            try:
+                file_rows = self._build_file_rows(parsed_in_batch, defined_table_registry)
+            except Exception as exc:
+                logger.warning(
+                    "%sFailed to build rows for %s: %s — skipping",
+                    warning_prefix,
+                    parsed_in_batch.path,
+                    exc,
+                )
+                nonlocal_counts["quality"]["failed"] += 1
+                continue
+            buf.extend(file_rows)
+            nonlocal_counts["tables"] += file_rows.counts["tables"]
+            nonlocal_counts["edges"] += file_rows.counts["edges"]
+            nonlocal_counts["star_sources"] += file_rows.counts.get("star_sources", 0)
+            nonlocal_counts["columns_defined"] += file_rows.counts.get("columns_defined", 0)
+            nonlocal_counts["quality"][file_rows.parse_quality_key] += 1
+        with db.transaction():
+            _flush_row_batch(db, buf)
+
+    def _upsert_parsed_file(
+        self,
+        parsed: ParsedFile,
+        db: GraphBackend,
+        defined_table_registry: dict[str, str] | None = None,
+    ) -> dict:
+        """Thin single-file wrapper used by reindex_file and sqlcg watch.
+
+        Builds one file's rows (pure) and flushes them immediately — semantically
+        equivalent to the old per-file upsert, but now via the shared builder/flusher
+        split. The caller (reindex_file at indexer.py:796) wraps this in its own
+        db.transaction(); this method does NOT open a transaction.
 
-        return counts
+        Args:
+            parsed: ParsedFile to upsert
+            db: GraphBackend instance
+            defined_table_registry: Optional cross-file DDL dedup registry
+
+        Returns:
+            Dict with keys: tables, edges, columns_defined, star_sources
+        """
+        file_rows = self._build_file_rows(parsed, defined_table_registry)
+        buf = BatchRowBuffer.from_single(file_rows)
+        _flush_row_batch(db, buf)
+        return file_rows.counts
 
     def _expand_star_sources(self, db: GraphBackend) -> int:
         """Run the post-ingestion star expansion query.
@@ -1134,21 +1298,73 @@ class Indexer:
         # Return the number of new edges created
         return max(0, after_count - before_count)
 
-    def _reindex_view_definition(
-        self, view_name: str, db: GraphBackend, dialect: str | None
-    ) -> None:
-        """Re-index the file that defines a view.
+    def _ingest_external_consumers(self, db: GraphBackend, path: Path) -> dict:
+        """Ingest declared external downstream consumers from .sqlcg.toml.
 
-        Args:
-            view_name: Qualified view name
-            db: GraphBackend instance
-            dialect: SQL dialect
+        Runs ONCE per index_repo call, after _expand_star_sources and before
+        set_indexed_sha. Uses upsert_nodes_bulk/upsert_edges_bulk exclusively
+        (never upsert_node/upsert_edge per-row — CLAUDE.md perf invariant).
+
+        Returns:
+            Dict with keys: consumers (int), edges (int), warnings (list[str])
         """
-        query = (
-            f"MATCH (t:{NodeLabel.TABLE} {{qualified: $name}})"
-            f"-[:{RelType.DEFINED_IN}]->(f:{NodeLabel.FILE}) "
-            "RETURN f.path AS path"
-        )
-        result = db.run_read(query, {"name": view_name})
-        for row in result:
-            self.reindex_file(row["path"], db, dialect)
+        specs = get_external_consumers(path)
+        if not specs:
+            return {"consumers": 0, "edges": 0, "warnings": []}
+
+        presentation_prefixes = get_presentation_prefixes(path)
+
+        # Gather all target table qualifieds across all specs for a single existence check
+        all_targets: list[str] = []
+        for spec in specs:
+            all_targets.extend(spec.consumes)
+
+        # Single UNWIND round-trip to check which targets exist as SqlTable nodes
+        if all_targets:
+            existing_rows = db.run_read(
+                "UNWIND $names AS n MATCH (t:SqlTable {qualified: n}) RETURN n AS qualified",
+                {"names": all_targets},
+            )
+            existing_tables: set[str] = {row["qualified"] for row in existing_rows}
+        else:
+            existing_tables = set()
+
+        warnings: list[str] = []
+        consumer_rows: list[dict] = []
+        edge_rows: list[dict] = []
+
+        for spec in specs:
+            consumer_rows.append({"name": spec.name, "consumer_type": spec.consumer_type})
+            for target in spec.consumes:
+                if target not in existing_tables:
+                    warnings.append(
+                        f"Warning: external consumer '{spec.name}' "
+                        f"references unknown table '{target}'"
+                    )
+                    continue
+                # Check if this target is presentation-facing
+                if presentation_prefixes and not any(
+                    target.startswith(p) for p in presentation_prefixes
+                ):
+                    warnings.append(
+                        f"Warning: external consumer '{spec.name}' "
+                        f"references non-presentation table '{target}'"
+                        f" — edge created anyway (advisory)"
+                    )
+                edge_rows.append({"src_key": target, "dst_key": spec.name})
+
+        # Bulk upsert nodes, then edges — never per-row
+        db.upsert_nodes_bulk(NodeLabel.EXTERNAL_CONSUMER, consumer_rows)
+        if edge_rows:
+            db.upsert_edges_bulk(
+                NodeLabel.TABLE,
+                NodeLabel.EXTERNAL_CONSUMER,
+                RelType.CONSUMED_BY,
+                edge_rows,
+            )
+
+        return {
+            "consumers": len(consumer_rows),
+            "edges": len(edge_rows),
+            "warnings": warnings,
+        }