src2sink 3.0.0__tar.gz → 3.2.0__tar.gz

{src2sink-3.0.0/src2sink.egg-info → src2sink-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: src2sink
-Version: 3.0.0
+Version: 3.2.0
 Summary: Cross-repository source-to-sink metabase for LLM-assisted SAST
 Author: Brett Crawley
 License-Expression: MIT

{src2sink-3.0.0 → src2sink-3.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "src2sink"
-version = "3.0.0"
+version = "3.2.0"
 description = "Cross-repository source-to-sink metabase for LLM-assisted SAST"
 authors = [{ name = "Brett Crawley" }]
 readme = "README.md"

{src2sink-3.0.0 → src2sink-3.2.0}/src2sink/aggregators/api_client_discovery.py

@@ -27,6 +27,7 @@ from __future__ import annotations
 
 import json
 import re
+import sys
 from collections import defaultdict
 from pathlib import Path
 from typing import Any
@@ -609,16 +610,108 @@ def _write_discovered(metabase_root: Path, entries: list[dict[str, Any]]) -> Non
     )
 
 
-def _load_bindings(path: Path) -> list[dict[str, Any]]:
-    """Return the authoritative bindings list, or [] if the file is missing/invalid."""
+def _load_bindings_file(path: Path) -> dict[str, Any]:
+    """Return the whole authoritative file, or an empty document if unreadable.
+
+    The *whole* document, not just its bindings. `promote` used to rewrite the
+    file as `{"bindings": [...]}` and drop every other top-level key — including
+    the `_comment` carrying the "never commit — internal topology" handling
+    notice on a gitignored, sensitivity-marked file. The marker disappeared
+    silently and the file still looked correct (`OI-42`).
+    """
     try:
         data = json.loads(path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError):
-        return []
-    bindings = data.get("bindings")
+    except OSError:
+        # Absent is ordinary — the first promote creates it.
+        if path.exists():
+            print(f"WARNING: could not read {path}; promoting into an empty file",
+                  file=sys.stderr)
+        return {}
+    except json.JSONDecodeError as exc:
+        # Malformed is not ordinary. Silently treating it as empty would discard
+        # every existing binding on the next write, which is exactly why
+        # `--allow-empty-api-clients` exists for the *load* path (`OI-36`).
+        print(f"WARNING: {path} is not valid JSON ({exc}); its existing bindings "
+              "will not be preserved by this promote", file=sys.stderr)
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def _load_bindings(path: Path) -> list[dict[str, Any]]:
+    """Return the authoritative bindings list, or [] if the file is missing/invalid."""
+    bindings = _load_bindings_file(path).get("bindings")
     return [b for b in bindings if isinstance(b, dict)] if isinstance(bindings, list) else []
 
 
+def _promotion_rejections(
+    candidate: dict[str, Any],
+    known_repos: frozenset[str],
+    endpoints: dict[str, set[str]],
+) -> list[str]:
+    """Why this candidate must not be promoted, or an empty list.
+
+    `promote` merged on trust and validated nothing, so every check was the
+    reviewer's to perform by hand. Two of the five documented gates are pure
+    computation the tool already does at discovery time — 50 of 191 candidates
+    in the first fleet review failed one of them (`OI-42`).
+
+    The reviewer keeps the judgement calls. This takes the arithmetic.
+    """
+    target = str(candidate.get("target_repo") or "")
+    artifact = candidate.get("maven_artifact") or "(none)"
+
+    if not target:
+        return [f"{artifact}: target_repo is empty — it resolved to no repo at all"]
+    if target not in known_repos:
+        return [
+            f"{artifact}: target_repo {target!r} is not a repo in this metabase; "
+            "promoting it would create an edge to a node that does not exist"
+        ]
+    service = _service_for_client_repo(target, endpoints)
+    if service:
+        return [
+            f"{artifact}: target_repo {target!r} declares no inbound endpoints and "
+            f"looks like a client library fronting {service!r} — a binding's "
+            "target is the service that receives the calls, not the library"
+        ]
+    return []
+
+
+def _screen_for_promotion(
+    metabase_root: Path, accepted: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    """Return the candidates that pass the mechanical gates, naming the rest."""
+    records = load_v2_repo_records(metabase_root)
+    known_repos = frozenset(repo_id(d) for d in records)
+    endpoints = _http_in_paths_by_repo(records)
+
+    mergeable: list[dict[str, Any]] = []
+    for cand in accepted:
+        reasons = _promotion_rejections(cand, known_repos, endpoints)
+        for reason in reasons:
+            print(f"REFUSED: {reason}", file=sys.stderr)
+        if not reasons:
+            mergeable.append(cand)
+    return mergeable
+
+
+def _merge_bindings(
+    bindings: list[dict[str, Any]],
+    index: dict[tuple[Any, Any], list[dict[str, Any]]],
+    mergeable: list[dict[str, Any]],
+) -> None:
+    """Merge candidates into ``bindings`` in place, refreshing every duplicate."""
+    for c in mergeable:
+        binding = {k: c[k] for k in _TUNABLE_FIELDS if k in c}
+        key = (binding.get("target_repo"), binding.get("maven_artifact"))
+        if key in index:
+            for existing in index[key]:
+                existing.update(binding)
+        else:
+            bindings.append(binding)
+            index[key].append(binding)
+
+
 def promote_api_clients(metabase_root: Path, target_path: Path) -> int:
     """Merge ``accepted`` candidates into the authoritative ``api-clients.json``.
 
@@ -626,25 +719,32 @@ def promote_api_clients(metabase_root: Path, target_path: Path) -> int:
     (by ``(target_repo, maven_artifact)``): a new coordinate is appended, an
     existing one is updated in place. Idempotent — re-running merges the same set
     without creating duplicates. Returns the number of accepted candidates merged.
+
+    Candidates failing a mechanically-checkable gate are **refused and named**
+    rather than merged, and the file's other top-level keys survive the rewrite
+    (`OI-42`).
     """
     candidates = _load_discovered(metabase_root / DISCOVERED_FILE).values()
     accepted = [c for c in candidates if c.get("status") == STATUS_ACCEPTED]
     if not accepted:
         return 0
 
-    bindings = _load_bindings(target_path)
-    index = {(b.get("target_repo"), b.get("maven_artifact")): b for b in bindings}
-    for c in accepted:
-        binding = {k: c[k] for k in _TUNABLE_FIELDS if k in c}
-        key = (binding.get("target_repo"), binding.get("maven_artifact"))
-        if key in index:
-            index[key].update(binding)
-        else:
-            bindings.append(binding)
-            index[key] = binding
+    mergeable = _screen_for_promotion(metabase_root, accepted)
 
+    document = _load_bindings_file(target_path)
+    bindings = _load_bindings(target_path)
+    # Every copy of a key, not just the last. Indexing a list that may hold
+    # duplicates into a dict keeps only the final one, so earlier copies were
+    # left at their old values — still present, still loaded, now inconsistent.
+    index: dict[tuple[Any, Any], list[dict[str, Any]]] = defaultdict(list)
+    for b in bindings:
+        index[(b.get("target_repo"), b.get("maven_artifact"))].append(b)
+    _merge_bindings(bindings, index, mergeable)
+
+    # The whole document, so `_comment` — which carries the "never commit"
+    # handling notice on a gitignored, sensitivity-marked file — survives.
+    document["bindings"] = bindings
     target_path.write_text(
-        json.dumps({"bindings": bindings}, indent=2, sort_keys=False) + "\n",
-        encoding="utf-8",
+        json.dumps(document, indent=2, sort_keys=False) + "\n", encoding="utf-8",
     )
-    return len(accepted)
+    return len(mergeable)

{src2sink-3.0.0 → src2sink-3.2.0}/src2sink/aggregators/auth_cards.py

@@ -7,7 +7,8 @@ from collections import Counter
 from pathlib import Path
 from typing import Any
 
-from ..graph_common import iter_nodes, load_v2_repo_records, repo_id
+from .fleet_pass import records_or_load
+from ..graph_common import iter_nodes, repo_id
 from ..renderers.markdown import md_table
 from ..sanitize import UNTRUSTED_CONTENT_NOTICE
 
@@ -60,10 +61,12 @@ def _collect_repo_auth(data: dict[str, Any]) -> dict[str, Any]:
 def write_auth_models_catalog(
     metabase_root: Path,
     repo_jsons: list[Path] | None = None,
+    *,
+    cards: list[dict[str, Any]] | None = None,
 ) -> list[dict[str, Any]]:
     """Write auth-model cards (JSONL + markdown) to conventions/ and return them."""
-    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
-    cards = [_collect_repo_auth(data) for data in records]
+    if cards is None:
+        cards = [_collect_repo_auth(d) for d in records_or_load(None, metabase_root, repo_jsons)]
 
     conv_dir = metabase_root / "conventions"
     conv_dir.mkdir(parents=True, exist_ok=True)

{src2sink-3.0.0 → src2sink-3.2.0}/src2sink/aggregators/crypto_cards.py

@@ -68,10 +68,15 @@ def _collect_repo_crypto(data: dict[str, Any]) -> dict[str, Any]:
 def write_crypto_agility_catalog(
     metabase_root: Path,
     repo_jsons: list[Path] | None = None,
+    *,
+    cards: list[dict[str, Any]] | None = None,
 ) -> list[dict[str, Any]]:
     """Write crypto-agility cards (JSONL + markdown) to conventions/ and return them."""
-    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
-    cards = [_collect_repo_crypto(data) for data in records]
+    if cards is None:
+        cards = [
+            _collect_repo_crypto(data)
+            for data in load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+        ]
 
     conv_dir = metabase_root / "conventions"
     conv_dir.mkdir(parents=True, exist_ok=True)

{src2sink-3.0.0 → src2sink-3.2.0}/src2sink/aggregators/data_stores.py

@@ -12,31 +12,56 @@ from ..sanitize import UNTRUSTED_CONTENT_NOTICE, for_markdown
 from ..renderers.markdown import md_table
 
 
-def _collect_stores(
-    records: list[dict[str, Any]],
-) -> tuple[dict[str, set[str]], dict[str, dict[str, Any]], "Counter[str]"]:
-    """Collect store->repos, store metadata, and per-repo SQL execution sink counts."""
-    store_repos: dict[str, set[str]] = defaultdict(set)
-    store_meta: dict[str, dict[str, Any]] = {}
-    sql_execution_by_repo: Counter[str] = Counter()
-    for data in records:
-        rid = repo_id(data)
-        for node in iter_nodes(data):
+class StoreCollector:
+    """Store->repos, store metadata, and per-repo SQL execution sink counts.
+
+    The reduction `_collect_stores` performed, turned inside out so it can run
+    inside a shared streamed pass (`OI-41`). It retains only what it derives —
+    a few thousand store keys — never the records it derived them from.
+    """
+
+    def __init__(self) -> None:
+        """Start an empty reduction."""
+        self.store_repos: dict[str, set[str]] = defaultdict(set)
+        self.store_meta: dict[str, dict[str, Any]] = {}
+        self.sql_execution_by_repo: Counter[str] = Counter()
+
+    def consume(self, record: dict[str, Any]) -> None:
+        """Fold one repo record in."""
+        rid = repo_id(record)
+        for node in iter_nodes(record):
             family = node.get("family", "")
             kind = node.get("kind")
             if family == "data-store" and kind == "store":
                 key = store_key_from_node(node)
                 if not key:
                     continue
-                store_repos[key].add(rid)
+                self.store_repos[key].add(rid)
                 detail = node.get("detail") or {}
-                store_meta.setdefault(key, {
+                self.store_meta.setdefault(key, {
                     "vendor": detail.get("vendor", "?"),
                     "sample_file": f"{node.get('file')}:{node.get('line')}",
                 })
             elif family == "sql" and kind == "sink" and (node.get("detail") or {}).get("execution", True):
-                sql_execution_by_repo[rid] += 1
-    return store_repos, store_meta, sql_execution_by_repo
+                self.sql_execution_by_repo[rid] += 1
+
+    def result(self) -> tuple[dict[str, set[str]], dict[str, dict[str, Any]], "Counter[str]"]:
+        """The finished reduction, in the shape the renderer expects."""
+        return self.store_repos, self.store_meta, self.sql_execution_by_repo
+
+
+def _collect_stores(
+    records: list[dict[str, Any]],
+) -> tuple[dict[str, set[str]], dict[str, dict[str, Any]], "Counter[str]"]:
+    """Collect store->repos, store metadata, and per-repo SQL execution sink counts.
+
+    Retained as the non-streaming entry point; the reduction itself lives in
+    :class:`StoreCollector` so one pass can drive it alongside the others.
+    """
+    collector = StoreCollector()
+    for data in records:
+        collector.consume(data)
+    return collector.result()
 
 
 def _stores_by_vendor(
@@ -65,10 +90,22 @@ def _write_data_store_jsonl(
             }, ensure_ascii=False) + "\n")
 
 
-def write_data_store_graph(metabase_root: Path, repo_jsons: list[Path]) -> None:
-    """Write the bipartite data-store graph markdown + jsonl from v2 store nodes."""
-    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
-    store_repos, store_meta, sql_execution_by_repo = _collect_stores(records)
+def write_data_store_graph(
+    metabase_root: Path,
+    repo_jsons: list[Path],
+    *,
+    collected: tuple[dict[str, set[str]], dict[str, dict[str, Any]], "Counter[str]"] | None = None,
+) -> None:
+    """Write the bipartite data-store graph markdown + jsonl from v2 store nodes.
+
+    ``collected`` lets a caller that has already streamed the fleet hand the
+    reduction in, rather than this parsing the metabase again (`OI-41`).
+    """
+    if collected is None:
+        collected = _collect_stores(
+            load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+        )
+    store_repos, store_meta, sql_execution_by_repo = collected
 
     out_dir = metabase_root / "graphs"
     out_dir.mkdir(parents=True, exist_ok=True)

src2sink-3.2.0/src2sink/aggregators/fleet_pass.py

@@ -0,0 +1,120 @@
+"""One streamed pass over the metabase, shared by every aggregator (`OI-41`).
+
+Aggregation parsed the whole metabase **fourteen times per run** — 2.2 GB each on
+the observed fleet, so ~31 GB of JSON decoded to read the same bytes over and
+over. Measured A/B, that repetition is **67% of aggregation time**, and
+aggregation is 78% of the run.
+
+The one-line fix is the wrong line. Memoising the load removes the repetition and
+keeps one *held* copy of the fleet where re-parsing created and discarded:
+
+```
+peak RSS, 14 parses  : 148 MB
+peak RSS, 1 parse    : 266 MB     (+118 MB on a 29 MB metabase)
+```
+
+At the measured ~6.5x expansion that is several more GB resident on a fleet whose
+aggregation already peaks at 5.75 GiB, on a host observed swapping — `OI-15`'s
+ceiling, reached through the fix.
+
+So: **stream once, and reduce as you go.** Each collector sees every record in
+turn and retains only its own reduced state, which is orders smaller than the
+records it came from. Time is one parse; memory is one record plus the results.
+
+Every aggregator's `_collect_*` was already a pure `for data in records:`
+reduction, so a collector is that loop turned inside out — accumulators in
+`__init__`, body in `consume`, return in `result`. The conversion is mechanical,
+and `tests/test_aggregate_output_golden.py` pins all 27 generated artefacts, so
+one that changes any output fails immediately.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+from pathlib import Path
+from typing import Any, Generic, Protocol, TypeVar
+
+from ..graph_common import iter_v2_repo_records, load_v2_repo_records
+
+# Covariant: a collector only ever *produces* its result type, so a
+# `RecordCollector[QueueGraph]` is usable wherever a `RecordCollector[object]`
+# is expected.
+T_co = TypeVar("T_co", covariant=True)
+
+
+class RecordCollector(Protocol[T_co]):
+    """A reduction over the fleet that sees one record at a time.
+
+    The contract is deliberately narrow: `consume` may not retain the record it
+    is given, because retaining records is the memory cost this exists to avoid.
+    It retains whatever it derives from them instead.
+    """
+
+    def consume(self, record: dict[str, Any]) -> None:
+        """Fold one repo record into the accumulating result.
+
+        Must not retain ``record``: holding records is the memory cost this
+        whole mechanism exists to avoid.
+        """
+
+    def result(self) -> T_co:
+        """The finished reduction. Called once, after the pass."""
+
+
+def run_fleet_pass(
+    metabase_root: Path,
+    collectors: Iterable[RecordCollector[Any]],
+    *,
+    json_paths: list[Path] | None = None,
+) -> None:
+    """Stream every record once, offering it to every collector.
+
+    The ordering guarantee matters: records arrive in the same order
+    `load_v2_repo_records` produced them, so a collector whose output depends on
+    record order — several sort only at the end — is unaffected by the move.
+    """
+    live = list(collectors)
+    if not live:
+        return
+    for record in iter_v2_repo_records(metabase_root, json_paths=json_paths):
+        for collector in live:
+            collector.consume(record)
+
+
+class MapCollector(Generic[T_co]):
+    """Applies a per-record function and keeps the results, in record order.
+
+    Several aggregators reduce with a plain `[fn(data) for data in records]` —
+    one card per repo. They share this rather than each restating the loop, and
+    it keeps the derived cards while the records that produced them are released.
+    """
+
+    def __init__(self, fn: Callable[[dict[str, Any]], T_co]) -> None:
+        """Collect ``fn`` applied to every record."""
+        self._fn = fn
+        self._results: list[T_co] = []
+
+    def consume(self, record: dict[str, Any]) -> None:
+        """Map one record and keep only what came back."""
+        self._results.append(self._fn(record))
+
+    def result(self) -> list[T_co]:
+        """The mapped results, in the order the records arrived."""
+        return self._results
+
+
+def records_or_load(
+    records: list[dict[str, Any]] | None,
+    metabase_root: Path,
+    json_paths: list[Path] | None = None,
+) -> list[dict[str, Any]]:
+    """Return the caller's records, loading the fleet only if it supplied none.
+
+    Every converted aggregator keeps a non-streaming entry point so it still
+    works when called directly. Writing that fallback as a guard clause inside
+    each `write_*` added a branch to functions already at the complexity limit,
+    for no reader benefit — the interesting code is below it.
+    """
+    if records is not None:
+        return records
+    return load_v2_repo_records(metabase_root, json_paths=json_paths)

src2sink-3.2.0/src2sink/aggregators/graphs.py

@@ -0,0 +1,121 @@
+"""Orchestrate v2 cross-repo graph generation."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from ..graph_common import (
+    iter_v2_repo_records,
+    load_v2_repo_records,
+    v2_record_paths,
+)
+from ..index_store import build_index
+from .. import run_timing
+from .data_stores import StoreCollector, write_data_store_graph
+from .index_v2 import _row_from_record, write_index_v2
+from .openapi_edges import write_openapi_artifacts
+from .payload_producers import build_producer_indices, write_payload_producer_index
+from .phase3 import aggregate_phase3_v2
+from .pii_flow_v2 import PiiFlowCollector, write_pii_flow_v2
+from .fleet_pass import MapCollector, run_fleet_pass
+from .queues import QueueCollector, write_queue_graph
+from .service_calls import collect_service_edges, write_service_call_graph
+from .traces_index import write_traces_index
+
+
+def write_fleet_index(
+    metabase_root: Path,
+    repo_jsons: list[Path],
+    *,
+    repos_root: Path | None = None,
+    producer_indices: list[Any] | None = None,
+    call_edges: list[Any] | None = None,
+) -> Path:
+    """Persist the four things a trace consults, keyed by target repo (`OI-15`).
+
+    Built here because aggregation already walks every repo, so the index is one
+    more pass over data that is being read anyway rather than a second crawl.
+
+    The two fleet-wide derivations are computed the same way `trace` computed
+    them, from the same functions — the index is a cache of this code's output,
+    not a reimplementation of it, which is what keeps the two from drifting.
+
+    ``producer_indices`` is taken from the caller when aggregation has already
+    built it. Rebuilding it here meant the fleet's source tree was scanned a
+    second time, which doubled the slowest step of the whole run (`OI-30`).
+    """
+    paths = v2_record_paths(metabase_root, json_paths=repo_jsons)
+    records: list[dict[str, Any]] | None = None
+    if call_edges is None:
+        # Fleet-wide and target-independent, and three consumers each recomputed
+        # it — the derivation `OI-14` identified as dominating cost (`OI-41`).
+        records = load_v2_repo_records(metabase_root, json_paths=paths)
+        call_edges, _unmatched = collect_service_edges(records)
+    if producer_indices is None:
+        producer_indices = build_producer_indices(
+            metabase_root, repos_root=repos_root, json_paths=paths,
+        )
+    del records  # the streamed pass below must not be served from this list
+
+    return build_index(
+        metabase_root,
+        paths,
+        call_edges=call_edges,
+        producer_indices=producer_indices,
+        records=iter_v2_repo_records(metabase_root, json_paths=paths),
+    )
+
+
+def aggregate_graphs_v2(
+    metabase_root: Path,
+    repo_jsons: list[Path],
+    *,
+    repos_root: Path | None = None,
+    phase3: bool = True,
+    fleet_index: bool = True,
+) -> None:
+    """Run all v2 cross-repo graph writers (service, queue, data-store, PII, phase 3)."""
+    # One load serves the report and the index, and the edges it computes are
+    # handed on rather than recomputed (`OI-41`).
+    with run_timing.phase("shared-load"):
+        shared = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+    with run_timing.phase("service-call-graph"):
+        call_edges, _unmatched = write_service_call_graph(
+            metabase_root, repo_jsons, repos_root=repos_root, records=shared,
+        )
+    if repos_root:
+        with run_timing.phase("openapi"):
+            write_openapi_artifacts(metabase_root, repos_root, repo_jsons)
+    # One streamed pass drives every converted aggregator, instead of each
+    # parsing the whole metabase for itself (`OI-41`). Collectors retain their
+    # reductions, never the records — so this buys the time without the resident
+    # copy that memoising would cost (`OI-15`).
+    queues, stores = QueueCollector(), StoreCollector()
+    index_rows = MapCollector(_row_from_record)
+    pii_flow = PiiFlowCollector()
+    with run_timing.phase("streamed-pass"):
+        run_fleet_pass(
+            metabase_root, (queues, stores, index_rows, pii_flow), json_paths=repo_jsons,
+        )
+    with run_timing.phase("queue-and-store-graphs"):
+        write_queue_graph(metabase_root, repo_jsons, graph=queues.result())
+        write_data_store_graph(metabase_root, repo_jsons, collected=stores.result())
+    with run_timing.phase("payload-producers"):
+        producer_indices = write_payload_producer_index(
+            metabase_root, repo_jsons, repos_root=repos_root, records=shared,
+        )
+    del shared   # released before the streamed pass below
+    with run_timing.phase("index-and-pii"):
+        write_index_v2(metabase_root, repo_jsons, rows=index_rows.result())
+        write_pii_flow_v2(metabase_root, repo_jsons, counts=pii_flow.result())
+        write_traces_index(metabase_root)
+    if phase3:
+        with run_timing.phase("phase3"):
+            aggregate_phase3_v2(metabase_root, repo_jsons, edges=call_edges)
+    if fleet_index:
+        with run_timing.phase("fleet-index"):
+            write_fleet_index(
+                metabase_root, repo_jsons, repos_root=repos_root,
+                producer_indices=producer_indices, call_edges=call_edges,
+            )

{src2sink-3.0.0 → src2sink-3.2.0}/src2sink/aggregators/index_v2.py

@@ -8,7 +8,8 @@ from collections import Counter, defaultdict
 from pathlib import Path
 from typing import Any
 
-from ..graph_common import iter_nodes, load_v2_repo_records
+from .fleet_pass import records_or_load
+from ..graph_common import iter_nodes
 from ..renderers.markdown import md_table
 
 
@@ -43,10 +44,19 @@ def _row_from_record(data: dict[str, Any]) -> dict[str, Any]:
     }
 
 
-def write_index_v2(metabase_root: Path, repo_jsons: list[Path]) -> None:
-    """Write the v2 repo index JSON and the all/by-group/by-language markdown pages."""
-    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
-    rows = [_row_from_record(data) for data in records]
+def write_index_v2(
+    metabase_root: Path,
+    repo_jsons: list[Path],
+    *,
+    rows: list[dict[str, Any]] | None = None,
+) -> None:
+    """Write the v2 repo index JSON and the all/by-group/by-language markdown pages.
+
+    ``rows`` lets a caller that has already streamed the fleet hand the mapped
+    rows in, rather than this parsing the metabase again (`OI-41`).
+    """
+    if rows is None:
+        rows = [_row_from_record(d) for d in records_or_load(None, metabase_root, repo_jsons)]
     index_dir = metabase_root / "index"
     index_dir.mkdir(parents=True, exist_ok=True)
 

{src2sink-3.0.0 → src2sink-3.2.0}/src2sink/aggregators/payload_producers.py

@@ -329,6 +329,7 @@ def build_producer_indices(
     *,
     repos_root: Path | None = None,
     json_paths: list[Path] | None = None,
+    records: list[dict[str, Any]] | None = None,
 ) -> list[ProducerIndex]:
     """Build a producer index per known API-client binding.
 
@@ -343,7 +344,8 @@ def build_producer_indices(
     Returns:
         One ProducerIndex per binding, hits sorted by (source_repo, kind).
     """
-    records = load_v2_repo_records(metabase_root, json_paths=json_paths)
+    if records is None:
+        records = load_v2_repo_records(metabase_root, json_paths=json_paths)
     bindings = list(get_bindings())
 
     # One walk of the fleet for every binding, rather than one walk each. The
@@ -403,6 +405,7 @@ def write_payload_producer_index(
     repo_jsons: list[Path],
     *,
     repos_root: Path | None = None,
+    records: list[dict[str, Any]] | None = None,
 ) -> list[ProducerIndex]:
     """Write the payload-endpoint-producers markdown + jsonl catalogue.
 
@@ -419,6 +422,7 @@ def write_payload_producer_index(
         metabase_root,
         repos_root=root,
         json_paths=repo_jsons,
+        records=records,
     )
 
     out_dir = metabase_root / "graphs"