src2sink 1.0.0__py3-none-any.whl

src2sink/__init__.py

@@ -0,0 +1 @@
+"""src2sink — extract and sync source-repo metabase records."""

src2sink/aggregators/__init__.py

@@ -0,0 +1,3 @@
+"""Cross-repo aggregators (service calls, queues, data stores) — Phase 2."""
+
+from __future__ import annotations

src2sink/aggregators/api_client_discovery.py

@@ -0,0 +1,259 @@
+"""Discover candidate ``api-clients.json`` bindings from the metabase (plan B11).
+
+``api-clients.json`` maps a first-party HTTP *client library* to the backend
+*service* it talks to (see ``docs/api-clients-json.md``). Those edges cannot be
+inferred from consumer source alone, so the file is normally authored by hand.
+
+This module drafts **candidates** — never authoritative entries — by mining data
+the scan already produced:
+
+* a consumer repo's ``dependencies_internal`` naming a ``*-client`` / ``*-sdk``
+  artifact identifies it as a *consumer* of that client library;
+* the component-identity index (``repo_utils._build_component_identity_index``)
+  resolves that artifact's coordinate to the *publishing* repo — the candidate
+  ``target_repo``;
+* the target repo's ``http-in`` nodes supply candidate ``paths``.
+
+Because it runs in the aggregation phase (whole fleet in memory) it needs no new
+extractor and is unaffected by the worker-init binding-load ordering. Candidates
+are written to ``metabase/api-clients.discovered.json`` with a ``confidence`` and
+an ``evidence`` block. A human sets each candidate's ``status`` to ``accepted``
+or ``rejected``; :func:`promote_api_clients` then merges only the accepted ones
+into the authoritative (gitignored) file. Nothing is ever auto-merged — an
+unverified binding poisons the taint graph with phantom cross-service edges.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from ..graph_common import load_v2_repo_records, repo_id
+
+DISCOVERED_FILE = "api-clients.discovered.json"
+
+# Human decisions on a candidate, preserved verbatim across regeneration.
+STATUS_PENDING = "pending"
+STATUS_ACCEPTED = "accepted"
+STATUS_REJECTED = "rejected"
+
+# Fields a reviewer may tune; preserved once they mark a candidate non-pending.
+_TUNABLE_FIELDS = (
+    "target_repo",
+    "maven_artifact",
+    "import_prefix",
+    "paths",
+    "payload_fields",
+    "service_aliases",
+    "class_patterns",
+)
+
+# Artifact-id suffixes that strongly signal a published HTTP client library.
+_CLIENT_SUFFIXES = (
+    "-client",
+    "-api-client",
+    "-rest-client",
+    "-client-java",
+    "-java-client",
+    "-sdk",
+    "-sdk-java",
+)
+
+
+def _looks_like_client_artifact(artifact: str) -> bool:
+    """True if an artifact id looks like a published HTTP client library."""
+    a = (artifact or "").lower()
+    return any(a.endswith(suffix) for suffix in _CLIENT_SUFFIXES)
+
+
+def _http_in_paths_by_repo(records: list[dict[str, Any]]) -> dict[str, set[str]]:
+    """Map each repo id to the set of ``http-in`` endpoint paths it declares."""
+    out: dict[str, set[str]] = defaultdict(set)
+    for data in records:
+        rid = repo_id(data)
+        for node in data.get("nodes", []):
+            if node.get("family") != "http-in":
+                continue
+            path = (node.get("detail") or {}).get("path")
+            if isinstance(path, str) and path:
+                out[rid].add(path)
+    return out
+
+
+def _key(target_repo: str, artifact: str) -> str:
+    """Stable candidate key used for de-dup and for preserving reviewer edits."""
+    return f"{target_repo}\t{artifact}"
+
+
+def _load_discovered(path: Path) -> dict[str, dict[str, Any]]:
+    """Return prior candidates keyed by (target_repo, artifact), or {} if absent."""
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    out: dict[str, dict[str, Any]] = {}
+    for c in data.get("candidates", []):
+        if isinstance(c, dict) and "maven_artifact" in c:
+            out[_key(c.get("target_repo", ""), c["maven_artifact"])] = c
+    return out
+
+
+def _confidence(target_repo: str, scanned: set[str], has_paths: bool) -> str:
+    """Rate a candidate: resolved-and-in-fleet-with-paths is strongest."""
+    if target_repo and target_repo in scanned:
+        return "high" if has_paths else "medium"
+    if target_repo:
+        return "medium"  # resolved to a path not present as a scanned repo
+    return "low"  # artifact looks like a client but coordinate did not resolve
+
+
+def _collect_candidates(
+    records: list[dict[str, Any]],
+    resolve: Any,
+) -> dict[str, dict[str, Any]]:
+    """Mine consumer ``dependencies_internal`` into per-(target, artifact) candidates."""
+    paths_by_repo = _http_in_paths_by_repo(records)
+    cands: dict[str, dict[str, Any]] = {}
+    for data in records:
+        consumer = repo_id(data)
+        for dep in data.get("dependencies_internal", []):
+            aid = dep.get("artifactId", "")
+            gid = dep.get("groupId", "")
+            if not _looks_like_client_artifact(aid):
+                continue
+            coord = f"{gid}:{aid}" if gid else aid
+            target = resolve(coord) or ""
+            key = _key(target, aid)
+            cand = cands.setdefault(
+                key,
+                {
+                    "target_repo": target,
+                    "maven_artifact": aid,
+                    "import_prefix": gid,  # best-effort; reviewer tightens
+                    "paths": sorted(paths_by_repo.get(target, ())),
+                    "payload_fields": ["sql"],  # documented default assumption
+                    "service_aliases": [target.split("/")[-1]] if target else [],
+                    "class_patterns": [],
+                    "coordinate": coord,
+                    "consumers": set(),
+                },
+            )
+            cand["consumers"].add(consumer)
+    return cands
+
+
+def _build_entry(
+    cand: dict[str, Any],
+    scanned: set[str],
+    prev: dict[str, Any] | None,
+) -> dict[str, Any]:
+    """Assemble a candidate entry, preserving a reviewer's non-pending decision."""
+    consumers = sorted(cand.pop("consumers"))
+    coordinate = cand.pop("coordinate")
+    has_paths = bool(cand["paths"])
+    evidence = {
+        "coordinate": coordinate,
+        "consumers": consumers,
+        "resolved": bool(cand["target_repo"]),
+        "paths_from_target_scan": has_paths and cand["target_repo"] in scanned,
+    }
+    # A reviewer who set status to accepted/rejected has tuned the binding; keep
+    # their edits and status, only refreshing evidence + confidence.
+    if prev and prev.get("status") in (STATUS_ACCEPTED, STATUS_REJECTED):
+        entry = {k: prev.get(k, cand.get(k)) for k in _TUNABLE_FIELDS}
+        entry["status"] = prev["status"]
+    else:
+        entry = {k: cand[k] for k in _TUNABLE_FIELDS}
+        entry["status"] = STATUS_PENDING
+    entry["confidence"] = _confidence(
+        str(entry["target_repo"] or ""), scanned, bool(entry["paths"])
+    )
+    entry["evidence"] = evidence
+    return entry
+
+
+def discover_api_clients(
+    metabase_root: Path,
+    repo_jsons: list[Path],
+    repos_root: Path,
+) -> int:
+    """Write ``metabase/api-clients.discovered.json`` and return the candidate count.
+
+    Regenerates every candidate from the current metabase, preserving the
+    ``status`` and tuned fields of any candidate a reviewer has already accepted
+    or rejected. Never touches the authoritative ``api-clients.json``.
+    """
+    from ..repo_utils import _build_component_identity_index
+    from .library_source_map import _resolve_clone_path
+
+    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+    by_coord, by_name, by_full = _build_component_identity_index(repos_root)
+
+    def resolve(coord: str) -> str | None:
+        return _resolve_clone_path(coord, by_coord, by_name, by_full)
+
+    scanned = {repo_id(d) for d in records}
+    cands = _collect_candidates(records, resolve)
+    prev = _load_discovered(metabase_root / DISCOVERED_FILE)
+
+    entries = [
+        _build_entry(cand, scanned, prev.get(key))
+        for key, cand in sorted(cands.items())
+    ]
+    out = {
+        "_comment": (
+            "CANDIDATE api-client bindings discovered by src2sink-build "
+            "--discover-api-clients. NOT authoritative. Review each entry, set "
+            "'status' to 'accepted' or 'rejected' (tightening import_prefix/paths/"
+            "payload_fields as needed), then run --promote-api-clients to merge the "
+            "accepted ones into api-clients.json. Same sensitivity as that file."
+        ),
+        "candidates": entries,
+    }
+    (metabase_root / DISCOVERED_FILE).write_text(
+        json.dumps(out, indent=2, sort_keys=False) + "\n", encoding="utf-8"
+    )
+    return len(entries)
+
+
+def _load_bindings(path: Path) -> list[dict[str, Any]]:
+    """Return the authoritative bindings list, or [] if the file is missing/invalid."""
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return []
+    bindings = data.get("bindings")
+    return [b for b in bindings if isinstance(b, dict)] if isinstance(bindings, list) else []
+
+
+def promote_api_clients(metabase_root: Path, target_path: Path) -> int:
+    """Merge ``accepted`` candidates into the authoritative ``api-clients.json``.
+
+    Only candidates whose ``status`` a reviewer set to ``accepted`` are merged
+    (by ``(target_repo, maven_artifact)``): a new coordinate is appended, an
+    existing one is updated in place. Idempotent — re-running merges the same set
+    without creating duplicates. Returns the number of accepted candidates merged.
+    """
+    candidates = _load_discovered(metabase_root / DISCOVERED_FILE).values()
+    accepted = [c for c in candidates if c.get("status") == STATUS_ACCEPTED]
+    if not accepted:
+        return 0
+
+    bindings = _load_bindings(target_path)
+    index = {(b.get("target_repo"), b.get("maven_artifact")): b for b in bindings}
+    for c in accepted:
+        binding = {k: c[k] for k in _TUNABLE_FIELDS if k in c}
+        key = (binding.get("target_repo"), binding.get("maven_artifact"))
+        if key in index:
+            index[key].update(binding)
+        else:
+            bindings.append(binding)
+            index[key] = binding
+
+    target_path.write_text(
+        json.dumps({"bindings": bindings}, indent=2, sort_keys=False) + "\n",
+        encoding="utf-8",
+    )
+    return len(accepted)

src2sink/aggregators/auth_cards.py

@@ -0,0 +1,116 @@
+"""Per-repo auth model cards from v2 auth + http-in nodes (Phase 3)."""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+from ..graph_common import iter_nodes, load_v2_repo_records, repo_id
+from ..renderers.markdown import md_table
+from ..sanitize import UNTRUSTED_CONTENT_NOTICE
+
+MAX_CARD_ROWS = 500
+
+
+def _auth_maturity(patterns: "Counter[str]", http_in: int) -> str:
+    """Classify a repo's auth posture from its pattern counts and http-in count."""
+    if patterns.get("secured-authenticated") or patterns.get("spring-pre-authorize"):
+        return "authenticated-default"
+    if patterns.get("permit-all-config") or patterns.get("is-anonymous"):
+        return "anonymous-or-permit-all"
+    if http_in and not patterns:
+        return "http-without-auth-annotations"
+    return "unknown"
+
+
+def _collect_repo_auth(data: dict[str, Any]) -> dict[str, Any]:
+    """Build one repo's auth card from its `auth` and `http-in` flow nodes."""
+    rid = repo_id(data)
+    patterns: Counter[str] = Counter()
+    http_in = 0
+    public_paths: list[str] = []
+
+    for node in iter_nodes(data):
+        family = node.get("family", "")
+        if family == "auth":
+            label = (node.get("detail") or {}).get("pattern", "unknown")
+            patterns[label] += 1
+        elif family == "http-in":
+            http_in += 1
+            path = (node.get("detail") or {}).get("path", "")
+            is_public = patterns.get("permit-all-config") or patterns.get("is-anonymous")
+            if is_public and path and len(public_paths) < 10:
+                public_paths.append(path)
+
+    frameworks = data.get("frameworks") or []
+    maturity = _auth_maturity(patterns, http_in)
+
+    return {
+        "repo": rid,
+        "frameworks": frameworks,
+        "http_in_count": http_in,
+        "auth_patterns": dict(patterns.most_common(12)),
+        "maturity": maturity,
+        "sample_public_paths": public_paths[:5],
+    }
+
+
+def write_auth_models_catalog(
+    metabase_root: Path,
+    repo_jsons: list[Path] | None = None,
+) -> list[dict[str, Any]]:
+    """Write auth-model cards (JSONL + markdown) to conventions/ and return them."""
+    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+    cards = [_collect_repo_auth(data) for data in records]
+
+    conv_dir = metabase_root / "conventions"
+    conv_dir.mkdir(parents=True, exist_ok=True)
+    jsonl_path = conv_dir / "auth-models.jsonl"
+    with jsonl_path.open("w", encoding="utf-8") as fh:
+        for card in cards:
+            fh.write(json.dumps(card, ensure_ascii=False) + "\n")
+
+    fleet_patterns: Counter[str] = Counter()
+    maturity_counts: Counter[str] = Counter()
+    for card in cards:
+        maturity_counts[card["maturity"]] += 1
+        for pat, count in card["auth_patterns"].items():
+            fleet_patterns[pat] += count
+
+    body_parts = [
+        "# Auth-model conventions (v2)\n",
+        UNTRUSTED_CONTENT_NOTICE,
+        "_Per-repo cards from `auth` and `http-in` flow nodes. High counts of "
+        "`permit-all` / `is-anonymous` warrant manual review._\n",
+        "\n## Fleet pattern frequency\n",
+        md_table(
+            ["Pattern", "Count"],
+            [[p, str(c)] for p, c in fleet_patterns.most_common(30)],
+        ),
+        "\n## Maturity buckets\n",
+        md_table(
+            ["Bucket", "Repos"],
+            [[k, str(v)] for k, v in maturity_counts.most_common()],
+        ),
+        "\n## Per-repo cards (sample)\n",
+        md_table(
+            ["Repo", "Maturity", "HTTP-in", "Top auth patterns"],
+            [
+                [
+                    c["repo"],
+                    c["maturity"],
+                    str(c["http_in_count"]),
+                    ", ".join(f"{k}({v})" for k, v in list(c["auth_patterns"].items())[:4]),
+                ]
+                for c in sorted(cards, key=lambda x: x["repo"])[:MAX_CARD_ROWS]
+            ],
+        ),
+    ]
+    tail = max(0, len(cards) - MAX_CARD_ROWS)
+    if tail:
+        body_parts.append(f"\n_{tail} more in `auth-models.jsonl`._\n")
+
+    (conv_dir / "auth-models.md").write_text("\n".join(body_parts), encoding="utf-8")
+    return cards

src2sink/aggregators/crypto_cards.py

@@ -0,0 +1,132 @@
+"""Per-repo crypto agility cards from v2 crypto nodes (Phase 3)."""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+from ..graph_common import iter_nodes, load_v2_repo_records, repo_id
+from ..renderers.markdown import md_table
+from ..sanitize import UNTRUSTED_CONTENT_NOTICE, for_markdown
+
+WEAK_ALGORITHMS = frozenset({
+    "MD5", "SHA1", "SHA-1", "DES", "RC4", "3DES", "ECB",
+})
+MAX_CARD_ROWS = 500
+
+
+def _crypto_maturity(has_algorithms: bool, has_key_sources: bool) -> str:
+    """Classify a repo's crypto posture from algorithm and key-source presence."""
+    if has_algorithms and has_key_sources:
+        return "config-driven"
+    if has_algorithms:
+        return "hardcoded-algorithms"
+    if has_key_sources:
+        return "config-keys-only"
+    return "none-detected"
+
+
+def _collect_repo_crypto(data: dict[str, Any]) -> dict[str, Any]:
+    """Build one repo's crypto card from its crypto-related flow nodes."""
+    rid = repo_id(data)
+    algorithms: Counter[str] = Counter()
+    key_sources: Counter[str] = Counter()
+    weak: list[str] = []
+
+    for node in iter_nodes(data):
+        family = node.get("family", "")
+        detail = node.get("detail") or {}
+        if family == "crypto-algorithm":
+            algo = (detail.get("algorithm") or "?").upper()
+            algorithms[algo] += 1
+            if algo in WEAK_ALGORITHMS or "MD5" in algo or "SHA1" in algo:
+                # raw is PII-redacted upstream; neutralise the markdown structure of
+                # both the sample and the untrusted file path (SAST finding 1 class).
+                weak.append(
+                    f"{for_markdown(detail.get('raw', algo), max_len=60)} @ "
+                    f"{for_markdown(node.get('file'), max_len=80)}:{node.get('line')}"
+                )
+        elif family == "crypto-config":
+            src = detail.get("key_source") or detail.get("source") or "config"
+            key_sources[str(src)] += 1
+        elif family == "config-security":
+            key_sources["config-security"] += 1
+
+    maturity = _crypto_maturity(bool(algorithms), bool(key_sources))
+
+    return {
+        "repo": rid,
+        "maturity": maturity,
+        "algorithms": dict(algorithms.most_common(15)),
+        "key_sources": dict(key_sources.most_common(10)),
+        "weak_samples": weak[:5],
+    }
+
+
+def write_crypto_agility_catalog(
+    metabase_root: Path,
+    repo_jsons: list[Path] | None = None,
+) -> list[dict[str, Any]]:
+    """Write crypto-agility cards (JSONL + markdown) to conventions/ and return them."""
+    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+    cards = [_collect_repo_crypto(data) for data in records]
+
+    conv_dir = metabase_root / "conventions"
+    conv_dir.mkdir(parents=True, exist_ok=True)
+    jsonl_path = conv_dir / "crypto-agility.jsonl"
+    with jsonl_path.open("w", encoding="utf-8") as fh:
+        for card in cards:
+            fh.write(json.dumps(card, ensure_ascii=False) + "\n")
+
+    algo_fleet: Counter[str] = Counter()
+    maturity_counts: Counter[str] = Counter()
+    weak_repos = 0
+    for card in cards:
+        maturity_counts[card["maturity"]] += 1
+        if card["weak_samples"]:
+            weak_repos += 1
+        for algo, count in card["algorithms"].items():
+            algo_fleet[algo] += count
+
+    body_parts = [
+        "# Crypto agility (v2)\n",
+        UNTRUSTED_CONTENT_NOTICE,
+        "_Algorithms from `crypto-algorithm` nodes; key sources from "
+        "`crypto-config` / `config-security`. Classify repos as hardcoded vs "
+        "config-driven for rotation planning._\n",
+        "\n## Fleet algorithms\n",
+        md_table(
+            ["Algorithm", "Count"],
+            [[a, str(c)] for a, c in algo_fleet.most_common(25)],
+        ),
+        "\n## Maturity buckets\n",
+        md_table(
+            ["Bucket", "Repos"],
+            [[k, str(v)] for k, v in maturity_counts.most_common()],
+        ),
+        f"\n_Repos with weak algorithm hits: **{weak_repos}**._\n",
+        "\n## Per-repo cards (sample)\n",
+        md_table(
+            ["Repo", "Maturity", "Algorithms", "Key sources", "Weak"],
+            [
+                [
+                    c["repo"],
+                    c["maturity"],
+                    ", ".join(list(c["algorithms"].keys())[:4]),
+                    ", ".join(list(c["key_sources"].keys())[:3]) or "—",
+                    "yes" if c["weak_samples"] else "—",
+                ]
+                for c in sorted(cards, key=lambda x: x["repo"])[:MAX_CARD_ROWS]
+            ],
+        ),
+    ]
+    tail = max(0, len(cards) - MAX_CARD_ROWS)
+    if tail:
+        body_parts.append(f"\n_{tail} more in `crypto-agility.jsonl`._\n")
+
+    md_path = conv_dir / "crypto-agility.md"
+    body = "\n".join(body_parts)
+    md_path.write_text(body, encoding="utf-8")
+    return cards

src2sink/aggregators/data_stores.py

@@ -0,0 +1,125 @@
+"""Data-store graph aggregator — v2 config + code store nodes."""
+
+from __future__ import annotations
+
+import json
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import Any
+
+from ..graph_common import iter_nodes, load_v2_repo_records, repo_id, store_key_from_node
+from ..sanitize import UNTRUSTED_CONTENT_NOTICE, for_markdown
+from ..renderers.markdown import md_table
+
+
+def _collect_stores(
+    records: list[dict[str, Any]],
+) -> tuple[dict[str, set[str]], dict[str, dict[str, Any]], "Counter[str]"]:
+    """Collect store->repos, store metadata, and per-repo SQL execution sink counts."""
+    store_repos: dict[str, set[str]] = defaultdict(set)
+    store_meta: dict[str, dict[str, Any]] = {}
+    sql_execution_by_repo: Counter[str] = Counter()
+    for data in records:
+        rid = repo_id(data)
+        for node in iter_nodes(data):
+            family = node.get("family", "")
+            kind = node.get("kind")
+            if family == "data-store" and kind == "store":
+                key = store_key_from_node(node)
+                if not key:
+                    continue
+                store_repos[key].add(rid)
+                detail = node.get("detail") or {}
+                store_meta.setdefault(key, {
+                    "vendor": detail.get("vendor", "?"),
+                    "sample_file": f"{node.get('file')}:{node.get('line')}",
+                })
+            elif family == "sql" and kind == "sink" and (node.get("detail") or {}).get("execution", True):
+                sql_execution_by_repo[rid] += 1
+    return store_repos, store_meta, sql_execution_by_repo
+
+
+def _stores_by_vendor(
+    store_repos: dict[str, set[str]], store_meta: dict[str, dict[str, Any]]
+) -> dict[str, list[tuple[str, list[str]]]]:
+    """Group stores by vendor into (store_key, sorted repos) entries."""
+    by_vendor: dict[str, list[tuple[str, list[str]]]] = defaultdict(list)
+    for key, repos in sorted(store_repos.items(), key=lambda x: (-len(x[1]), x[0])):
+        vendor = store_meta.get(key, {}).get("vendor", key.split(":")[0])
+        by_vendor[vendor].append((key, sorted(repos)))
+    return by_vendor
+
+
+def _write_data_store_jsonl(
+    path: Path, store_repos: dict[str, set[str]], store_meta: dict[str, dict[str, Any]]
+) -> None:
+    """Write one JSON line per store with vendor, referencing repos, and sample file."""
+    with path.open("w", encoding="utf-8") as fh:
+        for key, repos in sorted(store_repos.items()):
+            meta = store_meta.get(key, {})
+            fh.write(json.dumps({
+                "store_key": key,
+                "vendor": meta.get("vendor"),
+                "repos": sorted(repos),
+                "sample_file": meta.get("sample_file"),
+            }, ensure_ascii=False) + "\n")
+
+
+def write_data_store_graph(metabase_root: Path, repo_jsons: list[Path]) -> None:
+    """Write the bipartite data-store graph markdown + jsonl from v2 store nodes."""
+    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+    store_repos, store_meta, sql_execution_by_repo = _collect_stores(records)
+
+    out_dir = metabase_root / "graphs"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    by_vendor = _stores_by_vendor(store_repos, store_meta)
+
+    md: list[str] = [
+        "# Data-store graph (v2)\n",
+        UNTRUSTED_CONTENT_NOTICE,
+        "_Bipartite view: config-discovered stores (`data-store` nodes) and "
+        "repos that reference them. SQL execution sink counts show repos with "
+        "runtime DB access even when JDBC URL is only in deployed config._\n",
+        "\n## Summary\n",
+        md_table(
+            ["Vendor", "Store count", "Repos touching"],
+            [
+                [
+                    v,
+                    str(len(entries)),
+                    str(len({r for _, rs in entries for r in rs})),
+                ]
+                for v, entries in sorted(by_vendor.items())
+            ],
+        ),
+    ]
+
+    for vendor, entries in sorted(by_vendor.items()):
+        # vendor is an untrusted extracted config value written into a heading.
+        safe_vendor = for_markdown(vendor, max_len=80)
+        md.append(f"\n## {safe_vendor}\n\n")
+        md.append(md_table(
+            ["Store key", "Repos", "Sample config ref"],
+            [
+                [
+                    key,
+                    ", ".join(repos[:8]) + (" …" if len(repos) > 8 else ""),
+                    store_meta.get(key, {}).get("sample_file", ""),
+                ]
+                for key, repos in entries[:80]
+            ],
+        ))
+        if len(entries) > 80:
+            md.append(f"\n_{len(entries) - 80} more {safe_vendor} stores in jsonl._\n")
+
+    md.append("\n## Repos with SQL execution sinks (no config store required)\n\n")
+    md.append(md_table(
+        ["Repo", "SQL execution sink count"],
+        [
+            [repo, str(count)]
+            for repo, count in sql_execution_by_repo.most_common(100)
+        ],
+    ))
+
+    (out_dir / "data-store-graph.md").write_text("\n".join(md), encoding="utf-8")
+    _write_data_store_jsonl(out_dir / "data-store-graph.jsonl", store_repos, store_meta)

src2sink/aggregators/graphs.py

@@ -0,0 +1,36 @@
+"""Orchestrate v2 cross-repo graph generation."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .data_stores import write_data_store_graph
+from .index_v2 import write_index_v2
+from .openapi_edges import write_openapi_artifacts
+from .payload_producers import write_payload_producer_index
+from .phase3 import aggregate_phase3_v2
+from .pii_flow_v2 import write_pii_flow_v2
+from .queues import write_queue_graph
+from .service_calls import write_service_call_graph
+from .traces_index import write_traces_index
+
+
+def aggregate_graphs_v2(
+    metabase_root: Path,
+    repo_jsons: list[Path],
+    *,
+    repos_root: Path | None = None,
+    phase3: bool = True,
+) -> None:
+    """Run all v2 cross-repo graph writers (service, queue, data-store, PII, phase 3)."""
+    write_service_call_graph(metabase_root, repo_jsons, repos_root=repos_root)
+    if repos_root:
+        write_openapi_artifacts(metabase_root, repos_root, repo_jsons)
+    write_queue_graph(metabase_root, repo_jsons)
+    write_data_store_graph(metabase_root, repo_jsons)
+    write_payload_producer_index(metabase_root, repo_jsons, repos_root=repos_root)
+    write_index_v2(metabase_root, repo_jsons)
+    write_pii_flow_v2(metabase_root, repo_jsons)
+    write_traces_index(metabase_root)
+    if phase3:
+        aggregate_phase3_v2(metabase_root, repo_jsons)