src2sink 1.0.0__py3-none-any.whl

src2sink/aggregators/payload_producers.py

@@ -0,0 +1,369 @@
+"""Index repos that produce traffic to dangerous-payload HTTP endpoints."""
+
+from __future__ import annotations
+
+import json
+import re
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from ..graph_common import host_matches_repo, load_v2_repo_records, repo_id
+from ..known_api_clients import ApiClientBinding, get_bindings
+from ..renderers.markdown import md_table
+from ..sanitize import UNTRUSTED_CONTENT_NOTICE
+
+IMPORT_SCAN_RX = re.compile(
+    r"^\s*import\s+([\w.]+)|"
+    r"import\s+([\w.]+)",
+    re.MULTILINE,
+)
+
+
+@dataclass
+class ProducerHit:
+    """A single evidence that one repo produces traffic to a payload endpoint."""
+
+    source_repo: str
+    target_repo: str
+    path: str
+    kind: str
+    confidence: str
+    evidence: str
+    ref: str = ""
+
+
+@dataclass
+class ProducerIndex:
+    """Per-target index: an API-client binding and the producer hits found for it."""
+
+    binding: ApiClientBinding
+    hits: list[ProducerHit] = field(default_factory=list)
+
+
+def _internal_dep_hit(
+    rid: str, binding: ApiClientBinding, data: dict[str, Any]
+) -> ProducerHit | None:
+    """Return a hit if the repo declares an internal dependency on the client artifact."""
+    for dep in data.get("dependencies_internal", []):
+        aid = dep.get("artifactId", "")
+        if binding.maven_artifact in aid:
+            return ProducerHit(
+                source_repo=rid,
+                target_repo=binding.target_repo,
+                path="*",
+                kind="internal-dep",
+                confidence="high",
+                evidence=f"{dep.get('groupId')}:{aid}",
+            )
+    return None
+
+
+def _api_client_consumer_hit(
+    rid: str, binding: ApiClientBinding, detail: dict[str, Any], ref: str
+) -> ProducerHit | None:
+    """Return a hit if an `api-client-consumer` node targets the binding's repo."""
+    if detail.get("target_repo") != binding.target_repo:
+        return None
+    return ProducerHit(
+        source_repo=rid,
+        target_repo=binding.target_repo,
+        path=",".join(detail.get("paths", [])) or "*",
+        kind="import",
+        confidence="high",
+        evidence=detail.get("import", "")[:120],
+        ref=ref,
+    )
+
+
+def _http_out_hit(
+    rid: str, binding: ApiClientBinding, detail: dict[str, Any], ref: str
+) -> ProducerHit | None:
+    """Return a hit if an `http-out` node's host or path matches the binding's target."""
+    host = detail.get("host", "")
+    path = detail.get("path", "")
+    if host and host_matches_repo(host, binding.target_repo):
+        return ProducerHit(
+            source_repo=rid,
+            target_repo=binding.target_repo,
+            path=path or "*",
+            kind="http-out",
+            confidence="high" if path else "medium",
+            evidence=detail.get("url") or detail.get("raw", "")[:120],
+            ref=ref,
+        )
+    if path and any(
+        path.rstrip("/").endswith(p.rstrip("/").replace("{handle}", ""))
+        or p in (detail.get("paths") or [])
+        for p in binding.paths
+    ):
+        return ProducerHit(
+            source_repo=rid,
+            target_repo=binding.target_repo,
+            path=path,
+            kind="http-out-path",
+            confidence="medium",
+            evidence=detail.get("raw", "")[:120],
+            ref=ref,
+        )
+    return None
+
+
+def _hits_from_repo_json(data: dict[str, Any], binding: ApiClientBinding) -> list[ProducerHit]:
+    """Collect all producer hits for one binding from a single repo's v2 JSON."""
+    rid = repo_id(data)
+    if rid == binding.target_repo:
+        return []
+    hits: list[ProducerHit] = []
+    dep_hit = _internal_dep_hit(rid, binding, data)
+    if dep_hit:
+        hits.append(dep_hit)
+
+    for node in data.get("nodes", []):
+        family = node.get("family", "")
+        detail = node.get("detail") or {}
+        ref = f"{node.get('file')}:{node.get('line')}"
+        if family == "api-client-consumer":
+            hit = _api_client_consumer_hit(rid, binding, detail, ref)
+        elif family == "http-out":
+            hit = _http_out_hit(rid, binding, detail, ref)
+        else:
+            hit = None
+        if hit:
+            hits.append(hit)
+    return hits
+
+
+_BINDING_SCAN_SUFFIXES = frozenset({".java", ".kt", ".kts", ".xml", ".gradle"})
+_BINDING_SCAN_SKIP = frozenset({"node_modules", ".git", "target", "build"})
+_BUILD_FILE_NAMES = ("pom.xml", "build.gradle", "build.gradle.kts")
+
+
+def _read_capped(path: Path, cap: int = 400_000) -> str | None:
+    """Read a file as text, returning None if it exceeds the byte cap or errors."""
+    try:
+        if path.stat().st_size > cap:
+            return None
+        return path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return None
+
+
+def _import_hits_in_file(
+    text: str, path: Path, repo_dir: Path, src: str,
+    binding: ApiClientBinding, seen: set[tuple[str, str]],
+) -> list[ProducerHit]:
+    """Scan one file's imports for the binding's import prefix, deduped via `seen`."""
+    hits: list[ProducerHit] = []
+    for m in IMPORT_SCAN_RX.finditer(text):
+        pkg = m.group(1) or m.group(2) or ""
+        if binding.import_prefix in pkg and (src, "import") not in seen:
+            seen.add((src, "import"))
+            line = text.count("\n", 0, m.start()) + 1
+            hits.append(ProducerHit(
+                source_repo=src,
+                target_repo=binding.target_repo,
+                path="*",
+                kind="import-scan",
+                confidence="high",
+                evidence=f"import {pkg}",
+                ref=f"{path.relative_to(repo_dir)}:{line}",
+            ))
+    return hits
+
+
+def _scan_repo_for_binding(
+    repo_dir: Path, src: str, binding: ApiClientBinding,
+    seen: set[tuple[str, str]], pom_rx: re.Pattern[str], gradle_rx: re.Pattern[str],
+) -> list[ProducerHit]:
+    """Scan a repo directory tree for imports and build-file references to the binding."""
+    hits: list[ProducerHit] = []
+    found_pom = False
+    for path in repo_dir.rglob("*"):
+        if path.suffix.lower() not in _BINDING_SCAN_SUFFIXES:
+            continue
+        if any(p in path.parts for p in _BINDING_SCAN_SKIP):
+            continue
+        text = _read_capped(path)
+        if text is None:
+            continue
+        if path.name in _BUILD_FILE_NAMES and (pom_rx.search(text) or gradle_rx.search(text)):
+            found_pom = True
+        hits.extend(_import_hits_in_file(text, path, repo_dir, src, binding, seen))
+
+    if found_pom and (src, "pom") not in seen:
+        seen.add((src, "pom"))
+        hits.append(ProducerHit(
+            source_repo=src,
+            target_repo=binding.target_repo,
+            path="*",
+            kind="build-dep-scan",
+            confidence="high",
+            evidence=binding.maven_artifact,
+        ))
+    return hits
+
+
+def _scan_repos_for_binding(
+    repos_root: Path,
+    binding: ApiClientBinding,
+) -> list[ProducerHit]:
+    """Scan every repo under `repos_root` for source-level uses of the binding."""
+    if not repos_root.is_dir():
+        return []
+    hits: list[ProducerHit] = []
+    seen: set[tuple[str, str]] = set()
+    pom_rx = re.compile(
+        rf"<artifactId>({re.escape(binding.maven_artifact)})</artifactId>",
+        re.IGNORECASE,
+    )
+    gradle_rx = re.compile(re.escape(binding.maven_artifact), re.IGNORECASE)
+
+    for group_dir in repos_root.iterdir():
+        if not group_dir.is_dir():
+            continue
+        for repo_dir in group_dir.iterdir():
+            if not repo_dir.is_dir():
+                continue
+            src = f"{group_dir.name}/{repo_dir.name}"
+            if src != binding.target_repo:
+                hits.extend(
+                    _scan_repo_for_binding(repo_dir, src, binding, seen, pom_rx, gradle_rx)
+                )
+    return hits
+
+
+def build_producer_indices(
+    metabase_root: Path,
+    *,
+    repos_root: Path | None = None,
+    json_paths: list[Path] | None = None,
+) -> list[ProducerIndex]:
+    """Build a producer index per known API-client binding.
+
+    Merges v2-JSON hits with optional source-tree scan hits, keeping the
+    highest-confidence hit per (repo, kind).
+
+    Args:
+        metabase_root: Metabase root holding per-repo v2 JSON.
+        repos_root: Optional checkout root to scan for imports/build deps.
+        json_paths: Explicit v2 JSON paths (else discovered under metabase_root).
+
+    Returns:
+        One ProducerIndex per binding, hits sorted by (source_repo, kind).
+    """
+    records = load_v2_repo_records(metabase_root, json_paths=json_paths)
+    indices: list[ProducerIndex] = []
+
+    for binding in get_bindings():
+        by_repo: dict[str, list[ProducerHit]] = defaultdict(list)
+
+        for data in records:
+            for hit in _hits_from_repo_json(data, binding):
+                by_repo[hit.source_repo].append(hit)
+
+        if repos_root:
+            for hit in _scan_repos_for_binding(repos_root, binding):
+                by_repo[hit.source_repo].append(hit)
+
+        # Merge per repo: keep highest confidence per kind
+        merged: list[ProducerHit] = []
+        for src, repo_hits in sorted(by_repo.items()):
+            kinds: dict[str, ProducerHit] = {}
+            for h in repo_hits:
+                prev = kinds.get(h.kind)
+                if prev is None or _conf_rank(h.confidence) > _conf_rank(prev.confidence):
+                    kinds[h.kind] = h
+            merged.extend(kinds.values())
+
+        indices.append(ProducerIndex(binding=binding, hits=sorted(
+            merged,
+            key=lambda h: (h.source_repo, h.kind),
+        )))
+
+    return indices
+
+
+def _conf_rank(c: str) -> int:
+    """Map a confidence label to a sortable integer rank."""
+    return {"high": 3, "medium": 2, "low": 1}.get(c, 0)
+
+
+def write_payload_producer_index(
+    metabase_root: Path,
+    repo_jsons: list[Path],
+    *,
+    repos_root: Path | None = None,
+) -> None:
+    """Write the payload-endpoint-producers markdown + jsonl catalogue."""
+    root = repos_root
+    if root is None:
+        candidate = metabase_root.parent / "repos"
+        root = candidate if candidate.is_dir() else None
+
+    indices = build_producer_indices(
+        metabase_root,
+        repos_root=root,
+        json_paths=repo_jsons,
+    )
+
+    out_dir = metabase_root / "graphs"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    jsonl_path = out_dir / "payload-endpoint-producers.jsonl"
+    all_records: list[dict[str, Any]] = []
+
+    md: list[str] = [
+        "# Payload endpoint producers\n",
+        UNTRUSTED_CONTENT_NOTICE,
+        "_Repos that call dangerous-payload HTTP APIs via published clients, "
+        "declared dependencies, or enriched `http-out` nodes. Built from "
+        "`known_api_clients.py` + optional `repos/` import scan._\n",
+    ]
+
+    for index in indices:
+        binding = index.binding
+        md.append(f"\n## Target: `{binding.target_repo}`\n")
+        md.append(
+            f"Client: `{binding.maven_artifact}` · paths: "
+            + ", ".join(f"`{p}`" for p in binding.paths)
+            + "\n",
+        )
+        if not index.hits:
+            md.append("_No producers indexed._\n")
+            continue
+
+        md.append(md_table(
+            ["Producer repo", "Kind", "Path", "Confidence", "Evidence", "Ref"],
+            [
+                [
+                    h.source_repo,
+                    h.kind,
+                    h.path,
+                    h.confidence,
+                    h.evidence[:90],
+                    h.ref,
+                ]
+                for h in index.hits[:200]
+            ],
+        ))
+        if len(index.hits) > 200:
+            md.append(f"\n_{len(index.hits) - 200} more in jsonl._\n")
+
+        for h in index.hits:
+            rec = {
+                "target_repo": h.target_repo,
+                "source_repo": h.source_repo,
+                "path": h.path,
+                "kind": h.kind,
+                "confidence": h.confidence,
+                "evidence": h.evidence,
+                "ref": h.ref,
+                "client_artifact": binding.maven_artifact,
+            }
+            all_records.append(rec)
+
+    (out_dir / "payload-endpoint-producers.md").write_text("\n".join(md), encoding="utf-8")
+    with jsonl_path.open("w", encoding="utf-8") as fh:
+        for rec in all_records:
+            fh.write(json.dumps(rec, ensure_ascii=False) + "\n")

src2sink/aggregators/phase3.py

@@ -0,0 +1,30 @@
+"""Phase 3 aggregators: PII lifecycle, ROPA, auth/crypto conventions."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .auth_cards import write_auth_models_catalog
+from .crypto_cards import write_crypto_agility_catalog
+from .pii_cross_repo import write_pii_cross_repo_graph
+from ..models.ropa import ROPA_SHOWCASE_FIELDS
+from .pii_lifecycle import write_pii_lifecycle_graph
+from .ropa import write_ropa_view
+
+
+def aggregate_phase3_v2(
+    metabase_root: Path,
+    repo_jsons: list[Path],
+) -> None:
+    """Run the phase 3 aggregators: PII lifecycle/cross-repo, ROPA, auth, and crypto."""
+    touches = write_pii_lifecycle_graph(metabase_root, repo_jsons)
+    for field_key in sorted(ROPA_SHOWCASE_FIELDS):
+        write_pii_cross_repo_graph(
+            metabase_root,
+            touches,
+            repo_jsons,
+            field_key=field_key,
+        )
+    write_ropa_view(metabase_root, repo_jsons, touches=touches)
+    write_auth_models_catalog(metabase_root, repo_jsons)
+    write_crypto_agility_catalog(metabase_root, repo_jsons)

src2sink/aggregators/pii_cross_repo.py

@@ -0,0 +1,283 @@
+"""Cross-repo PII flow links (queue hops + HTTP edges) for showcase fields."""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from .service_calls import collect_service_edges
+from ..graph_common import iter_nodes, load_v2_repo_records, repo_id
+from ..models.pii_lifecycle import PiiTouchpoint
+from ..renderers.markdown import md_table
+from ..sanitize import UNTRUSTED_CONTENT_NOTICE, for_markdown
+
+MAX_FLOW_ROWS = 80
+
+
+def _repos_with_field(
+    touches: list[PiiTouchpoint],
+    field_key: str,
+) -> set[str]:
+    """Return repos with any touchpoint for the given field."""
+    return {t.repo for t in touches if t.field_key == field_key}
+
+
+# Cross-repo hops: repos must have material handling of the field (not merely
+# a pii-field declaration or a low-confidence "nearby http-out/sql" hint).
+_MATERIAL_FAMILIES = frozenset({"pii-field", "pii-log", "pii-storage", "queue-pub"})
+_MATERIAL_STAGES = frozenset({"store", "log", "collect", "delete", "encrypt"})
+_STRONG_FAMILIES = frozenset({"pii-log", "pii-storage", "queue-pub", "queue-sub"})
+# Generic routes shared by many services — not indicative of PII data flow.
+_HTTP_NOISE_PATHS = frozenset({
+    "/metrics",
+    "/health",
+    "/healthz",
+    "/ready",
+    "/live",
+    "/actuator",
+    "/swagger",
+    "/openapi",
+    "/api/health",
+    "/status",
+})
+
+
+def _material_repos_with_field(
+    touches: list[PiiTouchpoint],
+    field_key: str,
+) -> set[str]:
+    """Return repos that materially handle the field (not mere declarations/hints)."""
+    out: set[str] = set()
+    for t in touches:
+        if t.field_key != field_key:
+            continue
+        if t.family in _MATERIAL_FAMILIES:
+            out.add(t.repo)
+        elif t.stage in _MATERIAL_STAGES:
+            out.add(t.repo)
+        elif t.stage == "transmit" and t.confidence != "low":
+            out.add(t.repo)
+    return out
+
+
+def _strong_material_repos_with_field(
+    touches: list[PiiTouchpoint],
+    field_key: str,
+) -> set[str]:
+    """Repos with log/store/queue touchpoints — used for HTTP cross-repo hops."""
+    out: set[str] = set()
+    for t in touches:
+        if t.field_key != field_key:
+            continue
+        if t.family in _STRONG_FAMILIES:
+            out.add(t.repo)
+        elif t.stage in ("store", "log", "delete", "encrypt"):
+            out.add(t.repo)
+    return out
+
+
+def _is_noise_http_path(path: str) -> bool:
+    """Return True for generic infra paths (health/metrics/etc.) that aren't PII flow."""
+    norm = path.split("?")[0].rstrip("/") or "/"
+    if norm in _HTTP_NOISE_PATHS:
+        return True
+    for prefix in _HTTP_NOISE_PATHS:
+        if norm.startswith(prefix + "/"):
+            return True
+    return False
+
+
+def _queue_topics_by_repo(records: list[dict[str, Any]]) -> dict[str, dict[str, set[str]]]:
+    """repo -> {produce: topics, consume: topics}."""
+    out: dict[str, dict[str, set[str]]] = defaultdict(
+        lambda: {"produce": set(), "consume": set()},
+    )
+    for data in records:
+        rid = repo_id(data)
+        for node in iter_nodes(data):
+            family = node.get("family", "")
+            topic = (node.get("detail") or {}).get("topic", "")
+            if not topic or topic == "?":
+                continue
+            if family == "queue-pub":
+                out[rid]["produce"].add(topic)
+            elif family == "queue-sub":
+                out[rid]["consume"].add(topic)
+    return out
+
+
+def _build_queue_flows(
+    records: list[dict[str, Any]],
+    *,
+    field_key: str,
+    material_repos: set[str],
+) -> list[dict[str, Any]]:
+    """Link repos that share a topic and both materially touch the field."""
+    flows: list[dict[str, Any]] = []
+    queue_map = _queue_topics_by_repo(records)
+    topics: dict[str, dict[str, set[str]]] = defaultdict(
+        lambda: {"produce": set(), "consume": set()},
+    )
+    for rid, dirs in queue_map.items():
+        if rid not in material_repos:
+            continue
+        for topic in dirs["produce"]:
+            topics[topic]["produce"].add(rid)
+        for topic in dirs["consume"]:
+            topics[topic]["consume"].add(rid)
+
+    for topic, dirs in sorted(topics.items()):
+        producers = sorted(dirs["produce"] & material_repos)
+        consumers = sorted(dirs["consume"] & material_repos)
+        if not producers or not consumers:
+            continue
+        for p in producers:
+            for c in consumers:
+                if p == c:
+                    continue
+                flows.append({
+                    "field_key": field_key,
+                    "kind": "queue",
+                    "source_repo": p,
+                    "target_repo": c,
+                    "via": topic,
+                    "confidence": "medium",
+                    "evidence": f"Both repos touch `{field_key}`; topic `{topic}`",
+                })
+    return flows
+
+
+def _build_http_flows(
+    records: list[dict[str, Any]],
+    *,
+    field_key: str,
+    strong_repos: set[str],
+) -> list[dict[str, Any]]:
+    """Link high/openapi service-call edges where both repos strongly touch the field."""
+    flows: list[dict[str, Any]] = []
+    edges, _ = collect_service_edges(records)
+    for edge in edges:
+        if edge.confidence not in ("high", "openapi"):
+            continue
+        if _is_noise_http_path(edge.target_path):
+            continue
+        if edge.source_repo not in strong_repos or edge.target_repo not in strong_repos:
+            continue
+        flows.append({
+            "field_key": field_key,
+            "kind": "http",
+            "source_repo": edge.source_repo,
+            "target_repo": edge.target_repo,
+            "via": edge.target_path,
+            "confidence": edge.confidence,
+            "evidence": edge.evidence[:120],
+        })
+    return flows
+
+
+def build_cross_repo_flows(
+    records: list[dict[str, Any]],
+    touches: list[PiiTouchpoint],
+    *,
+    field_key: str = "phone",
+) -> list[dict[str, Any]]:
+    """Build queue + HTTP cross-repo flows for a field between repos that touch it."""
+    material_repos = _material_repos_with_field(touches, field_key)
+    if not material_repos:
+        return []
+    strong_repos = _strong_material_repos_with_field(touches, field_key)
+    return (
+        _build_queue_flows(records, field_key=field_key, material_repos=material_repos)
+        + _build_http_flows(records, field_key=field_key, strong_repos=strong_repos)
+    )
+
+
+def write_pii_cross_repo_graph(
+    metabase_root: Path,
+    touches: list[PiiTouchpoint],
+    repo_jsons: list[Path] | None = None,
+    *,
+    field_key: str = "phone",
+) -> list[dict[str, Any]]:
+    """Write the cross-repo field-flow markdown + jsonl and return the flows."""
+    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+    phone_repos = _material_repos_with_field(touches, field_key)
+    strong_repos = _strong_material_repos_with_field(touches, field_key)
+    flows = build_cross_repo_flows(records, touches, field_key=field_key)
+
+    graphs_dir = metabase_root / "graphs"
+    graphs_dir.mkdir(parents=True, exist_ok=True)
+
+    jsonl_path = graphs_dir / f"pii-{field_key}-cross-repo.jsonl"
+    with jsonl_path.open("w", encoding="utf-8") as fh:
+        for row in flows:
+            fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+    queue_rows = [f for f in flows if f["kind"] == "queue"]
+    http_rows = [f for f in flows if f["kind"] == "http"]
+
+    md: list[str] = [
+        f"# Cross-repo `{for_markdown(field_key, max_len=80)}` flows\n",
+        UNTRUSTED_CONTENT_NOTICE,
+        "_Messaging hops require a material field touchpoint; HTTP hops require "
+        "**strong** touchpoints (log/store/queue) on both sides plus a "
+        "non-generic path (`/metrics`, `/health`, etc. excluded). "
+        "**high**/**openapi** service-call edges only._\n",
+        f"\n**Repos with material `{field_key}` touchpoints:** "
+        f"{len(phone_repos)} "
+        f"(**strong** for HTTP: {len(strong_repos)}; "
+        f"of {len(_repos_with_field(touches, field_key))} with any touchpoint).\n",
+        f"\n**Cross-repo hops:** {len(flows)} "
+        f"({len(queue_rows)} queue, {len(http_rows)} HTTP).\n",
+    ]
+
+    if queue_rows:
+        md.append("\n## Messaging hops\n")
+        md.append(
+            md_table(
+                ["Producer", "Topic", "Consumer", "Confidence"],
+                [
+                    [
+                        r["source_repo"],
+                        r["via"],
+                        r["target_repo"],
+                        r["confidence"],
+                    ]
+                    for r in queue_rows[:MAX_FLOW_ROWS]
+                ],
+            ),
+        )
+    if http_rows:
+        md.append("\n## HTTP hops (both repos touch field)\n")
+        md.append(
+            md_table(
+                ["Caller", "Target path", "Callee", "Confidence"],
+                [
+                    [
+                        r["source_repo"],
+                        r["via"],
+                        r["target_repo"],
+                        r["confidence"],
+                    ]
+                    for r in http_rows[:MAX_FLOW_ROWS]
+                ],
+            ),
+        )
+
+    if not flows:
+        md.append(
+            "\n_No cross-repo hops found. Extend queue/http extraction or "
+            "re-run after more repos expose `queue-pub`/`queue-sub` nodes._\n",
+        )
+
+    tail = max(0, len(flows) - MAX_FLOW_ROWS)
+    if tail:
+        md.append(f"\n_{tail} more rows in `pii-{field_key}-cross-repo.jsonl`._\n")
+
+    (graphs_dir / f"pii-{field_key}-cross-repo.md").write_text(
+        "\n".join(md),
+        encoding="utf-8",
+    )
+    return flows