src2sink 1.0.0__py3-none-any.whl

src2sink/aggregators/ropa.py

@@ -0,0 +1,190 @@
+"""ROPA Article 30 view projected from PII lifecycle touchpoints (Phase 3)."""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from .pii_lifecycle import collect_pii_touchpoints
+from ..graph_common import load_v2_repo_records
+from ..models.ropa import ROPA_CATEGORY_BY_CLASSIFICATION, RopaProcessingActivity
+from ..renderers.markdown import md_table
+
+MAX_MD_ROWS = 300
+
+
+def _group_touches(touches: list[Any]) -> dict[tuple[str, str, str], dict[str, Any]]:
+    """Group PII touchpoints by (ROPA category, classification, repo) with stage flags."""
+    grouped: dict[tuple[str, str, str], dict[str, Any]] = {}
+    for touch in touches:
+        category = ROPA_CATEGORY_BY_CLASSIFICATION.get(
+            touch.pii_classification, "Unclassified personal data"
+        )
+        key = (category, touch.pii_classification, touch.repo)
+        bucket = grouped.setdefault(key, {
+            "field_keys": set(), "stages": set(),
+            "store": False, "log": False, "transmit": False,
+        })
+        bucket["field_keys"].add(touch.field_key)
+        bucket["stages"].add(touch.stage)
+        if touch.stage in ("store", "log", "transmit"):
+            bucket[touch.stage] = True
+    return grouped
+
+
+def _ropa_purposes(bucket: dict[str, Any]) -> list[str]:
+    """Infer ROPA processing purposes from the touchpoint bucket's stages."""
+    purposes: list[str] = []
+    if "collect" in bucket["stages"] or "process" in bucket["stages"]:
+        purposes.append("Service delivery / API processing")
+    if bucket["store"]:
+        purposes.append("Persistence")
+    if bucket["log"]:
+        purposes.append("Operations and diagnostics (logging)")
+    if bucket["transmit"]:
+        purposes.append("Inter-service transmission")
+    return purposes or ["Code references (stage unclear)"]
+
+
+def _ropa_security(bucket: dict[str, Any]) -> list[str]:
+    """Infer ROPA security-measure notes from the touchpoint bucket's stages."""
+    security: list[str] = []
+    if "encrypt" in bucket["stages"]:
+        security.append("Crypto operations near field (static)")
+    if bucket["log"]:
+        security.append("Review logging sinks for field")
+    else:
+        security.append("Logging exposure not detected in static scan")
+    return security
+
+
+def build_ropa_activities(
+    touches: list[Any],
+) -> list[RopaProcessingActivity]:
+    """Group touchpoints into repo × ROPA category processing activities."""
+    grouped = _group_touches(touches)
+
+    activities: list[RopaProcessingActivity] = []
+    for (category, pii_class, repo), bucket in sorted(grouped.items()):
+        purposes = _ropa_purposes(bucket)
+        security = _ropa_security(bucket)
+        retention = (
+            "See repo config / data-store graph"
+            if bucket["store"]
+            else "Not inferred from static scan"
+        )
+
+        activities.append(
+            RopaProcessingActivity(
+                category=category,
+                pii_classification=pii_class,
+                repo=repo,
+                purposes=purposes,
+                data_subjects="Customers and end users (inferred)",
+                recipients="Internal services" if bucket["transmit"] else "Not inferred",
+                retention_hint=retention,
+                security_measures=security,
+                field_keys=sorted(bucket["field_keys"])[:20],
+            ),
+        )
+    return activities
+
+
+def write_ropa_view(
+    metabase_root: Path,
+    repo_jsons: list[Path] | None = None,
+    *,
+    touches: list[Any] | None = None,
+) -> list[RopaProcessingActivity]:
+    """Write the ROPA Article 30 markdown + jsonl view and return the activities."""
+    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+    if touches is None:
+        touches = collect_pii_touchpoints(records)
+    activities = build_ropa_activities(touches)
+
+    ropa_dir = metabase_root / "ropa"
+    ropa_dir.mkdir(parents=True, exist_ok=True)
+
+    jsonl_path = ropa_dir / "processing-activities.jsonl"
+    with jsonl_path.open("w", encoding="utf-8") as fh:
+        for act in activities:
+            fh.write(json.dumps(act.to_dict(), ensure_ascii=False) + "\n")
+
+    by_category: dict[str, list[RopaProcessingActivity]] = defaultdict(list)
+    for act in activities:
+        by_category[act.category].append(act)
+
+    md: list[str] = [
+        "# Categories of personal data (ROPA Article 30 view)\n",
+        "_Projected from v2 PII lifecycle touchpoints. For legal ROPA, validate "
+        "purposes, lawful basis, and retention with product owners._\n",
+        f"\n_Total processing activities (repo × category): **{len(activities)}**._\n",
+    ]
+
+    phone_acts = [
+        a for a in activities
+        if "phone" in a.field_keys
+    ]
+    if phone_acts:
+        md.append("\n## Worked example: phone number processing\n")
+        md.append(
+            md_table(
+                ["Repo", "Category", "Purposes", "Store", "Log", "Transmit"],
+                [
+                    [
+                        a.repo,
+                        a.category,
+                        "; ".join(a.purposes[:2]),
+                        "yes" if "Persistence" in a.purposes else "—",
+                        "yes" if "logging" in " ".join(a.purposes).lower() else "—",
+                        "yes" if a.recipients != "Not inferred" else "—",
+                    ]
+                    for a in sorted(phone_acts, key=lambda x: x.repo)[:40]
+                ],
+            ),
+        )
+
+    md.append("\n## By category (summary)\n")
+    md.append(
+        md_table(
+            ["Category", "Activities", "Distinct repos"],
+            [
+                [
+                    cat,
+                    str(len(acts)),
+                    str(len({a.repo for a in acts})),
+                ]
+                for cat, acts in sorted(
+                    by_category.items(),
+                    key=lambda x: -len(x[1]),
+                )[:30]
+            ],
+        ),
+    )
+
+    md.append("\n## All activities (sample)\n")
+    md.append(
+        md_table(
+            ["Category", "Repo", "Fields", "Purposes"],
+            [
+                [
+                    a.category,
+                    a.repo,
+                    ", ".join(a.field_keys[:5]),
+                    "; ".join(a.purposes[:2]),
+                ]
+                for a in activities[:MAX_MD_ROWS]
+            ],
+        ),
+    )
+    tail = max(0, len(activities) - MAX_MD_ROWS)
+    if tail:
+        md.append(f"\n_{tail} more rows in `processing-activities.jsonl`._\n")
+
+    (ropa_dir / "categories-of-personal-data.md").write_text(
+        "\n".join(md),
+        encoding="utf-8",
+    )
+    return activities

src2sink/aggregators/service_call_collect.py

@@ -0,0 +1,240 @@
+"""Collect cross-repo service-call edges from v2 flow nodes."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..graph_common import (
+    build_repo_alias_index,
+    extract_urls_and_paths,
+    iter_nodes,
+    match_path_in_inbound_index,
+    repo_id,
+    resolve_repo_for_host,
+)
+from .service_call_index import InboundRow, build_inbound_index
+from .service_call_models import CallEdge
+
+
+def _append_path_edge(
+    edges: list[CallEdge],
+    seen: set[tuple[str, str, str, str]],
+    *,
+    src: str,
+    tgt_repo: str,
+    tgt_path: str,
+    conf: str,
+    evidence: str,
+    ref: str,
+) -> None:
+    """Append a deduplicated path-level call edge unless it is a self-edge."""
+    if tgt_repo == src:
+        return
+    key = (src, tgt_repo, tgt_path, conf)
+    if key in seen:
+        return
+    seen.add(key)
+    edges.append(CallEdge(
+        source_repo=src,
+        target_repo=tgt_repo,
+        target_path=tgt_path,
+        confidence=conf,
+        evidence=evidence,
+        refs=[ref],
+    ))
+
+
+def _append_host_edge(
+    edges: list[CallEdge],
+    seen: set[tuple[str, str, str, str]],
+    *,
+    src: str,
+    tgt_repo: str,
+    evidence: str,
+    ref: str,
+) -> None:
+    """Append a deduplicated low-confidence host-level call edge unless self-edge."""
+    if tgt_repo == src:
+        return
+    key = (src, tgt_repo, "*", "low")
+    if key in seen:
+        return
+    seen.add(key)
+    edges.append(CallEdge(
+        source_repo=src,
+        target_repo=tgt_repo,
+        target_path="*",
+        confidence="low",
+        evidence=evidence,
+        refs=[ref],
+    ))
+
+
+def _paths_from_http_out_detail(detail: dict[str, Any], raw: str) -> tuple[list[str], list[str]]:
+    """Extract (hosts, paths) from an http-out node detail plus its raw text."""
+    hosts = [detail["host"]] if detail.get("host") else []
+    paths: list[str] = list(detail.get("paths") or [])
+    if detail.get("path"):
+        paths.insert(0, detail["path"])
+    extra_hosts, extra_paths = extract_urls_and_paths(raw)
+    hosts.extend(h for h in extra_hosts if h not in hosts)
+    paths.extend(p for p in extra_paths if p not in paths)
+    return hosts, paths
+
+
+def _collect_http_out_edges(
+    records: list[dict[str, Any]],
+    inbound: dict[str, list[InboundRow]],
+    alias_to_repo: dict[str, str],
+    seen: set[tuple[str, str, str, str]],
+    edges: list[CallEdge],
+    broken: list[dict[str, str]],
+) -> None:
+    """Collect edges from http-out sink nodes, recording unmatched refs in broken."""
+    for data in records:
+        src = repo_id(data)
+        for node in iter_nodes(data):
+            if node.get("family") != "http-out" or node.get("kind") != "sink":
+                continue
+            detail = node.get("detail") or {}
+            raw = detail.get("raw", "")
+            ref = f"{node.get('file')}:{node.get('line')}"
+            hosts, paths = _paths_from_http_out_detail(detail, raw)
+
+            matched = False
+            for path in paths:
+                targets, conf = match_path_in_inbound_index(path, inbound)
+                for tgt_repo, tgt_path, _method, _in_ref in targets:
+                    _append_path_edge(
+                        edges,
+                        seen,
+                        src=src,
+                        tgt_repo=tgt_repo,
+                        tgt_path=tgt_path,
+                        conf=conf,
+                        evidence=f"path literal {path!r} in http-out",
+                        ref=ref,
+                    )
+                    matched = True
+
+            for host in hosts:
+                tgt_repo = resolve_repo_for_host(host, alias_to_repo)
+                if not tgt_repo:
+                    if not matched:
+                        broken.append({
+                            "source_repo": src,
+                            "host": host,
+                            "ref": ref,
+                            "raw": raw[:120],
+                        })
+                    continue
+                _append_host_edge(
+                    edges,
+                    seen,
+                    src=src,
+                    tgt_repo=tgt_repo,
+                    evidence=f"host {host!r} in http-out",
+                    ref=ref,
+                )
+
+
+def _collect_reference_edges(
+    records: list[dict[str, Any]],
+    inbound: dict[str, list[InboundRow]],
+    alias_to_repo: dict[str, str],
+    seen: set[tuple[str, str, str, str]],
+    edges: list[CallEdge],
+) -> None:
+    """Match path/URL literals in any node detail (config, tests, clients)."""
+    for data in records:
+        src = repo_id(data)
+        for node in iter_nodes(data):
+            detail = node.get("detail") or {}
+            blob = " ".join(
+                str(v) for v in detail.values() if isinstance(v, str)
+            )
+            if not blob:
+                continue
+            ref = f"{node.get('file')}:{node.get('line')}"
+            hosts, paths = extract_urls_and_paths(blob)
+            for path in paths:
+                targets, conf = match_path_in_inbound_index(path, inbound)
+                for tgt_repo, tgt_path, _, _ in targets:
+                    _append_path_edge(
+                        edges,
+                        seen,
+                        src=src,
+                        tgt_repo=tgt_repo,
+                        tgt_path=tgt_path,
+                        conf=conf,
+                        evidence=f"path/url reference in {node.get('family')}",
+                        ref=ref,
+                    )
+            for host in hosts:
+                tgt_repo = resolve_repo_for_host(host, alias_to_repo)
+                if tgt_repo:
+                    _append_host_edge(
+                        edges,
+                        seen,
+                        src=src,
+                        tgt_repo=tgt_repo,
+                        evidence=f"host reference in {node.get('family')}",
+                        ref=ref,
+                    )
+
+
+def collect_service_edges(
+    records: list[dict[str, Any]],
+) -> tuple[list[CallEdge], list[dict[str, str]]]:
+    """Collect all cross-repo call edges; returns (edges, unmatched outbound refs)."""
+    inbound = build_inbound_index(records)
+    alias_to_repo = build_repo_alias_index(records)
+    edges: list[CallEdge] = []
+    broken: list[dict[str, str]] = []
+    seen: set[tuple[str, str, str, str]] = set()
+
+    _collect_http_out_edges(records, inbound, alias_to_repo, seen, edges, broken)
+    _collect_reference_edges(records, inbound, alias_to_repo, seen, edges)
+    return edges, broken
+
+
+def merge_openapi_edges(
+    edges: list[CallEdge],
+    records: list[dict[str, Any]],
+    repos_root: Any,
+) -> int:
+    """Append OpenAPI-matched edges; returns count added."""
+    from pathlib import Path
+
+    root = Path(repos_root)
+    if not root.is_dir():
+        return 0
+    from .openapi_edges import (
+        build_openapi_inbound_index,
+        discover_openapi_specs,
+        match_http_out_to_openapi,
+    )
+
+    specs = discover_openapi_specs(root)
+    if not specs:
+        return 0
+    inbound = build_openapi_inbound_index(specs)
+    alias_to_repo = build_repo_alias_index(records)
+    rows = match_http_out_to_openapi(records, inbound, alias_to_repo)
+    seen = {(e.source_repo, e.target_repo, e.target_path, e.confidence) for e in edges}
+    added = 0
+    for row in rows:
+        key = (row["source_repo"], row["target_repo"], row["target_path"], "openapi")
+        if key in seen:
+            continue
+        seen.add(key)
+        edges.append(CallEdge(
+            source_repo=row["source_repo"],
+            target_repo=row["target_repo"],
+            target_path=row["target_path"],
+            confidence="openapi",
+            evidence=row.get("evidence", "OpenAPI path match"),
+            refs=row.get("refs", []),
+        ))
+        added += 1
+    return added

src2sink/aggregators/service_call_index.py

@@ -0,0 +1,31 @@
+"""Inbound route index for cross-repo service-call matching."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any
+
+from ..graph_common import iter_nodes, normalize_path_template as norm_path, repo_id
+
+# (target_repo, inbound_path, http_method, file:line)
+InboundRow = tuple[str, str, str, str]
+
+
+def build_inbound_index(
+    records: list[dict[str, Any]],
+) -> dict[str, list[InboundRow]]:
+    """Map normalised path template -> inbound http-in nodes."""
+    index: dict[str, list[InboundRow]] = defaultdict(list)
+    for data in records:
+        rid = repo_id(data)
+        for node in iter_nodes(data):
+            if node.get("family") != "http-in" or node.get("kind") != "source":
+                continue
+            path = (node.get("detail") or {}).get("path", "")
+            if not path or path == "?":
+                continue
+            key = norm_path(path)
+            ref = f"{node.get('file')}:{node.get('line')}"
+            method = (node.get("detail") or {}).get("method", "?")
+            index[key].append((rid, path, method, ref))
+    return index

src2sink/aggregators/service_call_models.py

@@ -0,0 +1,17 @@
+"""Data types for service-call graph edges."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class CallEdge:
+    """A cross-repo service-call edge from a source repo to a target route."""
+
+    source_repo: str
+    target_repo: str
+    target_path: str
+    confidence: str
+    evidence: str
+    refs: list[str] = field(default_factory=list)

src2sink/aggregators/service_call_report.py

@@ -0,0 +1,118 @@
+"""Render service-call graph markdown and JSONL."""
+
+from __future__ import annotations
+
+import json
+import re
+from collections import Counter
+from pathlib import Path
+
+from ..graph_common import load_v2_repo_records
+from ..renderers.markdown import md_table
+from ..sanitize import UNTRUSTED_CONTENT_NOTICE, for_mermaid_label
+
+from .service_call_collect import collect_service_edges, merge_openapi_edges
+from .service_call_models import CallEdge
+
+
+def _render_mermaid_section(edges: list[CallEdge]) -> list[str]:
+    """Return markdown lines for a Mermaid flowchart of the top cross-repo pairs."""
+    top = Counter((e.source_repo, e.target_repo) for e in edges).most_common(40)
+    if not top:
+        return []
+    lines = [
+        "\n## Mermaid (top cross-repo pairs)\n\n```mermaid\nflowchart LR\n",
+    ]
+    for (src, dst), _ in top:
+        sid = re.sub(r"[^A-Za-z0-9_]", "_", src)[:40]
+        did = re.sub(r"[^A-Za-z0-9_]", "_", dst)[:40]
+        # ids are slugified above; labels carry the raw repo id — neutralise them.
+        lines.append(f'  {sid}["{for_mermaid_label(src)}"] --> {did}["{for_mermaid_label(dst)}"]\n')
+    lines.append("```\n")
+    return lines
+
+
+def write_service_call_graph(
+    metabase_root: Path,
+    repo_jsons: list[Path],
+    *,
+    repos_root: Path | None = None,
+) -> None:
+    """Write the service-call graph markdown report and edges JSONL to graphs/."""
+    records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
+    edges, broken = collect_service_edges(records)
+    if repos_root:
+        merge_openapi_edges(edges, records, repos_root)
+
+    out_dir = metabase_root / "graphs"
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    by_conf = Counter(e.confidence for e in edges)
+    md: list[str] = [
+        "# Service-call graph (v2)\n",
+        UNTRUSTED_CONTENT_NOTICE,
+        "_Cross-repo edges from `http-out` call sites matched to `http-in` path "
+        "templates. Confidence: **high** = same normalised path; **medium** = "
+        "prefix/template overlap; **low** = hostname hints only; **openapi** = "
+        "path matched to a discovered OpenAPI/Swagger spec._\n",
+        "\n> Regex-derived; validate against OpenAPI / Helm before acting on "
+        "low-confidence edges.\n",
+        "\n## Summary\n",
+        md_table(
+            ["Confidence", "Edge count"],
+            [[k, str(v)] for k, v in by_conf.most_common()],
+        ),
+    ]
+
+    detail = sorted(
+        edges,
+        key=lambda e: (e.confidence, e.target_repo, e.source_repo),
+    )
+    md.append("\n## Matched edges (sampled)\n")
+    md.append(md_table(
+        ["Source", "Target", "Path", "Confidence", "Evidence"],
+        [
+            [
+                e.source_repo,
+                e.target_repo,
+                e.target_path,
+                e.confidence,
+                e.evidence[:100],
+            ]
+            for e in detail[:500]
+        ],
+    ))
+    if len(detail) > 500:
+        md.append(f"\n_{len(detail) - 500} additional edges omitted._\n")
+
+    md.extend(_render_mermaid_section(edges))
+
+    md.append("\n## Broken / unmatched outbound refs\n")
+    md.append(
+        "_http-out sites with a resolvable host or path that did not match any "
+        "indexed inbound route (sampled)._\n\n",
+    )
+    if broken:
+        md.append(md_table(
+            ["Source repo", "Host / hint", "File:line", "Raw (trimmed)"],
+            [
+                [b["source_repo"], b.get("host", "?"), b["ref"], b["raw"]]
+                for b in broken[:200]
+            ],
+        ))
+    else:
+        md.append("_No unmatched hosts in sample._\n")
+
+    (out_dir / "service-call-graph.md").write_text("\n".join(md), encoding="utf-8")
+
+    jsonl = out_dir / "service-call-edges.jsonl"
+    with jsonl.open("w", encoding="utf-8") as fh:
+        for e in edges:
+            fh.write(json.dumps({
+                "source_repo": e.source_repo,
+                "target_repo": e.target_repo,
+                "target_path": e.target_path,
+                "confidence": e.confidence,
+                "evidence": e.evidence,
+                "refs": e.refs,
+            }, ensure_ascii=False) + "\n")

src2sink/aggregators/service_calls.py

@@ -0,0 +1,14 @@
+"""Service-call graph aggregator — v2 http-in / http-out cross-repo matching."""
+
+from __future__ import annotations
+
+from .service_call_collect import collect_service_edges, merge_openapi_edges
+from .service_call_models import CallEdge
+from .service_call_report import write_service_call_graph
+
+__all__ = [
+    "CallEdge",
+    "collect_service_edges",
+    "merge_openapi_edges",
+    "write_service_call_graph",
+]

src2sink/aggregators/taint_buckets.py

@@ -0,0 +1,74 @@
+"""Collect v2 flow nodes into catalogue buckets before markdown/jsonl export."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class TaintCatalogueBuckets:
+    """Grouped node records keyed by taint family."""
+
+    sql_sources: list[dict[str, Any]] = field(default_factory=list)
+    sql_sinks: list[dict[str, Any]] = field(default_factory=list)
+    file_sinks: list[dict[str, Any]] = field(default_factory=list)
+    http_sinks: list[dict[str, Any]] = field(default_factory=list)
+    pii_sources: list[dict[str, Any]] = field(default_factory=list)
+    data_class_fields: list[dict[str, Any]] = field(default_factory=list)
+    pii_sinks: list[dict[str, Any]] = field(default_factory=list)
+    crypto_ops: list[dict[str, Any]] = field(default_factory=list)
+    raw_payload: list[dict[str, Any]] = field(default_factory=list)
+    config_stores: list[dict[str, Any]] = field(default_factory=list)
+    config_security: list[dict[str, Any]] = field(default_factory=list)
+    crypto_config: list[dict[str, Any]] = field(default_factory=list)
+
+
+def collect_taint_buckets(repo_jsons: list[Path]) -> TaintCatalogueBuckets:
+    """Walk per-repo v2 JSON files and partition nodes into export buckets."""
+    buckets = TaintCatalogueBuckets()
+    for jp in repo_jsons:
+        try:
+            data = json.loads(jp.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        repo_id = f"{data['group']}/{data['name']}"
+        for node in data.get("nodes", []):
+            _route_node(buckets, node, repo_id)
+    return buckets
+
+
+# Families that map 1:1 to a bucket regardless of kind.
+_SIMPLE_FAMILY_BUCKET = {
+    "file": "file_sinks",
+    "http-out": "http_sinks",
+    "pii-field": "pii_sources",
+    "data-class-field": "data_class_fields",
+    "crypto-algorithm": "crypto_ops",
+    "raw-code-payload": "raw_payload",
+    "config-security": "config_security",
+    "crypto-config": "crypto_config",
+}
+
+
+def _route_node(buckets: TaintCatalogueBuckets, node: dict[str, Any], repo_id: str) -> None:
+    """Append a repo-tagged copy of the node to the bucket for its family/kind."""
+    rec = {**node, "repo": repo_id}
+    family = node.get("family", "")
+    kind = node.get("kind", "")
+
+    if family == "sql":
+        if kind == "source":
+            buckets.sql_sources.append(rec)
+        elif kind == "sink" and node.get("detail", {}).get("execution", True):
+            buckets.sql_sinks.append(rec)
+    elif family in ("pii-log", "pii-storage"):
+        buckets.pii_sinks.append(rec)
+    elif family == "data-store" and kind == "store":
+        buckets.config_stores.append(rec)
+    else:
+        attr = _SIMPLE_FAMILY_BUCKET.get(family)
+        if attr:
+            getattr(buckets, attr).append(rec)