src2sink 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. src2sink/__init__.py +1 -0
  2. src2sink/aggregators/__init__.py +3 -0
  3. src2sink/aggregators/api_client_discovery.py +259 -0
  4. src2sink/aggregators/auth_cards.py +116 -0
  5. src2sink/aggregators/crypto_cards.py +132 -0
  6. src2sink/aggregators/data_stores.py +125 -0
  7. src2sink/aggregators/graphs.py +36 -0
  8. src2sink/aggregators/index_v2.py +138 -0
  9. src2sink/aggregators/library_source_map.py +188 -0
  10. src2sink/aggregators/openapi_discovery.py +113 -0
  11. src2sink/aggregators/openapi_edges.py +30 -0
  12. src2sink/aggregators/openapi_match.py +81 -0
  13. src2sink/aggregators/openapi_models.py +25 -0
  14. src2sink/aggregators/openapi_report.py +75 -0
  15. src2sink/aggregators/payload_producers.py +369 -0
  16. src2sink/aggregators/phase3.py +30 -0
  17. src2sink/aggregators/pii_cross_repo.py +283 -0
  18. src2sink/aggregators/pii_flow_v2.py +118 -0
  19. src2sink/aggregators/pii_lifecycle.py +19 -0
  20. src2sink/aggregators/pii_lifecycle_report.py +121 -0
  21. src2sink/aggregators/pii_touchpoint_collect.py +134 -0
  22. src2sink/aggregators/pii_touchpoint_utils.py +96 -0
  23. src2sink/aggregators/queues.py +141 -0
  24. src2sink/aggregators/ropa.py +190 -0
  25. src2sink/aggregators/service_call_collect.py +240 -0
  26. src2sink/aggregators/service_call_index.py +31 -0
  27. src2sink/aggregators/service_call_models.py +17 -0
  28. src2sink/aggregators/service_call_report.py +118 -0
  29. src2sink/aggregators/service_calls.py +14 -0
  30. src2sink/aggregators/taint_buckets.py +74 -0
  31. src2sink/aggregators/taint_catalogs.py +35 -0
  32. src2sink/aggregators/taint_writers.py +369 -0
  33. src2sink/aggregators/traces_index.py +103 -0
  34. src2sink/build_metabase_v2.py +807 -0
  35. src2sink/constants.py +44 -0
  36. src2sink/curate_internal_libraries.py +135 -0
  37. src2sink/extractors/__init__.py +17 -0
  38. src2sink/extractors/ast_walk.py +131 -0
  39. src2sink/extractors/base.py +59 -0
  40. src2sink/extractors/config.py +224 -0
  41. src2sink/extractors/file_context.py +35 -0
  42. src2sink/extractors/go.py +3 -0
  43. src2sink/extractors/http_out.py +108 -0
  44. src2sink/extractors/java.py +3 -0
  45. src2sink/extractors/javascript.py +3 -0
  46. src2sink/extractors/kotlin.py +3 -0
  47. src2sink/extractors/node_factory.py +65 -0
  48. src2sink/extractors/patterns.py +145 -0
  49. src2sink/extractors/python_extractor.py +3 -0
  50. src2sink/extractors/regex_extractors.py +378 -0
  51. src2sink/extractors/ts_extractors.py +123 -0
  52. src2sink/extractors/unified.py +64 -0
  53. src2sink/graph_common.py +213 -0
  54. src2sink/internal_groups.py +142 -0
  55. src2sink/known_api_clients.py +108 -0
  56. src2sink/library_taint_java.py +92 -0
  57. src2sink/limits.py +171 -0
  58. src2sink/models/__init__.py +3 -0
  59. src2sink/models/auth_model.py +3 -0
  60. src2sink/models/crypto_agility.py +3 -0
  61. src2sink/models/pii_lifecycle.py +92 -0
  62. src2sink/models/ropa.py +37 -0
  63. src2sink/prescreen.py +93 -0
  64. src2sink/py.typed +0 -0
  65. src2sink/record_fleet_baseline.py +81 -0
  66. src2sink/renderers/__init__.py +3 -0
  67. src2sink/renderers/markdown.py +149 -0
  68. src2sink/renderers/mermaid.py +3 -0
  69. src2sink/repo_utils.py +885 -0
  70. src2sink/safe_paths.py +54 -0
  71. src2sink/sanitize.py +107 -0
  72. src2sink/schema.py +57 -0
  73. src2sink/trace.py +558 -0
  74. src2sink/trace_batch.py +176 -0
  75. src2sink/vocabulary.py +152 -0
  76. src2sink-1.0.0.dist-info/METADATA +355 -0
  77. src2sink-1.0.0.dist-info/RECORD +81 -0
  78. src2sink-1.0.0.dist-info/WHEEL +5 -0
  79. src2sink-1.0.0.dist-info/entry_points.txt +6 -0
  80. src2sink-1.0.0.dist-info/licenses/LICENSE +21 -0
  81. src2sink-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,118 @@
1
+ """v2 PII flow roll-up (replaces v1 graphs/pii-flow.md)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections import Counter
6
+ from dataclasses import dataclass, field
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ from ..graph_common import iter_nodes, load_v2_repo_records
11
+ from ..renderers.markdown import md_table
12
+ from ..sanitize import UNTRUSTED_CONTENT_NOTICE
13
+
14
+ MAX_MD_ROWS = 200
15
+
16
+
17
+ @dataclass
18
+ class _PiiFlowCounts:
19
+ """Aggregated PII/HTTP/queue touchpoint counters across all repos."""
20
+
21
+ by_class: Counter[str] = field(default_factory=Counter)
22
+ by_repo: Counter[str] = field(default_factory=Counter)
23
+ field_counts: Counter[str] = field(default_factory=Counter)
24
+ sensitive_by_repo: Counter[str] = field(default_factory=Counter)
25
+ http_in_by_repo: Counter[str] = field(default_factory=Counter)
26
+ http_out_by_repo: Counter[str] = field(default_factory=Counter)
27
+ queue_by_repo: Counter[str] = field(default_factory=Counter)
28
+
29
+
30
+ def _collect_pii_flow(records: list[dict[str, Any]]) -> _PiiFlowCounts:
31
+ """Tally PII, HTTP, and queue nodes per classification and per repo."""
32
+ c = _PiiFlowCounts()
33
+ for data in records:
34
+ rid = f"{data['group']}/{data['name']}"
35
+ for node in iter_nodes(data):
36
+ family = node.get("family")
37
+ if family in ("pii-field", "pii-log", "pii-storage"):
38
+ cls = node.get("pii_classification") or "unknown"
39
+ c.by_class[cls] += 1
40
+ c.by_repo[rid] += 1
41
+ field_name = (node.get("detail") or {}).get("field_name")
42
+ if field_name:
43
+ c.field_counts[field_name.lower()] += 1
44
+ if cls in ("sensitive", "special-category-gdpr"):
45
+ c.sensitive_by_repo[rid] += 1
46
+ elif family == "http-in":
47
+ c.http_in_by_repo[rid] += 1
48
+ elif family == "http-out":
49
+ c.http_out_by_repo[rid] += 1
50
+ elif family in ("queue-in", "queue-out"):
51
+ c.queue_by_repo[rid] += 1
52
+ return c
53
+
54
+
55
+ def write_pii_flow_v2(metabase_root: Path, repo_jsons: list[Path]) -> None:
56
+ """Write the v2 PII flow roll-up markdown (graphs/pii-flow.md)."""
57
+ records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
58
+ c = _collect_pii_flow(records)
59
+ by_class = c.by_class
60
+ by_repo = c.by_repo
61
+ field_counts = c.field_counts
62
+ sensitive_by_repo = c.sensitive_by_repo
63
+ http_in_by_repo = c.http_in_by_repo
64
+ http_out_by_repo = c.http_out_by_repo
65
+ queue_by_repo = c.queue_by_repo
66
+
67
+ graphs_dir = metabase_root / "graphs"
68
+ graphs_dir.mkdir(parents=True, exist_ok=True)
69
+
70
+ md: list[str] = [
71
+ "# PII flow roll-up (v2)\n",
72
+ UNTRUSTED_CONTENT_NOTICE,
73
+ "_Aggregated from `pii-field`, `pii-log`, and `pii-storage` nodes. "
74
+ "For lifecycle stages see `pii-lifecycle.md`._\n",
75
+ "\n## By GDPR classification\n",
76
+ md_table(
77
+ ["Classification", "Touchpoints"],
78
+ [[k, str(v)] for k, v in by_class.most_common()],
79
+ ),
80
+ "\n## Top fields (all repos)\n",
81
+ md_table(
82
+ ["Field", "Count"],
83
+ [[k, str(v)] for k, v in field_counts.most_common(40)],
84
+ ),
85
+ "\n## Top repos by PII touchpoints\n",
86
+ md_table(
87
+ ["Repo", "Touchpoints"],
88
+ [[k, str(v)] for k, v in by_repo.most_common(MAX_MD_ROWS)],
89
+ ),
90
+ ]
91
+ tail = len(by_repo) - MAX_MD_ROWS
92
+ if tail > 0:
93
+ md.append(f"\n_{tail} more repos — see `taint/pii-sources.jsonl`._\n")
94
+
95
+ # Probable PII handlers: repos that have inbound HTTP endpoints AND
96
+ # reference at least one sensitive/special-category-gdpr PII node.
97
+ handlers = [
98
+ rid for rid in sensitive_by_repo
99
+ if http_in_by_repo.get(rid, 0) > 0
100
+ ]
101
+ handler_rows = sorted(
102
+ handlers,
103
+ key=lambda r: -sensitive_by_repo[r],
104
+ )
105
+ md.append(
106
+ "\n## Probable PII handlers\n"
107
+ "_Repos with inbound HTTP endpoints and `sensitive` or "
108
+ "`special-category-gdpr` PII nodes. Top candidates for PII & "
109
+ "data-minimisation review._\n"
110
+ )
111
+ md.append(md_table(
112
+ ["Repo", "HTTP-in", "HTTP-out", "Queue I/O", "Sensitive count"],
113
+ [[r, str(http_in_by_repo[r]), str(http_out_by_repo.get(r, 0)),
114
+ str(queue_by_repo.get(r, 0)), str(sensitive_by_repo[r])]
115
+ for r in handler_rows[:80]] or [["—", "—", "—", "—", "—"]],
116
+ ))
117
+
118
+ (graphs_dir / "pii-flow.md").write_text("\n".join(md), encoding="utf-8")
@@ -0,0 +1,19 @@
1
+ """Fleet-wide PII lifecycle graph from v2 flow nodes (Phase 3)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .pii_lifecycle_report import (
6
+ MAX_MD_ROWS,
7
+ aggregate_by_field,
8
+ write_pii_lifecycle_graph,
9
+ )
10
+ from .pii_touchpoint_collect import collect_pii_touchpoints
11
+ from ..models.pii_lifecycle import PiiTouchpoint
12
+
13
+ __all__ = [
14
+ "MAX_MD_ROWS",
15
+ "PiiTouchpoint",
16
+ "aggregate_by_field",
17
+ "collect_pii_touchpoints",
18
+ "write_pii_lifecycle_graph",
19
+ ]
@@ -0,0 +1,121 @@
1
+ """Render PII lifecycle graph markdown and JSONL."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections import Counter
7
+ from pathlib import Path
8
+
9
+ from ..graph_common import load_v2_repo_records
10
+ from ..models.pii_lifecycle import LIFECYCLE_STAGES, FieldLifecycleAggregate, PiiTouchpoint
11
+ from ..sanitize import UNTRUSTED_CONTENT_NOTICE
12
+ from ..renderers.markdown import md_table
13
+
14
+ from .pii_touchpoint_collect import collect_pii_touchpoints
15
+
16
+ MAX_MD_ROWS = 400
17
+
18
+
19
+ def aggregate_by_field(touches: list[PiiTouchpoint]) -> dict[str, FieldLifecycleAggregate]:
20
+ """Group touchpoints into per-(field, classification) lifecycle aggregates."""
21
+ buckets: dict[str, FieldLifecycleAggregate] = {}
22
+ for touch in touches:
23
+ bucket_key = f"{touch.field_key}|{touch.pii_classification}"
24
+ if bucket_key not in buckets:
25
+ buckets[bucket_key] = FieldLifecycleAggregate(
26
+ field_key=touch.field_key,
27
+ pii_classification=touch.pii_classification,
28
+ )
29
+ buckets[bucket_key].add(touch)
30
+ return buckets
31
+
32
+
33
+ def _append_phone_example(
34
+ md: list[str],
35
+ aggregates: dict[str, FieldLifecycleAggregate],
36
+ ) -> None:
37
+ """Append a worked `phone` lifecycle example section to the markdown lines."""
38
+ phone_agg = [a for a in aggregates.values() if a.field_key == "phone"]
39
+ if not phone_agg:
40
+ return
41
+ md.append("\n## Worked example: `phone` (direct PII)\n")
42
+ for agg in sorted(phone_agg, key=lambda a: -a.touchpoints)[:3]:
43
+ md.append(f"\n### Classification: `{agg.pii_classification}`\n")
44
+ md.append(
45
+ md_table(
46
+ ["Stage", "Repos (count)"],
47
+ [
48
+ [s, str(len(agg.repos_by_stage.get(s, set())))]
49
+ for s in LIFECYCLE_STAGES
50
+ ],
51
+ ),
52
+ )
53
+ if agg.sample_refs:
54
+ md.append("\nSample locations:\n")
55
+ for ref in agg.sample_refs:
56
+ md.append(f"- {ref}\n")
57
+
58
+
59
+ def write_pii_lifecycle_graph(
60
+ metabase_root: Path,
61
+ repo_jsons: list[Path] | None = None,
62
+ ) -> list[PiiTouchpoint]:
63
+ """Write the PII lifecycle graph (JSONL + markdown) to graphs/ and return touches."""
64
+ records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
65
+ touches = collect_pii_touchpoints(records)
66
+ aggregates = aggregate_by_field(touches)
67
+
68
+ graphs_dir = metabase_root / "graphs"
69
+ graphs_dir.mkdir(parents=True, exist_ok=True)
70
+
71
+ jsonl_path = graphs_dir / "pii-lifecycle.jsonl"
72
+ with jsonl_path.open("w", encoding="utf-8") as fh:
73
+ for touch in touches:
74
+ fh.write(json.dumps(touch.to_dict(), ensure_ascii=False) + "\n")
75
+
76
+ stage_counts = Counter(t.stage for t in touches)
77
+ class_counts = Counter(t.pii_classification for t in touches)
78
+
79
+ md: list[str] = [
80
+ "# PII lifecycle (fleet)\n",
81
+ UNTRUSTED_CONTENT_NOTICE,
82
+ "_Static analysis from v2 `pii-field`, `pii-storage`, `pii-log`, and "
83
+ "file-local proximity to SQL/HTTP/crypto. Not a runtime data-flow proof._\n",
84
+ "\n## Stage totals\n",
85
+ md_table(
86
+ ["Stage", "Touchpoints"],
87
+ [[s, str(stage_counts.get(s, 0))] for s in LIFECYCLE_STAGES],
88
+ ),
89
+ "\n## By GDPR classification\n",
90
+ md_table(
91
+ ["Classification", "Touchpoints"],
92
+ [[k, str(v)] for k, v in class_counts.most_common()],
93
+ ),
94
+ ]
95
+
96
+ _append_phone_example(md, aggregates)
97
+
98
+ md.append("\n## Field catalogue (top by touchpoints)\n")
99
+ ranked = sorted(aggregates.values(), key=lambda a: -a.touchpoints)
100
+ md.append(
101
+ md_table(
102
+ ["Field", "Classification", "Touchpoints", "Collect repos", "Store repos", "Log repos"],
103
+ [
104
+ [
105
+ a.field_key,
106
+ a.pii_classification,
107
+ str(a.touchpoints),
108
+ str(len(a.repos_by_stage.get("collect", set()))),
109
+ str(len(a.repos_by_stage.get("store", set()))),
110
+ str(len(a.repos_by_stage.get("log", set()))),
111
+ ]
112
+ for a in ranked[:MAX_MD_ROWS]
113
+ ],
114
+ ),
115
+ )
116
+ tail = max(0, len(ranked) - MAX_MD_ROWS)
117
+ if tail:
118
+ md.append(f"\n_{tail} more fields in `pii-lifecycle.jsonl`._\n")
119
+
120
+ (graphs_dir / "pii-lifecycle.md").write_text("\n".join(md), encoding="utf-8")
121
+ return touches
@@ -0,0 +1,134 @@
1
+ """Collect fleet-wide PII lifecycle touchpoints from v2 flow nodes."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ from ..graph_common import iter_nodes, repo_id
8
+ from ..models.pii_lifecycle import PiiTouchpoint
9
+
10
+ from .pii_touchpoint_utils import (
11
+ PROXIMITY_LINES,
12
+ DELETE_RX,
13
+ lines_by_file,
14
+ nearby,
15
+ proximity_touch,
16
+ touch_from_node,
17
+ )
18
+
19
+
20
+ def _maybe_add_proximity(
21
+ touches: list[PiiTouchpoint],
22
+ seen_intra: set[tuple[str, str, str, int]],
23
+ base: PiiTouchpoint,
24
+ *,
25
+ stage: str,
26
+ family: str,
27
+ evidence: str,
28
+ file: str,
29
+ line: int,
30
+ by_file: dict[str, list[tuple[int, str, dict[str, Any]]]],
31
+ probe_families: set[str],
32
+ ) -> None:
33
+ """Add a deduplicated proximity touchpoint if a probe family is near the line."""
34
+ if not nearby(by_file, file, line, probe_families, window=PROXIMITY_LINES):
35
+ return
36
+ sig = (base.repo, base.field_key, stage, line)
37
+ if sig in seen_intra:
38
+ return
39
+ seen_intra.add(sig)
40
+ touches.append(proximity_touch(base, stage=stage, family=family, evidence=evidence))
41
+
42
+
43
+ def _collect_pii_field_node(
44
+ rid: str,
45
+ node: dict[str, Any],
46
+ by_file: dict[str, list[tuple[int, str, dict[str, Any]]]],
47
+ touches: list[PiiTouchpoint],
48
+ seen_intra: set[tuple[str, str, str, int]],
49
+ ) -> None:
50
+ """Emit a collect touch for a pii-field node plus nearby process/transmit/encrypt touches."""
51
+ line = int(node.get("line") or 0)
52
+ file = node.get("file", "")
53
+ t = touch_from_node(rid, "collect", node)
54
+ if t:
55
+ touches.append(t)
56
+ if not t:
57
+ return
58
+ _maybe_add_proximity(
59
+ touches, seen_intra, t,
60
+ stage="process", family="sql+near-pii-field",
61
+ evidence="sql sink within file window",
62
+ file=file, line=line, by_file=by_file, probe_families={"sql"},
63
+ )
64
+ _maybe_add_proximity(
65
+ touches, seen_intra, t,
66
+ stage="transmit", family="http-out+near-pii-field",
67
+ evidence="http-out within file window",
68
+ file=file, line=line, by_file=by_file, probe_families={"http-out"},
69
+ )
70
+ _maybe_add_proximity(
71
+ touches, seen_intra, t,
72
+ stage="encrypt", family="crypto+near-pii-field",
73
+ evidence="crypto op within file window",
74
+ file=file, line=line, by_file=by_file,
75
+ probe_families={"crypto-algorithm", "crypto-config"},
76
+ )
77
+
78
+
79
+ def _collect_delete_hints(
80
+ rid: str,
81
+ data: dict[str, Any],
82
+ by_file: dict[str, list[tuple[int, str, dict[str, Any]]]],
83
+ touches: list[PiiTouchpoint],
84
+ ) -> None:
85
+ """Emit delete-stage touches where a delete/remove op sits near a PII field."""
86
+ for node in iter_nodes(data):
87
+ if node.get("family") not in ("pii-log", "pii-storage", "pii-field"):
88
+ continue
89
+ detail = node.get("detail") or {}
90
+ field_name = detail.get("field_name")
91
+ if not field_name:
92
+ continue
93
+ file = node.get("file", "")
94
+ line = int(node.get("line") or 0)
95
+ for ln, _, n in by_file.get(file, []):
96
+ if abs(ln - line) > 30:
97
+ continue
98
+ raw = (n.get("detail") or {}).get("raw", "")
99
+ if DELETE_RX.search(raw):
100
+ t = touch_from_node(
101
+ rid,
102
+ "delete",
103
+ node,
104
+ evidence="delete/remove near field",
105
+ )
106
+ if t:
107
+ touches.append(t)
108
+ break
109
+
110
+
111
+ def collect_pii_touchpoints(records: list[dict[str, Any]]) -> list[PiiTouchpoint]:
112
+ """Collect all PII lifecycle touchpoints across the given repo records."""
113
+ touches: list[PiiTouchpoint] = []
114
+ for data in records:
115
+ rid = repo_id(data)
116
+ by_file = lines_by_file(data)
117
+ seen_intra: set[tuple[str, str, str, int]] = set()
118
+
119
+ for node in iter_nodes(data):
120
+ family = node.get("family", "")
121
+ if family == "pii-field":
122
+ _collect_pii_field_node(rid, node, by_file, touches, seen_intra)
123
+ elif family == "pii-storage":
124
+ t = touch_from_node(rid, "store", node)
125
+ if t:
126
+ touches.append(t)
127
+ elif family == "pii-log":
128
+ t = touch_from_node(rid, "log", node)
129
+ if t:
130
+ touches.append(t)
131
+
132
+ _collect_delete_hints(rid, data, by_file, touches)
133
+
134
+ return touches
@@ -0,0 +1,96 @@
1
+ """Shared helpers for PII lifecycle touchpoint collection."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from collections import defaultdict
7
+ from typing import Any
8
+
9
+ from ..models.pii_lifecycle import PiiTouchpoint, normalize_field_key
10
+
11
+ PROXIMITY_LINES = 80
12
+
13
+ DELETE_RX = re.compile(
14
+ r"\b(delete|remove|erase|purge|forget)\w*\b",
15
+ re.IGNORECASE,
16
+ )
17
+
18
+
19
+ def lines_by_file(data: dict[str, Any]) -> dict[str, list[tuple[int, str, dict[str, Any]]]]:
20
+ """file -> [(line, family, node), ...] sorted by line."""
21
+ by_file: dict[str, list[tuple[int, str, dict[str, Any]]]] = defaultdict(list)
22
+ for node in data.get("nodes", []):
23
+ by_file[node.get("file", "")].append(
24
+ (int(node.get("line") or 0), node.get("family", ""), node),
25
+ )
26
+ for entries in by_file.values():
27
+ entries.sort(key=lambda x: x[0])
28
+ return by_file
29
+
30
+
31
+ def nearby(
32
+ by_file: dict[str, list[tuple[int, str, dict[str, Any]]]],
33
+ file: str,
34
+ line: int,
35
+ families: set[str],
36
+ *,
37
+ window: int = PROXIMITY_LINES,
38
+ ) -> bool:
39
+ """Return True if any node of the given families sits within window lines."""
40
+ for ln, fam, _ in by_file.get(file, []):
41
+ if fam in families and abs(ln - line) <= window:
42
+ return True
43
+ return False
44
+
45
+
46
+ def touch_from_node(
47
+ rid: str,
48
+ stage: str,
49
+ node: dict[str, Any],
50
+ *,
51
+ evidence: str = "",
52
+ ) -> PiiTouchpoint | None:
53
+ """Build a PiiTouchpoint from a flow node, or None if it carries no usable field."""
54
+ detail = node.get("detail") or {}
55
+ field_name = detail.get("field_name") or ""
56
+ if not field_name and stage not in ("encrypt", "delete"):
57
+ if node.get("family") not in ("pii-log", "pii-storage"):
58
+ return None
59
+ field_key = normalize_field_key(field_name or "unknown")
60
+ pii_c = node.get("pii_classification") or "unknown"
61
+ return PiiTouchpoint(
62
+ repo=rid,
63
+ stage=stage,
64
+ family=node.get("family", ""),
65
+ field_key=field_key,
66
+ field_name=field_name or field_key,
67
+ pii_classification=pii_c,
68
+ data_class=node.get("data_class"),
69
+ file=node.get("file", ""),
70
+ line=int(node.get("line") or 0),
71
+ confidence=node.get("confidence", "medium"),
72
+ evidence=evidence,
73
+ )
74
+
75
+
76
+ def proximity_touch(
77
+ base: PiiTouchpoint,
78
+ *,
79
+ stage: str,
80
+ family: str,
81
+ evidence: str,
82
+ ) -> PiiTouchpoint:
83
+ """Clone a collect-stage touch with a proximity-derived stage."""
84
+ return PiiTouchpoint(
85
+ repo=base.repo,
86
+ stage=stage,
87
+ family=family,
88
+ field_key=base.field_key,
89
+ field_name=base.field_name,
90
+ pii_classification=base.pii_classification,
91
+ data_class=base.data_class,
92
+ file=base.file,
93
+ line=base.line,
94
+ confidence="low",
95
+ evidence=evidence,
96
+ )
@@ -0,0 +1,141 @@
1
+ """Queue / messaging graph aggregator — v2 queue-pub / queue-sub nodes."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ from collections import defaultdict
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from ..graph_common import iter_nodes, load_v2_repo_records, repo_id
12
+ from ..sanitize import UNTRUSTED_CONTENT_NOTICE, for_markdown, for_mermaid_label
13
+ from ..renderers.markdown import md_table
14
+
15
+
16
+ def _collect_queue_topics(records: list[dict[str, Any]]) -> dict[str, dict[str, set[str]]]:
17
+ """Map each topic to producing/consuming repos and messaging systems."""
18
+ topics: dict[str, dict[str, set[str]]] = defaultdict(
19
+ lambda: {"produce": set(), "consume": set(), "systems": set()},
20
+ )
21
+ for data in records:
22
+ rid = repo_id(data)
23
+ for node in iter_nodes(data):
24
+ family = node.get("family", "")
25
+ detail = node.get("detail") or {}
26
+ topic = detail.get("topic", "")
27
+ if not topic or topic == "?":
28
+ continue
29
+ role = {"queue-pub": "produce", "queue-sub": "consume"}.get(family)
30
+ if not role:
31
+ continue
32
+ topics[topic][role].add(rid)
33
+ system = detail.get("system") or ""
34
+ if system:
35
+ topics[topic]["systems"].add(system)
36
+ return topics
37
+
38
+
39
+ def _queue_rows_and_orphans(
40
+ topics: dict[str, dict[str, set[str]]],
41
+ ) -> tuple[list[list[str]], list[str], list[str]]:
42
+ """Build topic table rows plus produce-only and consume-only orphan lists."""
43
+ rows: list[list[str]] = []
44
+ orphans_produce: list[str] = []
45
+ orphans_consume: list[str] = []
46
+ for topic, dirs in sorted(topics.items()):
47
+ producers = sorted(dirs["produce"])
48
+ consumers = sorted(dirs["consume"])
49
+ systems = sorted(dirs["systems"])
50
+ rows.append([
51
+ topic,
52
+ ", ".join(systems) if systems else "—",
53
+ ", ".join(producers) if producers else "—",
54
+ ", ".join(consumers) if consumers else "—",
55
+ ])
56
+ if producers and not consumers:
57
+ orphans_produce.append(topic)
58
+ if consumers and not producers:
59
+ orphans_consume.append(topic)
60
+ return rows, orphans_produce, orphans_consume
61
+
62
+
63
+ def _orphan_line(label: str, topics: list[str]) -> str:
64
+ """Format a labelled bullet listing orphan topics (capped at 30)."""
65
+ return (
66
+ f"- **{label}** ({len(topics)}): "
67
+ # topics are untrusted extracted literals emitted outside a table cell.
68
+ + ", ".join(for_markdown(t, max_len=80) for t in topics[:30])
69
+ + (" …" if len(topics) > 30 else "")
70
+ + "\n"
71
+ )
72
+
73
+
74
+ def _queue_mermaid(topics: dict[str, dict[str, set[str]]]) -> list[str]:
75
+ """Render a Mermaid flowchart for topics that have both producers and consumers."""
76
+ both = [t for t, d in topics.items() if d["produce"] and d["consume"]][:25]
77
+ if not both:
78
+ return []
79
+ md = ["\n## Mermaid (topics with producers and consumers)\n\n```mermaid\nflowchart LR\n"]
80
+ for topic in both:
81
+ d = topics[topic]
82
+ # Both the node id and the label are attacker-influenced: the id must be a
83
+ # strict slug (drop everything non-alphanumeric) or the topic's quotes /
84
+ # newlines break out of the node; the label is neutralised separately.
85
+ tid = re.sub(r"[^A-Za-z0-9_]", "_", topic)[:50]
86
+ topic_label = for_mermaid_label(topic)
87
+ for p in sorted(d["produce"])[:3]:
88
+ pid = re.sub(r"[^A-Za-z0-9_]", "_", p)[:30]
89
+ md.append(f' {pid}["{for_mermaid_label(p)}"] -->|{topic_label}| hub_{tid}["{topic_label}"]\n')
90
+ for c in sorted(d["consume"])[:3]:
91
+ cid = re.sub(r"[^A-Za-z0-9_]", "_", c)[:30]
92
+ md.append(f' hub_{tid} --> {cid}["{for_mermaid_label(c)}"]\n')
93
+ md.append("```\n")
94
+ return md
95
+
96
+
97
+ def _write_queue_jsonl(path: Path, topics: dict[str, dict[str, set[str]]]) -> None:
98
+ """Write one JSON line per topic with its systems, producers, and consumers."""
99
+ with path.open("w", encoding="utf-8") as fh:
100
+ for topic, dirs in sorted(topics.items()):
101
+ systems = sorted(dirs["systems"])
102
+ fh.write(json.dumps({
103
+ "topic": topic,
104
+ "queue_types": systems,
105
+ "queue_type": ", ".join(systems) if systems else None,
106
+ "producers": sorted(dirs["produce"]),
107
+ "consumers": sorted(dirs["consume"]),
108
+ }, ensure_ascii=False) + "\n")
109
+
110
+
111
+ def write_queue_graph(metabase_root: Path, repo_jsons: list[Path]) -> None:
112
+ """Write the queue/messaging graph markdown + jsonl from v2 queue nodes."""
113
+ records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
114
+ topics = _collect_queue_topics(records)
115
+
116
+ out_dir = metabase_root / "graphs"
117
+ out_dir.mkdir(parents=True, exist_ok=True)
118
+
119
+ rows, orphans_produce, orphans_consume = _queue_rows_and_orphans(topics)
120
+ md: list[str] = [
121
+ "# Queue / messaging graph (v2)\n",
122
+ UNTRUSTED_CONTENT_NOTICE,
123
+ "_Producers (`queue-pub`) and consumers (`queue-sub`) per topic from "
124
+ "v2 flow nodes. **Queue type** is the `detail.system` from the extractor "
125
+ "(kafka, rabbitmq, sqs, sns, redis-stream, nats, jms, …)._\n",
126
+ "\n## Topics\n",
127
+ md_table(
128
+ ["Topic", "Queue type", "Producers", "Consumers"],
129
+ rows or [["—", "—", "—", "—"]],
130
+ ),
131
+ ]
132
+ if orphans_produce or orphans_consume:
133
+ md.append("\n## Orphan topics\n")
134
+ if orphans_produce:
135
+ md.append(_orphan_line("Produce-only", orphans_produce))
136
+ if orphans_consume:
137
+ md.append(_orphan_line("Consume-only", orphans_consume))
138
+ md.extend(_queue_mermaid(topics))
139
+
140
+ (out_dir / "queue-graph.md").write_text("\n".join(md), encoding="utf-8")
141
+ _write_queue_jsonl(out_dir / "queue-graph.jsonl", topics)