src2sink 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- src2sink/__init__.py +1 -0
- src2sink/aggregators/__init__.py +3 -0
- src2sink/aggregators/api_client_discovery.py +259 -0
- src2sink/aggregators/auth_cards.py +116 -0
- src2sink/aggregators/crypto_cards.py +132 -0
- src2sink/aggregators/data_stores.py +125 -0
- src2sink/aggregators/graphs.py +36 -0
- src2sink/aggregators/index_v2.py +138 -0
- src2sink/aggregators/library_source_map.py +188 -0
- src2sink/aggregators/openapi_discovery.py +113 -0
- src2sink/aggregators/openapi_edges.py +30 -0
- src2sink/aggregators/openapi_match.py +81 -0
- src2sink/aggregators/openapi_models.py +25 -0
- src2sink/aggregators/openapi_report.py +75 -0
- src2sink/aggregators/payload_producers.py +369 -0
- src2sink/aggregators/phase3.py +30 -0
- src2sink/aggregators/pii_cross_repo.py +283 -0
- src2sink/aggregators/pii_flow_v2.py +118 -0
- src2sink/aggregators/pii_lifecycle.py +19 -0
- src2sink/aggregators/pii_lifecycle_report.py +121 -0
- src2sink/aggregators/pii_touchpoint_collect.py +134 -0
- src2sink/aggregators/pii_touchpoint_utils.py +96 -0
- src2sink/aggregators/queues.py +141 -0
- src2sink/aggregators/ropa.py +190 -0
- src2sink/aggregators/service_call_collect.py +240 -0
- src2sink/aggregators/service_call_index.py +31 -0
- src2sink/aggregators/service_call_models.py +17 -0
- src2sink/aggregators/service_call_report.py +118 -0
- src2sink/aggregators/service_calls.py +14 -0
- src2sink/aggregators/taint_buckets.py +74 -0
- src2sink/aggregators/taint_catalogs.py +35 -0
- src2sink/aggregators/taint_writers.py +369 -0
- src2sink/aggregators/traces_index.py +103 -0
- src2sink/build_metabase_v2.py +807 -0
- src2sink/constants.py +44 -0
- src2sink/curate_internal_libraries.py +135 -0
- src2sink/extractors/__init__.py +17 -0
- src2sink/extractors/ast_walk.py +131 -0
- src2sink/extractors/base.py +59 -0
- src2sink/extractors/config.py +224 -0
- src2sink/extractors/file_context.py +35 -0
- src2sink/extractors/go.py +3 -0
- src2sink/extractors/http_out.py +108 -0
- src2sink/extractors/java.py +3 -0
- src2sink/extractors/javascript.py +3 -0
- src2sink/extractors/kotlin.py +3 -0
- src2sink/extractors/node_factory.py +65 -0
- src2sink/extractors/patterns.py +145 -0
- src2sink/extractors/python_extractor.py +3 -0
- src2sink/extractors/regex_extractors.py +378 -0
- src2sink/extractors/ts_extractors.py +123 -0
- src2sink/extractors/unified.py +64 -0
- src2sink/graph_common.py +213 -0
- src2sink/internal_groups.py +142 -0
- src2sink/known_api_clients.py +108 -0
- src2sink/library_taint_java.py +92 -0
- src2sink/limits.py +171 -0
- src2sink/models/__init__.py +3 -0
- src2sink/models/auth_model.py +3 -0
- src2sink/models/crypto_agility.py +3 -0
- src2sink/models/pii_lifecycle.py +92 -0
- src2sink/models/ropa.py +37 -0
- src2sink/prescreen.py +93 -0
- src2sink/py.typed +0 -0
- src2sink/record_fleet_baseline.py +81 -0
- src2sink/renderers/__init__.py +3 -0
- src2sink/renderers/markdown.py +149 -0
- src2sink/renderers/mermaid.py +3 -0
- src2sink/repo_utils.py +885 -0
- src2sink/safe_paths.py +54 -0
- src2sink/sanitize.py +107 -0
- src2sink/schema.py +57 -0
- src2sink/trace.py +558 -0
- src2sink/trace_batch.py +176 -0
- src2sink/vocabulary.py +152 -0
- src2sink-1.0.0.dist-info/METADATA +355 -0
- src2sink-1.0.0.dist-info/RECORD +81 -0
- src2sink-1.0.0.dist-info/WHEEL +5 -0
- src2sink-1.0.0.dist-info/entry_points.txt +6 -0
- src2sink-1.0.0.dist-info/licenses/LICENSE +21 -0
- src2sink-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""v2 PII flow roll-up (replaces v1 graphs/pii-flow.md)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from ..graph_common import iter_nodes, load_v2_repo_records
|
|
11
|
+
from ..renderers.markdown import md_table
|
|
12
|
+
from ..sanitize import UNTRUSTED_CONTENT_NOTICE
|
|
13
|
+
|
|
14
|
+
MAX_MD_ROWS = 200
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class _PiiFlowCounts:
|
|
19
|
+
"""Aggregated PII/HTTP/queue touchpoint counters across all repos."""
|
|
20
|
+
|
|
21
|
+
by_class: Counter[str] = field(default_factory=Counter)
|
|
22
|
+
by_repo: Counter[str] = field(default_factory=Counter)
|
|
23
|
+
field_counts: Counter[str] = field(default_factory=Counter)
|
|
24
|
+
sensitive_by_repo: Counter[str] = field(default_factory=Counter)
|
|
25
|
+
http_in_by_repo: Counter[str] = field(default_factory=Counter)
|
|
26
|
+
http_out_by_repo: Counter[str] = field(default_factory=Counter)
|
|
27
|
+
queue_by_repo: Counter[str] = field(default_factory=Counter)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _collect_pii_flow(records: list[dict[str, Any]]) -> _PiiFlowCounts:
|
|
31
|
+
"""Tally PII, HTTP, and queue nodes per classification and per repo."""
|
|
32
|
+
c = _PiiFlowCounts()
|
|
33
|
+
for data in records:
|
|
34
|
+
rid = f"{data['group']}/{data['name']}"
|
|
35
|
+
for node in iter_nodes(data):
|
|
36
|
+
family = node.get("family")
|
|
37
|
+
if family in ("pii-field", "pii-log", "pii-storage"):
|
|
38
|
+
cls = node.get("pii_classification") or "unknown"
|
|
39
|
+
c.by_class[cls] += 1
|
|
40
|
+
c.by_repo[rid] += 1
|
|
41
|
+
field_name = (node.get("detail") or {}).get("field_name")
|
|
42
|
+
if field_name:
|
|
43
|
+
c.field_counts[field_name.lower()] += 1
|
|
44
|
+
if cls in ("sensitive", "special-category-gdpr"):
|
|
45
|
+
c.sensitive_by_repo[rid] += 1
|
|
46
|
+
elif family == "http-in":
|
|
47
|
+
c.http_in_by_repo[rid] += 1
|
|
48
|
+
elif family == "http-out":
|
|
49
|
+
c.http_out_by_repo[rid] += 1
|
|
50
|
+
elif family in ("queue-in", "queue-out"):
|
|
51
|
+
c.queue_by_repo[rid] += 1
|
|
52
|
+
return c
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def write_pii_flow_v2(metabase_root: Path, repo_jsons: list[Path]) -> None:
|
|
56
|
+
"""Write the v2 PII flow roll-up markdown (graphs/pii-flow.md)."""
|
|
57
|
+
records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
|
|
58
|
+
c = _collect_pii_flow(records)
|
|
59
|
+
by_class = c.by_class
|
|
60
|
+
by_repo = c.by_repo
|
|
61
|
+
field_counts = c.field_counts
|
|
62
|
+
sensitive_by_repo = c.sensitive_by_repo
|
|
63
|
+
http_in_by_repo = c.http_in_by_repo
|
|
64
|
+
http_out_by_repo = c.http_out_by_repo
|
|
65
|
+
queue_by_repo = c.queue_by_repo
|
|
66
|
+
|
|
67
|
+
graphs_dir = metabase_root / "graphs"
|
|
68
|
+
graphs_dir.mkdir(parents=True, exist_ok=True)
|
|
69
|
+
|
|
70
|
+
md: list[str] = [
|
|
71
|
+
"# PII flow roll-up (v2)\n",
|
|
72
|
+
UNTRUSTED_CONTENT_NOTICE,
|
|
73
|
+
"_Aggregated from `pii-field`, `pii-log`, and `pii-storage` nodes. "
|
|
74
|
+
"For lifecycle stages see `pii-lifecycle.md`._\n",
|
|
75
|
+
"\n## By GDPR classification\n",
|
|
76
|
+
md_table(
|
|
77
|
+
["Classification", "Touchpoints"],
|
|
78
|
+
[[k, str(v)] for k, v in by_class.most_common()],
|
|
79
|
+
),
|
|
80
|
+
"\n## Top fields (all repos)\n",
|
|
81
|
+
md_table(
|
|
82
|
+
["Field", "Count"],
|
|
83
|
+
[[k, str(v)] for k, v in field_counts.most_common(40)],
|
|
84
|
+
),
|
|
85
|
+
"\n## Top repos by PII touchpoints\n",
|
|
86
|
+
md_table(
|
|
87
|
+
["Repo", "Touchpoints"],
|
|
88
|
+
[[k, str(v)] for k, v in by_repo.most_common(MAX_MD_ROWS)],
|
|
89
|
+
),
|
|
90
|
+
]
|
|
91
|
+
tail = len(by_repo) - MAX_MD_ROWS
|
|
92
|
+
if tail > 0:
|
|
93
|
+
md.append(f"\n_{tail} more repos — see `taint/pii-sources.jsonl`._\n")
|
|
94
|
+
|
|
95
|
+
# Probable PII handlers: repos that have inbound HTTP endpoints AND
|
|
96
|
+
# reference at least one sensitive/special-category-gdpr PII node.
|
|
97
|
+
handlers = [
|
|
98
|
+
rid for rid in sensitive_by_repo
|
|
99
|
+
if http_in_by_repo.get(rid, 0) > 0
|
|
100
|
+
]
|
|
101
|
+
handler_rows = sorted(
|
|
102
|
+
handlers,
|
|
103
|
+
key=lambda r: -sensitive_by_repo[r],
|
|
104
|
+
)
|
|
105
|
+
md.append(
|
|
106
|
+
"\n## Probable PII handlers\n"
|
|
107
|
+
"_Repos with inbound HTTP endpoints and `sensitive` or "
|
|
108
|
+
"`special-category-gdpr` PII nodes. Top candidates for PII & "
|
|
109
|
+
"data-minimisation review._\n"
|
|
110
|
+
)
|
|
111
|
+
md.append(md_table(
|
|
112
|
+
["Repo", "HTTP-in", "HTTP-out", "Queue I/O", "Sensitive count"],
|
|
113
|
+
[[r, str(http_in_by_repo[r]), str(http_out_by_repo.get(r, 0)),
|
|
114
|
+
str(queue_by_repo.get(r, 0)), str(sensitive_by_repo[r])]
|
|
115
|
+
for r in handler_rows[:80]] or [["—", "—", "—", "—", "—"]],
|
|
116
|
+
))
|
|
117
|
+
|
|
118
|
+
(graphs_dir / "pii-flow.md").write_text("\n".join(md), encoding="utf-8")
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Fleet-wide PII lifecycle graph from v2 flow nodes (Phase 3)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .pii_lifecycle_report import (
|
|
6
|
+
MAX_MD_ROWS,
|
|
7
|
+
aggregate_by_field,
|
|
8
|
+
write_pii_lifecycle_graph,
|
|
9
|
+
)
|
|
10
|
+
from .pii_touchpoint_collect import collect_pii_touchpoints
|
|
11
|
+
from ..models.pii_lifecycle import PiiTouchpoint
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"MAX_MD_ROWS",
|
|
15
|
+
"PiiTouchpoint",
|
|
16
|
+
"aggregate_by_field",
|
|
17
|
+
"collect_pii_touchpoints",
|
|
18
|
+
"write_pii_lifecycle_graph",
|
|
19
|
+
]
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Render PII lifecycle graph markdown and JSONL."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections import Counter
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from ..graph_common import load_v2_repo_records
|
|
10
|
+
from ..models.pii_lifecycle import LIFECYCLE_STAGES, FieldLifecycleAggregate, PiiTouchpoint
|
|
11
|
+
from ..sanitize import UNTRUSTED_CONTENT_NOTICE
|
|
12
|
+
from ..renderers.markdown import md_table
|
|
13
|
+
|
|
14
|
+
from .pii_touchpoint_collect import collect_pii_touchpoints
|
|
15
|
+
|
|
16
|
+
MAX_MD_ROWS = 400
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def aggregate_by_field(touches: list[PiiTouchpoint]) -> dict[str, FieldLifecycleAggregate]:
|
|
20
|
+
"""Group touchpoints into per-(field, classification) lifecycle aggregates."""
|
|
21
|
+
buckets: dict[str, FieldLifecycleAggregate] = {}
|
|
22
|
+
for touch in touches:
|
|
23
|
+
bucket_key = f"{touch.field_key}|{touch.pii_classification}"
|
|
24
|
+
if bucket_key not in buckets:
|
|
25
|
+
buckets[bucket_key] = FieldLifecycleAggregate(
|
|
26
|
+
field_key=touch.field_key,
|
|
27
|
+
pii_classification=touch.pii_classification,
|
|
28
|
+
)
|
|
29
|
+
buckets[bucket_key].add(touch)
|
|
30
|
+
return buckets
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _append_phone_example(
|
|
34
|
+
md: list[str],
|
|
35
|
+
aggregates: dict[str, FieldLifecycleAggregate],
|
|
36
|
+
) -> None:
|
|
37
|
+
"""Append a worked `phone` lifecycle example section to the markdown lines."""
|
|
38
|
+
phone_agg = [a for a in aggregates.values() if a.field_key == "phone"]
|
|
39
|
+
if not phone_agg:
|
|
40
|
+
return
|
|
41
|
+
md.append("\n## Worked example: `phone` (direct PII)\n")
|
|
42
|
+
for agg in sorted(phone_agg, key=lambda a: -a.touchpoints)[:3]:
|
|
43
|
+
md.append(f"\n### Classification: `{agg.pii_classification}`\n")
|
|
44
|
+
md.append(
|
|
45
|
+
md_table(
|
|
46
|
+
["Stage", "Repos (count)"],
|
|
47
|
+
[
|
|
48
|
+
[s, str(len(agg.repos_by_stage.get(s, set())))]
|
|
49
|
+
for s in LIFECYCLE_STAGES
|
|
50
|
+
],
|
|
51
|
+
),
|
|
52
|
+
)
|
|
53
|
+
if agg.sample_refs:
|
|
54
|
+
md.append("\nSample locations:\n")
|
|
55
|
+
for ref in agg.sample_refs:
|
|
56
|
+
md.append(f"- {ref}\n")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def write_pii_lifecycle_graph(
|
|
60
|
+
metabase_root: Path,
|
|
61
|
+
repo_jsons: list[Path] | None = None,
|
|
62
|
+
) -> list[PiiTouchpoint]:
|
|
63
|
+
"""Write the PII lifecycle graph (JSONL + markdown) to graphs/ and return touches."""
|
|
64
|
+
records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
|
|
65
|
+
touches = collect_pii_touchpoints(records)
|
|
66
|
+
aggregates = aggregate_by_field(touches)
|
|
67
|
+
|
|
68
|
+
graphs_dir = metabase_root / "graphs"
|
|
69
|
+
graphs_dir.mkdir(parents=True, exist_ok=True)
|
|
70
|
+
|
|
71
|
+
jsonl_path = graphs_dir / "pii-lifecycle.jsonl"
|
|
72
|
+
with jsonl_path.open("w", encoding="utf-8") as fh:
|
|
73
|
+
for touch in touches:
|
|
74
|
+
fh.write(json.dumps(touch.to_dict(), ensure_ascii=False) + "\n")
|
|
75
|
+
|
|
76
|
+
stage_counts = Counter(t.stage for t in touches)
|
|
77
|
+
class_counts = Counter(t.pii_classification for t in touches)
|
|
78
|
+
|
|
79
|
+
md: list[str] = [
|
|
80
|
+
"# PII lifecycle (fleet)\n",
|
|
81
|
+
UNTRUSTED_CONTENT_NOTICE,
|
|
82
|
+
"_Static analysis from v2 `pii-field`, `pii-storage`, `pii-log`, and "
|
|
83
|
+
"file-local proximity to SQL/HTTP/crypto. Not a runtime data-flow proof._\n",
|
|
84
|
+
"\n## Stage totals\n",
|
|
85
|
+
md_table(
|
|
86
|
+
["Stage", "Touchpoints"],
|
|
87
|
+
[[s, str(stage_counts.get(s, 0))] for s in LIFECYCLE_STAGES],
|
|
88
|
+
),
|
|
89
|
+
"\n## By GDPR classification\n",
|
|
90
|
+
md_table(
|
|
91
|
+
["Classification", "Touchpoints"],
|
|
92
|
+
[[k, str(v)] for k, v in class_counts.most_common()],
|
|
93
|
+
),
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
_append_phone_example(md, aggregates)
|
|
97
|
+
|
|
98
|
+
md.append("\n## Field catalogue (top by touchpoints)\n")
|
|
99
|
+
ranked = sorted(aggregates.values(), key=lambda a: -a.touchpoints)
|
|
100
|
+
md.append(
|
|
101
|
+
md_table(
|
|
102
|
+
["Field", "Classification", "Touchpoints", "Collect repos", "Store repos", "Log repos"],
|
|
103
|
+
[
|
|
104
|
+
[
|
|
105
|
+
a.field_key,
|
|
106
|
+
a.pii_classification,
|
|
107
|
+
str(a.touchpoints),
|
|
108
|
+
str(len(a.repos_by_stage.get("collect", set()))),
|
|
109
|
+
str(len(a.repos_by_stage.get("store", set()))),
|
|
110
|
+
str(len(a.repos_by_stage.get("log", set()))),
|
|
111
|
+
]
|
|
112
|
+
for a in ranked[:MAX_MD_ROWS]
|
|
113
|
+
],
|
|
114
|
+
),
|
|
115
|
+
)
|
|
116
|
+
tail = max(0, len(ranked) - MAX_MD_ROWS)
|
|
117
|
+
if tail:
|
|
118
|
+
md.append(f"\n_{tail} more fields in `pii-lifecycle.jsonl`._\n")
|
|
119
|
+
|
|
120
|
+
(graphs_dir / "pii-lifecycle.md").write_text("\n".join(md), encoding="utf-8")
|
|
121
|
+
return touches
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Collect fleet-wide PII lifecycle touchpoints from v2 flow nodes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ..graph_common import iter_nodes, repo_id
|
|
8
|
+
from ..models.pii_lifecycle import PiiTouchpoint
|
|
9
|
+
|
|
10
|
+
from .pii_touchpoint_utils import (
|
|
11
|
+
PROXIMITY_LINES,
|
|
12
|
+
DELETE_RX,
|
|
13
|
+
lines_by_file,
|
|
14
|
+
nearby,
|
|
15
|
+
proximity_touch,
|
|
16
|
+
touch_from_node,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _maybe_add_proximity(
|
|
21
|
+
touches: list[PiiTouchpoint],
|
|
22
|
+
seen_intra: set[tuple[str, str, str, int]],
|
|
23
|
+
base: PiiTouchpoint,
|
|
24
|
+
*,
|
|
25
|
+
stage: str,
|
|
26
|
+
family: str,
|
|
27
|
+
evidence: str,
|
|
28
|
+
file: str,
|
|
29
|
+
line: int,
|
|
30
|
+
by_file: dict[str, list[tuple[int, str, dict[str, Any]]]],
|
|
31
|
+
probe_families: set[str],
|
|
32
|
+
) -> None:
|
|
33
|
+
"""Add a deduplicated proximity touchpoint if a probe family is near the line."""
|
|
34
|
+
if not nearby(by_file, file, line, probe_families, window=PROXIMITY_LINES):
|
|
35
|
+
return
|
|
36
|
+
sig = (base.repo, base.field_key, stage, line)
|
|
37
|
+
if sig in seen_intra:
|
|
38
|
+
return
|
|
39
|
+
seen_intra.add(sig)
|
|
40
|
+
touches.append(proximity_touch(base, stage=stage, family=family, evidence=evidence))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _collect_pii_field_node(
|
|
44
|
+
rid: str,
|
|
45
|
+
node: dict[str, Any],
|
|
46
|
+
by_file: dict[str, list[tuple[int, str, dict[str, Any]]]],
|
|
47
|
+
touches: list[PiiTouchpoint],
|
|
48
|
+
seen_intra: set[tuple[str, str, str, int]],
|
|
49
|
+
) -> None:
|
|
50
|
+
"""Emit a collect touch for a pii-field node plus nearby process/transmit/encrypt touches."""
|
|
51
|
+
line = int(node.get("line") or 0)
|
|
52
|
+
file = node.get("file", "")
|
|
53
|
+
t = touch_from_node(rid, "collect", node)
|
|
54
|
+
if t:
|
|
55
|
+
touches.append(t)
|
|
56
|
+
if not t:
|
|
57
|
+
return
|
|
58
|
+
_maybe_add_proximity(
|
|
59
|
+
touches, seen_intra, t,
|
|
60
|
+
stage="process", family="sql+near-pii-field",
|
|
61
|
+
evidence="sql sink within file window",
|
|
62
|
+
file=file, line=line, by_file=by_file, probe_families={"sql"},
|
|
63
|
+
)
|
|
64
|
+
_maybe_add_proximity(
|
|
65
|
+
touches, seen_intra, t,
|
|
66
|
+
stage="transmit", family="http-out+near-pii-field",
|
|
67
|
+
evidence="http-out within file window",
|
|
68
|
+
file=file, line=line, by_file=by_file, probe_families={"http-out"},
|
|
69
|
+
)
|
|
70
|
+
_maybe_add_proximity(
|
|
71
|
+
touches, seen_intra, t,
|
|
72
|
+
stage="encrypt", family="crypto+near-pii-field",
|
|
73
|
+
evidence="crypto op within file window",
|
|
74
|
+
file=file, line=line, by_file=by_file,
|
|
75
|
+
probe_families={"crypto-algorithm", "crypto-config"},
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _collect_delete_hints(
|
|
80
|
+
rid: str,
|
|
81
|
+
data: dict[str, Any],
|
|
82
|
+
by_file: dict[str, list[tuple[int, str, dict[str, Any]]]],
|
|
83
|
+
touches: list[PiiTouchpoint],
|
|
84
|
+
) -> None:
|
|
85
|
+
"""Emit delete-stage touches where a delete/remove op sits near a PII field."""
|
|
86
|
+
for node in iter_nodes(data):
|
|
87
|
+
if node.get("family") not in ("pii-log", "pii-storage", "pii-field"):
|
|
88
|
+
continue
|
|
89
|
+
detail = node.get("detail") or {}
|
|
90
|
+
field_name = detail.get("field_name")
|
|
91
|
+
if not field_name:
|
|
92
|
+
continue
|
|
93
|
+
file = node.get("file", "")
|
|
94
|
+
line = int(node.get("line") or 0)
|
|
95
|
+
for ln, _, n in by_file.get(file, []):
|
|
96
|
+
if abs(ln - line) > 30:
|
|
97
|
+
continue
|
|
98
|
+
raw = (n.get("detail") or {}).get("raw", "")
|
|
99
|
+
if DELETE_RX.search(raw):
|
|
100
|
+
t = touch_from_node(
|
|
101
|
+
rid,
|
|
102
|
+
"delete",
|
|
103
|
+
node,
|
|
104
|
+
evidence="delete/remove near field",
|
|
105
|
+
)
|
|
106
|
+
if t:
|
|
107
|
+
touches.append(t)
|
|
108
|
+
break
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def collect_pii_touchpoints(records: list[dict[str, Any]]) -> list[PiiTouchpoint]:
|
|
112
|
+
"""Collect all PII lifecycle touchpoints across the given repo records."""
|
|
113
|
+
touches: list[PiiTouchpoint] = []
|
|
114
|
+
for data in records:
|
|
115
|
+
rid = repo_id(data)
|
|
116
|
+
by_file = lines_by_file(data)
|
|
117
|
+
seen_intra: set[tuple[str, str, str, int]] = set()
|
|
118
|
+
|
|
119
|
+
for node in iter_nodes(data):
|
|
120
|
+
family = node.get("family", "")
|
|
121
|
+
if family == "pii-field":
|
|
122
|
+
_collect_pii_field_node(rid, node, by_file, touches, seen_intra)
|
|
123
|
+
elif family == "pii-storage":
|
|
124
|
+
t = touch_from_node(rid, "store", node)
|
|
125
|
+
if t:
|
|
126
|
+
touches.append(t)
|
|
127
|
+
elif family == "pii-log":
|
|
128
|
+
t = touch_from_node(rid, "log", node)
|
|
129
|
+
if t:
|
|
130
|
+
touches.append(t)
|
|
131
|
+
|
|
132
|
+
_collect_delete_hints(rid, data, by_file, touches)
|
|
133
|
+
|
|
134
|
+
return touches
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Shared helpers for PII lifecycle touchpoint collection."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections import defaultdict
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from ..models.pii_lifecycle import PiiTouchpoint, normalize_field_key
|
|
10
|
+
|
|
11
|
+
PROXIMITY_LINES = 80
|
|
12
|
+
|
|
13
|
+
DELETE_RX = re.compile(
|
|
14
|
+
r"\b(delete|remove|erase|purge|forget)\w*\b",
|
|
15
|
+
re.IGNORECASE,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def lines_by_file(data: dict[str, Any]) -> dict[str, list[tuple[int, str, dict[str, Any]]]]:
|
|
20
|
+
"""file -> [(line, family, node), ...] sorted by line."""
|
|
21
|
+
by_file: dict[str, list[tuple[int, str, dict[str, Any]]]] = defaultdict(list)
|
|
22
|
+
for node in data.get("nodes", []):
|
|
23
|
+
by_file[node.get("file", "")].append(
|
|
24
|
+
(int(node.get("line") or 0), node.get("family", ""), node),
|
|
25
|
+
)
|
|
26
|
+
for entries in by_file.values():
|
|
27
|
+
entries.sort(key=lambda x: x[0])
|
|
28
|
+
return by_file
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def nearby(
|
|
32
|
+
by_file: dict[str, list[tuple[int, str, dict[str, Any]]]],
|
|
33
|
+
file: str,
|
|
34
|
+
line: int,
|
|
35
|
+
families: set[str],
|
|
36
|
+
*,
|
|
37
|
+
window: int = PROXIMITY_LINES,
|
|
38
|
+
) -> bool:
|
|
39
|
+
"""Return True if any node of the given families sits within window lines."""
|
|
40
|
+
for ln, fam, _ in by_file.get(file, []):
|
|
41
|
+
if fam in families and abs(ln - line) <= window:
|
|
42
|
+
return True
|
|
43
|
+
return False
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def touch_from_node(
|
|
47
|
+
rid: str,
|
|
48
|
+
stage: str,
|
|
49
|
+
node: dict[str, Any],
|
|
50
|
+
*,
|
|
51
|
+
evidence: str = "",
|
|
52
|
+
) -> PiiTouchpoint | None:
|
|
53
|
+
"""Build a PiiTouchpoint from a flow node, or None if it carries no usable field."""
|
|
54
|
+
detail = node.get("detail") or {}
|
|
55
|
+
field_name = detail.get("field_name") or ""
|
|
56
|
+
if not field_name and stage not in ("encrypt", "delete"):
|
|
57
|
+
if node.get("family") not in ("pii-log", "pii-storage"):
|
|
58
|
+
return None
|
|
59
|
+
field_key = normalize_field_key(field_name or "unknown")
|
|
60
|
+
pii_c = node.get("pii_classification") or "unknown"
|
|
61
|
+
return PiiTouchpoint(
|
|
62
|
+
repo=rid,
|
|
63
|
+
stage=stage,
|
|
64
|
+
family=node.get("family", ""),
|
|
65
|
+
field_key=field_key,
|
|
66
|
+
field_name=field_name or field_key,
|
|
67
|
+
pii_classification=pii_c,
|
|
68
|
+
data_class=node.get("data_class"),
|
|
69
|
+
file=node.get("file", ""),
|
|
70
|
+
line=int(node.get("line") or 0),
|
|
71
|
+
confidence=node.get("confidence", "medium"),
|
|
72
|
+
evidence=evidence,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def proximity_touch(
|
|
77
|
+
base: PiiTouchpoint,
|
|
78
|
+
*,
|
|
79
|
+
stage: str,
|
|
80
|
+
family: str,
|
|
81
|
+
evidence: str,
|
|
82
|
+
) -> PiiTouchpoint:
|
|
83
|
+
"""Clone a collect-stage touch with a proximity-derived stage."""
|
|
84
|
+
return PiiTouchpoint(
|
|
85
|
+
repo=base.repo,
|
|
86
|
+
stage=stage,
|
|
87
|
+
family=family,
|
|
88
|
+
field_key=base.field_key,
|
|
89
|
+
field_name=base.field_name,
|
|
90
|
+
pii_classification=base.pii_classification,
|
|
91
|
+
data_class=base.data_class,
|
|
92
|
+
file=base.file,
|
|
93
|
+
line=base.line,
|
|
94
|
+
confidence="low",
|
|
95
|
+
evidence=evidence,
|
|
96
|
+
)
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Queue / messaging graph aggregator — v2 queue-pub / queue-sub nodes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from collections import defaultdict
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from ..graph_common import iter_nodes, load_v2_repo_records, repo_id
|
|
12
|
+
from ..sanitize import UNTRUSTED_CONTENT_NOTICE, for_markdown, for_mermaid_label
|
|
13
|
+
from ..renderers.markdown import md_table
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _collect_queue_topics(records: list[dict[str, Any]]) -> dict[str, dict[str, set[str]]]:
|
|
17
|
+
"""Map each topic to producing/consuming repos and messaging systems."""
|
|
18
|
+
topics: dict[str, dict[str, set[str]]] = defaultdict(
|
|
19
|
+
lambda: {"produce": set(), "consume": set(), "systems": set()},
|
|
20
|
+
)
|
|
21
|
+
for data in records:
|
|
22
|
+
rid = repo_id(data)
|
|
23
|
+
for node in iter_nodes(data):
|
|
24
|
+
family = node.get("family", "")
|
|
25
|
+
detail = node.get("detail") or {}
|
|
26
|
+
topic = detail.get("topic", "")
|
|
27
|
+
if not topic or topic == "?":
|
|
28
|
+
continue
|
|
29
|
+
role = {"queue-pub": "produce", "queue-sub": "consume"}.get(family)
|
|
30
|
+
if not role:
|
|
31
|
+
continue
|
|
32
|
+
topics[topic][role].add(rid)
|
|
33
|
+
system = detail.get("system") or ""
|
|
34
|
+
if system:
|
|
35
|
+
topics[topic]["systems"].add(system)
|
|
36
|
+
return topics
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _queue_rows_and_orphans(
|
|
40
|
+
topics: dict[str, dict[str, set[str]]],
|
|
41
|
+
) -> tuple[list[list[str]], list[str], list[str]]:
|
|
42
|
+
"""Build topic table rows plus produce-only and consume-only orphan lists."""
|
|
43
|
+
rows: list[list[str]] = []
|
|
44
|
+
orphans_produce: list[str] = []
|
|
45
|
+
orphans_consume: list[str] = []
|
|
46
|
+
for topic, dirs in sorted(topics.items()):
|
|
47
|
+
producers = sorted(dirs["produce"])
|
|
48
|
+
consumers = sorted(dirs["consume"])
|
|
49
|
+
systems = sorted(dirs["systems"])
|
|
50
|
+
rows.append([
|
|
51
|
+
topic,
|
|
52
|
+
", ".join(systems) if systems else "—",
|
|
53
|
+
", ".join(producers) if producers else "—",
|
|
54
|
+
", ".join(consumers) if consumers else "—",
|
|
55
|
+
])
|
|
56
|
+
if producers and not consumers:
|
|
57
|
+
orphans_produce.append(topic)
|
|
58
|
+
if consumers and not producers:
|
|
59
|
+
orphans_consume.append(topic)
|
|
60
|
+
return rows, orphans_produce, orphans_consume
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _orphan_line(label: str, topics: list[str]) -> str:
|
|
64
|
+
"""Format a labelled bullet listing orphan topics (capped at 30)."""
|
|
65
|
+
return (
|
|
66
|
+
f"- **{label}** ({len(topics)}): "
|
|
67
|
+
# topics are untrusted extracted literals emitted outside a table cell.
|
|
68
|
+
+ ", ".join(for_markdown(t, max_len=80) for t in topics[:30])
|
|
69
|
+
+ (" …" if len(topics) > 30 else "")
|
|
70
|
+
+ "\n"
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _queue_mermaid(topics: dict[str, dict[str, set[str]]]) -> list[str]:
|
|
75
|
+
"""Render a Mermaid flowchart for topics that have both producers and consumers."""
|
|
76
|
+
both = [t for t, d in topics.items() if d["produce"] and d["consume"]][:25]
|
|
77
|
+
if not both:
|
|
78
|
+
return []
|
|
79
|
+
md = ["\n## Mermaid (topics with producers and consumers)\n\n```mermaid\nflowchart LR\n"]
|
|
80
|
+
for topic in both:
|
|
81
|
+
d = topics[topic]
|
|
82
|
+
# Both the node id and the label are attacker-influenced: the id must be a
|
|
83
|
+
# strict slug (drop everything non-alphanumeric) or the topic's quotes /
|
|
84
|
+
# newlines break out of the node; the label is neutralised separately.
|
|
85
|
+
tid = re.sub(r"[^A-Za-z0-9_]", "_", topic)[:50]
|
|
86
|
+
topic_label = for_mermaid_label(topic)
|
|
87
|
+
for p in sorted(d["produce"])[:3]:
|
|
88
|
+
pid = re.sub(r"[^A-Za-z0-9_]", "_", p)[:30]
|
|
89
|
+
md.append(f' {pid}["{for_mermaid_label(p)}"] -->|{topic_label}| hub_{tid}["{topic_label}"]\n')
|
|
90
|
+
for c in sorted(d["consume"])[:3]:
|
|
91
|
+
cid = re.sub(r"[^A-Za-z0-9_]", "_", c)[:30]
|
|
92
|
+
md.append(f' hub_{tid} --> {cid}["{for_mermaid_label(c)}"]\n')
|
|
93
|
+
md.append("```\n")
|
|
94
|
+
return md
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _write_queue_jsonl(path: Path, topics: dict[str, dict[str, set[str]]]) -> None:
|
|
98
|
+
"""Write one JSON line per topic with its systems, producers, and consumers."""
|
|
99
|
+
with path.open("w", encoding="utf-8") as fh:
|
|
100
|
+
for topic, dirs in sorted(topics.items()):
|
|
101
|
+
systems = sorted(dirs["systems"])
|
|
102
|
+
fh.write(json.dumps({
|
|
103
|
+
"topic": topic,
|
|
104
|
+
"queue_types": systems,
|
|
105
|
+
"queue_type": ", ".join(systems) if systems else None,
|
|
106
|
+
"producers": sorted(dirs["produce"]),
|
|
107
|
+
"consumers": sorted(dirs["consume"]),
|
|
108
|
+
}, ensure_ascii=False) + "\n")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def write_queue_graph(metabase_root: Path, repo_jsons: list[Path]) -> None:
|
|
112
|
+
"""Write the queue/messaging graph markdown + jsonl from v2 queue nodes."""
|
|
113
|
+
records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
|
|
114
|
+
topics = _collect_queue_topics(records)
|
|
115
|
+
|
|
116
|
+
out_dir = metabase_root / "graphs"
|
|
117
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
118
|
+
|
|
119
|
+
rows, orphans_produce, orphans_consume = _queue_rows_and_orphans(topics)
|
|
120
|
+
md: list[str] = [
|
|
121
|
+
"# Queue / messaging graph (v2)\n",
|
|
122
|
+
UNTRUSTED_CONTENT_NOTICE,
|
|
123
|
+
"_Producers (`queue-pub`) and consumers (`queue-sub`) per topic from "
|
|
124
|
+
"v2 flow nodes. **Queue type** is the `detail.system` from the extractor "
|
|
125
|
+
"(kafka, rabbitmq, sqs, sns, redis-stream, nats, jms, …)._\n",
|
|
126
|
+
"\n## Topics\n",
|
|
127
|
+
md_table(
|
|
128
|
+
["Topic", "Queue type", "Producers", "Consumers"],
|
|
129
|
+
rows or [["—", "—", "—", "—"]],
|
|
130
|
+
),
|
|
131
|
+
]
|
|
132
|
+
if orphans_produce or orphans_consume:
|
|
133
|
+
md.append("\n## Orphan topics\n")
|
|
134
|
+
if orphans_produce:
|
|
135
|
+
md.append(_orphan_line("Produce-only", orphans_produce))
|
|
136
|
+
if orphans_consume:
|
|
137
|
+
md.append(_orphan_line("Consume-only", orphans_consume))
|
|
138
|
+
md.extend(_queue_mermaid(topics))
|
|
139
|
+
|
|
140
|
+
(out_dir / "queue-graph.md").write_text("\n".join(md), encoding="utf-8")
|
|
141
|
+
_write_queue_jsonl(out_dir / "queue-graph.jsonl", topics)
|