src2sink 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- src2sink/__init__.py +1 -0
- src2sink/aggregators/__init__.py +3 -0
- src2sink/aggregators/api_client_discovery.py +259 -0
- src2sink/aggregators/auth_cards.py +116 -0
- src2sink/aggregators/crypto_cards.py +132 -0
- src2sink/aggregators/data_stores.py +125 -0
- src2sink/aggregators/graphs.py +36 -0
- src2sink/aggregators/index_v2.py +138 -0
- src2sink/aggregators/library_source_map.py +188 -0
- src2sink/aggregators/openapi_discovery.py +113 -0
- src2sink/aggregators/openapi_edges.py +30 -0
- src2sink/aggregators/openapi_match.py +81 -0
- src2sink/aggregators/openapi_models.py +25 -0
- src2sink/aggregators/openapi_report.py +75 -0
- src2sink/aggregators/payload_producers.py +369 -0
- src2sink/aggregators/phase3.py +30 -0
- src2sink/aggregators/pii_cross_repo.py +283 -0
- src2sink/aggregators/pii_flow_v2.py +118 -0
- src2sink/aggregators/pii_lifecycle.py +19 -0
- src2sink/aggregators/pii_lifecycle_report.py +121 -0
- src2sink/aggregators/pii_touchpoint_collect.py +134 -0
- src2sink/aggregators/pii_touchpoint_utils.py +96 -0
- src2sink/aggregators/queues.py +141 -0
- src2sink/aggregators/ropa.py +190 -0
- src2sink/aggregators/service_call_collect.py +240 -0
- src2sink/aggregators/service_call_index.py +31 -0
- src2sink/aggregators/service_call_models.py +17 -0
- src2sink/aggregators/service_call_report.py +118 -0
- src2sink/aggregators/service_calls.py +14 -0
- src2sink/aggregators/taint_buckets.py +74 -0
- src2sink/aggregators/taint_catalogs.py +35 -0
- src2sink/aggregators/taint_writers.py +369 -0
- src2sink/aggregators/traces_index.py +103 -0
- src2sink/build_metabase_v2.py +807 -0
- src2sink/constants.py +44 -0
- src2sink/curate_internal_libraries.py +135 -0
- src2sink/extractors/__init__.py +17 -0
- src2sink/extractors/ast_walk.py +131 -0
- src2sink/extractors/base.py +59 -0
- src2sink/extractors/config.py +224 -0
- src2sink/extractors/file_context.py +35 -0
- src2sink/extractors/go.py +3 -0
- src2sink/extractors/http_out.py +108 -0
- src2sink/extractors/java.py +3 -0
- src2sink/extractors/javascript.py +3 -0
- src2sink/extractors/kotlin.py +3 -0
- src2sink/extractors/node_factory.py +65 -0
- src2sink/extractors/patterns.py +145 -0
- src2sink/extractors/python_extractor.py +3 -0
- src2sink/extractors/regex_extractors.py +378 -0
- src2sink/extractors/ts_extractors.py +123 -0
- src2sink/extractors/unified.py +64 -0
- src2sink/graph_common.py +213 -0
- src2sink/internal_groups.py +142 -0
- src2sink/known_api_clients.py +108 -0
- src2sink/library_taint_java.py +92 -0
- src2sink/limits.py +171 -0
- src2sink/models/__init__.py +3 -0
- src2sink/models/auth_model.py +3 -0
- src2sink/models/crypto_agility.py +3 -0
- src2sink/models/pii_lifecycle.py +92 -0
- src2sink/models/ropa.py +37 -0
- src2sink/prescreen.py +93 -0
- src2sink/py.typed +0 -0
- src2sink/record_fleet_baseline.py +81 -0
- src2sink/renderers/__init__.py +3 -0
- src2sink/renderers/markdown.py +149 -0
- src2sink/renderers/mermaid.py +3 -0
- src2sink/repo_utils.py +885 -0
- src2sink/safe_paths.py +54 -0
- src2sink/sanitize.py +107 -0
- src2sink/schema.py +57 -0
- src2sink/trace.py +558 -0
- src2sink/trace_batch.py +176 -0
- src2sink/vocabulary.py +152 -0
- src2sink-1.0.0.dist-info/METADATA +355 -0
- src2sink-1.0.0.dist-info/RECORD +81 -0
- src2sink-1.0.0.dist-info/WHEEL +5 -0
- src2sink-1.0.0.dist-info/entry_points.txt +6 -0
- src2sink-1.0.0.dist-info/licenses/LICENSE +21 -0
- src2sink-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""v2 repo index (replaces v1 index/ for flow-graph metabase)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as dt
|
|
6
|
+
import json
|
|
7
|
+
from collections import Counter, defaultdict
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from ..graph_common import iter_nodes, load_v2_repo_records
|
|
12
|
+
from ..renderers.markdown import md_table
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _row_from_record(data: dict[str, Any]) -> dict[str, Any]:
|
|
16
|
+
"""Summarise one repo record into an index row of per-family node counts."""
|
|
17
|
+
families: Counter[str] = Counter()
|
|
18
|
+
weak_algos: list[str] = []
|
|
19
|
+
for node in iter_nodes(data):
|
|
20
|
+
families[node.get("family", "")] += 1
|
|
21
|
+
if node.get("family") == "crypto-algorithm":
|
|
22
|
+
algo = (node.get("detail") or {}).get("algorithm", "")
|
|
23
|
+
if algo and algo.upper() in {"MD5", "SHA1", "SHA-1", "DES", "RC4"}:
|
|
24
|
+
weak_algos.append(algo)
|
|
25
|
+
return {
|
|
26
|
+
"group": data["group"],
|
|
27
|
+
"name": data["name"],
|
|
28
|
+
"path": data.get("path", ""),
|
|
29
|
+
"primary_language": data.get("primary_language", "unknown"),
|
|
30
|
+
"frameworks": data.get("frameworks") or [],
|
|
31
|
+
"nodes": len(data.get("nodes", [])),
|
|
32
|
+
"edges": len(data.get("edges", [])),
|
|
33
|
+
"http_in": families.get("http-in", 0),
|
|
34
|
+
"http_out": families.get("http-out", 0),
|
|
35
|
+
"sql": families.get("sql", 0),
|
|
36
|
+
"raw_payload": families.get("raw-code-payload", 0),
|
|
37
|
+
"pii_field": families.get("pii-field", 0),
|
|
38
|
+
"pii_log": families.get("pii-log", 0),
|
|
39
|
+
"queue": families.get("queue-pub", 0) + families.get("queue-sub", 0),
|
|
40
|
+
"crypto": families.get("crypto-algorithm", 0),
|
|
41
|
+
"weak_algos": sorted(set(weak_algos)),
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def write_index_v2(metabase_root: Path, repo_jsons: list[Path]) -> None:
|
|
46
|
+
"""Write the v2 repo index JSON and the all/by-group/by-language markdown pages."""
|
|
47
|
+
records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
|
|
48
|
+
rows = [_row_from_record(data) for data in records]
|
|
49
|
+
index_dir = metabase_root / "index"
|
|
50
|
+
index_dir.mkdir(parents=True, exist_ok=True)
|
|
51
|
+
|
|
52
|
+
(index_dir / "repos.json").write_text(
|
|
53
|
+
json.dumps(rows, indent=2, sort_keys=True),
|
|
54
|
+
encoding="utf-8",
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
ts = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
58
|
+
md = [
|
|
59
|
+
"# All repos (v2 index)\n",
|
|
60
|
+
f"_Generated {ts} from `schema_version: 2` JSON summaries._\n",
|
|
61
|
+
f"_{len(rows)} repos._\n\n",
|
|
62
|
+
md_table(
|
|
63
|
+
[
|
|
64
|
+
"Group / Repo",
|
|
65
|
+
"Lang",
|
|
66
|
+
"Frameworks",
|
|
67
|
+
"Nodes",
|
|
68
|
+
"HTTP-in",
|
|
69
|
+
"HTTP-out",
|
|
70
|
+
"SQL",
|
|
71
|
+
"Raw payload",
|
|
72
|
+
"PII fields",
|
|
73
|
+
],
|
|
74
|
+
sorted(
|
|
75
|
+
[
|
|
76
|
+
[
|
|
77
|
+
f"[{r['group']}/{r['name']}](../repos/{r['group']}/{r['name']}.md)",
|
|
78
|
+
r["primary_language"],
|
|
79
|
+
", ".join(r["frameworks"]) or "—",
|
|
80
|
+
str(r["nodes"]),
|
|
81
|
+
str(r["http_in"]),
|
|
82
|
+
str(r["http_out"]),
|
|
83
|
+
str(r["sql"]),
|
|
84
|
+
str(r["raw_payload"]),
|
|
85
|
+
str(r["pii_field"]),
|
|
86
|
+
]
|
|
87
|
+
for r in rows
|
|
88
|
+
],
|
|
89
|
+
key=lambda row: row[0].lower(),
|
|
90
|
+
),
|
|
91
|
+
),
|
|
92
|
+
]
|
|
93
|
+
(index_dir / "repos.md").write_text("\n".join(md), encoding="utf-8")
|
|
94
|
+
|
|
95
|
+
by_group: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
96
|
+
for r in rows:
|
|
97
|
+
by_group[r["group"]].append(r)
|
|
98
|
+
out = [f"# Repos by group (v2)\n\n_Generated {ts}._\n\n"]
|
|
99
|
+
for g in sorted(by_group):
|
|
100
|
+
out.append(f"## {g} ({len(by_group[g])} repos)\n")
|
|
101
|
+
out.append(
|
|
102
|
+
md_table(
|
|
103
|
+
["Repo", "Lang", "Nodes", "Raw payload", "PII"],
|
|
104
|
+
[
|
|
105
|
+
[
|
|
106
|
+
f"[{r['name']}](../repos/{g}/{r['name']}.md)",
|
|
107
|
+
r["primary_language"],
|
|
108
|
+
str(r["nodes"]),
|
|
109
|
+
str(r["raw_payload"]),
|
|
110
|
+
str(r["pii_field"]),
|
|
111
|
+
]
|
|
112
|
+
for r in sorted(by_group[g], key=lambda x: x["name"].lower())
|
|
113
|
+
],
|
|
114
|
+
),
|
|
115
|
+
)
|
|
116
|
+
(index_dir / "by-group.md").write_text("\n".join(out), encoding="utf-8")
|
|
117
|
+
|
|
118
|
+
by_lang: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
119
|
+
for r in rows:
|
|
120
|
+
by_lang[r["primary_language"]].append(r)
|
|
121
|
+
out = ["# Repos by primary language (v2)\n\n"]
|
|
122
|
+
for lang in sorted(by_lang):
|
|
123
|
+
out.append(f"## {lang} ({len(by_lang[lang])} repos)\n")
|
|
124
|
+
out.append(
|
|
125
|
+
md_table(
|
|
126
|
+
["Group", "Repo", "Nodes", "HTTP-in"],
|
|
127
|
+
[
|
|
128
|
+
[
|
|
129
|
+
r["group"],
|
|
130
|
+
f"[{r['name']}](../repos/{r['group']}/{r['name']}.md)",
|
|
131
|
+
str(r["nodes"]),
|
|
132
|
+
str(r["http_in"]),
|
|
133
|
+
]
|
|
134
|
+
for r in sorted(by_lang[lang], key=lambda x: (x["group"], x["name"]))
|
|
135
|
+
],
|
|
136
|
+
),
|
|
137
|
+
)
|
|
138
|
+
(index_dir / "by-language.md").write_text("\n".join(out), encoding="utf-8")
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Generator and loader for library-source-map.json.
|
|
2
|
+
|
|
3
|
+
Maps Maven/npm coordinates to source-repo clone paths. Generated by
|
|
4
|
+
src2sink-build from per-repo dependency data; manual edits to
|
|
5
|
+
``status`` and ``clone_path`` fields are preserved on regeneration.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
import json
|
|
10
|
+
from collections import defaultdict
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
STATUS_PENDING = "pending"
|
|
15
|
+
STATUS_CLONED = "cloned"
|
|
16
|
+
STATUS_EXCLUDED = "excluded"
|
|
17
|
+
STATUS_AMBIGUOUS = "ambiguous"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def load_library_source_map(path: Path) -> dict[str, dict[str, Any]]:
|
|
21
|
+
"""Return coord -> entry dict, or {} if the file is missing/invalid."""
|
|
22
|
+
try:
|
|
23
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
24
|
+
except (OSError, json.JSONDecodeError):
|
|
25
|
+
return {}
|
|
26
|
+
return {m["coordinate"]: m for m in data.get("mappings", []) if "coordinate" in m}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def generate_library_source_map(metabase_root: Path, repo_jsons: list[Path]) -> None:
|
|
30
|
+
"""Write/update library-source-map.json from per-repo internal dependency data."""
|
|
31
|
+
map_path = metabase_root / "library-source-map.json"
|
|
32
|
+
existing = load_library_source_map(map_path)
|
|
33
|
+
|
|
34
|
+
# Collect all internal deps from per-repo JSONs
|
|
35
|
+
coord_consumers: dict[str, int] = defaultdict(int)
|
|
36
|
+
coord_versions: dict[str, set[str]] = defaultdict(set)
|
|
37
|
+
|
|
38
|
+
for jp in repo_jsons:
|
|
39
|
+
try:
|
|
40
|
+
data = json.loads(jp.read_text(encoding="utf-8"))
|
|
41
|
+
except (OSError, json.JSONDecodeError):
|
|
42
|
+
continue
|
|
43
|
+
for dep in data.get("dependencies_internal", []):
|
|
44
|
+
gid = dep.get("groupId", "")
|
|
45
|
+
aid = dep.get("artifactId", "")
|
|
46
|
+
ver = dep.get("version", "")
|
|
47
|
+
if not aid:
|
|
48
|
+
continue
|
|
49
|
+
coord = f"{gid}:{aid}" if gid else aid
|
|
50
|
+
coord_consumers[coord] += 1
|
|
51
|
+
if ver:
|
|
52
|
+
coord_versions[coord].add(ver)
|
|
53
|
+
|
|
54
|
+
# Build updated mappings, preserving manual overrides
|
|
55
|
+
mappings: list[dict[str, Any]] = []
|
|
56
|
+
for coord, count in sorted(coord_consumers.items(), key=lambda x: -x[1]):
|
|
57
|
+
prev = existing.get(coord, {})
|
|
58
|
+
# Preserve manually-set status and clone_path overrides
|
|
59
|
+
status = prev.get("status", STATUS_PENDING)
|
|
60
|
+
clone_path = prev.get("clone_path") or _infer_clone_path(coord)
|
|
61
|
+
entry: dict[str, Any] = {
|
|
62
|
+
"coordinate": coord,
|
|
63
|
+
"clone_path": clone_path or "",
|
|
64
|
+
"status": status,
|
|
65
|
+
"consumers": count,
|
|
66
|
+
}
|
|
67
|
+
versions = sorted(coord_versions.get(coord, set()))
|
|
68
|
+
if versions:
|
|
69
|
+
entry["versions"] = versions
|
|
70
|
+
if prev.get("notes"):
|
|
71
|
+
entry["notes"] = prev["notes"]
|
|
72
|
+
mappings.append(entry)
|
|
73
|
+
|
|
74
|
+
out = {
|
|
75
|
+
"_comment": (
|
|
76
|
+
"Map of Maven/npm coordinates to source-repo clone paths. "
|
|
77
|
+
"Generated by src2sink-build from per-repo dependency data. "
|
|
78
|
+
"Edit 'status' ('pending'|'cloned'|'excluded') and 'clone_path' to "
|
|
79
|
+
"guide curate_internal_libraries.py. Changes are preserved on regeneration."
|
|
80
|
+
),
|
|
81
|
+
"mappings": mappings,
|
|
82
|
+
}
|
|
83
|
+
map_path.write_text(json.dumps(out, indent=2, sort_keys=False) + "\n", encoding="utf-8")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _resolve_clone_path(
|
|
87
|
+
coord: str,
|
|
88
|
+
by_coord: dict[tuple[str, str], str],
|
|
89
|
+
by_name: dict[str, list[str]],
|
|
90
|
+
by_full: dict[str, str],
|
|
91
|
+
) -> str | None:
|
|
92
|
+
"""Resolve a dependency coordinate to a clone path via the identity index."""
|
|
93
|
+
# 1. Exact canonical-coordinate match.
|
|
94
|
+
if coord in by_full:
|
|
95
|
+
return by_full[coord]
|
|
96
|
+
if ":" in coord:
|
|
97
|
+
group, artifact = coord.split(":", 1)
|
|
98
|
+
# 2. group:artifact pair.
|
|
99
|
+
hit = by_coord.get((group, artifact))
|
|
100
|
+
if hit:
|
|
101
|
+
return hit
|
|
102
|
+
# 3. The artifact half may itself be a full name (scoped npm, vendor/pkg).
|
|
103
|
+
if artifact in by_full:
|
|
104
|
+
return by_full[artifact]
|
|
105
|
+
candidates = by_name.get(artifact, [])
|
|
106
|
+
return candidates[0] if len(candidates) == 1 else None
|
|
107
|
+
# No group: try a group-less pair, then a unique bare-name match.
|
|
108
|
+
hit = by_coord.get(("", coord))
|
|
109
|
+
if hit:
|
|
110
|
+
return hit
|
|
111
|
+
candidates = by_name.get(coord, [])
|
|
112
|
+
return candidates[0] if len(candidates) == 1 else None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _entry_needs_resolution(entry: dict[str, Any]) -> bool:
|
|
116
|
+
"""True if a source-map entry is still flagged and worth resolving."""
|
|
117
|
+
if not entry.get("coordinate"):
|
|
118
|
+
return False
|
|
119
|
+
status = entry.get("status")
|
|
120
|
+
if status == STATUS_EXCLUDED:
|
|
121
|
+
return False
|
|
122
|
+
if status == STATUS_CLONED and entry.get("clone_path"):
|
|
123
|
+
return False
|
|
124
|
+
return True
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def fix_flagged_mappings(metabase_root: Path, repos_root: Path) -> int:
|
|
128
|
+
"""Resolve flagged source-map entries from on-disk project manifests.
|
|
129
|
+
|
|
130
|
+
Walks every recognised project-definition file under ``repos_root`` —
|
|
131
|
+
Maven (``pom.xml``), Gradle (``build.gradle``/``settings.gradle`` ±kts),
|
|
132
|
+
npm (``package.json``), Python (``pyproject.toml``/``setup.cfg``), Rust
|
|
133
|
+
(``Cargo.toml``), Go (``go.mod``), PHP (``composer.json``), .NET
|
|
134
|
+
(``*.csproj``/``*.fsproj``/``*.vbproj``) and Ruby (``*.gemspec``) — and
|
|
135
|
+
reads each project's declared identity. For any mapping still flagged
|
|
136
|
+
(status ``pending`` / ``ambiguous``, or missing a ``clone_path``) whose
|
|
137
|
+
coordinate matches a discovered project, sets ``clone_path`` to that repo's
|
|
138
|
+
path (relative to ``repos_root``) and marks the status ``cloned``.
|
|
139
|
+
``excluded`` and already-resolved ``cloned`` entries are left untouched.
|
|
140
|
+
|
|
141
|
+
Matching tries, in order: the exact coordinate string (npm ``@scope/pkg``,
|
|
142
|
+
Composer ``vendor/pkg``, Go module path, bare names), then the
|
|
143
|
+
``group:artifact`` pair, then a unique bare-name match.
|
|
144
|
+
|
|
145
|
+
Returns the number of mappings corrected.
|
|
146
|
+
"""
|
|
147
|
+
map_path = metabase_root / "library-source-map.json"
|
|
148
|
+
try:
|
|
149
|
+
data = json.loads(map_path.read_text(encoding="utf-8"))
|
|
150
|
+
except (OSError, json.JSONDecodeError):
|
|
151
|
+
return 0
|
|
152
|
+
mappings = data.get("mappings")
|
|
153
|
+
if not isinstance(mappings, list) or not mappings:
|
|
154
|
+
return 0
|
|
155
|
+
|
|
156
|
+
from ..repo_utils import _build_component_identity_index
|
|
157
|
+
|
|
158
|
+
by_coord, by_name, by_full = _build_component_identity_index(repos_root)
|
|
159
|
+
|
|
160
|
+
fixed = 0
|
|
161
|
+
for entry in mappings:
|
|
162
|
+
if not isinstance(entry, dict) or not _entry_needs_resolution(entry):
|
|
163
|
+
continue
|
|
164
|
+
clone_path = _resolve_clone_path(entry["coordinate"], by_coord, by_name, by_full)
|
|
165
|
+
if not clone_path:
|
|
166
|
+
continue
|
|
167
|
+
entry["clone_path"] = clone_path
|
|
168
|
+
entry["status"] = STATUS_CLONED
|
|
169
|
+
entry.pop("notes", None)
|
|
170
|
+
fixed += 1
|
|
171
|
+
|
|
172
|
+
if fixed:
|
|
173
|
+
map_path.write_text(
|
|
174
|
+
json.dumps(data, indent=2, sort_keys=False) + "\n", encoding="utf-8"
|
|
175
|
+
)
|
|
176
|
+
return fixed
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _infer_clone_path(coord: str) -> str | None:
|
|
180
|
+
"""Best-effort inference of group/artifact from a Maven coordinate."""
|
|
181
|
+
if ":" not in coord:
|
|
182
|
+
return None
|
|
183
|
+
group, artifact = coord.split(":", 1)
|
|
184
|
+
# Strip common org prefix patterns to get a GitLab group name
|
|
185
|
+
parts = group.split(".")
|
|
186
|
+
# Heuristic: last meaningful segment of the group + artifact name
|
|
187
|
+
group_hint = parts[-1] if len(parts) > 2 else parts[-1]
|
|
188
|
+
return f"{group_hint}/{artifact}"
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Discover OpenAPI specs and Helm ingress hosts under repos/."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from ..graph_common import normalize_path_template as norm_path
|
|
9
|
+
|
|
10
|
+
from .openapi_models import OpenApiSpec
|
|
11
|
+
|
|
12
|
+
OPENAPI_GLOBS = ("openapi.yaml", "openapi.yml", "swagger.yaml", "swagger.yml")
|
|
13
|
+
PATH_LINE_RX = re.compile(
|
|
14
|
+
r"^\s{2,}(/[/\w{}.:$-]+)\s*:\s*$",
|
|
15
|
+
re.MULTILINE,
|
|
16
|
+
)
|
|
17
|
+
SERVERS_URL_RX = re.compile(
|
|
18
|
+
r"^\s*-\s*url:\s*['\"]?(https?://[^'\"\s]+)",
|
|
19
|
+
re.MULTILINE | re.IGNORECASE,
|
|
20
|
+
)
|
|
21
|
+
HELM_VALUES_NAMES = ("values.yaml", "values.yml")
|
|
22
|
+
HELM_HOST_RX = re.compile(
|
|
23
|
+
r"^\s*(?:host|hostname|externalHost|ingress\.host)\s*:\s*['\"]?([^\s#'\"]+)",
|
|
24
|
+
re.MULTILINE | re.IGNORECASE,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
_SKIP_DIR_PARTS = frozenset({".git", "node_modules", "target", "build", "charts"})
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def repo_from_under_repos(spec_path: Path, repos_root: Path) -> str | None:
|
|
31
|
+
"""Return the ``org/repo`` id for a path under repos_root, or None."""
|
|
32
|
+
try:
|
|
33
|
+
rel = spec_path.relative_to(repos_root)
|
|
34
|
+
except ValueError:
|
|
35
|
+
return None
|
|
36
|
+
parts = rel.parts
|
|
37
|
+
if len(parts) < 2:
|
|
38
|
+
return None
|
|
39
|
+
return f"{parts[0]}/{parts[1]}"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _should_skip_path(path: Path) -> bool:
|
|
43
|
+
"""True if the path lies within a skipped directory (.git, node_modules, ...)."""
|
|
44
|
+
return any(p in path.parts for p in _SKIP_DIR_PARTS)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def discover_openapi_specs(repos_root: Path) -> list[OpenApiSpec]:
|
|
48
|
+
"""Scan repos_root for OpenAPI/Swagger specs and return parsed OpenApiSpec rows."""
|
|
49
|
+
if not repos_root.is_dir():
|
|
50
|
+
return []
|
|
51
|
+
specs: list[OpenApiSpec] = []
|
|
52
|
+
seen: set[tuple[str, str]] = set()
|
|
53
|
+
for name in OPENAPI_GLOBS:
|
|
54
|
+
for path in repos_root.rglob(name):
|
|
55
|
+
if _should_skip_path(path):
|
|
56
|
+
continue
|
|
57
|
+
try:
|
|
58
|
+
if path.stat().st_size > 2_000_000:
|
|
59
|
+
continue
|
|
60
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
61
|
+
except OSError:
|
|
62
|
+
continue
|
|
63
|
+
target = repo_from_under_repos(path, repos_root)
|
|
64
|
+
if not target:
|
|
65
|
+
continue
|
|
66
|
+
key = (target, str(path))
|
|
67
|
+
if key in seen:
|
|
68
|
+
continue
|
|
69
|
+
seen.add(key)
|
|
70
|
+
paths = sorted({norm_path(m.group(1)) for m in PATH_LINE_RX.finditer(text)})
|
|
71
|
+
servers = [m.group(1) for m in SERVERS_URL_RX.finditer(text)]
|
|
72
|
+
specs.append(OpenApiSpec(
|
|
73
|
+
target_repo=target,
|
|
74
|
+
spec_path=str(path.relative_to(repos_root)),
|
|
75
|
+
paths=[p for p in paths if p and p != "/"],
|
|
76
|
+
servers=servers[:5],
|
|
77
|
+
))
|
|
78
|
+
return specs
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def discover_helm_hosts(repos_root: Path) -> list[dict[str, str]]:
|
|
82
|
+
"""Scan Helm values for declared ingress/service hostnames."""
|
|
83
|
+
if not repos_root.is_dir():
|
|
84
|
+
return []
|
|
85
|
+
rows: list[dict[str, str]] = []
|
|
86
|
+
seen: set[tuple[str, str]] = set()
|
|
87
|
+
for name in HELM_VALUES_NAMES:
|
|
88
|
+
for path in repos_root.rglob(name):
|
|
89
|
+
if _should_skip_path(path):
|
|
90
|
+
continue
|
|
91
|
+
try:
|
|
92
|
+
if path.stat().st_size > 500_000:
|
|
93
|
+
continue
|
|
94
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
95
|
+
except OSError:
|
|
96
|
+
continue
|
|
97
|
+
target = repo_from_under_repos(path, repos_root)
|
|
98
|
+
if not target:
|
|
99
|
+
continue
|
|
100
|
+
for m in HELM_HOST_RX.finditer(text):
|
|
101
|
+
host = m.group(1).strip().strip("'\"")
|
|
102
|
+
if not host or host.startswith("{{"):
|
|
103
|
+
continue
|
|
104
|
+
key = (target, host)
|
|
105
|
+
if key in seen:
|
|
106
|
+
continue
|
|
107
|
+
seen.add(key)
|
|
108
|
+
rows.append({
|
|
109
|
+
"target_repo": target,
|
|
110
|
+
"host": host,
|
|
111
|
+
"values_path": str(path.relative_to(repos_root)),
|
|
112
|
+
})
|
|
113
|
+
return rows
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""OpenAPI / Swagger path ingestion for authoritative inbound route index."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .openapi_discovery import (
|
|
6
|
+
OPENAPI_GLOBS,
|
|
7
|
+
PATH_LINE_RX,
|
|
8
|
+
discover_helm_hosts,
|
|
9
|
+
discover_openapi_specs,
|
|
10
|
+
repo_from_under_repos,
|
|
11
|
+
)
|
|
12
|
+
from .openapi_match import (
|
|
13
|
+
build_openapi_inbound_index,
|
|
14
|
+
match_http_out_to_openapi,
|
|
15
|
+
)
|
|
16
|
+
from .openapi_models import OpenApiInbound, OpenApiSpec
|
|
17
|
+
from .openapi_report import write_openapi_artifacts
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"OPENAPI_GLOBS",
|
|
21
|
+
"PATH_LINE_RX",
|
|
22
|
+
"OpenApiInbound",
|
|
23
|
+
"OpenApiSpec",
|
|
24
|
+
"build_openapi_inbound_index",
|
|
25
|
+
"discover_helm_hosts",
|
|
26
|
+
"discover_openapi_specs",
|
|
27
|
+
"match_http_out_to_openapi",
|
|
28
|
+
"repo_from_under_repos",
|
|
29
|
+
"write_openapi_artifacts",
|
|
30
|
+
]
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Match http-out nodes to OpenAPI-discovered inbound paths."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ..graph_common import (
|
|
8
|
+
build_repo_alias_index,
|
|
9
|
+
extract_urls_and_paths,
|
|
10
|
+
iter_nodes,
|
|
11
|
+
match_path_in_inbound_index,
|
|
12
|
+
normalize_path_template as norm_path,
|
|
13
|
+
repo_id,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
from .openapi_models import OpenApiSpec
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def build_openapi_inbound_index(
|
|
20
|
+
specs: list[OpenApiSpec],
|
|
21
|
+
) -> dict[str, list[tuple[str, str, str]]]:
|
|
22
|
+
"""normalized_path -> [(repo, original_path, spec_path)]."""
|
|
23
|
+
index: dict[str, list[tuple[str, str, str]]] = {}
|
|
24
|
+
for spec in specs:
|
|
25
|
+
for path in spec.paths:
|
|
26
|
+
key = norm_path(path)
|
|
27
|
+
index.setdefault(key, []).append(
|
|
28
|
+
(spec.target_repo, path, spec.spec_path),
|
|
29
|
+
)
|
|
30
|
+
return index
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _paths_from_http_out(detail: dict[str, Any], raw: str) -> list[str]:
|
|
34
|
+
"""Collect candidate request paths from an http-out node detail and raw text."""
|
|
35
|
+
paths: list[str] = list(detail.get("paths") or [])
|
|
36
|
+
if detail.get("path"):
|
|
37
|
+
paths.insert(0, detail["path"])
|
|
38
|
+
_, extra_paths = extract_urls_and_paths(raw)
|
|
39
|
+
paths.extend(p for p in extra_paths if p not in paths)
|
|
40
|
+
return paths
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def match_http_out_to_openapi(
|
|
44
|
+
records: list[dict[str, Any]],
|
|
45
|
+
inbound: dict[str, list[tuple[str, str, str]]],
|
|
46
|
+
alias_to_repo: dict[str, str] | None = None,
|
|
47
|
+
) -> list[dict[str, Any]]:
|
|
48
|
+
"""Return edges from http-out call sites that match an OpenAPI path."""
|
|
49
|
+
_ = alias_to_repo or build_repo_alias_index(records)
|
|
50
|
+
edges: list[dict[str, Any]] = []
|
|
51
|
+
seen: set[tuple[str, str, str]] = set()
|
|
52
|
+
|
|
53
|
+
for data in records:
|
|
54
|
+
src = repo_id(data)
|
|
55
|
+
for node in iter_nodes(data):
|
|
56
|
+
if node.get("family") != "http-out":
|
|
57
|
+
continue
|
|
58
|
+
detail = node.get("detail") or {}
|
|
59
|
+
raw = detail.get("raw", "")
|
|
60
|
+
ref = f"{node.get('file')}:{node.get('line')}"
|
|
61
|
+
|
|
62
|
+
for path in _paths_from_http_out(detail, raw):
|
|
63
|
+
targets, _conf = match_path_in_inbound_index(path, inbound)
|
|
64
|
+
for tgt_repo, tgt_path, spec_path in targets:
|
|
65
|
+
if tgt_repo == src:
|
|
66
|
+
continue
|
|
67
|
+
key = (src, tgt_repo, norm_path(tgt_path))
|
|
68
|
+
if key in seen:
|
|
69
|
+
continue
|
|
70
|
+
seen.add(key)
|
|
71
|
+
edges.append({
|
|
72
|
+
"source_repo": src,
|
|
73
|
+
"target_repo": tgt_repo,
|
|
74
|
+
"target_path": tgt_path,
|
|
75
|
+
"confidence": "openapi",
|
|
76
|
+
"evidence": f"http-out path matches OpenAPI in {spec_path}",
|
|
77
|
+
"refs": [ref],
|
|
78
|
+
"spec_path": spec_path,
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
return edges
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Data types for OpenAPI graph ingestion."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class OpenApiSpec:
|
|
10
|
+
"""A discovered OpenAPI/Swagger spec: its owning repo, file, paths and servers."""
|
|
11
|
+
|
|
12
|
+
target_repo: str
|
|
13
|
+
spec_path: str
|
|
14
|
+
paths: list[str] = field(default_factory=list)
|
|
15
|
+
servers: list[str] = field(default_factory=list)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class OpenApiInbound:
|
|
20
|
+
"""A single inbound OpenAPI route: repo, path, HTTP method and spec file."""
|
|
21
|
+
|
|
22
|
+
target_repo: str
|
|
23
|
+
path: str
|
|
24
|
+
method: str
|
|
25
|
+
spec_path: str
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Write OpenAPI / Helm discovery artefacts under metabase/graphs/."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from ..graph_common import build_repo_alias_index, load_v2_repo_records
|
|
9
|
+
from ..renderers.markdown import md_table
|
|
10
|
+
|
|
11
|
+
from .openapi_discovery import OPENAPI_GLOBS, discover_helm_hosts, discover_openapi_specs
|
|
12
|
+
from .openapi_match import build_openapi_inbound_index, match_http_out_to_openapi
|
|
13
|
+
from .openapi_models import OpenApiSpec
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def write_openapi_artifacts(
|
|
17
|
+
metabase_root: Path,
|
|
18
|
+
repos_root: Path | None,
|
|
19
|
+
repo_jsons: list[Path] | None = None,
|
|
20
|
+
) -> list[OpenApiSpec]:
|
|
21
|
+
"""Discover specs/Helm hosts, write JSONL + markdown artefacts, return the specs."""
|
|
22
|
+
graphs_dir = metabase_root / "graphs"
|
|
23
|
+
graphs_dir.mkdir(parents=True, exist_ok=True)
|
|
24
|
+
|
|
25
|
+
specs: list[OpenApiSpec] = []
|
|
26
|
+
if repos_root and repos_root.is_dir():
|
|
27
|
+
specs = discover_openapi_specs(repos_root)
|
|
28
|
+
|
|
29
|
+
jsonl_path = graphs_dir / "openapi-specs.jsonl"
|
|
30
|
+
with jsonl_path.open("w", encoding="utf-8") as fh:
|
|
31
|
+
for spec in specs:
|
|
32
|
+
fh.write(json.dumps({
|
|
33
|
+
"target_repo": spec.target_repo,
|
|
34
|
+
"spec_path": spec.spec_path,
|
|
35
|
+
"paths": spec.paths,
|
|
36
|
+
"servers": spec.servers,
|
|
37
|
+
}, ensure_ascii=False) + "\n")
|
|
38
|
+
|
|
39
|
+
inbound = build_openapi_inbound_index(specs)
|
|
40
|
+
openapi_edges: list[dict[str, object]] = []
|
|
41
|
+
if repo_jsons and inbound:
|
|
42
|
+
records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
|
|
43
|
+
alias_to_repo = build_repo_alias_index(records)
|
|
44
|
+
openapi_edges = match_http_out_to_openapi(records, inbound, alias_to_repo)
|
|
45
|
+
|
|
46
|
+
edges_path = graphs_dir / "openapi-service-edges.jsonl"
|
|
47
|
+
with edges_path.open("w", encoding="utf-8") as fh:
|
|
48
|
+
for row in openapi_edges:
|
|
49
|
+
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
50
|
+
|
|
51
|
+
helm_hosts = discover_helm_hosts(repos_root) if repos_root else []
|
|
52
|
+
helm_path = graphs_dir / "helm-service-hosts.jsonl"
|
|
53
|
+
with helm_path.open("w", encoding="utf-8") as fh:
|
|
54
|
+
for helm_row in helm_hosts:
|
|
55
|
+
fh.write(json.dumps(helm_row, ensure_ascii=False) + "\n")
|
|
56
|
+
|
|
57
|
+
md: list[str] = [
|
|
58
|
+
"# OpenAPI / Swagger specs (discovered)\n",
|
|
59
|
+
f"_Scanned under `repos/` for `{'`, `'.join(OPENAPI_GLOBS)}`._\n",
|
|
60
|
+
f"\n**Specs:** {len(specs)}. **Cross-repo edges (openapi confidence):** "
|
|
61
|
+
f"{len(openapi_edges)}. **Helm host hints:** {len(helm_hosts)}.\n",
|
|
62
|
+
"\n## Specs by service\n",
|
|
63
|
+
md_table(
|
|
64
|
+
["Target repo", "Path count", "Spec file"],
|
|
65
|
+
[
|
|
66
|
+
[spec.target_repo, str(len(spec.paths)), spec.spec_path]
|
|
67
|
+
for spec in sorted(specs, key=lambda s: -len(s.paths))[:80]
|
|
68
|
+
],
|
|
69
|
+
),
|
|
70
|
+
]
|
|
71
|
+
if len(specs) > 80:
|
|
72
|
+
md.append(f"\n_{len(specs) - 80} more in `openapi-specs.jsonl`._\n")
|
|
73
|
+
|
|
74
|
+
(graphs_dir / "openapi-specs.md").write_text("\n".join(md), encoding="utf-8")
|
|
75
|
+
return specs
|