src2sink 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. src2sink/__init__.py +1 -0
  2. src2sink/aggregators/__init__.py +3 -0
  3. src2sink/aggregators/api_client_discovery.py +259 -0
  4. src2sink/aggregators/auth_cards.py +116 -0
  5. src2sink/aggregators/crypto_cards.py +132 -0
  6. src2sink/aggregators/data_stores.py +125 -0
  7. src2sink/aggregators/graphs.py +36 -0
  8. src2sink/aggregators/index_v2.py +138 -0
  9. src2sink/aggregators/library_source_map.py +188 -0
  10. src2sink/aggregators/openapi_discovery.py +113 -0
  11. src2sink/aggregators/openapi_edges.py +30 -0
  12. src2sink/aggregators/openapi_match.py +81 -0
  13. src2sink/aggregators/openapi_models.py +25 -0
  14. src2sink/aggregators/openapi_report.py +75 -0
  15. src2sink/aggregators/payload_producers.py +369 -0
  16. src2sink/aggregators/phase3.py +30 -0
  17. src2sink/aggregators/pii_cross_repo.py +283 -0
  18. src2sink/aggregators/pii_flow_v2.py +118 -0
  19. src2sink/aggregators/pii_lifecycle.py +19 -0
  20. src2sink/aggregators/pii_lifecycle_report.py +121 -0
  21. src2sink/aggregators/pii_touchpoint_collect.py +134 -0
  22. src2sink/aggregators/pii_touchpoint_utils.py +96 -0
  23. src2sink/aggregators/queues.py +141 -0
  24. src2sink/aggregators/ropa.py +190 -0
  25. src2sink/aggregators/service_call_collect.py +240 -0
  26. src2sink/aggregators/service_call_index.py +31 -0
  27. src2sink/aggregators/service_call_models.py +17 -0
  28. src2sink/aggregators/service_call_report.py +118 -0
  29. src2sink/aggregators/service_calls.py +14 -0
  30. src2sink/aggregators/taint_buckets.py +74 -0
  31. src2sink/aggregators/taint_catalogs.py +35 -0
  32. src2sink/aggregators/taint_writers.py +369 -0
  33. src2sink/aggregators/traces_index.py +103 -0
  34. src2sink/build_metabase_v2.py +807 -0
  35. src2sink/constants.py +44 -0
  36. src2sink/curate_internal_libraries.py +135 -0
  37. src2sink/extractors/__init__.py +17 -0
  38. src2sink/extractors/ast_walk.py +131 -0
  39. src2sink/extractors/base.py +59 -0
  40. src2sink/extractors/config.py +224 -0
  41. src2sink/extractors/file_context.py +35 -0
  42. src2sink/extractors/go.py +3 -0
  43. src2sink/extractors/http_out.py +108 -0
  44. src2sink/extractors/java.py +3 -0
  45. src2sink/extractors/javascript.py +3 -0
  46. src2sink/extractors/kotlin.py +3 -0
  47. src2sink/extractors/node_factory.py +65 -0
  48. src2sink/extractors/patterns.py +145 -0
  49. src2sink/extractors/python_extractor.py +3 -0
  50. src2sink/extractors/regex_extractors.py +378 -0
  51. src2sink/extractors/ts_extractors.py +123 -0
  52. src2sink/extractors/unified.py +64 -0
  53. src2sink/graph_common.py +213 -0
  54. src2sink/internal_groups.py +142 -0
  55. src2sink/known_api_clients.py +108 -0
  56. src2sink/library_taint_java.py +92 -0
  57. src2sink/limits.py +171 -0
  58. src2sink/models/__init__.py +3 -0
  59. src2sink/models/auth_model.py +3 -0
  60. src2sink/models/crypto_agility.py +3 -0
  61. src2sink/models/pii_lifecycle.py +92 -0
  62. src2sink/models/ropa.py +37 -0
  63. src2sink/prescreen.py +93 -0
  64. src2sink/py.typed +0 -0
  65. src2sink/record_fleet_baseline.py +81 -0
  66. src2sink/renderers/__init__.py +3 -0
  67. src2sink/renderers/markdown.py +149 -0
  68. src2sink/renderers/mermaid.py +3 -0
  69. src2sink/repo_utils.py +885 -0
  70. src2sink/safe_paths.py +54 -0
  71. src2sink/sanitize.py +107 -0
  72. src2sink/schema.py +57 -0
  73. src2sink/trace.py +558 -0
  74. src2sink/trace_batch.py +176 -0
  75. src2sink/vocabulary.py +152 -0
  76. src2sink-1.0.0.dist-info/METADATA +355 -0
  77. src2sink-1.0.0.dist-info/RECORD +81 -0
  78. src2sink-1.0.0.dist-info/WHEEL +5 -0
  79. src2sink-1.0.0.dist-info/entry_points.txt +6 -0
  80. src2sink-1.0.0.dist-info/licenses/LICENSE +21 -0
  81. src2sink-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,138 @@
1
+ """v2 repo index (replaces v1 index/ for flow-graph metabase)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import datetime as dt
6
+ import json
7
+ from collections import Counter, defaultdict
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from ..graph_common import iter_nodes, load_v2_repo_records
12
+ from ..renderers.markdown import md_table
13
+
14
+
15
+ def _row_from_record(data: dict[str, Any]) -> dict[str, Any]:
16
+ """Summarise one repo record into an index row of per-family node counts."""
17
+ families: Counter[str] = Counter()
18
+ weak_algos: list[str] = []
19
+ for node in iter_nodes(data):
20
+ families[node.get("family", "")] += 1
21
+ if node.get("family") == "crypto-algorithm":
22
+ algo = (node.get("detail") or {}).get("algorithm", "")
23
+ if algo and algo.upper() in {"MD5", "SHA1", "SHA-1", "DES", "RC4"}:
24
+ weak_algos.append(algo)
25
+ return {
26
+ "group": data["group"],
27
+ "name": data["name"],
28
+ "path": data.get("path", ""),
29
+ "primary_language": data.get("primary_language", "unknown"),
30
+ "frameworks": data.get("frameworks") or [],
31
+ "nodes": len(data.get("nodes", [])),
32
+ "edges": len(data.get("edges", [])),
33
+ "http_in": families.get("http-in", 0),
34
+ "http_out": families.get("http-out", 0),
35
+ "sql": families.get("sql", 0),
36
+ "raw_payload": families.get("raw-code-payload", 0),
37
+ "pii_field": families.get("pii-field", 0),
38
+ "pii_log": families.get("pii-log", 0),
39
+ "queue": families.get("queue-pub", 0) + families.get("queue-sub", 0),
40
+ "crypto": families.get("crypto-algorithm", 0),
41
+ "weak_algos": sorted(set(weak_algos)),
42
+ }
43
+
44
+
45
+ def write_index_v2(metabase_root: Path, repo_jsons: list[Path]) -> None:
46
+ """Write the v2 repo index JSON and the all/by-group/by-language markdown pages."""
47
+ records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
48
+ rows = [_row_from_record(data) for data in records]
49
+ index_dir = metabase_root / "index"
50
+ index_dir.mkdir(parents=True, exist_ok=True)
51
+
52
+ (index_dir / "repos.json").write_text(
53
+ json.dumps(rows, indent=2, sort_keys=True),
54
+ encoding="utf-8",
55
+ )
56
+
57
+ ts = dt.datetime.now(dt.timezone.utc).isoformat()
58
+ md = [
59
+ "# All repos (v2 index)\n",
60
+ f"_Generated {ts} from `schema_version: 2` JSON summaries._\n",
61
+ f"_{len(rows)} repos._\n\n",
62
+ md_table(
63
+ [
64
+ "Group / Repo",
65
+ "Lang",
66
+ "Frameworks",
67
+ "Nodes",
68
+ "HTTP-in",
69
+ "HTTP-out",
70
+ "SQL",
71
+ "Raw payload",
72
+ "PII fields",
73
+ ],
74
+ sorted(
75
+ [
76
+ [
77
+ f"[{r['group']}/{r['name']}](../repos/{r['group']}/{r['name']}.md)",
78
+ r["primary_language"],
79
+ ", ".join(r["frameworks"]) or "—",
80
+ str(r["nodes"]),
81
+ str(r["http_in"]),
82
+ str(r["http_out"]),
83
+ str(r["sql"]),
84
+ str(r["raw_payload"]),
85
+ str(r["pii_field"]),
86
+ ]
87
+ for r in rows
88
+ ],
89
+ key=lambda row: row[0].lower(),
90
+ ),
91
+ ),
92
+ ]
93
+ (index_dir / "repos.md").write_text("\n".join(md), encoding="utf-8")
94
+
95
+ by_group: dict[str, list[dict[str, Any]]] = defaultdict(list)
96
+ for r in rows:
97
+ by_group[r["group"]].append(r)
98
+ out = [f"# Repos by group (v2)\n\n_Generated {ts}._\n\n"]
99
+ for g in sorted(by_group):
100
+ out.append(f"## {g} ({len(by_group[g])} repos)\n")
101
+ out.append(
102
+ md_table(
103
+ ["Repo", "Lang", "Nodes", "Raw payload", "PII"],
104
+ [
105
+ [
106
+ f"[{r['name']}](../repos/{g}/{r['name']}.md)",
107
+ r["primary_language"],
108
+ str(r["nodes"]),
109
+ str(r["raw_payload"]),
110
+ str(r["pii_field"]),
111
+ ]
112
+ for r in sorted(by_group[g], key=lambda x: x["name"].lower())
113
+ ],
114
+ ),
115
+ )
116
+ (index_dir / "by-group.md").write_text("\n".join(out), encoding="utf-8")
117
+
118
+ by_lang: dict[str, list[dict[str, Any]]] = defaultdict(list)
119
+ for r in rows:
120
+ by_lang[r["primary_language"]].append(r)
121
+ out = ["# Repos by primary language (v2)\n\n"]
122
+ for lang in sorted(by_lang):
123
+ out.append(f"## {lang} ({len(by_lang[lang])} repos)\n")
124
+ out.append(
125
+ md_table(
126
+ ["Group", "Repo", "Nodes", "HTTP-in"],
127
+ [
128
+ [
129
+ r["group"],
130
+ f"[{r['name']}](../repos/{r['group']}/{r['name']}.md)",
131
+ str(r["nodes"]),
132
+ str(r["http_in"]),
133
+ ]
134
+ for r in sorted(by_lang[lang], key=lambda x: (x["group"], x["name"]))
135
+ ],
136
+ ),
137
+ )
138
+ (index_dir / "by-language.md").write_text("\n".join(out), encoding="utf-8")
@@ -0,0 +1,188 @@
1
+ """Generator and loader for library-source-map.json.
2
+
3
+ Maps Maven/npm coordinates to source-repo clone paths. Generated by
4
+ src2sink-build from per-repo dependency data; manual edits to
5
+ ``status`` and ``clone_path`` fields are preserved on regeneration.
6
+ """
7
+
8
+ from __future__ import annotations
9
+ import json
10
+ from collections import defaultdict
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ STATUS_PENDING = "pending"
15
+ STATUS_CLONED = "cloned"
16
+ STATUS_EXCLUDED = "excluded"
17
+ STATUS_AMBIGUOUS = "ambiguous"
18
+
19
+
20
+ def load_library_source_map(path: Path) -> dict[str, dict[str, Any]]:
21
+ """Return coord -> entry dict, or {} if the file is missing/invalid."""
22
+ try:
23
+ data = json.loads(path.read_text(encoding="utf-8"))
24
+ except (OSError, json.JSONDecodeError):
25
+ return {}
26
+ return {m["coordinate"]: m for m in data.get("mappings", []) if "coordinate" in m}
27
+
28
+
29
+ def generate_library_source_map(metabase_root: Path, repo_jsons: list[Path]) -> None:
30
+ """Write/update library-source-map.json from per-repo internal dependency data."""
31
+ map_path = metabase_root / "library-source-map.json"
32
+ existing = load_library_source_map(map_path)
33
+
34
+ # Collect all internal deps from per-repo JSONs
35
+ coord_consumers: dict[str, int] = defaultdict(int)
36
+ coord_versions: dict[str, set[str]] = defaultdict(set)
37
+
38
+ for jp in repo_jsons:
39
+ try:
40
+ data = json.loads(jp.read_text(encoding="utf-8"))
41
+ except (OSError, json.JSONDecodeError):
42
+ continue
43
+ for dep in data.get("dependencies_internal", []):
44
+ gid = dep.get("groupId", "")
45
+ aid = dep.get("artifactId", "")
46
+ ver = dep.get("version", "")
47
+ if not aid:
48
+ continue
49
+ coord = f"{gid}:{aid}" if gid else aid
50
+ coord_consumers[coord] += 1
51
+ if ver:
52
+ coord_versions[coord].add(ver)
53
+
54
+ # Build updated mappings, preserving manual overrides
55
+ mappings: list[dict[str, Any]] = []
56
+ for coord, count in sorted(coord_consumers.items(), key=lambda x: -x[1]):
57
+ prev = existing.get(coord, {})
58
+ # Preserve manually-set status and clone_path overrides
59
+ status = prev.get("status", STATUS_PENDING)
60
+ clone_path = prev.get("clone_path") or _infer_clone_path(coord)
61
+ entry: dict[str, Any] = {
62
+ "coordinate": coord,
63
+ "clone_path": clone_path or "",
64
+ "status": status,
65
+ "consumers": count,
66
+ }
67
+ versions = sorted(coord_versions.get(coord, set()))
68
+ if versions:
69
+ entry["versions"] = versions
70
+ if prev.get("notes"):
71
+ entry["notes"] = prev["notes"]
72
+ mappings.append(entry)
73
+
74
+ out = {
75
+ "_comment": (
76
+ "Map of Maven/npm coordinates to source-repo clone paths. "
77
+ "Generated by src2sink-build from per-repo dependency data. "
78
+ "Edit 'status' ('pending'|'cloned'|'excluded') and 'clone_path' to "
79
+ "guide curate_internal_libraries.py. Changes are preserved on regeneration."
80
+ ),
81
+ "mappings": mappings,
82
+ }
83
+ map_path.write_text(json.dumps(out, indent=2, sort_keys=False) + "\n", encoding="utf-8")
84
+
85
+
86
+ def _resolve_clone_path(
87
+ coord: str,
88
+ by_coord: dict[tuple[str, str], str],
89
+ by_name: dict[str, list[str]],
90
+ by_full: dict[str, str],
91
+ ) -> str | None:
92
+ """Resolve a dependency coordinate to a clone path via the identity index."""
93
+ # 1. Exact canonical-coordinate match.
94
+ if coord in by_full:
95
+ return by_full[coord]
96
+ if ":" in coord:
97
+ group, artifact = coord.split(":", 1)
98
+ # 2. group:artifact pair.
99
+ hit = by_coord.get((group, artifact))
100
+ if hit:
101
+ return hit
102
+ # 3. The artifact half may itself be a full name (scoped npm, vendor/pkg).
103
+ if artifact in by_full:
104
+ return by_full[artifact]
105
+ candidates = by_name.get(artifact, [])
106
+ return candidates[0] if len(candidates) == 1 else None
107
+ # No group: try a group-less pair, then a unique bare-name match.
108
+ hit = by_coord.get(("", coord))
109
+ if hit:
110
+ return hit
111
+ candidates = by_name.get(coord, [])
112
+ return candidates[0] if len(candidates) == 1 else None
113
+
114
+
115
+ def _entry_needs_resolution(entry: dict[str, Any]) -> bool:
116
+ """True if a source-map entry is still flagged and worth resolving."""
117
+ if not entry.get("coordinate"):
118
+ return False
119
+ status = entry.get("status")
120
+ if status == STATUS_EXCLUDED:
121
+ return False
122
+ if status == STATUS_CLONED and entry.get("clone_path"):
123
+ return False
124
+ return True
125
+
126
+
127
+ def fix_flagged_mappings(metabase_root: Path, repos_root: Path) -> int:
128
+ """Resolve flagged source-map entries from on-disk project manifests.
129
+
130
+ Walks every recognised project-definition file under ``repos_root`` —
131
+ Maven (``pom.xml``), Gradle (``build.gradle``/``settings.gradle`` ±kts),
132
+ npm (``package.json``), Python (``pyproject.toml``/``setup.cfg``), Rust
133
+ (``Cargo.toml``), Go (``go.mod``), PHP (``composer.json``), .NET
134
+ (``*.csproj``/``*.fsproj``/``*.vbproj``) and Ruby (``*.gemspec``) — and
135
+ reads each project's declared identity. For any mapping still flagged
136
+ (status ``pending`` / ``ambiguous``, or missing a ``clone_path``) whose
137
+ coordinate matches a discovered project, sets ``clone_path`` to that repo's
138
+ path (relative to ``repos_root``) and marks the status ``cloned``.
139
+ ``excluded`` and already-resolved ``cloned`` entries are left untouched.
140
+
141
+ Matching tries, in order: the exact coordinate string (npm ``@scope/pkg``,
142
+ Composer ``vendor/pkg``, Go module path, bare names), then the
143
+ ``group:artifact`` pair, then a unique bare-name match.
144
+
145
+ Returns the number of mappings corrected.
146
+ """
147
+ map_path = metabase_root / "library-source-map.json"
148
+ try:
149
+ data = json.loads(map_path.read_text(encoding="utf-8"))
150
+ except (OSError, json.JSONDecodeError):
151
+ return 0
152
+ mappings = data.get("mappings")
153
+ if not isinstance(mappings, list) or not mappings:
154
+ return 0
155
+
156
+ from ..repo_utils import _build_component_identity_index
157
+
158
+ by_coord, by_name, by_full = _build_component_identity_index(repos_root)
159
+
160
+ fixed = 0
161
+ for entry in mappings:
162
+ if not isinstance(entry, dict) or not _entry_needs_resolution(entry):
163
+ continue
164
+ clone_path = _resolve_clone_path(entry["coordinate"], by_coord, by_name, by_full)
165
+ if not clone_path:
166
+ continue
167
+ entry["clone_path"] = clone_path
168
+ entry["status"] = STATUS_CLONED
169
+ entry.pop("notes", None)
170
+ fixed += 1
171
+
172
+ if fixed:
173
+ map_path.write_text(
174
+ json.dumps(data, indent=2, sort_keys=False) + "\n", encoding="utf-8"
175
+ )
176
+ return fixed
177
+
178
+
179
+ def _infer_clone_path(coord: str) -> str | None:
180
+ """Best-effort inference of group/artifact from a Maven coordinate."""
181
+ if ":" not in coord:
182
+ return None
183
+ group, artifact = coord.split(":", 1)
184
+ # Strip common org prefix patterns to get a GitLab group name
185
+ parts = group.split(".")
186
+ # Heuristic: last meaningful segment of the group + artifact name
187
+ group_hint = parts[-1] if len(parts) > 2 else parts[-1]
188
+ return f"{group_hint}/{artifact}"
@@ -0,0 +1,113 @@
1
+ """Discover OpenAPI specs and Helm ingress hosts under repos/."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from pathlib import Path
7
+
8
+ from ..graph_common import normalize_path_template as norm_path
9
+
10
+ from .openapi_models import OpenApiSpec
11
+
12
+ OPENAPI_GLOBS = ("openapi.yaml", "openapi.yml", "swagger.yaml", "swagger.yml")
13
+ PATH_LINE_RX = re.compile(
14
+ r"^\s{2,}(/[/\w{}.:$-]+)\s*:\s*$",
15
+ re.MULTILINE,
16
+ )
17
+ SERVERS_URL_RX = re.compile(
18
+ r"^\s*-\s*url:\s*['\"]?(https?://[^'\"\s]+)",
19
+ re.MULTILINE | re.IGNORECASE,
20
+ )
21
+ HELM_VALUES_NAMES = ("values.yaml", "values.yml")
22
+ HELM_HOST_RX = re.compile(
23
+ r"^\s*(?:host|hostname|externalHost|ingress\.host)\s*:\s*['\"]?([^\s#'\"]+)",
24
+ re.MULTILINE | re.IGNORECASE,
25
+ )
26
+
27
+ _SKIP_DIR_PARTS = frozenset({".git", "node_modules", "target", "build", "charts"})
28
+
29
+
30
+ def repo_from_under_repos(spec_path: Path, repos_root: Path) -> str | None:
31
+ """Return the ``org/repo`` id for a path under repos_root, or None."""
32
+ try:
33
+ rel = spec_path.relative_to(repos_root)
34
+ except ValueError:
35
+ return None
36
+ parts = rel.parts
37
+ if len(parts) < 2:
38
+ return None
39
+ return f"{parts[0]}/{parts[1]}"
40
+
41
+
42
+ def _should_skip_path(path: Path) -> bool:
43
+ """True if the path lies within a skipped directory (.git, node_modules, ...)."""
44
+ return any(p in path.parts for p in _SKIP_DIR_PARTS)
45
+
46
+
47
+ def discover_openapi_specs(repos_root: Path) -> list[OpenApiSpec]:
48
+ """Scan repos_root for OpenAPI/Swagger specs and return parsed OpenApiSpec rows."""
49
+ if not repos_root.is_dir():
50
+ return []
51
+ specs: list[OpenApiSpec] = []
52
+ seen: set[tuple[str, str]] = set()
53
+ for name in OPENAPI_GLOBS:
54
+ for path in repos_root.rglob(name):
55
+ if _should_skip_path(path):
56
+ continue
57
+ try:
58
+ if path.stat().st_size > 2_000_000:
59
+ continue
60
+ text = path.read_text(encoding="utf-8", errors="replace")
61
+ except OSError:
62
+ continue
63
+ target = repo_from_under_repos(path, repos_root)
64
+ if not target:
65
+ continue
66
+ key = (target, str(path))
67
+ if key in seen:
68
+ continue
69
+ seen.add(key)
70
+ paths = sorted({norm_path(m.group(1)) for m in PATH_LINE_RX.finditer(text)})
71
+ servers = [m.group(1) for m in SERVERS_URL_RX.finditer(text)]
72
+ specs.append(OpenApiSpec(
73
+ target_repo=target,
74
+ spec_path=str(path.relative_to(repos_root)),
75
+ paths=[p for p in paths if p and p != "/"],
76
+ servers=servers[:5],
77
+ ))
78
+ return specs
79
+
80
+
81
+ def discover_helm_hosts(repos_root: Path) -> list[dict[str, str]]:
82
+ """Scan Helm values for declared ingress/service hostnames."""
83
+ if not repos_root.is_dir():
84
+ return []
85
+ rows: list[dict[str, str]] = []
86
+ seen: set[tuple[str, str]] = set()
87
+ for name in HELM_VALUES_NAMES:
88
+ for path in repos_root.rglob(name):
89
+ if _should_skip_path(path):
90
+ continue
91
+ try:
92
+ if path.stat().st_size > 500_000:
93
+ continue
94
+ text = path.read_text(encoding="utf-8", errors="replace")
95
+ except OSError:
96
+ continue
97
+ target = repo_from_under_repos(path, repos_root)
98
+ if not target:
99
+ continue
100
+ for m in HELM_HOST_RX.finditer(text):
101
+ host = m.group(1).strip().strip("'\"")
102
+ if not host or host.startswith("{{"):
103
+ continue
104
+ key = (target, host)
105
+ if key in seen:
106
+ continue
107
+ seen.add(key)
108
+ rows.append({
109
+ "target_repo": target,
110
+ "host": host,
111
+ "values_path": str(path.relative_to(repos_root)),
112
+ })
113
+ return rows
@@ -0,0 +1,30 @@
1
+ """OpenAPI / Swagger path ingestion for authoritative inbound route index."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .openapi_discovery import (
6
+ OPENAPI_GLOBS,
7
+ PATH_LINE_RX,
8
+ discover_helm_hosts,
9
+ discover_openapi_specs,
10
+ repo_from_under_repos,
11
+ )
12
+ from .openapi_match import (
13
+ build_openapi_inbound_index,
14
+ match_http_out_to_openapi,
15
+ )
16
+ from .openapi_models import OpenApiInbound, OpenApiSpec
17
+ from .openapi_report import write_openapi_artifacts
18
+
19
+ __all__ = [
20
+ "OPENAPI_GLOBS",
21
+ "PATH_LINE_RX",
22
+ "OpenApiInbound",
23
+ "OpenApiSpec",
24
+ "build_openapi_inbound_index",
25
+ "discover_helm_hosts",
26
+ "discover_openapi_specs",
27
+ "match_http_out_to_openapi",
28
+ "repo_from_under_repos",
29
+ "write_openapi_artifacts",
30
+ ]
@@ -0,0 +1,81 @@
1
+ """Match http-out nodes to OpenAPI-discovered inbound paths."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ from ..graph_common import (
8
+ build_repo_alias_index,
9
+ extract_urls_and_paths,
10
+ iter_nodes,
11
+ match_path_in_inbound_index,
12
+ normalize_path_template as norm_path,
13
+ repo_id,
14
+ )
15
+
16
+ from .openapi_models import OpenApiSpec
17
+
18
+
19
+ def build_openapi_inbound_index(
20
+ specs: list[OpenApiSpec],
21
+ ) -> dict[str, list[tuple[str, str, str]]]:
22
+ """normalized_path -> [(repo, original_path, spec_path)]."""
23
+ index: dict[str, list[tuple[str, str, str]]] = {}
24
+ for spec in specs:
25
+ for path in spec.paths:
26
+ key = norm_path(path)
27
+ index.setdefault(key, []).append(
28
+ (spec.target_repo, path, spec.spec_path),
29
+ )
30
+ return index
31
+
32
+
33
+ def _paths_from_http_out(detail: dict[str, Any], raw: str) -> list[str]:
34
+ """Collect candidate request paths from an http-out node detail and raw text."""
35
+ paths: list[str] = list(detail.get("paths") or [])
36
+ if detail.get("path"):
37
+ paths.insert(0, detail["path"])
38
+ _, extra_paths = extract_urls_and_paths(raw)
39
+ paths.extend(p for p in extra_paths if p not in paths)
40
+ return paths
41
+
42
+
43
+ def match_http_out_to_openapi(
44
+ records: list[dict[str, Any]],
45
+ inbound: dict[str, list[tuple[str, str, str]]],
46
+ alias_to_repo: dict[str, str] | None = None,
47
+ ) -> list[dict[str, Any]]:
48
+ """Return edges from http-out call sites that match an OpenAPI path."""
49
+ _ = alias_to_repo or build_repo_alias_index(records)
50
+ edges: list[dict[str, Any]] = []
51
+ seen: set[tuple[str, str, str]] = set()
52
+
53
+ for data in records:
54
+ src = repo_id(data)
55
+ for node in iter_nodes(data):
56
+ if node.get("family") != "http-out":
57
+ continue
58
+ detail = node.get("detail") or {}
59
+ raw = detail.get("raw", "")
60
+ ref = f"{node.get('file')}:{node.get('line')}"
61
+
62
+ for path in _paths_from_http_out(detail, raw):
63
+ targets, _conf = match_path_in_inbound_index(path, inbound)
64
+ for tgt_repo, tgt_path, spec_path in targets:
65
+ if tgt_repo == src:
66
+ continue
67
+ key = (src, tgt_repo, norm_path(tgt_path))
68
+ if key in seen:
69
+ continue
70
+ seen.add(key)
71
+ edges.append({
72
+ "source_repo": src,
73
+ "target_repo": tgt_repo,
74
+ "target_path": tgt_path,
75
+ "confidence": "openapi",
76
+ "evidence": f"http-out path matches OpenAPI in {spec_path}",
77
+ "refs": [ref],
78
+ "spec_path": spec_path,
79
+ })
80
+
81
+ return edges
@@ -0,0 +1,25 @@
1
+ """Data types for OpenAPI graph ingestion."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field
6
+
7
+
8
+ @dataclass
9
+ class OpenApiSpec:
10
+ """A discovered OpenAPI/Swagger spec: its owning repo, file, paths and servers."""
11
+
12
+ target_repo: str
13
+ spec_path: str
14
+ paths: list[str] = field(default_factory=list)
15
+ servers: list[str] = field(default_factory=list)
16
+
17
+
18
+ @dataclass
19
+ class OpenApiInbound:
20
+ """A single inbound OpenAPI route: repo, path, HTTP method and spec file."""
21
+
22
+ target_repo: str
23
+ path: str
24
+ method: str
25
+ spec_path: str
@@ -0,0 +1,75 @@
1
+ """Write OpenAPI / Helm discovery artefacts under metabase/graphs/."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+
8
+ from ..graph_common import build_repo_alias_index, load_v2_repo_records
9
+ from ..renderers.markdown import md_table
10
+
11
+ from .openapi_discovery import OPENAPI_GLOBS, discover_helm_hosts, discover_openapi_specs
12
+ from .openapi_match import build_openapi_inbound_index, match_http_out_to_openapi
13
+ from .openapi_models import OpenApiSpec
14
+
15
+
16
+ def write_openapi_artifacts(
17
+ metabase_root: Path,
18
+ repos_root: Path | None,
19
+ repo_jsons: list[Path] | None = None,
20
+ ) -> list[OpenApiSpec]:
21
+ """Discover specs/Helm hosts, write JSONL + markdown artefacts, return the specs."""
22
+ graphs_dir = metabase_root / "graphs"
23
+ graphs_dir.mkdir(parents=True, exist_ok=True)
24
+
25
+ specs: list[OpenApiSpec] = []
26
+ if repos_root and repos_root.is_dir():
27
+ specs = discover_openapi_specs(repos_root)
28
+
29
+ jsonl_path = graphs_dir / "openapi-specs.jsonl"
30
+ with jsonl_path.open("w", encoding="utf-8") as fh:
31
+ for spec in specs:
32
+ fh.write(json.dumps({
33
+ "target_repo": spec.target_repo,
34
+ "spec_path": spec.spec_path,
35
+ "paths": spec.paths,
36
+ "servers": spec.servers,
37
+ }, ensure_ascii=False) + "\n")
38
+
39
+ inbound = build_openapi_inbound_index(specs)
40
+ openapi_edges: list[dict[str, object]] = []
41
+ if repo_jsons and inbound:
42
+ records = load_v2_repo_records(metabase_root, json_paths=repo_jsons)
43
+ alias_to_repo = build_repo_alias_index(records)
44
+ openapi_edges = match_http_out_to_openapi(records, inbound, alias_to_repo)
45
+
46
+ edges_path = graphs_dir / "openapi-service-edges.jsonl"
47
+ with edges_path.open("w", encoding="utf-8") as fh:
48
+ for row in openapi_edges:
49
+ fh.write(json.dumps(row, ensure_ascii=False) + "\n")
50
+
51
+ helm_hosts = discover_helm_hosts(repos_root) if repos_root else []
52
+ helm_path = graphs_dir / "helm-service-hosts.jsonl"
53
+ with helm_path.open("w", encoding="utf-8") as fh:
54
+ for helm_row in helm_hosts:
55
+ fh.write(json.dumps(helm_row, ensure_ascii=False) + "\n")
56
+
57
+ md: list[str] = [
58
+ "# OpenAPI / Swagger specs (discovered)\n",
59
+ f"_Scanned under `repos/` for `{'`, `'.join(OPENAPI_GLOBS)}`._\n",
60
+ f"\n**Specs:** {len(specs)}. **Cross-repo edges (openapi confidence):** "
61
+ f"{len(openapi_edges)}. **Helm host hints:** {len(helm_hosts)}.\n",
62
+ "\n## Specs by service\n",
63
+ md_table(
64
+ ["Target repo", "Path count", "Spec file"],
65
+ [
66
+ [spec.target_repo, str(len(spec.paths)), spec.spec_path]
67
+ for spec in sorted(specs, key=lambda s: -len(s.paths))[:80]
68
+ ],
69
+ ),
70
+ ]
71
+ if len(specs) > 80:
72
+ md.append(f"\n_{len(specs) - 80} more in `openapi-specs.jsonl`._\n")
73
+
74
+ (graphs_dir / "openapi-specs.md").write_text("\n".join(md), encoding="utf-8")
75
+ return specs