src2sink 2.0.0__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {src2sink-2.0.0/src2sink.egg-info → src2sink-3.0.0}/PKG-INFO +1 -1
- {src2sink-2.0.0 → src2sink-3.0.0}/pyproject.toml +1 -1
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/api_client_discovery.py +247 -25
- src2sink-3.0.0/src2sink/aggregators/graphs.py +89 -0
- src2sink-3.0.0/src2sink/aggregators/openapi_discovery.py +125 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/payload_producers.py +175 -61
- src2sink-3.0.0/src2sink/aggregators/queues.py +215 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/build_metabase_v2.py +191 -11
- src2sink-3.0.0/src2sink/checkout_scan.py +187 -0
- src2sink-3.0.0/src2sink/constants.py +76 -0
- src2sink-3.0.0/src2sink/dependencies.py +214 -0
- src2sink-3.0.0/src2sink/derive.py +388 -0
- src2sink-3.0.0/src2sink/extractors/ast_walk.py +539 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/patterns.py +104 -2
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/regex_extractors.py +53 -2
- src2sink-3.0.0/src2sink/extractors/ts_extractors.py +376 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/unified.py +18 -4
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/graph_common.py +129 -13
- src2sink-3.0.0/src2sink/index_store.py +370 -0
- src2sink-3.0.0/src2sink/maven.py +334 -0
- src2sink-3.0.0/src2sink/paths.py +361 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/repo_utils.py +58 -24
- src2sink-3.0.0/src2sink/resolve.py +250 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/schema.py +7 -1
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/trace.py +263 -84
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/trace_batch.py +8 -1
- {src2sink-2.0.0 → src2sink-3.0.0/src2sink.egg-info}/PKG-INFO +1 -1
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink.egg-info/SOURCES.txt +33 -0
- src2sink-3.0.0/tests/test_aggregate_output_golden.py +114 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_ast_walk.py +107 -19
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_build_internals.py +6 -1
- src2sink-3.0.0/tests/test_call_observations.py +153 -0
- src2sink-3.0.0/tests/test_call_resolution.py +341 -0
- src2sink-3.0.0/tests/test_checkout_scan.py +261 -0
- src2sink-3.0.0/tests/test_derive_pass.py +164 -0
- src2sink-3.0.0/tests/test_detection_input_coverage.py +142 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_detection_version.py +6 -1
- src2sink-3.0.0/tests/test_discovery_scan_cost.py +214 -0
- src2sink-3.0.0/tests/test_entry_points.py +165 -0
- src2sink-3.0.0/tests/test_fleet_index.py +369 -0
- src2sink-3.0.0/tests/test_maven_resolution.py +239 -0
- src2sink-3.0.0/tests/test_method_structure.py +180 -0
- src2sink-3.0.0/tests/test_oi13_kotlin_parity.py +113 -0
- src2sink-3.0.0/tests/test_oi26_receiver_scope.py +141 -0
- src2sink-3.0.0/tests/test_oi28_index_fast_path.py +126 -0
- src2sink-3.0.0/tests/test_oi33_canonical_repo_id.py +259 -0
- src2sink-3.0.0/tests/test_oi37_express_anchor.py +174 -0
- src2sink-3.0.0/tests/test_oi38_trace_index_freshness.py +171 -0
- src2sink-3.0.0/tests/test_oi39_test_path_classification.py +160 -0
- src2sink-3.0.0/tests/test_oi40_client_repo_target.py +206 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_output_neutralisation.py +15 -3
- src2sink-3.0.0/tests/test_path_match_significance.py +110 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase4_extractor_snapshots.py +9 -0
- src2sink-3.0.0/tests/test_polyglot_dependencies.py +164 -0
- src2sink-3.0.0/tests/test_producer_scan_single_pass.py +250 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_redos_bounds.py +15 -1
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_repo_utils_helpers.py +0 -9
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_security_input_handling.py +5 -4
- src2sink-3.0.0/tests/test_silent_failure_gate.py +285 -0
- src2sink-3.0.0/tests/test_sql_classifier.py +114 -0
- src2sink-3.0.0/tests/test_tainted_paths.py +410 -0
- src2sink-3.0.0/tests/test_type_declarations.py +206 -0
- src2sink-2.0.0/src2sink/aggregators/graphs.py +0 -36
- src2sink-2.0.0/src2sink/aggregators/openapi_discovery.py +0 -113
- src2sink-2.0.0/src2sink/aggregators/queues.py +0 -141
- src2sink-2.0.0/src2sink/constants.py +0 -44
- src2sink-2.0.0/src2sink/extractors/ast_walk.py +0 -158
- src2sink-2.0.0/src2sink/extractors/ts_extractors.py +0 -212
- {src2sink-2.0.0 → src2sink-3.0.0}/LICENSE +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/README.md +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/setup.cfg +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/__init__.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/__init__.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/auth_cards.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/crypto_cards.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/data_stores.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/index_v2.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/library_source_map.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/openapi_edges.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/openapi_match.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/openapi_models.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/openapi_report.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/phase3.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_cross_repo.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_flow_v2.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_lifecycle.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_lifecycle_report.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_touchpoint_collect.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_touchpoint_utils.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/ropa.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/service_call_collect.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/service_call_index.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/service_call_models.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/service_call_report.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/service_calls.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/taint_buckets.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/taint_catalogs.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/taint_writers.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/traces_index.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/curate_internal_libraries.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/__init__.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/base.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/config.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/file_context.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/go.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/http_out.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/java.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/javascript.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/kotlin.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/node_factory.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/python_extractor.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/symbols.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/internal_groups.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/known_api_clients.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/library_taint_java.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/limits.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/models/__init__.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/models/auth_model.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/models/crypto_agility.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/models/pii_lifecycle.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/models/ropa.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/prescreen.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/py.typed +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/record_fleet_baseline.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/renderers/__init__.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/renderers/markdown.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/renderers/mermaid.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/safe_paths.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/sanitize.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/vocabulary.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink.egg-info/dependency_links.txt +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink.egg-info/entry_points.txt +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink.egg-info/requires.txt +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/src2sink.egg-info/top_level.txt +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_aggregators_extra.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_api_client_discovery.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_build_main.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_characterization.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_config_hygiene.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_cross_repo_caller_coverage.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_curate.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_demand_side_discovery.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_dependency_pinning.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_detection_fingerprint_gate.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_field_precision.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_gradle_version_catalogs.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_graph_common.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_http_guard_evidence.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_import_graph.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_internal_groups.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_limits.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_module_imports.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_openapi.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase0_smoke.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase1_config.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase1_extractors.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase2_graphs.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase3_pii.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase4_aggregators.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase4_fleet_regression.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase4_regression.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase4_taint_caps.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_post_phase4_backlog.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_prescreen.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_renderers.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_run_manifest.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_sanitize.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_source_map_fix.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_sql_payload_out.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_sql_sink_evidence.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_sql_source_construction.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_trace_batch.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_trace_fleet_scaling.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_trace_render.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_zz_security_coverage_gate.py +0 -0
- {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_zz_untrusted_interpolation_guard.py +0 -0
|
@@ -26,6 +26,7 @@ unverified binding poisons the taint graph with phantom cross-service edges.
|
|
|
26
26
|
from __future__ import annotations
|
|
27
27
|
|
|
28
28
|
import json
|
|
29
|
+
import re
|
|
29
30
|
from collections import defaultdict
|
|
30
31
|
from pathlib import Path
|
|
31
32
|
from typing import Any
|
|
@@ -85,21 +86,107 @@ def _http_in_paths_by_repo(records: list[dict[str, Any]]) -> dict[str, set[str]]
|
|
|
85
86
|
return out
|
|
86
87
|
|
|
87
88
|
|
|
89
|
+
# Suffixes a client library's repo conventionally carries. Used only to derive
|
|
90
|
+
# the *stem to search for* — never as the test itself. The test is whether the
|
|
91
|
+
# repo declares inbound endpoints, which is name-independent and catches estates
|
|
92
|
+
# that name their libraries differently (`OI-40`).
|
|
93
|
+
_CLIENT_SUFFIX_RX = re.compile(r"[-_](?:clients?|sdk)$", re.IGNORECASE)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _service_for_client_repo(
|
|
97
|
+
declaring_repo: str, endpoints: dict[str, set[str]]
|
|
98
|
+
) -> str | None:
|
|
99
|
+
"""Map a client-library repo to the service repo it fronts, or None.
|
|
100
|
+
|
|
101
|
+
`resolve(coord)` names the repo that *declares* the artifact, and
|
|
102
|
+
`_canonical_repo_id` maps that to the repo owning it. Both are correct. But
|
|
103
|
+
when a client library is published from **its own repository** rather than as
|
|
104
|
+
a submodule of the service, the repo owning the declaration *is* the library
|
|
105
|
+
— so the binding points at the client instead of the service it calls
|
|
106
|
+
(`OI-40`).
|
|
107
|
+
|
|
108
|
+
`OI-33` fixed the *shape* of the identity; this is the *referent* being
|
|
109
|
+
wrong, and no amount of path normalisation reaches it. The service is not
|
|
110
|
+
named anywhere in the consumer's dependency declaration.
|
|
111
|
+
|
|
112
|
+
A service has inbound endpoints; a client library does not. So the test is
|
|
113
|
+
the endpoint count, and the name only supplies the stem to search for.
|
|
114
|
+
Returns None rather than guessing — 31 of 42 in the observed fleet had no
|
|
115
|
+
sibling because the service is not in the scanned set at all, and inventing
|
|
116
|
+
one would manufacture the broken edges `OI-33` was about.
|
|
117
|
+
"""
|
|
118
|
+
if "/" not in declaring_repo:
|
|
119
|
+
return None
|
|
120
|
+
if endpoints.get(declaring_repo):
|
|
121
|
+
return None # it receives calls, so it is a service; leave it alone
|
|
122
|
+
stem = _CLIENT_SUFFIX_RX.sub("", declaring_repo.split("/")[-1])
|
|
123
|
+
if not stem:
|
|
124
|
+
return None
|
|
125
|
+
hits = sorted(
|
|
126
|
+
repo for repo, paths in endpoints.items()
|
|
127
|
+
if paths and repo.split("/")[-1] in (stem, f"{stem}-service")
|
|
128
|
+
)
|
|
129
|
+
return hits[0] if len(hits) == 1 else None
|
|
130
|
+
|
|
131
|
+
|
|
88
132
|
def _key(target_repo: str, artifact: str) -> str:
|
|
89
133
|
"""Stable candidate key used for de-dup and for preserving reviewer edits."""
|
|
90
134
|
return f"{target_repo}\t{artifact}"
|
|
91
135
|
|
|
92
136
|
|
|
93
|
-
def
|
|
94
|
-
"""
|
|
137
|
+
def _canonical_repo_id(resolved: str, known: frozenset[str]) -> str | None:
|
|
138
|
+
"""Map a resolved coordinate path to the repo id that owns it.
|
|
139
|
+
|
|
140
|
+
The identity index resolves a coordinate to the directory that *declares* it,
|
|
141
|
+
which may be a build module inside a repo — `group/repo/some-client`. The
|
|
142
|
+
demand-side pass names its target `group/repo`, from `repo_id()`. The two
|
|
143
|
+
never met, so `discovery_method: "both"` — the strongest signal the design
|
|
144
|
+
produces — could not occur even once (`OI-33`).
|
|
145
|
+
|
|
146
|
+
Truncating to two segments would be wrong: `group/subgroup/repo` is a valid
|
|
147
|
+
GitLab path and this estate contains them (`OI-34`). So match the longest
|
|
148
|
+
known repo id instead. That handles in-repo modules and nested repos alike,
|
|
149
|
+
and depends on neither path depth nor on `.git` being present — 65 of 746
|
|
150
|
+
repos in the observed fleet have no `.git` at all.
|
|
151
|
+
|
|
152
|
+
Returns None when nothing matches, which is a resolver failure worth
|
|
153
|
+
surfacing rather than a repo id worth guessing.
|
|
154
|
+
"""
|
|
155
|
+
if not resolved:
|
|
156
|
+
return None
|
|
157
|
+
parts = resolved.split("/")
|
|
158
|
+
for n in range(len(parts), 0, -1):
|
|
159
|
+
candidate = "/".join(parts[:n])
|
|
160
|
+
if candidate in known:
|
|
161
|
+
return candidate
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _load_discovered(
|
|
166
|
+
path: Path, known: frozenset[str] | None = None
|
|
167
|
+
) -> dict[str, dict[str, Any]]:
|
|
168
|
+
"""Return prior candidates keyed by (target_repo, artifact), or {} if absent.
|
|
169
|
+
|
|
170
|
+
``known`` lets a candidate stored under a pre-`OI-33` module path also be
|
|
171
|
+
found under its canonical repo id, so reviewer decisions survive the fix.
|
|
172
|
+
"""
|
|
95
173
|
try:
|
|
96
174
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
97
175
|
except (OSError, json.JSONDecodeError):
|
|
98
176
|
return {}
|
|
99
177
|
out: dict[str, dict[str, Any]] = {}
|
|
100
178
|
for c in data.get("candidates", []):
|
|
101
|
-
if isinstance(c, dict)
|
|
102
|
-
|
|
179
|
+
if not isinstance(c, dict) or "maven_artifact" not in c:
|
|
180
|
+
continue
|
|
181
|
+
stored = str(c.get("target_repo", ""))
|
|
182
|
+
artifact = c["maven_artifact"]
|
|
183
|
+
out[_key(stored, artifact)] = c
|
|
184
|
+
# A candidate reviewed before `OI-33` was keyed on the *module* path.
|
|
185
|
+
# Index it under its canonical repo id too, or the fix would silently
|
|
186
|
+
# discard every accept/reject a reviewer has already made.
|
|
187
|
+
canonical = _canonical_repo_id(stored, known) if known else None
|
|
188
|
+
if canonical and canonical != stored:
|
|
189
|
+
out.setdefault(_key(canonical, artifact), c)
|
|
103
190
|
return out
|
|
104
191
|
|
|
105
192
|
|
|
@@ -112,12 +199,40 @@ def _confidence(target_repo: str, scanned: set[str], has_paths: bool) -> str:
|
|
|
112
199
|
return "low" # artifact looks like a client but coordinate did not resolve
|
|
113
200
|
|
|
114
201
|
|
|
202
|
+
def _resolve_target(
|
|
203
|
+
resolve: Any, coord: str, known: frozenset[str], endpoints: dict[str, set[str]],
|
|
204
|
+
) -> tuple[str, str, str]:
|
|
205
|
+
"""Resolve a coordinate to (declaring path, service repo, client repo if corrected).
|
|
206
|
+
|
|
207
|
+
Two corrections in sequence, both needed and neither sufficient alone:
|
|
208
|
+
`_canonical_repo_id` fixes the identity's *shape* (`OI-33`), and
|
|
209
|
+
`_service_for_client_repo` fixes its *referent* when the declaring repo turns
|
|
210
|
+
out to be the client library rather than the service (`OI-40`).
|
|
211
|
+
|
|
212
|
+
The client repo is returned rather than discarded, because a substitution a
|
|
213
|
+
reviewer cannot see is one they cannot check.
|
|
214
|
+
"""
|
|
215
|
+
resolved = resolve(coord) or ""
|
|
216
|
+
target = _canonical_repo_id(resolved, known) or ""
|
|
217
|
+
corrected = _service_for_client_repo(target, endpoints)
|
|
218
|
+
if corrected:
|
|
219
|
+
return resolved, corrected, target
|
|
220
|
+
return resolved, target, ""
|
|
221
|
+
|
|
222
|
+
|
|
115
223
|
def _collect_candidates(
|
|
116
224
|
records: list[dict[str, Any]],
|
|
117
225
|
resolve: Any,
|
|
118
226
|
) -> dict[str, dict[str, Any]]:
|
|
119
|
-
"""Mine consumer ``dependencies_internal`` into per-(target, artifact) candidates.
|
|
227
|
+
"""Mine consumer ``dependencies_internal`` into per-(target, artifact) candidates.
|
|
228
|
+
|
|
229
|
+
``target_repo`` is normalised to a repo id the metabase knows (`OI-33`). The
|
|
230
|
+
resolver returns the directory that *declares* a coordinate, which for a
|
|
231
|
+
multi-module build is deeper than the repo — and the demand-side pass names
|
|
232
|
+
the same service by its repo id, so without this the two can never agree.
|
|
233
|
+
"""
|
|
120
234
|
paths_by_repo = _http_in_paths_by_repo(records)
|
|
235
|
+
known = frozenset(repo_id(d) for d in records)
|
|
121
236
|
cands: dict[str, dict[str, Any]] = {}
|
|
122
237
|
for data in records:
|
|
123
238
|
consumer = repo_id(data)
|
|
@@ -127,12 +242,18 @@ def _collect_candidates(
|
|
|
127
242
|
if not _looks_like_client_artifact(aid):
|
|
128
243
|
continue
|
|
129
244
|
coord = f"{gid}:{aid}" if gid else aid
|
|
130
|
-
|
|
245
|
+
# The declaring directory, then the repo that owns it. Both are kept:
|
|
246
|
+
# the module path is strictly more information and `OI-34` may want it.
|
|
247
|
+
resolved, target, client_repo = _resolve_target(
|
|
248
|
+
resolve, coord, known, paths_by_repo,
|
|
249
|
+
)
|
|
131
250
|
key = _key(target, aid)
|
|
132
251
|
cand = cands.setdefault(
|
|
133
252
|
key,
|
|
134
253
|
{
|
|
135
254
|
"target_repo": target,
|
|
255
|
+
"target_module": resolved,
|
|
256
|
+
"client_repo": client_repo,
|
|
136
257
|
"maven_artifact": aid,
|
|
137
258
|
"import_prefix": gid, # best-effort; reviewer tightens
|
|
138
259
|
"paths": sorted(paths_by_repo.get(target, ())),
|
|
@@ -176,16 +297,29 @@ def _enclosing_class(file_path: str) -> str:
|
|
|
176
297
|
return Path(file_path).stem if file_path else ""
|
|
177
298
|
|
|
178
299
|
|
|
179
|
-
def
|
|
180
|
-
"""
|
|
181
|
-
|
|
300
|
+
def _repos_by_class(records: list[dict[str, Any]]) -> dict[str, set[str]]:
|
|
301
|
+
"""Map each class name to the repos containing a file of that name.
|
|
302
|
+
|
|
303
|
+
Built in **one** pass and queried by key. It replaces a per-class rescan of
|
|
304
|
+
the whole fleet: the caller iterates targets, and each target's classes, and
|
|
305
|
+
asked the corpus about every one — so the cost was
|
|
306
|
+
``targets x classes x records x nodes``, and on a synthetic fleet where all
|
|
307
|
+
four scale together the node visits grew ~15x for every doubling of the
|
|
308
|
+
repository count.
|
|
309
|
+
|
|
310
|
+
Nothing about the answer needed that. The question is corpus-wide and
|
|
311
|
+
target-independent: which repos hold a file called `StockClient`. Asked once
|
|
312
|
+
per class instead of once per (target, class), and answered from an index
|
|
313
|
+
instead of a scan.
|
|
314
|
+
"""
|
|
315
|
+
by_class: dict[str, set[str]] = defaultdict(set)
|
|
182
316
|
for data in records:
|
|
183
317
|
rid = repo_id(data)
|
|
184
318
|
for node in data.get("nodes", []):
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
return
|
|
319
|
+
cls = _enclosing_class(str(node.get("file", "")))
|
|
320
|
+
if cls:
|
|
321
|
+
by_class[cls].add(rid)
|
|
322
|
+
return by_class
|
|
189
323
|
|
|
190
324
|
|
|
191
325
|
def _demand_side_observations(
|
|
@@ -229,6 +363,80 @@ def _demand_side_observations(
|
|
|
229
363
|
return observed
|
|
230
364
|
|
|
231
365
|
|
|
366
|
+
def _unresolved_warning(target_repo: str, coordinate: str, module: str) -> str | None:
|
|
367
|
+
"""Explain an unresolvable coordinate, or None when it resolved.
|
|
368
|
+
|
|
369
|
+
An empty `target_repo` used to arrive as an ordinary candidate with nothing
|
|
370
|
+
saying why — 8 of them in the first fleet run. Promoting one creates an edge
|
|
371
|
+
pointing at nothing, so it must not read like a merely weak candidate
|
|
372
|
+
(`OI-33`, and the class in `OI-36`).
|
|
373
|
+
"""
|
|
374
|
+
if target_repo:
|
|
375
|
+
return None
|
|
376
|
+
where = f"; declared at {module!r}" if module else "; not found in the checkout"
|
|
377
|
+
return (
|
|
378
|
+
f"coordinate {coordinate or '(none)'} did not resolve to a known repo"
|
|
379
|
+
f"{where} — the target cannot be trusted, and promoting this would "
|
|
380
|
+
"create an edge to nothing"
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _target_warnings(
|
|
385
|
+
target_repo: str, coordinate: str, module: str, client_repo: str,
|
|
386
|
+
) -> list[str]:
|
|
387
|
+
"""Everything worth saying about how this candidate's target was arrived at."""
|
|
388
|
+
out: list[str] = []
|
|
389
|
+
unresolved = _unresolved_warning(target_repo, coordinate, module)
|
|
390
|
+
if unresolved:
|
|
391
|
+
out.append(unresolved)
|
|
392
|
+
if client_repo:
|
|
393
|
+
out.append(
|
|
394
|
+
f"target corrected from {client_repo!r}, which declares the client "
|
|
395
|
+
"library and no inbound endpoints, to the service it appears to "
|
|
396
|
+
"front — check the substitution before accepting (OI-40)"
|
|
397
|
+
)
|
|
398
|
+
return out
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _endpointless_warning(target_repo: str, has_paths: bool, scanned: set[str]) -> str | None:
|
|
402
|
+
"""Flag a target that receives no calls, without dropping it.
|
|
403
|
+
|
|
404
|
+
A binding's target is by definition a service that receives calls, so a
|
|
405
|
+
target with no detected inbound endpoint cannot fulfil that. But **zero
|
|
406
|
+
endpoints is also what a detection gap looks like** — `OI-17` left an entire
|
|
407
|
+
language half of a fleet returning none — and the two causes are
|
|
408
|
+
indistinguishable from the outside.
|
|
409
|
+
|
|
410
|
+
So this warns and never filters. Dropping these would turn `OI-17`-class
|
|
411
|
+
blindness into an invisible data-quality filter, which is `OI-36` with the
|
|
412
|
+
tool doing it to itself.
|
|
413
|
+
"""
|
|
414
|
+
if not target_repo or has_paths or target_repo not in scanned:
|
|
415
|
+
return None
|
|
416
|
+
return (
|
|
417
|
+
f"{target_repo} declares no inbound endpoints: it may be a client "
|
|
418
|
+
"library whose service is outside the scanned fleet, or a service whose "
|
|
419
|
+
"endpoints were not detected — the two look identical from here"
|
|
420
|
+
)
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _entry_fields(
|
|
424
|
+
cand: dict[str, Any], prev: dict[str, Any] | None
|
|
425
|
+
) -> dict[str, Any]:
|
|
426
|
+
"""The tunable fields and status, preserving a reviewer's decision.
|
|
427
|
+
|
|
428
|
+
A reviewer who set accepted/rejected has tuned the binding, so their edits
|
|
429
|
+
and status are kept and only evidence and confidence are refreshed.
|
|
430
|
+
"""
|
|
431
|
+
if prev and prev.get("status") in (STATUS_ACCEPTED, STATUS_REJECTED):
|
|
432
|
+
entry = {k: prev.get(k, cand.get(k)) for k in _TUNABLE_FIELDS}
|
|
433
|
+
entry["status"] = prev["status"]
|
|
434
|
+
return entry
|
|
435
|
+
entry = {k: cand[k] for k in _TUNABLE_FIELDS}
|
|
436
|
+
entry["status"] = STATUS_PENDING
|
|
437
|
+
return entry
|
|
438
|
+
|
|
439
|
+
|
|
232
440
|
def _build_entry(
|
|
233
441
|
cand: dict[str, Any],
|
|
234
442
|
scanned: set[str],
|
|
@@ -237,22 +445,22 @@ def _build_entry(
|
|
|
237
445
|
"""Assemble a candidate entry, preserving a reviewer's non-pending decision."""
|
|
238
446
|
consumers = sorted(cand.pop("consumers"))
|
|
239
447
|
coordinate = cand.pop("coordinate")
|
|
240
|
-
warnings = cand.pop("warnings", [])
|
|
448
|
+
warnings = list(cand.pop("warnings", []))
|
|
449
|
+
module = cand.pop("target_module", "")
|
|
450
|
+
client_repo = cand.pop("client_repo", "")
|
|
451
|
+
warnings += _target_warnings(
|
|
452
|
+
str(cand.get("target_repo", "")), coordinate, module, client_repo,
|
|
453
|
+
)
|
|
241
454
|
has_paths = bool(cand["paths"])
|
|
242
455
|
evidence = {
|
|
243
456
|
"coordinate": coordinate,
|
|
244
457
|
"consumers": consumers,
|
|
245
458
|
"resolved": bool(cand["target_repo"]),
|
|
459
|
+
"declared_at": module,
|
|
460
|
+
"client_repo": client_repo,
|
|
246
461
|
"paths_from_target_scan": has_paths and cand["target_repo"] in scanned,
|
|
247
462
|
}
|
|
248
|
-
|
|
249
|
-
# their edits and status, only refreshing evidence + confidence.
|
|
250
|
-
if prev and prev.get("status") in (STATUS_ACCEPTED, STATUS_REJECTED):
|
|
251
|
-
entry = {k: prev.get(k, cand.get(k)) for k in _TUNABLE_FIELDS}
|
|
252
|
-
entry["status"] = prev["status"]
|
|
253
|
-
else:
|
|
254
|
-
entry = {k: cand[k] for k in _TUNABLE_FIELDS}
|
|
255
|
-
entry["status"] = STATUS_PENDING
|
|
463
|
+
entry = _entry_fields(cand, prev)
|
|
256
464
|
# How the candidate was found. `both` is materially stronger than either
|
|
257
465
|
# alone: a declared dependency and an observed call site are independent
|
|
258
466
|
# lines of evidence. `call-site` sorts lowest, since it rests on the path
|
|
@@ -260,6 +468,12 @@ def _build_entry(
|
|
|
260
468
|
entry["discovery_method"] = cand.get("discovery_method", "dependency")
|
|
261
469
|
if warnings:
|
|
262
470
|
entry["warnings"] = warnings
|
|
471
|
+
endpointless = _endpointless_warning(
|
|
472
|
+
str(entry["target_repo"] or ""), bool(entry["paths"]), scanned,
|
|
473
|
+
)
|
|
474
|
+
if endpointless:
|
|
475
|
+
existing: list[str] = list(entry.get("warnings") or [])
|
|
476
|
+
entry["warnings"] = [*existing, endpointless]
|
|
263
477
|
entry["confidence"] = _confidence(
|
|
264
478
|
str(entry["target_repo"] or ""), scanned, bool(entry["paths"])
|
|
265
479
|
)
|
|
@@ -282,12 +496,14 @@ def _apply_demand_side(
|
|
|
282
496
|
"""
|
|
283
497
|
observed = _demand_side_observations(records)
|
|
284
498
|
by_target = {c["target_repo"]: (k, c) for k, c in cands.items() if c["target_repo"]}
|
|
499
|
+
# One pass over the fleet, before the loop rather than inside it.
|
|
500
|
+
repos_by_class = _repos_by_class(records)
|
|
285
501
|
|
|
286
502
|
for target, seen in sorted(observed.items()):
|
|
287
503
|
classes = sorted(seen["classes"])
|
|
288
504
|
warnings = []
|
|
289
505
|
for cls in classes:
|
|
290
|
-
repos =
|
|
506
|
+
repos = repos_by_class.get(cls, set())
|
|
291
507
|
if len(repos) > MAX_PATTERN_REPOS:
|
|
292
508
|
warnings.append(
|
|
293
509
|
f"class_pattern {cls!r} appears in {len(repos)} repos; too "
|
|
@@ -338,7 +554,7 @@ def discover_api_clients_from_records(
|
|
|
338
554
|
for cand in cands.values():
|
|
339
555
|
cand.setdefault("discovery_method", "dependency")
|
|
340
556
|
_apply_demand_side(cands, records)
|
|
341
|
-
prev = _load_discovered(metabase_root / DISCOVERED_FILE)
|
|
557
|
+
prev = _load_discovered(metabase_root / DISCOVERED_FILE, frozenset(scanned))
|
|
342
558
|
|
|
343
559
|
entries = [
|
|
344
560
|
_build_entry(cand, scanned, prev.get(key))
|
|
@@ -364,7 +580,13 @@ def discover_api_clients(
|
|
|
364
580
|
by_coord, by_name, by_full = _build_component_identity_index(repos_root)
|
|
365
581
|
|
|
366
582
|
def resolve(coord: str) -> str | None:
|
|
367
|
-
"""Resolve a
|
|
583
|
+
"""Resolve a coordinate to the clone path that *declares* it, or None.
|
|
584
|
+
|
|
585
|
+
Not the repo id — this docstring used to claim it was, and that claim is
|
|
586
|
+
how `OI-33` went unnoticed: `_resolve_clone_path` returns the declaring
|
|
587
|
+
directory, which for a multi-module build sits inside the repo. The
|
|
588
|
+
caller normalises via `_canonical_repo_id`.
|
|
589
|
+
"""
|
|
368
590
|
return _resolve_clone_path(coord, by_coord, by_name, by_full)
|
|
369
591
|
|
|
370
592
|
return discover_api_clients_from_records(metabase_root, records, resolve)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Orchestrate v2 cross-repo graph generation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from ..graph_common import (
|
|
9
|
+
iter_v2_repo_records,
|
|
10
|
+
load_v2_repo_records,
|
|
11
|
+
v2_record_paths,
|
|
12
|
+
)
|
|
13
|
+
from ..index_store import build_index
|
|
14
|
+
from .data_stores import write_data_store_graph
|
|
15
|
+
from .index_v2 import write_index_v2
|
|
16
|
+
from .openapi_edges import write_openapi_artifacts
|
|
17
|
+
from .payload_producers import build_producer_indices, write_payload_producer_index
|
|
18
|
+
from .phase3 import aggregate_phase3_v2
|
|
19
|
+
from .pii_flow_v2 import write_pii_flow_v2
|
|
20
|
+
from .queues import write_queue_graph
|
|
21
|
+
from .service_calls import collect_service_edges, write_service_call_graph
|
|
22
|
+
from .traces_index import write_traces_index
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def write_fleet_index(
|
|
26
|
+
metabase_root: Path,
|
|
27
|
+
repo_jsons: list[Path],
|
|
28
|
+
*,
|
|
29
|
+
repos_root: Path | None = None,
|
|
30
|
+
producer_indices: list[Any] | None = None,
|
|
31
|
+
) -> Path:
|
|
32
|
+
"""Persist the four things a trace consults, keyed by target repo (`OI-15`).
|
|
33
|
+
|
|
34
|
+
Built here because aggregation already walks every repo, so the index is one
|
|
35
|
+
more pass over data that is being read anyway rather than a second crawl.
|
|
36
|
+
|
|
37
|
+
The two fleet-wide derivations are computed the same way `trace` computed
|
|
38
|
+
them, from the same functions — the index is a cache of this code's output,
|
|
39
|
+
not a reimplementation of it, which is what keeps the two from drifting.
|
|
40
|
+
|
|
41
|
+
``producer_indices`` is taken from the caller when aggregation has already
|
|
42
|
+
built it. Rebuilding it here meant the fleet's source tree was scanned a
|
|
43
|
+
second time, which doubled the slowest step of the whole run (`OI-30`).
|
|
44
|
+
"""
|
|
45
|
+
paths = v2_record_paths(metabase_root, json_paths=repo_jsons)
|
|
46
|
+
records = load_v2_repo_records(metabase_root, json_paths=paths)
|
|
47
|
+
call_edges, _unmatched = collect_service_edges(records)
|
|
48
|
+
if producer_indices is None:
|
|
49
|
+
producer_indices = build_producer_indices(
|
|
50
|
+
metabase_root, repos_root=repos_root, json_paths=paths,
|
|
51
|
+
)
|
|
52
|
+
del records # the streamed pass below must not be served from this list
|
|
53
|
+
|
|
54
|
+
return build_index(
|
|
55
|
+
metabase_root,
|
|
56
|
+
paths,
|
|
57
|
+
call_edges=call_edges,
|
|
58
|
+
producer_indices=producer_indices,
|
|
59
|
+
records=iter_v2_repo_records(metabase_root, json_paths=paths),
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def aggregate_graphs_v2(
|
|
64
|
+
metabase_root: Path,
|
|
65
|
+
repo_jsons: list[Path],
|
|
66
|
+
*,
|
|
67
|
+
repos_root: Path | None = None,
|
|
68
|
+
phase3: bool = True,
|
|
69
|
+
fleet_index: bool = True,
|
|
70
|
+
) -> None:
|
|
71
|
+
"""Run all v2 cross-repo graph writers (service, queue, data-store, PII, phase 3)."""
|
|
72
|
+
write_service_call_graph(metabase_root, repo_jsons, repos_root=repos_root)
|
|
73
|
+
if repos_root:
|
|
74
|
+
write_openapi_artifacts(metabase_root, repos_root, repo_jsons)
|
|
75
|
+
write_queue_graph(metabase_root, repo_jsons)
|
|
76
|
+
write_data_store_graph(metabase_root, repo_jsons)
|
|
77
|
+
producer_indices = write_payload_producer_index(
|
|
78
|
+
metabase_root, repo_jsons, repos_root=repos_root,
|
|
79
|
+
)
|
|
80
|
+
write_index_v2(metabase_root, repo_jsons)
|
|
81
|
+
write_pii_flow_v2(metabase_root, repo_jsons)
|
|
82
|
+
write_traces_index(metabase_root)
|
|
83
|
+
if phase3:
|
|
84
|
+
aggregate_phase3_v2(metabase_root, repo_jsons)
|
|
85
|
+
if fleet_index:
|
|
86
|
+
write_fleet_index(
|
|
87
|
+
metabase_root, repo_jsons, repos_root=repos_root,
|
|
88
|
+
producer_indices=producer_indices,
|
|
89
|
+
)
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Discover OpenAPI specs and Helm ingress hosts under repos/."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from ..checkout_scan import paths_by_name
|
|
9
|
+
from ..graph_common import normalize_path_template as norm_path
|
|
10
|
+
|
|
11
|
+
from .openapi_models import OpenApiSpec
|
|
12
|
+
|
|
13
|
+
OPENAPI_GLOBS = ("openapi.yaml", "openapi.yml", "swagger.yaml", "swagger.yml")
|
|
14
|
+
PATH_LINE_RX = re.compile(
|
|
15
|
+
r"^\s{2,}(/[/\w{}.:$-]+)\s*:\s*$",
|
|
16
|
+
re.MULTILINE,
|
|
17
|
+
)
|
|
18
|
+
SERVERS_URL_RX = re.compile(
|
|
19
|
+
r"^\s*-\s*url:\s*['\"]?(https?://[^'\"\s]+)",
|
|
20
|
+
re.MULTILINE | re.IGNORECASE,
|
|
21
|
+
)
|
|
22
|
+
HELM_VALUES_NAMES = ("values.yaml", "values.yml")
|
|
23
|
+
HELM_HOST_RX = re.compile(
|
|
24
|
+
r"^\s*(?:host|hostname|externalHost|ingress\.host)\s*:\s*['\"]?([^\s#'\"]+)",
|
|
25
|
+
re.MULTILINE | re.IGNORECASE,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
_SKIP_DIR_PARTS = frozenset({".git", "node_modules", "target", "build", "charts"})
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def repo_from_under_repos(spec_path: Path, repos_root: Path) -> str | None:
|
|
32
|
+
"""Return the ``org/repo`` id for a path under repos_root, or None."""
|
|
33
|
+
try:
|
|
34
|
+
rel = spec_path.relative_to(repos_root)
|
|
35
|
+
except ValueError:
|
|
36
|
+
return None
|
|
37
|
+
parts = rel.parts
|
|
38
|
+
if len(parts) < 2:
|
|
39
|
+
return None
|
|
40
|
+
return f"{parts[0]}/{parts[1]}"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _should_skip_path(path: Path) -> bool:
|
|
44
|
+
"""True if the path lies within a skipped directory (.git, node_modules, ...)."""
|
|
45
|
+
return any(p in path.parts for p in _SKIP_DIR_PARTS)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _walk_once(repos_root: Path, names: tuple[str, ...]) -> list[Path]:
|
|
49
|
+
"""Every file under the checkout carrying one of ``names``, from a single walk.
|
|
50
|
+
|
|
51
|
+
`Path.rglob(name)` traverses the whole tree and filters by name, so the
|
|
52
|
+
per-pattern loop this replaces cost one full traversal of a 34 GB checkout
|
|
53
|
+
*per filename* — four for the OpenAPI globs, two for Helm, and again for
|
|
54
|
+
every call site. The same defect as `OI-30` in the producer scan: the loop
|
|
55
|
+
over what to look for sat outside the loop over where to look.
|
|
56
|
+
"""
|
|
57
|
+
found = paths_by_name(repos_root, frozenset(names))
|
|
58
|
+
return [path for name in names for path in found[name]]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def discover_openapi_specs(repos_root: Path) -> list[OpenApiSpec]:
|
|
62
|
+
"""Scan repos_root for OpenAPI/Swagger specs and return parsed OpenApiSpec rows."""
|
|
63
|
+
if not repos_root.is_dir():
|
|
64
|
+
return []
|
|
65
|
+
specs: list[OpenApiSpec] = []
|
|
66
|
+
seen: set[tuple[str, str]] = set()
|
|
67
|
+
for path in _walk_once(repos_root, OPENAPI_GLOBS):
|
|
68
|
+
if _should_skip_path(path):
|
|
69
|
+
continue
|
|
70
|
+
try:
|
|
71
|
+
if path.stat().st_size > 2_000_000:
|
|
72
|
+
continue
|
|
73
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
74
|
+
except OSError:
|
|
75
|
+
continue
|
|
76
|
+
target = repo_from_under_repos(path, repos_root)
|
|
77
|
+
if not target:
|
|
78
|
+
continue
|
|
79
|
+
key = (target, str(path))
|
|
80
|
+
if key in seen:
|
|
81
|
+
continue
|
|
82
|
+
seen.add(key)
|
|
83
|
+
paths = sorted({norm_path(m.group(1)) for m in PATH_LINE_RX.finditer(text)})
|
|
84
|
+
servers = [m.group(1) for m in SERVERS_URL_RX.finditer(text)]
|
|
85
|
+
specs.append(OpenApiSpec(
|
|
86
|
+
target_repo=target,
|
|
87
|
+
spec_path=str(path.relative_to(repos_root)),
|
|
88
|
+
paths=[p for p in paths if p and p != "/"],
|
|
89
|
+
servers=servers[:5],
|
|
90
|
+
))
|
|
91
|
+
return specs
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def discover_helm_hosts(repos_root: Path) -> list[dict[str, str]]:
|
|
95
|
+
"""Scan Helm values for declared ingress/service hostnames."""
|
|
96
|
+
if not repos_root.is_dir():
|
|
97
|
+
return []
|
|
98
|
+
rows: list[dict[str, str]] = []
|
|
99
|
+
seen: set[tuple[str, str]] = set()
|
|
100
|
+
for path in _walk_once(repos_root, HELM_VALUES_NAMES):
|
|
101
|
+
if _should_skip_path(path):
|
|
102
|
+
continue
|
|
103
|
+
try:
|
|
104
|
+
if path.stat().st_size > 500_000:
|
|
105
|
+
continue
|
|
106
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
107
|
+
except OSError:
|
|
108
|
+
continue
|
|
109
|
+
target = repo_from_under_repos(path, repos_root)
|
|
110
|
+
if not target:
|
|
111
|
+
continue
|
|
112
|
+
for m in HELM_HOST_RX.finditer(text):
|
|
113
|
+
host = m.group(1).strip().strip("'\"")
|
|
114
|
+
if not host or host.startswith("{{"):
|
|
115
|
+
continue
|
|
116
|
+
key = (target, host)
|
|
117
|
+
if key in seen:
|
|
118
|
+
continue
|
|
119
|
+
seen.add(key)
|
|
120
|
+
rows.append({
|
|
121
|
+
"target_repo": target,
|
|
122
|
+
"host": host,
|
|
123
|
+
"values_path": str(path.relative_to(repos_root)),
|
|
124
|
+
})
|
|
125
|
+
return rows
|