src2sink 2.0.0__tar.gz → 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. {src2sink-2.0.0/src2sink.egg-info → src2sink-3.0.0}/PKG-INFO +1 -1
  2. {src2sink-2.0.0 → src2sink-3.0.0}/pyproject.toml +1 -1
  3. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/api_client_discovery.py +247 -25
  4. src2sink-3.0.0/src2sink/aggregators/graphs.py +89 -0
  5. src2sink-3.0.0/src2sink/aggregators/openapi_discovery.py +125 -0
  6. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/payload_producers.py +175 -61
  7. src2sink-3.0.0/src2sink/aggregators/queues.py +215 -0
  8. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/build_metabase_v2.py +191 -11
  9. src2sink-3.0.0/src2sink/checkout_scan.py +187 -0
  10. src2sink-3.0.0/src2sink/constants.py +76 -0
  11. src2sink-3.0.0/src2sink/dependencies.py +214 -0
  12. src2sink-3.0.0/src2sink/derive.py +388 -0
  13. src2sink-3.0.0/src2sink/extractors/ast_walk.py +539 -0
  14. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/patterns.py +104 -2
  15. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/regex_extractors.py +53 -2
  16. src2sink-3.0.0/src2sink/extractors/ts_extractors.py +376 -0
  17. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/unified.py +18 -4
  18. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/graph_common.py +129 -13
  19. src2sink-3.0.0/src2sink/index_store.py +370 -0
  20. src2sink-3.0.0/src2sink/maven.py +334 -0
  21. src2sink-3.0.0/src2sink/paths.py +361 -0
  22. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/repo_utils.py +58 -24
  23. src2sink-3.0.0/src2sink/resolve.py +250 -0
  24. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/schema.py +7 -1
  25. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/trace.py +263 -84
  26. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/trace_batch.py +8 -1
  27. {src2sink-2.0.0 → src2sink-3.0.0/src2sink.egg-info}/PKG-INFO +1 -1
  28. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink.egg-info/SOURCES.txt +33 -0
  29. src2sink-3.0.0/tests/test_aggregate_output_golden.py +114 -0
  30. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_ast_walk.py +107 -19
  31. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_build_internals.py +6 -1
  32. src2sink-3.0.0/tests/test_call_observations.py +153 -0
  33. src2sink-3.0.0/tests/test_call_resolution.py +341 -0
  34. src2sink-3.0.0/tests/test_checkout_scan.py +261 -0
  35. src2sink-3.0.0/tests/test_derive_pass.py +164 -0
  36. src2sink-3.0.0/tests/test_detection_input_coverage.py +142 -0
  37. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_detection_version.py +6 -1
  38. src2sink-3.0.0/tests/test_discovery_scan_cost.py +214 -0
  39. src2sink-3.0.0/tests/test_entry_points.py +165 -0
  40. src2sink-3.0.0/tests/test_fleet_index.py +369 -0
  41. src2sink-3.0.0/tests/test_maven_resolution.py +239 -0
  42. src2sink-3.0.0/tests/test_method_structure.py +180 -0
  43. src2sink-3.0.0/tests/test_oi13_kotlin_parity.py +113 -0
  44. src2sink-3.0.0/tests/test_oi26_receiver_scope.py +141 -0
  45. src2sink-3.0.0/tests/test_oi28_index_fast_path.py +126 -0
  46. src2sink-3.0.0/tests/test_oi33_canonical_repo_id.py +259 -0
  47. src2sink-3.0.0/tests/test_oi37_express_anchor.py +174 -0
  48. src2sink-3.0.0/tests/test_oi38_trace_index_freshness.py +171 -0
  49. src2sink-3.0.0/tests/test_oi39_test_path_classification.py +160 -0
  50. src2sink-3.0.0/tests/test_oi40_client_repo_target.py +206 -0
  51. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_output_neutralisation.py +15 -3
  52. src2sink-3.0.0/tests/test_path_match_significance.py +110 -0
  53. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase4_extractor_snapshots.py +9 -0
  54. src2sink-3.0.0/tests/test_polyglot_dependencies.py +164 -0
  55. src2sink-3.0.0/tests/test_producer_scan_single_pass.py +250 -0
  56. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_redos_bounds.py +15 -1
  57. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_repo_utils_helpers.py +0 -9
  58. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_security_input_handling.py +5 -4
  59. src2sink-3.0.0/tests/test_silent_failure_gate.py +285 -0
  60. src2sink-3.0.0/tests/test_sql_classifier.py +114 -0
  61. src2sink-3.0.0/tests/test_tainted_paths.py +410 -0
  62. src2sink-3.0.0/tests/test_type_declarations.py +206 -0
  63. src2sink-2.0.0/src2sink/aggregators/graphs.py +0 -36
  64. src2sink-2.0.0/src2sink/aggregators/openapi_discovery.py +0 -113
  65. src2sink-2.0.0/src2sink/aggregators/queues.py +0 -141
  66. src2sink-2.0.0/src2sink/constants.py +0 -44
  67. src2sink-2.0.0/src2sink/extractors/ast_walk.py +0 -158
  68. src2sink-2.0.0/src2sink/extractors/ts_extractors.py +0 -212
  69. {src2sink-2.0.0 → src2sink-3.0.0}/LICENSE +0 -0
  70. {src2sink-2.0.0 → src2sink-3.0.0}/README.md +0 -0
  71. {src2sink-2.0.0 → src2sink-3.0.0}/setup.cfg +0 -0
  72. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/__init__.py +0 -0
  73. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/__init__.py +0 -0
  74. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/auth_cards.py +0 -0
  75. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/crypto_cards.py +0 -0
  76. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/data_stores.py +0 -0
  77. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/index_v2.py +0 -0
  78. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/library_source_map.py +0 -0
  79. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/openapi_edges.py +0 -0
  80. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/openapi_match.py +0 -0
  81. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/openapi_models.py +0 -0
  82. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/openapi_report.py +0 -0
  83. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/phase3.py +0 -0
  84. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_cross_repo.py +0 -0
  85. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_flow_v2.py +0 -0
  86. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_lifecycle.py +0 -0
  87. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_lifecycle_report.py +0 -0
  88. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_touchpoint_collect.py +0 -0
  89. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/pii_touchpoint_utils.py +0 -0
  90. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/ropa.py +0 -0
  91. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/service_call_collect.py +0 -0
  92. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/service_call_index.py +0 -0
  93. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/service_call_models.py +0 -0
  94. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/service_call_report.py +0 -0
  95. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/service_calls.py +0 -0
  96. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/taint_buckets.py +0 -0
  97. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/taint_catalogs.py +0 -0
  98. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/taint_writers.py +0 -0
  99. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/aggregators/traces_index.py +0 -0
  100. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/curate_internal_libraries.py +0 -0
  101. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/__init__.py +0 -0
  102. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/base.py +0 -0
  103. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/config.py +0 -0
  104. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/file_context.py +0 -0
  105. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/go.py +0 -0
  106. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/http_out.py +0 -0
  107. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/java.py +0 -0
  108. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/javascript.py +0 -0
  109. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/kotlin.py +0 -0
  110. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/node_factory.py +0 -0
  111. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/python_extractor.py +0 -0
  112. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/extractors/symbols.py +0 -0
  113. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/internal_groups.py +0 -0
  114. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/known_api_clients.py +0 -0
  115. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/library_taint_java.py +0 -0
  116. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/limits.py +0 -0
  117. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/models/__init__.py +0 -0
  118. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/models/auth_model.py +0 -0
  119. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/models/crypto_agility.py +0 -0
  120. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/models/pii_lifecycle.py +0 -0
  121. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/models/ropa.py +0 -0
  122. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/prescreen.py +0 -0
  123. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/py.typed +0 -0
  124. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/record_fleet_baseline.py +0 -0
  125. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/renderers/__init__.py +0 -0
  126. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/renderers/markdown.py +0 -0
  127. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/renderers/mermaid.py +0 -0
  128. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/safe_paths.py +0 -0
  129. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/sanitize.py +0 -0
  130. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink/vocabulary.py +0 -0
  131. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink.egg-info/dependency_links.txt +0 -0
  132. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink.egg-info/entry_points.txt +0 -0
  133. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink.egg-info/requires.txt +0 -0
  134. {src2sink-2.0.0 → src2sink-3.0.0}/src2sink.egg-info/top_level.txt +0 -0
  135. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_aggregators_extra.py +0 -0
  136. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_api_client_discovery.py +0 -0
  137. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_build_main.py +0 -0
  138. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_characterization.py +0 -0
  139. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_config_hygiene.py +0 -0
  140. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_cross_repo_caller_coverage.py +0 -0
  141. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_curate.py +0 -0
  142. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_demand_side_discovery.py +0 -0
  143. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_dependency_pinning.py +0 -0
  144. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_detection_fingerprint_gate.py +0 -0
  145. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_field_precision.py +0 -0
  146. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_gradle_version_catalogs.py +0 -0
  147. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_graph_common.py +0 -0
  148. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_http_guard_evidence.py +0 -0
  149. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_import_graph.py +0 -0
  150. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_internal_groups.py +0 -0
  151. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_limits.py +0 -0
  152. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_module_imports.py +0 -0
  153. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_openapi.py +0 -0
  154. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase0_smoke.py +0 -0
  155. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase1_config.py +0 -0
  156. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase1_extractors.py +0 -0
  157. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase2_graphs.py +0 -0
  158. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase3_pii.py +0 -0
  159. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase4_aggregators.py +0 -0
  160. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase4_fleet_regression.py +0 -0
  161. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase4_regression.py +0 -0
  162. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_phase4_taint_caps.py +0 -0
  163. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_post_phase4_backlog.py +0 -0
  164. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_prescreen.py +0 -0
  165. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_renderers.py +0 -0
  166. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_run_manifest.py +0 -0
  167. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_sanitize.py +0 -0
  168. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_source_map_fix.py +0 -0
  169. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_sql_payload_out.py +0 -0
  170. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_sql_sink_evidence.py +0 -0
  171. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_sql_source_construction.py +0 -0
  172. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_trace_batch.py +0 -0
  173. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_trace_fleet_scaling.py +0 -0
  174. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_trace_render.py +0 -0
  175. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_zz_security_coverage_gate.py +0 -0
  176. {src2sink-2.0.0 → src2sink-3.0.0}/tests/test_zz_untrusted_interpolation_guard.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: src2sink
3
- Version: 2.0.0
3
+ Version: 3.0.0
4
4
  Summary: Cross-repository source-to-sink metabase for LLM-assisted SAST
5
5
  Author: Brett Crawley
6
6
  License-Expression: MIT
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "src2sink"
3
- version = "2.0.0"
3
+ version = "3.0.0"
4
4
  description = "Cross-repository source-to-sink metabase for LLM-assisted SAST"
5
5
  authors = [{ name = "Brett Crawley" }]
6
6
  readme = "README.md"
@@ -26,6 +26,7 @@ unverified binding poisons the taint graph with phantom cross-service edges.
26
26
  from __future__ import annotations
27
27
 
28
28
  import json
29
+ import re
29
30
  from collections import defaultdict
30
31
  from pathlib import Path
31
32
  from typing import Any
@@ -85,21 +86,107 @@ def _http_in_paths_by_repo(records: list[dict[str, Any]]) -> dict[str, set[str]]
85
86
  return out
86
87
 
87
88
 
89
+ # Suffixes a client library's repo conventionally carries. Used only to derive
90
+ # the *stem to search for* — never as the test itself. The test is whether the
91
+ # repo declares inbound endpoints, which is name-independent and catches estates
92
+ # that name their libraries differently (`OI-40`).
93
+ _CLIENT_SUFFIX_RX = re.compile(r"[-_](?:clients?|sdk)$", re.IGNORECASE)
94
+
95
+
96
+ def _service_for_client_repo(
97
+ declaring_repo: str, endpoints: dict[str, set[str]]
98
+ ) -> str | None:
99
+ """Map a client-library repo to the service repo it fronts, or None.
100
+
101
+ `resolve(coord)` names the repo that *declares* the artifact, and
102
+ `_canonical_repo_id` maps that to the repo owning it. Both are correct. But
103
+ when a client library is published from **its own repository** rather than as
104
+ a submodule of the service, the repo owning the declaration *is* the library
105
+ — so the binding points at the client instead of the service it calls
106
+ (`OI-40`).
107
+
108
+ `OI-33` fixed the *shape* of the identity; this is the *referent* being
109
+ wrong, and no amount of path normalisation reaches it. The service is not
110
+ named anywhere in the consumer's dependency declaration.
111
+
112
+ A service has inbound endpoints; a client library does not. So the test is
113
+ the endpoint count, and the name only supplies the stem to search for.
114
+ Returns None rather than guessing — 31 of 42 in the observed fleet had no
115
+ sibling because the service is not in the scanned set at all, and inventing
116
+ one would manufacture the broken edges `OI-33` was about.
117
+ """
118
+ if "/" not in declaring_repo:
119
+ return None
120
+ if endpoints.get(declaring_repo):
121
+ return None # it receives calls, so it is a service; leave it alone
122
+ stem = _CLIENT_SUFFIX_RX.sub("", declaring_repo.split("/")[-1])
123
+ if not stem:
124
+ return None
125
+ hits = sorted(
126
+ repo for repo, paths in endpoints.items()
127
+ if paths and repo.split("/")[-1] in (stem, f"{stem}-service")
128
+ )
129
+ return hits[0] if len(hits) == 1 else None
130
+
131
+
88
132
  def _key(target_repo: str, artifact: str) -> str:
89
133
  """Stable candidate key used for de-dup and for preserving reviewer edits."""
90
134
  return f"{target_repo}\t{artifact}"
91
135
 
92
136
 
93
- def _load_discovered(path: Path) -> dict[str, dict[str, Any]]:
94
- """Return prior candidates keyed by (target_repo, artifact), or {} if absent."""
137
+ def _canonical_repo_id(resolved: str, known: frozenset[str]) -> str | None:
138
+ """Map a resolved coordinate path to the repo id that owns it.
139
+
140
+ The identity index resolves a coordinate to the directory that *declares* it,
141
+ which may be a build module inside a repo — `group/repo/some-client`. The
142
+ demand-side pass names its target `group/repo`, from `repo_id()`. The two
143
+ never met, so `discovery_method: "both"` — the strongest signal the design
144
+ produces — could not occur even once (`OI-33`).
145
+
146
+ Truncating to two segments would be wrong: `group/subgroup/repo` is a valid
147
+ GitLab path and this estate contains them (`OI-34`). So match the longest
148
+ known repo id instead. That handles in-repo modules and nested repos alike,
149
+ and depends on neither path depth nor on `.git` being present — 65 of 746
150
+ repos in the observed fleet have no `.git` at all.
151
+
152
+ Returns None when nothing matches, which is a resolver failure worth
153
+ surfacing rather than a repo id worth guessing.
154
+ """
155
+ if not resolved:
156
+ return None
157
+ parts = resolved.split("/")
158
+ for n in range(len(parts), 0, -1):
159
+ candidate = "/".join(parts[:n])
160
+ if candidate in known:
161
+ return candidate
162
+ return None
163
+
164
+
165
+ def _load_discovered(
166
+ path: Path, known: frozenset[str] | None = None
167
+ ) -> dict[str, dict[str, Any]]:
168
+ """Return prior candidates keyed by (target_repo, artifact), or {} if absent.
169
+
170
+ ``known`` lets a candidate stored under a pre-`OI-33` module path also be
171
+ found under its canonical repo id, so reviewer decisions survive the fix.
172
+ """
95
173
  try:
96
174
  data = json.loads(path.read_text(encoding="utf-8"))
97
175
  except (OSError, json.JSONDecodeError):
98
176
  return {}
99
177
  out: dict[str, dict[str, Any]] = {}
100
178
  for c in data.get("candidates", []):
101
- if isinstance(c, dict) and "maven_artifact" in c:
102
- out[_key(c.get("target_repo", ""), c["maven_artifact"])] = c
179
+ if not isinstance(c, dict) or "maven_artifact" not in c:
180
+ continue
181
+ stored = str(c.get("target_repo", ""))
182
+ artifact = c["maven_artifact"]
183
+ out[_key(stored, artifact)] = c
184
+ # A candidate reviewed before `OI-33` was keyed on the *module* path.
185
+ # Index it under its canonical repo id too, or the fix would silently
186
+ # discard every accept/reject a reviewer has already made.
187
+ canonical = _canonical_repo_id(stored, known) if known else None
188
+ if canonical and canonical != stored:
189
+ out.setdefault(_key(canonical, artifact), c)
103
190
  return out
104
191
 
105
192
 
@@ -112,12 +199,40 @@ def _confidence(target_repo: str, scanned: set[str], has_paths: bool) -> str:
112
199
  return "low" # artifact looks like a client but coordinate did not resolve
113
200
 
114
201
 
202
+ def _resolve_target(
203
+ resolve: Any, coord: str, known: frozenset[str], endpoints: dict[str, set[str]],
204
+ ) -> tuple[str, str, str]:
205
+ """Resolve a coordinate to (declaring path, service repo, client repo if corrected).
206
+
207
+ Two corrections in sequence, both needed and neither sufficient alone:
208
+ `_canonical_repo_id` fixes the identity's *shape* (`OI-33`), and
209
+ `_service_for_client_repo` fixes its *referent* when the declaring repo turns
210
+ out to be the client library rather than the service (`OI-40`).
211
+
212
+ The client repo is returned rather than discarded, because a substitution a
213
+ reviewer cannot see is one they cannot check.
214
+ """
215
+ resolved = resolve(coord) or ""
216
+ target = _canonical_repo_id(resolved, known) or ""
217
+ corrected = _service_for_client_repo(target, endpoints)
218
+ if corrected:
219
+ return resolved, corrected, target
220
+ return resolved, target, ""
221
+
222
+
115
223
  def _collect_candidates(
116
224
  records: list[dict[str, Any]],
117
225
  resolve: Any,
118
226
  ) -> dict[str, dict[str, Any]]:
119
- """Mine consumer ``dependencies_internal`` into per-(target, artifact) candidates."""
227
+ """Mine consumer ``dependencies_internal`` into per-(target, artifact) candidates.
228
+
229
+ ``target_repo`` is normalised to a repo id the metabase knows (`OI-33`). The
230
+ resolver returns the directory that *declares* a coordinate, which for a
231
+ multi-module build is deeper than the repo — and the demand-side pass names
232
+ the same service by its repo id, so without this the two can never agree.
233
+ """
120
234
  paths_by_repo = _http_in_paths_by_repo(records)
235
+ known = frozenset(repo_id(d) for d in records)
121
236
  cands: dict[str, dict[str, Any]] = {}
122
237
  for data in records:
123
238
  consumer = repo_id(data)
@@ -127,12 +242,18 @@ def _collect_candidates(
127
242
  if not _looks_like_client_artifact(aid):
128
243
  continue
129
244
  coord = f"{gid}:{aid}" if gid else aid
130
- target = resolve(coord) or ""
245
+ # The declaring directory, then the repo that owns it. Both are kept:
246
+ # the module path is strictly more information and `OI-34` may want it.
247
+ resolved, target, client_repo = _resolve_target(
248
+ resolve, coord, known, paths_by_repo,
249
+ )
131
250
  key = _key(target, aid)
132
251
  cand = cands.setdefault(
133
252
  key,
134
253
  {
135
254
  "target_repo": target,
255
+ "target_module": resolved,
256
+ "client_repo": client_repo,
136
257
  "maven_artifact": aid,
137
258
  "import_prefix": gid, # best-effort; reviewer tightens
138
259
  "paths": sorted(paths_by_repo.get(target, ())),
@@ -176,16 +297,29 @@ def _enclosing_class(file_path: str) -> str:
176
297
  return Path(file_path).stem if file_path else ""
177
298
 
178
299
 
179
- def _repos_containing_class(records: list[dict[str, Any]], class_name: str) -> set[str]:
180
- """Repos with a file of this name, as a proxy for corpus-wide occurrence."""
181
- hits: set[str] = set()
300
+ def _repos_by_class(records: list[dict[str, Any]]) -> dict[str, set[str]]:
301
+ """Map each class name to the repos containing a file of that name.
302
+
303
+ Built in **one** pass and queried by key. It replaces a per-class rescan of
304
+ the whole fleet: the caller iterates targets, and each target's classes, and
305
+ asked the corpus about every one — so the cost was
306
+ ``targets x classes x records x nodes``, and on a synthetic fleet where all
307
+ four scale together the node visits grew ~15x for every doubling of the
308
+ repository count.
309
+
310
+ Nothing about the answer needed that. The question is corpus-wide and
311
+ target-independent: which repos hold a file called `StockClient`. Asked once
312
+ per class instead of once per (target, class), and answered from an index
313
+ instead of a scan.
314
+ """
315
+ by_class: dict[str, set[str]] = defaultdict(set)
182
316
  for data in records:
183
317
  rid = repo_id(data)
184
318
  for node in data.get("nodes", []):
185
- if _enclosing_class(str(node.get("file", ""))) == class_name:
186
- hits.add(rid)
187
- break
188
- return hits
319
+ cls = _enclosing_class(str(node.get("file", "")))
320
+ if cls:
321
+ by_class[cls].add(rid)
322
+ return by_class
189
323
 
190
324
 
191
325
  def _demand_side_observations(
@@ -229,6 +363,80 @@ def _demand_side_observations(
229
363
  return observed
230
364
 
231
365
 
366
+ def _unresolved_warning(target_repo: str, coordinate: str, module: str) -> str | None:
367
+ """Explain an unresolvable coordinate, or None when it resolved.
368
+
369
+ An empty `target_repo` used to arrive as an ordinary candidate with nothing
370
+ saying why — 8 of them in the first fleet run. Promoting one creates an edge
371
+ pointing at nothing, so it must not read like a merely weak candidate
372
+ (`OI-33`, and the class in `OI-36`).
373
+ """
374
+ if target_repo:
375
+ return None
376
+ where = f"; declared at {module!r}" if module else "; not found in the checkout"
377
+ return (
378
+ f"coordinate {coordinate or '(none)'} did not resolve to a known repo"
379
+ f"{where} — the target cannot be trusted, and promoting this would "
380
+ "create an edge to nothing"
381
+ )
382
+
383
+
384
+ def _target_warnings(
385
+ target_repo: str, coordinate: str, module: str, client_repo: str,
386
+ ) -> list[str]:
387
+ """Everything worth saying about how this candidate's target was arrived at."""
388
+ out: list[str] = []
389
+ unresolved = _unresolved_warning(target_repo, coordinate, module)
390
+ if unresolved:
391
+ out.append(unresolved)
392
+ if client_repo:
393
+ out.append(
394
+ f"target corrected from {client_repo!r}, which declares the client "
395
+ "library and no inbound endpoints, to the service it appears to "
396
+ "front — check the substitution before accepting (OI-40)"
397
+ )
398
+ return out
399
+
400
+
401
+ def _endpointless_warning(target_repo: str, has_paths: bool, scanned: set[str]) -> str | None:
402
+ """Flag a target that receives no calls, without dropping it.
403
+
404
+ A binding's target is by definition a service that receives calls, so a
405
+ target with no detected inbound endpoint cannot fulfil that. But **zero
406
+ endpoints is also what a detection gap looks like** — `OI-17` left an entire
407
+ language half of a fleet returning none — and the two causes are
408
+ indistinguishable from the outside.
409
+
410
+ So this warns and never filters. Dropping these would turn `OI-17`-class
411
+ blindness into an invisible data-quality filter, which is `OI-36` with the
412
+ tool doing it to itself.
413
+ """
414
+ if not target_repo or has_paths or target_repo not in scanned:
415
+ return None
416
+ return (
417
+ f"{target_repo} declares no inbound endpoints: it may be a client "
418
+ "library whose service is outside the scanned fleet, or a service whose "
419
+ "endpoints were not detected — the two look identical from here"
420
+ )
421
+
422
+
423
+ def _entry_fields(
424
+ cand: dict[str, Any], prev: dict[str, Any] | None
425
+ ) -> dict[str, Any]:
426
+ """The tunable fields and status, preserving a reviewer's decision.
427
+
428
+ A reviewer who set accepted/rejected has tuned the binding, so their edits
429
+ and status are kept and only evidence and confidence are refreshed.
430
+ """
431
+ if prev and prev.get("status") in (STATUS_ACCEPTED, STATUS_REJECTED):
432
+ entry = {k: prev.get(k, cand.get(k)) for k in _TUNABLE_FIELDS}
433
+ entry["status"] = prev["status"]
434
+ return entry
435
+ entry = {k: cand[k] for k in _TUNABLE_FIELDS}
436
+ entry["status"] = STATUS_PENDING
437
+ return entry
438
+
439
+
232
440
  def _build_entry(
233
441
  cand: dict[str, Any],
234
442
  scanned: set[str],
@@ -237,22 +445,22 @@ def _build_entry(
237
445
  """Assemble a candidate entry, preserving a reviewer's non-pending decision."""
238
446
  consumers = sorted(cand.pop("consumers"))
239
447
  coordinate = cand.pop("coordinate")
240
- warnings = cand.pop("warnings", [])
448
+ warnings = list(cand.pop("warnings", []))
449
+ module = cand.pop("target_module", "")
450
+ client_repo = cand.pop("client_repo", "")
451
+ warnings += _target_warnings(
452
+ str(cand.get("target_repo", "")), coordinate, module, client_repo,
453
+ )
241
454
  has_paths = bool(cand["paths"])
242
455
  evidence = {
243
456
  "coordinate": coordinate,
244
457
  "consumers": consumers,
245
458
  "resolved": bool(cand["target_repo"]),
459
+ "declared_at": module,
460
+ "client_repo": client_repo,
246
461
  "paths_from_target_scan": has_paths and cand["target_repo"] in scanned,
247
462
  }
248
- # A reviewer who set status to accepted/rejected has tuned the binding; keep
249
- # their edits and status, only refreshing evidence + confidence.
250
- if prev and prev.get("status") in (STATUS_ACCEPTED, STATUS_REJECTED):
251
- entry = {k: prev.get(k, cand.get(k)) for k in _TUNABLE_FIELDS}
252
- entry["status"] = prev["status"]
253
- else:
254
- entry = {k: cand[k] for k in _TUNABLE_FIELDS}
255
- entry["status"] = STATUS_PENDING
463
+ entry = _entry_fields(cand, prev)
256
464
  # How the candidate was found. `both` is materially stronger than either
257
465
  # alone: a declared dependency and an observed call site are independent
258
466
  # lines of evidence. `call-site` sorts lowest, since it rests on the path
@@ -260,6 +468,12 @@ def _build_entry(
260
468
  entry["discovery_method"] = cand.get("discovery_method", "dependency")
261
469
  if warnings:
262
470
  entry["warnings"] = warnings
471
+ endpointless = _endpointless_warning(
472
+ str(entry["target_repo"] or ""), bool(entry["paths"]), scanned,
473
+ )
474
+ if endpointless:
475
+ existing: list[str] = list(entry.get("warnings") or [])
476
+ entry["warnings"] = [*existing, endpointless]
263
477
  entry["confidence"] = _confidence(
264
478
  str(entry["target_repo"] or ""), scanned, bool(entry["paths"])
265
479
  )
@@ -282,12 +496,14 @@ def _apply_demand_side(
282
496
  """
283
497
  observed = _demand_side_observations(records)
284
498
  by_target = {c["target_repo"]: (k, c) for k, c in cands.items() if c["target_repo"]}
499
+ # One pass over the fleet, before the loop rather than inside it.
500
+ repos_by_class = _repos_by_class(records)
285
501
 
286
502
  for target, seen in sorted(observed.items()):
287
503
  classes = sorted(seen["classes"])
288
504
  warnings = []
289
505
  for cls in classes:
290
- repos = _repos_containing_class(records, cls)
506
+ repos = repos_by_class.get(cls, set())
291
507
  if len(repos) > MAX_PATTERN_REPOS:
292
508
  warnings.append(
293
509
  f"class_pattern {cls!r} appears in {len(repos)} repos; too "
@@ -338,7 +554,7 @@ def discover_api_clients_from_records(
338
554
  for cand in cands.values():
339
555
  cand.setdefault("discovery_method", "dependency")
340
556
  _apply_demand_side(cands, records)
341
- prev = _load_discovered(metabase_root / DISCOVERED_FILE)
557
+ prev = _load_discovered(metabase_root / DISCOVERED_FILE, frozenset(scanned))
342
558
 
343
559
  entries = [
344
560
  _build_entry(cand, scanned, prev.get(key))
@@ -364,7 +580,13 @@ def discover_api_clients(
364
580
  by_coord, by_name, by_full = _build_component_identity_index(repos_root)
365
581
 
366
582
  def resolve(coord: str) -> str | None:
367
- """Resolve a maven coordinate to the repo id that publishes it, or None."""
583
+ """Resolve a coordinate to the clone path that *declares* it, or None.
584
+
585
+ Not the repo id — this docstring used to claim it was, and that claim is
586
+ how `OI-33` went unnoticed: `_resolve_clone_path` returns the declaring
587
+ directory, which for a multi-module build sits inside the repo. The
588
+ caller normalises via `_canonical_repo_id`.
589
+ """
368
590
  return _resolve_clone_path(coord, by_coord, by_name, by_full)
369
591
 
370
592
  return discover_api_clients_from_records(metabase_root, records, resolve)
@@ -0,0 +1,89 @@
1
+ """Orchestrate v2 cross-repo graph generation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ from ..graph_common import (
9
+ iter_v2_repo_records,
10
+ load_v2_repo_records,
11
+ v2_record_paths,
12
+ )
13
+ from ..index_store import build_index
14
+ from .data_stores import write_data_store_graph
15
+ from .index_v2 import write_index_v2
16
+ from .openapi_edges import write_openapi_artifacts
17
+ from .payload_producers import build_producer_indices, write_payload_producer_index
18
+ from .phase3 import aggregate_phase3_v2
19
+ from .pii_flow_v2 import write_pii_flow_v2
20
+ from .queues import write_queue_graph
21
+ from .service_calls import collect_service_edges, write_service_call_graph
22
+ from .traces_index import write_traces_index
23
+
24
+
25
+ def write_fleet_index(
26
+ metabase_root: Path,
27
+ repo_jsons: list[Path],
28
+ *,
29
+ repos_root: Path | None = None,
30
+ producer_indices: list[Any] | None = None,
31
+ ) -> Path:
32
+ """Persist the four things a trace consults, keyed by target repo (`OI-15`).
33
+
34
+ Built here because aggregation already walks every repo, so the index is one
35
+ more pass over data that is being read anyway rather than a second crawl.
36
+
37
+ The two fleet-wide derivations are computed the same way `trace` computed
38
+ them, from the same functions — the index is a cache of this code's output,
39
+ not a reimplementation of it, which is what keeps the two from drifting.
40
+
41
+ ``producer_indices`` is taken from the caller when aggregation has already
42
+ built it. Rebuilding it here meant the fleet's source tree was scanned a
43
+ second time, which doubled the slowest step of the whole run (`OI-30`).
44
+ """
45
+ paths = v2_record_paths(metabase_root, json_paths=repo_jsons)
46
+ records = load_v2_repo_records(metabase_root, json_paths=paths)
47
+ call_edges, _unmatched = collect_service_edges(records)
48
+ if producer_indices is None:
49
+ producer_indices = build_producer_indices(
50
+ metabase_root, repos_root=repos_root, json_paths=paths,
51
+ )
52
+ del records # the streamed pass below must not be served from this list
53
+
54
+ return build_index(
55
+ metabase_root,
56
+ paths,
57
+ call_edges=call_edges,
58
+ producer_indices=producer_indices,
59
+ records=iter_v2_repo_records(metabase_root, json_paths=paths),
60
+ )
61
+
62
+
63
+ def aggregate_graphs_v2(
64
+ metabase_root: Path,
65
+ repo_jsons: list[Path],
66
+ *,
67
+ repos_root: Path | None = None,
68
+ phase3: bool = True,
69
+ fleet_index: bool = True,
70
+ ) -> None:
71
+ """Run all v2 cross-repo graph writers (service, queue, data-store, PII, phase 3)."""
72
+ write_service_call_graph(metabase_root, repo_jsons, repos_root=repos_root)
73
+ if repos_root:
74
+ write_openapi_artifacts(metabase_root, repos_root, repo_jsons)
75
+ write_queue_graph(metabase_root, repo_jsons)
76
+ write_data_store_graph(metabase_root, repo_jsons)
77
+ producer_indices = write_payload_producer_index(
78
+ metabase_root, repo_jsons, repos_root=repos_root,
79
+ )
80
+ write_index_v2(metabase_root, repo_jsons)
81
+ write_pii_flow_v2(metabase_root, repo_jsons)
82
+ write_traces_index(metabase_root)
83
+ if phase3:
84
+ aggregate_phase3_v2(metabase_root, repo_jsons)
85
+ if fleet_index:
86
+ write_fleet_index(
87
+ metabase_root, repo_jsons, repos_root=repos_root,
88
+ producer_indices=producer_indices,
89
+ )
@@ -0,0 +1,125 @@
1
+ """Discover OpenAPI specs and Helm ingress hosts under repos/."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from pathlib import Path
7
+
8
+ from ..checkout_scan import paths_by_name
9
+ from ..graph_common import normalize_path_template as norm_path
10
+
11
+ from .openapi_models import OpenApiSpec
12
+
13
+ OPENAPI_GLOBS = ("openapi.yaml", "openapi.yml", "swagger.yaml", "swagger.yml")
14
+ PATH_LINE_RX = re.compile(
15
+ r"^\s{2,}(/[/\w{}.:$-]+)\s*:\s*$",
16
+ re.MULTILINE,
17
+ )
18
+ SERVERS_URL_RX = re.compile(
19
+ r"^\s*-\s*url:\s*['\"]?(https?://[^'\"\s]+)",
20
+ re.MULTILINE | re.IGNORECASE,
21
+ )
22
+ HELM_VALUES_NAMES = ("values.yaml", "values.yml")
23
+ HELM_HOST_RX = re.compile(
24
+ r"^\s*(?:host|hostname|externalHost|ingress\.host)\s*:\s*['\"]?([^\s#'\"]+)",
25
+ re.MULTILINE | re.IGNORECASE,
26
+ )
27
+
28
+ _SKIP_DIR_PARTS = frozenset({".git", "node_modules", "target", "build", "charts"})
29
+
30
+
31
+ def repo_from_under_repos(spec_path: Path, repos_root: Path) -> str | None:
32
+ """Return the ``org/repo`` id for a path under repos_root, or None."""
33
+ try:
34
+ rel = spec_path.relative_to(repos_root)
35
+ except ValueError:
36
+ return None
37
+ parts = rel.parts
38
+ if len(parts) < 2:
39
+ return None
40
+ return f"{parts[0]}/{parts[1]}"
41
+
42
+
43
+ def _should_skip_path(path: Path) -> bool:
44
+ """True if the path lies within a skipped directory (.git, node_modules, ...)."""
45
+ return any(p in path.parts for p in _SKIP_DIR_PARTS)
46
+
47
+
48
+ def _walk_once(repos_root: Path, names: tuple[str, ...]) -> list[Path]:
49
+ """Every file under the checkout carrying one of ``names``, from a single walk.
50
+
51
+ `Path.rglob(name)` traverses the whole tree and filters by name, so the
52
+ per-pattern loop this replaces cost one full traversal of a 34 GB checkout
53
+ *per filename* — four for the OpenAPI globs, two for Helm, and again for
54
+ every call site. The same defect as `OI-30` in the producer scan: the loop
55
+ over what to look for sat outside the loop over where to look.
56
+ """
57
+ found = paths_by_name(repos_root, frozenset(names))
58
+ return [path for name in names for path in found[name]]
59
+
60
+
61
+ def discover_openapi_specs(repos_root: Path) -> list[OpenApiSpec]:
62
+ """Scan repos_root for OpenAPI/Swagger specs and return parsed OpenApiSpec rows."""
63
+ if not repos_root.is_dir():
64
+ return []
65
+ specs: list[OpenApiSpec] = []
66
+ seen: set[tuple[str, str]] = set()
67
+ for path in _walk_once(repos_root, OPENAPI_GLOBS):
68
+ if _should_skip_path(path):
69
+ continue
70
+ try:
71
+ if path.stat().st_size > 2_000_000:
72
+ continue
73
+ text = path.read_text(encoding="utf-8", errors="replace")
74
+ except OSError:
75
+ continue
76
+ target = repo_from_under_repos(path, repos_root)
77
+ if not target:
78
+ continue
79
+ key = (target, str(path))
80
+ if key in seen:
81
+ continue
82
+ seen.add(key)
83
+ paths = sorted({norm_path(m.group(1)) for m in PATH_LINE_RX.finditer(text)})
84
+ servers = [m.group(1) for m in SERVERS_URL_RX.finditer(text)]
85
+ specs.append(OpenApiSpec(
86
+ target_repo=target,
87
+ spec_path=str(path.relative_to(repos_root)),
88
+ paths=[p for p in paths if p and p != "/"],
89
+ servers=servers[:5],
90
+ ))
91
+ return specs
92
+
93
+
94
+ def discover_helm_hosts(repos_root: Path) -> list[dict[str, str]]:
95
+ """Scan Helm values for declared ingress/service hostnames."""
96
+ if not repos_root.is_dir():
97
+ return []
98
+ rows: list[dict[str, str]] = []
99
+ seen: set[tuple[str, str]] = set()
100
+ for path in _walk_once(repos_root, HELM_VALUES_NAMES):
101
+ if _should_skip_path(path):
102
+ continue
103
+ try:
104
+ if path.stat().st_size > 500_000:
105
+ continue
106
+ text = path.read_text(encoding="utf-8", errors="replace")
107
+ except OSError:
108
+ continue
109
+ target = repo_from_under_repos(path, repos_root)
110
+ if not target:
111
+ continue
112
+ for m in HELM_HOST_RX.finditer(text):
113
+ host = m.group(1).strip().strip("'\"")
114
+ if not host or host.startswith("{{"):
115
+ continue
116
+ key = (target, host)
117
+ if key in seen:
118
+ continue
119
+ seen.add(key)
120
+ rows.append({
121
+ "target_repo": target,
122
+ "host": host,
123
+ "values_path": str(path.relative_to(repos_root)),
124
+ })
125
+ return rows