intentumdiff-python 0.0.1__py3-none-manylinux_2_39_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. intentumdiff/__init__.py +134 -0
  2. intentumdiff/_differ_gate.py +210 -0
  3. intentumdiff/_differ_presentation.py +786 -0
  4. intentumdiff/_differ_runtime.py +320 -0
  5. intentumdiff/analysis/__init__.py +1 -0
  6. intentumdiff/analysis/compile_commands.py +151 -0
  7. intentumdiff/analysis/cross_file.py +71 -0
  8. intentumdiff/analysis/diagnostics.py +309 -0
  9. intentumdiff/analysis/guardrail_reports.py +254 -0
  10. intentumdiff/analysis/guardrails.py +318 -0
  11. intentumdiff/analysis/invariances.py +1207 -0
  12. intentumdiff/analysis/keyed_profiles.py +25 -0
  13. intentumdiff/analysis/resource_profiles.py +14 -0
  14. intentumdiff/analysis/schema_resolver.py +816 -0
  15. intentumdiff/analysis/text_review.py +46 -0
  16. intentumdiff/analysis/user_schemas.py +430 -0
  17. intentumdiff/assets.py +45 -0
  18. intentumdiff/cache/__init__.py +37 -0
  19. intentumdiff/cache/duckdb_store.py +98 -0
  20. intentumdiff/cache/sqlite_store.py +149 -0
  21. intentumdiff/cache/store.py +190 -0
  22. intentumdiff/cli/__init__.py +199 -0
  23. intentumdiff/cli/_commands.py +2141 -0
  24. intentumdiff/cli/_parser.py +1061 -0
  25. intentumdiff/cli/_shared.py +644 -0
  26. intentumdiff/content_type.py +59 -0
  27. intentumdiff/core/__init__.py +1 -0
  28. intentumdiff/core/commit_differ.py +599 -0
  29. intentumdiff/core/config.py +51 -0
  30. intentumdiff/core/cst_serializer.py +101 -0
  31. intentumdiff/core/diffignore.py +87 -0
  32. intentumdiff/core/index.py +189 -0
  33. intentumdiff/core/indexer.py +792 -0
  34. intentumdiff/core/models.py +837 -0
  35. intentumdiff/differ.py +2195 -0
  36. intentumdiff/github_app.py +287 -0
  37. intentumdiff/intentumdiff_rust_core/__init__.py +7 -0
  38. intentumdiff/intentumdiff_rust_core/ffi.py +8 -0
  39. intentumdiff/intentumdiff_rust_core/libintentumdiff_rust_core.so +0 -0
  40. intentumdiff/invariances/__init__.py +2 -0
  41. intentumdiff/invariances/rules.schema.json +283 -0
  42. intentumdiff/invariances/rules.yaml +349 -0
  43. intentumdiff/live_server.py +1314 -0
  44. intentumdiff/lsp/__init__.py +67 -0
  45. intentumdiff/lsp/client.py +544 -0
  46. intentumdiff/lsp/config.py +96 -0
  47. intentumdiff/lsp/enricher.py +227 -0
  48. intentumdiff/lsp/exceptions.py +27 -0
  49. intentumdiff/lsp/launcher.py +273 -0
  50. intentumdiff/lsp/servers.py +356 -0
  51. intentumdiff/lsp_server/__init__.py +14 -0
  52. intentumdiff/lsp_server/_codelens.py +67 -0
  53. intentumdiff/lsp_server/_diagnostics.py +62 -0
  54. intentumdiff/lsp_server/_handlers.py +175 -0
  55. intentumdiff/lsp_server/server.py +205 -0
  56. intentumdiff/plugins/__init__.py +1 -0
  57. intentumdiff/plugins/adapter.py +397 -0
  58. intentumdiff/plugins/builtins.py +323 -0
  59. intentumdiff/plugins/exceptions.py +93 -0
  60. intentumdiff/plugins/hub.py +1155 -0
  61. intentumdiff/plugins/language_metadata.py +250 -0
  62. intentumdiff/plugins/loader.py +1252 -0
  63. intentumdiff/plugins/registry.py +1079 -0
  64. intentumdiff/plugins/registry.schema.json +99 -0
  65. intentumdiff/plugins/registry_schema.py +280 -0
  66. intentumdiff/plugins/wit/plugin.wit +351 -0
  67. intentumdiff/rust_core.py +2370 -0
  68. intentumdiff/serve/__init__.py +19 -0
  69. intentumdiff/serve/_app.py +485 -0
  70. intentumdiff/serve/_static/index.html +326 -0
  71. intentumdiff/serve/_static/playground.js +567 -0
  72. intentumdiff/sources/__init__.py +1 -0
  73. intentumdiff/sources/base.py +28 -0
  74. intentumdiff/sources/file_source.py +48 -0
  75. intentumdiff/sources/git_source.py +375 -0
  76. intentumdiff/sources/live_buffer_source.py +113 -0
  77. intentumdiff/sources/patch_source.py +126 -0
  78. intentumdiff/sources/string_source.py +44 -0
  79. intentumdiff/testing.py +165 -0
  80. intentumdiff/vcs/__init__.py +37 -0
  81. intentumdiff/vcs/_safe_path.py +50 -0
  82. intentumdiff/vcs/base.py +136 -0
  83. intentumdiff/vcs/git_backend.py +97 -0
  84. intentumdiff/vcs/git_cli.py +100 -0
  85. intentumdiff/vcs/hg_backend.py +94 -0
  86. intentumdiff/vcs/perforce_backend.py +109 -0
  87. intentumdiff/vcs/svn_backend.py +105 -0
  88. intentumdiff/wasm/.gitkeep +2 -0
  89. intentumdiff/wasm/abap_parser.wasm +0 -0
  90. intentumdiff/wasm/adf_parser.wasm +0 -0
  91. intentumdiff/wasm/asciidoc_parser.wasm +0 -0
  92. intentumdiff/wasm/asm_parser.wasm +0 -0
  93. intentumdiff/wasm/assemblyscript_parser.wasm +0 -0
  94. intentumdiff/wasm/astro_parser.wasm +0 -0
  95. intentumdiff/wasm/bash_parser.wasm +0 -0
  96. intentumdiff/wasm/clojure_parser.wasm +0 -0
  97. intentumdiff/wasm/cmake_parser.wasm +0 -0
  98. intentumdiff/wasm/cpp_parser.wasm +0 -0
  99. intentumdiff/wasm/csharp_parser.wasm +0 -0
  100. intentumdiff/wasm/css_parser.wasm +0 -0
  101. intentumdiff/wasm/dart_parser.wasm +0 -0
  102. intentumdiff/wasm/databricks_parser.wasm +0 -0
  103. intentumdiff/wasm/dax_parser.wasm +0 -0
  104. intentumdiff/wasm/delphi_parser.wasm +0 -0
  105. intentumdiff/wasm/dockerfile_parser.wasm +0 -0
  106. intentumdiff/wasm/elixir_parser.wasm +0 -0
  107. intentumdiff/wasm/freebasic_parser.wasm +0 -0
  108. intentumdiff/wasm/generic_parser.wasm +0 -0
  109. intentumdiff/wasm/gitignore_parser.wasm +0 -0
  110. intentumdiff/wasm/go_parser.wasm +0 -0
  111. intentumdiff/wasm/gomod_parser.wasm +0 -0
  112. intentumdiff/wasm/graphql_parser.wasm +0 -0
  113. intentumdiff/wasm/groovy_parser.wasm +0 -0
  114. intentumdiff/wasm/haskell_parser.wasm +0 -0
  115. intentumdiff/wasm/html_parser.wasm +0 -0
  116. intentumdiff/wasm/ini_parser.wasm +0 -0
  117. intentumdiff/wasm/java_parser.wasm +0 -0
  118. intentumdiff/wasm/js_ts_parser.wasm +0 -0
  119. intentumdiff/wasm/json_parser.wasm +0 -0
  120. intentumdiff/wasm/kotlin_parser.wasm +0 -0
  121. intentumdiff/wasm/latex_parser.wasm +0 -0
  122. intentumdiff/wasm/lua_parser.wasm +0 -0
  123. intentumdiff/wasm/make_parser.wasm +0 -0
  124. intentumdiff/wasm/markdown_parser.wasm +0 -0
  125. intentumdiff/wasm/mdx_parser.wasm +0 -0
  126. intentumdiff/wasm/ocaml_parser.wasm +0 -0
  127. intentumdiff/wasm/odin_parser.wasm +0 -0
  128. intentumdiff/wasm/parser_manifest.json +590 -0
  129. intentumdiff/wasm/perl_parser.wasm +0 -0
  130. intentumdiff/wasm/php_parser.wasm +0 -0
  131. intentumdiff/wasm/plsql_parser.wasm +0 -0
  132. intentumdiff/wasm/po_parser.wasm +0 -0
  133. intentumdiff/wasm/postscript_parser.wasm +0 -0
  134. intentumdiff/wasm/powershell_parser.wasm +0 -0
  135. intentumdiff/wasm/proto_parser.wasm +0 -0
  136. intentumdiff/wasm/puppet_parser.wasm +0 -0
  137. intentumdiff/wasm/python_parser.wasm +0 -0
  138. intentumdiff/wasm/qsharp_parser.wasm +0 -0
  139. intentumdiff/wasm/r_parser.wasm +0 -0
  140. intentumdiff/wasm/reasonml_parser.wasm +0 -0
  141. intentumdiff/wasm/ruby_parser.wasm +0 -0
  142. intentumdiff/wasm/rust_parser.wasm +0 -0
  143. intentumdiff/wasm/sas_parser.wasm +0 -0
  144. intentumdiff/wasm/scala_parser.wasm +0 -0
  145. intentumdiff/wasm/scss_parser.wasm +0 -0
  146. intentumdiff/wasm/sql_parser.wasm +0 -0
  147. intentumdiff/wasm/squirrel_parser.wasm +0 -0
  148. intentumdiff/wasm/svelte_parser.wasm +0 -0
  149. intentumdiff/wasm/swift_parser.wasm +0 -0
  150. intentumdiff/wasm/terraform_parser.wasm +0 -0
  151. intentumdiff/wasm/toml_parser.wasm +0 -0
  152. intentumdiff/wasm/tsql_parser.wasm +0 -0
  153. intentumdiff/wasm/vbnet_parser.wasm +0 -0
  154. intentumdiff/wasm/vue_parser.wasm +0 -0
  155. intentumdiff/wasm/wat_parser.wasm +0 -0
  156. intentumdiff/wasm/xml_parser.wasm +0 -0
  157. intentumdiff/wasm/yaml_parser.wasm +0 -0
  158. intentumdiff/wasm/zig_parser.wasm +0 -0
  159. intentumdiff/watcher.py +259 -0
  160. intentumdiff_python-0.0.1.dist-info/METADATA +112 -0
  161. intentumdiff_python-0.0.1.dist-info/RECORD +165 -0
  162. intentumdiff_python-0.0.1.dist-info/WHEEL +4 -0
  163. intentumdiff_python-0.0.1.dist-info/entry_points.txt +80 -0
  164. intentumdiff_python-0.0.1.dist-info/licenses/LICENSE +21 -0
  165. intentumdiff_python-0.0.1.dist-info/sboms/intentumdiff-rust-core.cyclonedx.json +10968 -0
intentumdiff/differ.py ADDED
@@ -0,0 +1,2195 @@
1
+ """
2
+ intentumdiff.differ
3
+ ~~~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Pipeline orchestrator — the central coordinator.
6
+
7
+ Pipeline stages
8
+ ───────────────
9
+ 1. Source retrieval — call ``Source.get_content()``
10
+ 2. Language detection — ``PluginRegistry.detect_parser(...)``
11
+ 3. Parser dispatch - full-parse Wasm parser consumes raw source
12
+ 4. Parser output - parser returns semantic tree JSON
13
+ 5. Trivia stripping — ``_strip_trivia_impl`` removes comment / whitespace nodes
14
+ 6. Style-only shortcut — compare root structural hashes; short-circuit if equal
15
+ 7. Plugin dispatch - call ``ParserAdapter.process(source, ...)`` (Wasm)
16
+ 8. ID validation — ``_validate_tree_ids`` catches duplicate IDs from plugins
17
+ 9. Rust finalize routing — ``finalize_review_json`` (matching, edit script,
18
+ refinement, presentation, and grouping all run in the Rust core; the
19
+ transitional python stages 9-13 were retired — issue #57 payoff)
20
+ 13.5 Diff-analyzer pass — ``DiffAnalyzerAdapter.analyze_diff(...)`` (Wasm, optional)
21
+
22
+ The ``SemanticDiffer`` class is the primary public API object.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import hashlib
28
+ import logging
29
+ import os
30
+ import threading
31
+ import time
32
+ from collections.abc import Iterator
33
+ from concurrent.futures import ThreadPoolExecutor, as_completed
34
+ from pathlib import Path
35
+ from typing import Any
36
+
37
+ from pydantic import ValidationError
38
+
39
+ from intentumdiff._differ_gate import (
40
+ RUST_CERTIFIED_LANGUAGES as RUST_CERTIFIED_LANGUAGES,
41
+ )
42
+ from intentumdiff._differ_gate import (
43
+ RUST_FINALIZE_LANGUAGES as RUST_FINALIZE_LANGUAGES,
44
+ )
45
+ from intentumdiff._differ_gate import (
46
+ STRICT_RUST_GATE_ENV as STRICT_RUST_GATE_ENV,
47
+ )
48
+ from intentumdiff._differ_gate import (
49
+ RustOnlyGateError as RustOnlyGateError,
50
+ )
51
+ from intentumdiff._differ_gate import (
52
+ _raise_rust_only_gate_error as _raise_rust_only_gate_error,
53
+ )
54
+ from intentumdiff._differ_gate import (
55
+ _rust_certified_languages as _rust_certified_languages,
56
+ )
57
+ from intentumdiff._differ_gate import (
58
+ _rust_finalize_languages as _rust_finalize_languages,
59
+ )
60
+ from intentumdiff._differ_gate import (
61
+ _strict_rust_only_enabled as _strict_rust_only_enabled,
62
+ )
63
+ from intentumdiff._differ_gate import (
64
+ _validate_git_ref as _validate_git_ref,
65
+ )
66
+ from intentumdiff._differ_presentation import (
67
+ _GENERIC_STRING_LABELS as _GENERIC_STRING_LABELS,
68
+ )
69
+ from intentumdiff._differ_presentation import (
70
+ _NAMED_ENTITY_NODE_TYPES as _NAMED_ENTITY_NODE_TYPES,
71
+ )
72
+ from intentumdiff._differ_presentation import (
73
+ _all_semantic_nodes as _all_semantic_nodes,
74
+ )
75
+ from intentumdiff._differ_presentation import (
76
+ _annotate_text_diffs as _annotate_text_diffs,
77
+ )
78
+ from intentumdiff._differ_presentation import (
79
+ _changes_to_stream_events as _changes_to_stream_events,
80
+ )
81
+ from intentumdiff._differ_presentation import (
82
+ _clean_string_literal_label as _clean_string_literal_label,
83
+ )
84
+ from intentumdiff._differ_presentation import (
85
+ _compute_structural_hash_for_tree as _compute_structural_hash_for_tree,
86
+ )
87
+ from intentumdiff._differ_presentation import (
88
+ _count_cst_nodes as _count_cst_nodes,
89
+ )
90
+ from intentumdiff._differ_presentation import (
91
+ _count_semantic_nodes as _count_semantic_nodes,
92
+ )
93
+ from intentumdiff._differ_presentation import (
94
+ _empty_semantic_tree as _empty_semantic_tree,
95
+ )
96
+ from intentumdiff._differ_presentation import (
97
+ _enrich_literal_labels as _enrich_literal_labels,
98
+ )
99
+ from intentumdiff._differ_presentation import (
100
+ _has_error_node as _has_error_node,
101
+ )
102
+ from intentumdiff._differ_presentation import (
103
+ _is_markdown_filename as _is_markdown_filename,
104
+ )
105
+ from intentumdiff._differ_presentation import (
106
+ _is_named_entity_node as _is_named_entity_node,
107
+ )
108
+ from intentumdiff._differ_presentation import (
109
+ _markdown_section_body_hashes as _markdown_section_body_hashes,
110
+ )
111
+ from intentumdiff._differ_presentation import (
112
+ _markdown_section_heading_rename_presentation as _markdown_section_heading_rename_presentation,
113
+ )
114
+ from intentumdiff._differ_presentation import (
115
+ _markdown_section_move_presentation as _markdown_section_move_presentation,
116
+ )
117
+ from intentumdiff._differ_presentation import (
118
+ _markdown_sections as _markdown_sections,
119
+ )
120
+ from intentumdiff._differ_presentation import (
121
+ _node_to_dict as _node_to_dict,
122
+ )
123
+ from intentumdiff._differ_presentation import (
124
+ _root_structural_hash as _root_structural_hash,
125
+ )
126
+ from intentumdiff._differ_presentation import (
127
+ _semantic_parent_map as _semantic_parent_map,
128
+ )
129
+ from intentumdiff._differ_presentation import (
130
+ _slice_source_text as _slice_source_text,
131
+ )
132
+ from intentumdiff._differ_presentation import (
133
+ _surface_changed_in_place_entities as _surface_changed_in_place_entities,
134
+ )
135
+ from intentumdiff._differ_presentation import (
136
+ _token_fallback_diff as _token_fallback_diff,
137
+ )
138
+ from intentumdiff._differ_presentation import (
139
+ _validate_tree_ids as _validate_tree_ids,
140
+ )
141
+ from intentumdiff._differ_runtime import (
142
+ _ADDED_FILE_STATUSES as _ADDED_FILE_STATUSES,
143
+ )
144
+ from intentumdiff._differ_runtime import (
145
+ _DEFAULT_PLUGIN_FUEL as _DEFAULT_PLUGIN_FUEL,
146
+ )
147
+ from intentumdiff._differ_runtime import (
148
+ _DELETED_FILE_STATUSES as _DELETED_FILE_STATUSES,
149
+ )
150
+ from intentumdiff._differ_runtime import (
151
+ _EXPLICIT_FUEL_EXHAUSTION_TEST_CAP as _EXPLICIT_FUEL_EXHAUSTION_TEST_CAP,
152
+ )
153
+ from intentumdiff._differ_runtime import (
154
+ _FUEL_HOTSPOT_ABSOLUTE as _FUEL_HOTSPOT_ABSOLUTE,
155
+ )
156
+ from intentumdiff._differ_runtime import (
157
+ _FUEL_HOTSPOT_PER_KB as _FUEL_HOTSPOT_PER_KB,
158
+ )
159
+ from intentumdiff._differ_runtime import (
160
+ _FUEL_HOTSPOT_PER_LINE as _FUEL_HOTSPOT_PER_LINE,
161
+ )
162
+ from intentumdiff._differ_runtime import (
163
+ _apply_file_lifecycle_to_diff as _apply_file_lifecycle_to_diff,
164
+ )
165
+ from intentumdiff._differ_runtime import (
166
+ _attach_content_type_metadata as _attach_content_type_metadata,
167
+ )
168
+ from intentumdiff._differ_runtime import (
169
+ _attach_run_telemetry as _attach_run_telemetry,
170
+ )
171
+ from intentumdiff._differ_runtime import (
172
+ _drain_plugin_telemetry as _drain_plugin_telemetry,
173
+ )
174
+ from intentumdiff._differ_runtime import (
175
+ _fuel_budget as _fuel_budget,
176
+ )
177
+ from intentumdiff._differ_runtime import (
178
+ _fuel_hotspot_for_call as _fuel_hotspot_for_call,
179
+ )
180
+ from intentumdiff._differ_runtime import (
181
+ _infer_file_lifecycle as _infer_file_lifecycle,
182
+ )
183
+ from intentumdiff._differ_runtime import (
184
+ _PhaseProfiler as _PhaseProfiler,
185
+ )
186
+ from intentumdiff._differ_runtime import (
187
+ _record_engine_telemetry as _record_engine_telemetry,
188
+ )
189
+ from intentumdiff._differ_runtime import (
190
+ _summarize_engine_telemetry as _summarize_engine_telemetry,
191
+ )
192
+ from intentumdiff.analysis.compile_commands import compile_commands_metadata
193
+ from intentumdiff.analysis.diagnostics import (
194
+ DiagnosticsRecorder,
195
+ )
196
+ from intentumdiff.analysis.guardrails import (
197
+ apply_guardrails_to_diff,
198
+ guardrails_may_apply,
199
+ )
200
+
201
+ # All five profile-enrichment families (keyed / path / query / statement /
202
+ # resource) are now Rust-authoritative — no Python enricher on the diff path (#90).
203
+ from intentumdiff.analysis.schema_resolver import (
204
+ resolve_schema,
205
+ schema_cache_fingerprint,
206
+ schema_resolution_metadata,
207
+ )
208
+ from intentumdiff.analysis.text_review import (
209
+ PresentationResult,
210
+ normalize_generic_text_for_review,
211
+ )
212
+ from intentumdiff.analysis.user_schemas import (
213
+ load_user_schema_profiles,
214
+ user_xml_dialects_payload,
215
+ )
216
+ from intentumdiff.cache.store import CacheStore
217
+ from intentumdiff.core.models import (
218
+ Change,
219
+ ChangeGroupKind,
220
+ ChangeStreamEvent,
221
+ ChangeStreamPhase,
222
+ ChangeType,
223
+ DetectionResult,
224
+ DiffConfig,
225
+ LanguageInfoGroup,
226
+ SemanticDiff,
227
+ SemanticNode,
228
+ )
229
+ from intentumdiff.plugins.exceptions import (
230
+ PluginFuelExhausted,
231
+ PluginNotFoundError,
232
+ PluginOutputError,
233
+ )
234
+ from intentumdiff.plugins.loader import _strip_trivia_impl
235
+ from intentumdiff.plugins.registry import PluginRegistry
236
+ from intentumdiff.rust_core import (
237
+ RustCoreCommitJsonAttempt,
238
+ apply_invariances,
239
+ build_style_only_evidence,
240
+ enrich_node_facts,
241
+ try_register_user_xml_dialects,
242
+ try_rust_core_batch_diff,
243
+ try_rust_core_batch_diffs,
244
+ try_rust_core_commit_json,
245
+ try_rust_core_working_tree_commit_json,
246
+ try_rust_profile_label_enrichment,
247
+ )
248
+ from intentumdiff.sources.base import Source
249
+
250
+ logger = logging.getLogger(__name__)
251
+
252
+
253
+ class SemanticDiffer:
254
+ """
255
+ High-level semantic differ.
256
+
257
+ Usage::
258
+
259
+ differ = SemanticDiffer()
260
+ diff = differ.diff(GitSource(repo_path, "src/main.py", "HEAD~1", "HEAD"))
261
+ print(diff.has_semantic_changes)
262
+ """
263
+
264
+ def __init__(
265
+ self,
266
+ config: DiffConfig | None = None,
267
+ registry: PluginRegistry | None = None,
268
+ cache: CacheStore | None = None,
269
+ ) -> None:
270
+ # Defensive copy so callers cannot mutate config after construction.
271
+ self._config = config.model_copy() if config is not None else DiffConfig()
272
+ self._registry = registry or PluginRegistry(self._config)
273
+ self._parallel_executor: ThreadPoolExecutor | None = None
274
+ self._parallel_executor_workers: int | None = None
275
+ self._parallel_executor_lock = threading.Lock()
276
+ self._parallel_worker_local = threading.local()
277
+ # Per-grammar wasm binary hash, computed once and memoised.
278
+ self._wasm_hash_cache: dict[str, str] = {}
279
+
280
+ # ── Cache / analytics setup ───────────────────────────────────────
281
+ if cache is not None:
282
+ # Explicitly injected store (e.g. for tests)
283
+ self._cache: CacheStore | None = cache
284
+ self._analytics = None
285
+ elif self._config.cache_path is not None:
286
+ from intentumdiff.cache.sqlite_store import SqliteCacheStore # noqa: PLC0415
287
+
288
+ self._cache = SqliteCacheStore(
289
+ self._config.cache_path / "cache.db",
290
+ ttl_days=self._config.cache_ttl_days,
291
+ max_mb=self._config.cache_max_mb,
292
+ )
293
+ if self._config.analytics_path is not None:
294
+ from intentumdiff.cache.duckdb_store import (
295
+ DuckDBAnalyticsStore, # noqa: PLC0415
296
+ )
297
+
298
+ self._analytics = DuckDBAnalyticsStore(
299
+ self._config.analytics_path / "analytics.duckdb"
300
+ )
301
+ else:
302
+ self._analytics = None
303
+ else:
304
+ self._cache = None
305
+ self._analytics = None
306
+
307
+ # ── Public API ────────────────────────────────────────────────────────────
308
+
309
+ def close(self) -> None:
310
+ """Release background resources owned by this differ."""
311
+ with self._parallel_executor_lock:
312
+ executor = self._parallel_executor
313
+ self._parallel_executor = None
314
+ self._parallel_executor_workers = None
315
+ self._parallel_worker_local = threading.local()
316
+ if executor is not None:
317
+ executor.shutdown(wait=False, cancel_futures=True)
318
+
319
+ def __del__(self) -> None:
320
+ executor = getattr(self, "_parallel_executor", None)
321
+ if executor is not None:
322
+ executor.shutdown(wait=False, cancel_futures=True)
323
+
324
+ def _new_profiler(self) -> _PhaseProfiler:
325
+ return _PhaseProfiler(bool(self._config.profile_phases))
326
+
327
+ def _parallel_executor_for(
328
+ self,
329
+ max_workers: int,
330
+ ) -> tuple[ThreadPoolExecutor, threading.local]:
331
+ with self._parallel_executor_lock:
332
+ if self._parallel_executor is None or self._parallel_executor_workers != max_workers:
333
+ old_executor = self._parallel_executor
334
+ self._parallel_worker_local = threading.local()
335
+ self._parallel_executor = ThreadPoolExecutor(max_workers=max_workers)
336
+ self._parallel_executor_workers = max_workers
337
+ else:
338
+ old_executor = None
339
+ executor = self._parallel_executor
340
+ worker_local = self._parallel_worker_local
341
+ if old_executor is not None:
342
+ old_executor.shutdown(wait=True, cancel_futures=False)
343
+ return executor, worker_local
344
+
345
+ def _attach_phase_timings(
346
+ self,
347
+ diff: SemanticDiff,
348
+ profiler: _PhaseProfiler,
349
+ ) -> SemanticDiff:
350
+ if not profiler.enabled:
351
+ return diff
352
+ metadata = dict(diff.metadata)
353
+ metadata["phase_timings"] = profiler.snapshot()
354
+ return diff.model_copy(update={"metadata": metadata})
355
+
356
+ def _attach_commit_batch_phase_timings(
357
+ self,
358
+ diff: SemanticDiff,
359
+ *,
360
+ source_collection_ms: float,
361
+ rust_batch_ms: float,
362
+ rust_commit_merge_ms: float = 0.0,
363
+ rust_adapter_phase_timings: list[dict[str, Any]] | None = None,
364
+ rust_batch_parallel_auto: bool,
365
+ rust_batch_parallel_workers: int | None,
366
+ rust_batch_size: int,
367
+ rust_batch_fallback_count: int,
368
+ ) -> SemanticDiff:
369
+ metadata = dict(diff.metadata)
370
+ batch_metadata = dict(metadata.get("rust_core_batch") or {})
371
+ batch_metadata.update(
372
+ {
373
+ "rust_core_batch_parallel_auto": rust_batch_parallel_auto,
374
+ "parallel_workers": rust_batch_parallel_workers,
375
+ "batch_size": rust_batch_size,
376
+ "fallback_count": rust_batch_fallback_count,
377
+ }
378
+ )
379
+ adapter_phases = [
380
+ {
381
+ **phase,
382
+ "shared": True,
383
+ }
384
+ for phase in rust_adapter_phase_timings or []
385
+ if isinstance(phase, dict)
386
+ ]
387
+ if adapter_phases:
388
+ batch_metadata["adapter_phase_timings"] = adapter_phases
389
+ metadata["rust_core_batch"] = batch_metadata
390
+ if not self._config.profile_phases:
391
+ return diff.model_copy(update={"metadata": metadata})
392
+ phases: list[dict[str, Any]] = [
393
+ {
394
+ "name": "source_collection",
395
+ "duration_ms": round(source_collection_ms, 3),
396
+ "shared": True,
397
+ }
398
+ ]
399
+ if adapter_phases:
400
+ phases.extend(adapter_phases)
401
+ else:
402
+ phases.append(
403
+ {
404
+ "name": "rust_core_batch_execution",
405
+ "duration_ms": round(rust_batch_ms, 3),
406
+ "shared": True,
407
+ }
408
+ )
409
+ metadata["phase_timings"] = {
410
+ "schema_version": 1,
411
+ "total_ms": round(
412
+ source_collection_ms + rust_batch_ms + rust_commit_merge_ms,
413
+ 3,
414
+ ),
415
+ "phases": phases,
416
+ }
417
+ if rust_commit_merge_ms:
418
+ metadata["phase_timings"]["phases"].append(
419
+ {
420
+ "name": "commit_result_merge",
421
+ "duration_ms": round(rust_commit_merge_ms, 3),
422
+ "shared": True,
423
+ }
424
+ )
425
+ return diff.model_copy(update={"metadata": metadata})
426
+
427
+ def _rust_core_python_batch_skip_reason(self) -> str | None:
428
+ if not self._config.experimental_rust_core:
429
+ return "disabled"
430
+ if self._config.diagnostics:
431
+ return "diagnostics require Python pipeline"
432
+ if self._config.extra_trivia_types:
433
+ return "extra trivia types require Python pipeline"
434
+ if self._config.extra_grammars.get("python"):
435
+ return "custom Python grammar requires Python pipeline"
436
+ allowed = self._config.allowed_plugins
437
+ if allowed is not None and not any(
438
+ plugin_id in allowed
439
+ for plugin_id in (
440
+ "python",
441
+ "python-parser",
442
+ "python_parser",
443
+ "intentumdiff:python:python",
444
+ )
445
+ ):
446
+ return "Python parser is not allowed by plugin policy"
447
+ if self._registry.get_enrichers("python"):
448
+ return "enrichers require Python pipeline"
449
+ if self._registry.get_diff_analyzers("python"):
450
+ return "diff analyzers require Python pipeline"
451
+ return None
452
+
453
+ @staticmethod
454
+ def _rust_core_parallel_workers(parallel: int | bool) -> int | None:
455
+ if parallel is True:
456
+ return os.cpu_count() or 1
457
+ if isinstance(parallel, int) and parallel > 0:
458
+ return parallel
459
+ return None
460
+
461
+ @staticmethod
462
+ def _rust_core_batch_parallel_plan(
463
+ file_count: int,
464
+ parallel: int | bool,
465
+ ) -> tuple[bool, int | None, bool]:
466
+ """Return Rust batch parallel settings for the opt-in Rust product lane."""
467
+ if file_count <= 1:
468
+ return False, 1 if file_count == 1 else None, False
469
+ cpu_count = os.cpu_count() or 1
470
+ if parallel is True:
471
+ return True, min(cpu_count, file_count), False
472
+ if isinstance(parallel, int) and not isinstance(parallel, bool) and parallel > 0:
473
+ return True, min(parallel, file_count), False
474
+ return True, min(cpu_count, file_count), True
475
+
476
+ def diff(self, source: Source) -> SemanticDiff:
477
+ """
478
+ Run the full semantic diff pipeline on a ``Source``.
479
+
480
+ Parameters
481
+ ----------
482
+ source:
483
+ Any ``Source`` subclass (``GitSource``, ``FileSource``,
484
+ ``StringSource``, ``PatchSource``).
485
+
486
+ Returns
487
+ -------
488
+ SemanticDiff
489
+ Immutable result object.
490
+ """
491
+ profiler = self._new_profiler()
492
+ with profiler.phase("source_loading"):
493
+ old_content, new_content, filename, language_hint = source.get_content()
494
+ return self._run_pipeline(
495
+ old_content,
496
+ new_content,
497
+ filename,
498
+ language_hint,
499
+ _profiler=profiler,
500
+ )
501
+
502
+ def diff_strings(
503
+ self,
504
+ old: str,
505
+ new: str,
506
+ filename: str,
507
+ language_hint: str | None = None,
508
+ parser_plugin_id: str | None = None,
509
+ ) -> SemanticDiff:
510
+ """Convenience method for in-memory diffs."""
511
+ return self._run_pipeline(
512
+ old,
513
+ new,
514
+ filename,
515
+ language_hint,
516
+ parser_plugin_id=parser_plugin_id,
517
+ )
518
+
519
+ def detect_all(
520
+ self,
521
+ content: str,
522
+ candidates: list[str] | None = None,
523
+ preferred_plugins: dict[str, str] | None = None,
524
+ plugin_id: str | None = None,
525
+ ) -> list[DetectionResult]:
526
+ """Return every parser that claims to handle *content*, ranked by priority.
527
+
528
+ Each registered parser is asked via its own Wasm ``detect-language``
529
+ function — no regex heuristics in Python.
530
+
531
+ Parameters
532
+ ----------
533
+ content:
534
+ Source snippet to identify (first 4 KB are inspected).
535
+ candidates:
536
+ Optional shortlist of language IDs. When given, only parsers that
537
+ handle at least one of those IDs are consulted.
538
+
539
+ Returns
540
+ -------
541
+ list[DetectionResult]
542
+ Possibly empty. First element is the most confident match.
543
+ """
544
+ return self._registry.detect_by_content(
545
+ content,
546
+ candidates,
547
+ preferred_plugins,
548
+ plugin_id,
549
+ )
550
+
551
+ def supported_languages(self) -> list[str]:
552
+ """Return language IDs reported by registered parser plugins.
553
+
554
+ This is the API used by the playground language picker. The values
555
+ come from each parser plugin's ``language-ids`` export, so plugin
556
+ authors own aliases and displayable IDs in one place.
557
+ """
558
+ return self._registry.language_ids()
559
+
560
+ def language_info(self) -> list[LanguageInfoGroup]:
561
+ """Return rich parser-plugin metadata grouped by language ID."""
562
+ return self._registry.language_info()
563
+
564
+ def detect_language(
565
+ self,
566
+ content: str,
567
+ candidates: list[str] | None = None,
568
+ preferred_plugins: dict[str, str] | None = None,
569
+ plugin_id: str | None = None,
570
+ ) -> DetectionResult | None:
571
+ """Return the best-matching language for *content*, or ``None``.
572
+
573
+ Convenience wrapper around :meth:`detect_all` that returns only the
574
+ top result.
575
+ """
576
+ results = self.detect_all(content, candidates, preferred_plugins, plugin_id)
577
+ return results[0] if results else None
578
+
579
+ def playground_example(
580
+ self,
581
+ language: str,
582
+ plugin_id: str | None = None,
583
+ ) -> dict[str, str] | None:
584
+ """Return the ``{\"old\": ..., \"new\": ...}`` demo pair from the plugin
585
+ that owns *language*, or ``None`` if unavailable."""
586
+ return self._registry.example_for(language, plugin_id)
587
+
588
+ def parse(
589
+ self,
590
+ content: str,
591
+ filename: str,
592
+ language_hint: str | None = None,
593
+ ) -> tuple[SemanticNode, str]:
594
+ """
595
+ Parse *content* as *filename* and return ``(SemanticNode, language)``.
596
+
597
+ This runs pipeline stages 1–7 (language detection, preprocessing,
598
+ tree-sitter parsing, trivia stripping, and Wasm plugin dispatch) and
599
+ respects the parse-tree cache if one is configured.
600
+
601
+ Raises :class:`~intentumdiff.plugins.exceptions.PluginNotFoundError`
602
+ when no registered parser supports *filename*.
603
+
604
+ Parameters
605
+ ----------
606
+ content:
607
+ Raw source text of the file.
608
+ filename:
609
+ File name used for language detection and debug messages.
610
+ language_hint:
611
+ Optional language override; if omitted the registry detects it.
612
+
613
+ Returns
614
+ -------
615
+ tuple[SemanticNode, str]
616
+ ``(tree, language)`` where *tree* is the fully processed semantic
617
+ tree and *language* is the detected language identifier.
618
+ """
619
+ parser, language = self._registry.detect_parser(filename, content, language_hint)
620
+ adaptive_fuel = self._config.plugin_fuel
621
+
622
+ if parser.parser_mode == "interpret-cst":
623
+ content = parser.preprocess_source(content)
624
+ cst_json = self._parse(content, parser, language, filename)
625
+ if len(cst_json) > self._config.max_cst_bytes:
626
+ raise ValueError(
627
+ f"CST JSON for {filename!r} is {len(cst_json):,} characters, "
628
+ f"exceeding the limit of {self._config.max_cst_bytes:,} "
629
+ "(DiffConfig.max_cst_bytes)."
630
+ )
631
+ trivia_types = list(parser.trivia_node_types) + self._config.extra_trivia_types
632
+ filtered = _strip_trivia_impl(cst_json, trivia_types)
633
+ cst_nodes = _count_cst_nodes(filtered)
634
+ adaptive_fuel = _fuel_budget(self._config.plugin_fuel, 20_000_000 + cst_nodes * 200_000)
635
+ input_ = filtered
636
+ else:
637
+ input_ = content
638
+
639
+ if self._cache is not None:
640
+ node = self._cached_process(parser, input_, language, filename, adaptive_fuel)
641
+ else:
642
+ node = parser.process(input_, language, filename, fuel=adaptive_fuel)
643
+
644
+ return node, language
645
+
646
+ def render(self, diff: SemanticDiff, output_format: str = "terminal") -> str:
647
+ """
648
+ Render a ``SemanticDiff`` to a plain string.
649
+
650
+ Parameters
651
+ ----------
652
+ diff:
653
+ The diff result to render.
654
+ output_format:
655
+ Output format. Built-in: ``"terminal"`` (default, no ANSI),
656
+ ``"terminal-color"`` (ANSI via renderer plugin), ``"json"``.
657
+ Plugin-provided: ``"patch"``, ``"html"``, ``"llm"``, or any
658
+ format name registered by a third-party renderer plugin.
659
+
660
+ Returns
661
+ -------
662
+ str
663
+ The rendered string. ``"terminal"`` output contains no ANSI codes.
664
+ """
665
+ if output_format == "json":
666
+ return diff.model_dump_json(indent=2)
667
+
668
+ if output_format == "terminal":
669
+ # Inline Python renderer — always available, no ANSI codes.
670
+ from io import StringIO
671
+
672
+ from rich.console import Console as _Console
673
+
674
+ buf = StringIO()
675
+ console = _Console(file=buf, highlight=False, no_color=True)
676
+ console.print(
677
+ f"Semantic diff: {diff.old_filename} -> {diff.new_filename} ({diff.language})"
678
+ )
679
+ for change in diff.changes:
680
+ ct_str = (
681
+ change.change_type.value
682
+ if isinstance(change.change_type, ChangeType)
683
+ else str(change.change_type)
684
+ )
685
+ console.print(f" {ct_str:14} {change.description}")
686
+ return buf.getvalue()
687
+
688
+ try:
689
+ renderer = self._registry.get_renderer(output_format)
690
+ except KeyError as exc:
691
+ raise ValueError(str(exc)) from exc
692
+ diff_json = diff.model_dump_json()
693
+ adaptive_fuel = _fuel_budget(self._config.plugin_fuel, 20_000_000 + len(diff_json) * 100)
694
+ return renderer.render(diff_json, fuel=adaptive_fuel)
695
+
696
+ def _diff_commit_certified_json(
697
+ self,
698
+ repo_path: str | os.PathLike[str] = ".",
699
+ old_ref: str = "HEAD",
700
+ new_ref: str = "",
701
+ ) -> RustCoreCommitJsonAttempt:
702
+ """Return Rust-certified CommitDiff JSON bytes for narrow Python batches."""
703
+ _validate_git_ref(old_ref)
704
+ _validate_git_ref(new_ref)
705
+ rust_batch_skip_reason = self._rust_core_python_batch_skip_reason()
706
+ if rust_batch_skip_reason is not None:
707
+ _raise_rust_only_gate_error(rust_batch_skip_reason)
708
+ return RustCoreCommitJsonAttempt(fallback_reason=rust_batch_skip_reason)
709
+
710
+ if not new_ref:
711
+ rust_batch_parallel, rust_batch_workers, _rust_batch_parallel_auto = (
712
+ True,
713
+ self._rust_core_parallel_workers(self._config.parallel),
714
+ True,
715
+ )
716
+ working_tree_attempt = try_rust_core_working_tree_commit_json(
717
+ repo_path=repo_path,
718
+ old_ref=old_ref,
719
+ config=self._config,
720
+ parallel=rust_batch_parallel,
721
+ max_workers=rust_batch_workers,
722
+ )
723
+ if working_tree_attempt.used:
724
+ return working_tree_attempt
725
+ if working_tree_attempt.fallback_reason and not (
726
+ "does not expose diff_working_tree_python_commit_json"
727
+ in working_tree_attempt.fallback_reason
728
+ or working_tree_attempt.fallback_reason.startswith("unavailable:")
729
+ ):
730
+ _raise_rust_only_gate_error(working_tree_attempt.fallback_reason)
731
+ return working_tree_attempt
732
+
733
+ from intentumdiff.plugins.builtins import python_parser_entry
734
+ from intentumdiff.sources.git_source import (
735
+ collect_working_tree_python_sources_fast,
736
+ iter_changed_sources,
737
+ )
738
+
739
+ source_started = time.perf_counter()
740
+ fast_collection = (
741
+ collect_working_tree_python_sources_fast(repo_path, old_ref) if not new_ref else None
742
+ )
743
+ if fast_collection is None:
744
+ sources = list(iter_changed_sources(repo_path, old_ref, new_ref))
745
+ fast_fallback_reason = None
746
+ else:
747
+ sources, fast_fallback_reason = fast_collection
748
+ source_collection_ms = (time.perf_counter() - source_started) * 1000
749
+ if fast_fallback_reason is not None:
750
+ _raise_rust_only_gate_error(fast_fallback_reason)
751
+ return RustCoreCommitJsonAttempt(fallback_reason=fast_fallback_reason)
752
+ if not sources:
753
+ return RustCoreCommitJsonAttempt()
754
+
755
+ parser_wasm_path = python_parser_entry()
756
+ request_files: list[dict[str, Any]] = []
757
+ source_filter_started = time.perf_counter()
758
+ for old_content, new_content, old_path, new_path, staging_status in sources:
759
+ effective_new_path = new_path if new_path is not None else old_path
760
+ file_lifecycle = _infer_file_lifecycle(old_content, new_content, staging_status)
761
+ old_path_lower = old_path.lower()
762
+ effective_new_path_lower = effective_new_path.lower()
763
+ if not (
764
+ old_path_lower.endswith((".py", ".pyi"))
765
+ or effective_new_path_lower.endswith((".py", ".pyi"))
766
+ ):
767
+ fallback_attempt = RustCoreCommitJsonAttempt(
768
+ fallback_reason="certified commit JSON requires all changed files to be Python"
769
+ )
770
+ _raise_rust_only_gate_error(fallback_attempt.fallback_reason)
771
+ return fallback_attempt
772
+ # Protected guardrail rules currently apply only to config/profile
773
+ # languages, not Python. Avoid a per-file policy search here; any
774
+ # actual policy-file change is excluded by the Python-only gate.
775
+ request_file: dict[str, Any] = {
776
+ "old_source": old_content,
777
+ "new_source": new_content,
778
+ "old_filename": old_path,
779
+ "new_filename": effective_new_path,
780
+ "language": "python",
781
+ "parser_plugin_id": "python",
782
+ "parser_wasm_path": parser_wasm_path,
783
+ "file_lifecycle": file_lifecycle,
784
+ }
785
+ if staging_status is not None:
786
+ request_file["staging_status"] = staging_status
787
+ request_files.append(request_file)
788
+ source_filter_ms = (time.perf_counter() - source_filter_started) * 1000
789
+
790
+ rust_batch_parallel, rust_batch_workers, _rust_batch_parallel_auto = (
791
+ self._rust_core_batch_parallel_plan(len(request_files), self._config.parallel)
792
+ )
793
+ attempt = try_rust_core_commit_json(
794
+ files=request_files,
795
+ old_ref=old_ref,
796
+ new_ref=new_ref,
797
+ config=self._config,
798
+ parallel=rust_batch_parallel,
799
+ max_workers=rust_batch_workers,
800
+ )
801
+ if not attempt.used:
802
+ _raise_rust_only_gate_error(attempt.fallback_reason)
803
+ return attempt
804
+ adapter_phase_timings = [
805
+ {
806
+ "name": "source_collection",
807
+ "duration_ms": round(source_collection_ms, 3),
808
+ "shared": True,
809
+ },
810
+ {
811
+ "name": "rust_core_commit_source_filtering",
812
+ "duration_ms": round(source_filter_ms, 3),
813
+ "shared": True,
814
+ },
815
+ *attempt.adapter_phase_timings,
816
+ ]
817
+ control = dict(attempt.control)
818
+ control["source_collection_ms"] = round(source_collection_ms, 3)
819
+ control["source_filter_ms"] = round(source_filter_ms, 3)
820
+ return RustCoreCommitJsonAttempt(
821
+ control=control,
822
+ commit_diff_json=attempt.commit_diff_json,
823
+ adapter_phase_timings=adapter_phase_timings,
824
+ backend_version=attempt.backend_version,
825
+ )
826
+
827
+ def diff_commit(
828
+ self,
829
+ repo_path: str | os.PathLike[str] = ".",
830
+ old_ref: str = "HEAD",
831
+ new_ref: str = "",
832
+ ) -> list[SemanticDiff]:
833
+ _validate_git_ref(old_ref)
834
+ _validate_git_ref(new_ref)
835
+ """
836
+ Diff every file that changed between two git refs.
837
+
838
+ Files that have no registered parser are silently skipped.
839
+
840
+ Parameters
841
+ ----------
842
+ repo_path:
843
+ Path to (any directory inside) the git repository.
844
+ old_ref:
845
+ Old git ref (commit SHA, tag, branch). Defaults to ``"HEAD"``.
846
+ new_ref:
847
+ New git ref. Defaults to ``""`` (working tree — diff against
848
+ current unsaved/uncommitted files).
849
+
850
+ Returns
851
+ -------
852
+ list[SemanticDiff]
853
+ One entry per changed file that could be parsed.
854
+ """
855
+ from intentumdiff.sources.git_source import iter_changed_sources
856
+
857
+ source_started = time.perf_counter()
858
+ sources = list(iter_changed_sources(repo_path, old_ref, new_ref))
859
+ source_collection_ms = (time.perf_counter() - source_started) * 1000
860
+ rust_batch_results: dict[int, SemanticDiff] = {}
861
+ rust_batch_attempted: set[int] = set()
862
+ rust_batch_skip_reason = self._rust_core_python_batch_skip_reason()
863
+ if rust_batch_skip_reason is None and sources:
864
+ from intentumdiff.plugins.builtins import python_parser_entry
865
+
866
+ parser_wasm_path = python_parser_entry()
867
+ request_files: list[dict[str, Any]] = []
868
+ request_to_source_index: list[int] = []
869
+ rust_batch_source_filter_started = time.perf_counter()
870
+ for idx, item in enumerate(sources):
871
+ old_content, new_content, old_path, new_path, _staging_status = item
872
+ effective_new_path = new_path if new_path is not None else old_path
873
+ file_lifecycle = _infer_file_lifecycle(old_content, new_content, _staging_status)
874
+ old_path_lower = old_path.lower()
875
+ effective_new_path_lower = effective_new_path.lower()
876
+ if not (
877
+ old_path_lower.endswith((".py", ".pyi"))
878
+ or effective_new_path_lower.endswith((".py", ".pyi"))
879
+ ):
880
+ continue
881
+ # Protected guardrail rules currently apply only to
882
+ # config/profile languages. Certified Python batches can skip
883
+ # the expensive policy lookup; policy-file changes are not
884
+ # Python files and therefore remain on the fallback path.
885
+ request_to_source_index.append(idx)
886
+ request_files.append(
887
+ {
888
+ "old_source": old_content,
889
+ "new_source": new_content,
890
+ "old_filename": old_path,
891
+ "new_filename": effective_new_path,
892
+ "language": "python",
893
+ "parser_plugin_id": "python",
894
+ "parser_wasm_path": parser_wasm_path,
895
+ "file_lifecycle": file_lifecycle,
896
+ }
897
+ )
898
+ rust_batch_source_filter_ms = (
899
+ time.perf_counter() - rust_batch_source_filter_started
900
+ ) * 1000
901
+ if request_files:
902
+ rust_batch_attempted = set(request_to_source_index)
903
+ (
904
+ rust_batch_parallel,
905
+ rust_batch_workers,
906
+ rust_batch_parallel_auto,
907
+ ) = self._rust_core_batch_parallel_plan(
908
+ len(request_files),
909
+ self._config.parallel,
910
+ )
911
+ rust_batch_started = time.perf_counter()
912
+ rust_batch = try_rust_core_batch_diffs(
913
+ files=request_files,
914
+ config=self._config,
915
+ parallel=rust_batch_parallel,
916
+ max_workers=rust_batch_workers,
917
+ )
918
+ rust_batch_ms = (time.perf_counter() - rust_batch_started) * 1000
919
+ rust_batch_fallback_count = int(
920
+ rust_batch.batch_metadata.get("fallback_count")
921
+ or len(rust_batch.fallback_reasons)
922
+ )
923
+ # A per-item batch fallback is NOT a Python fallback: the declined
924
+ # files re-run through ``_run_one`` -> ``_run_pipeline`` below, whose
925
+ # per-stage Rust finalize serves them natively. The RUST_ONLY gate
926
+ # therefore does not fire here; it fires inside ``_run_pipeline`` only
927
+ # if an item genuinely reaches a token-level fallback (and the fan-out
928
+ # re-raises that RustOnlyGateError instead of swallowing it).
929
+ rust_merge_started = time.perf_counter()
930
+ rust_merged_results: list[tuple[int, SemanticDiff]] = []
931
+ for request_index, diff in rust_batch.diffs.items():
932
+ if request_index < 0 or request_index >= len(request_to_source_index):
933
+ continue
934
+ source_index = request_to_source_index[request_index]
935
+ _old_content, _new_content, _old_path, _new_path, staging_status = sources[
936
+ source_index
937
+ ]
938
+ diff = _apply_file_lifecycle_to_diff(
939
+ diff,
940
+ _infer_file_lifecycle(_old_content, _new_content, staging_status),
941
+ )
942
+ if staging_status is not None:
943
+ diff = diff.model_copy(update={"staging_status": staging_status})
944
+ rust_merged_results.append((source_index, diff))
945
+ rust_commit_merge_ms = (time.perf_counter() - rust_merge_started) * 1000
946
+ for source_index, diff in rust_merged_results:
947
+ rust_batch_results[source_index] = self._attach_commit_batch_phase_timings(
948
+ diff,
949
+ source_collection_ms=source_collection_ms,
950
+ rust_batch_ms=rust_batch_ms,
951
+ rust_commit_merge_ms=rust_commit_merge_ms,
952
+ rust_adapter_phase_timings=[
953
+ {
954
+ "name": "rust_core_commit_source_filtering",
955
+ "duration_ms": round(rust_batch_source_filter_ms, 3),
956
+ "shared": True,
957
+ },
958
+ *rust_batch.adapter_phase_timings,
959
+ ],
960
+ rust_batch_parallel_auto=rust_batch_parallel_auto,
961
+ rust_batch_parallel_workers=rust_batch_workers,
962
+ rust_batch_size=len(request_files),
963
+ rust_batch_fallback_count=rust_batch_fallback_count,
964
+ )
965
+ if rust_batch.fallback_reasons or rust_batch.fallback_reason:
966
+ logger.debug(
967
+ "Rust core commit batch used %d/%d files; fallbacks=%s reason=%s",
968
+ len(rust_batch_results),
969
+ len(request_files),
970
+ rust_batch.fallback_reasons,
971
+ rust_batch.fallback_reason,
972
+ )
973
+ elif rust_batch_skip_reason is not None:
974
+ # The commit BATCH shortcut was skipped (guardrails / enrichers / analyzers
975
+ # / diagnostics / trivia / policy / rust-core disabled). Every source now
976
+ # runs through ``_run_one`` -> ``_run_pipeline`` below, which is the native
977
+ # per-stage Rust finalize path; the RUST_ONLY gate fires there, and only if
978
+ # an item genuinely reaches a token-level fallback. A batch skip is not
979
+ # itself a Python fallback, so no gate fires here.
980
+ logger.debug("Rust core commit batch skipped: %s", rust_batch_skip_reason)
981
+
982
+ def _run_one(
983
+ differ: SemanticDiffer,
984
+ item: tuple[str, str, str, str | None, str | None],
985
+ *,
986
+ include_source_collection: bool = False,
987
+ ) -> SemanticDiff:
988
+ old_content, new_content, old_path, new_path, staging_status = item
989
+ profiler = differ._new_profiler()
990
+ if include_source_collection:
991
+ profiler.record("source_collection", source_collection_ms, shared=True)
992
+ result = differ._run_pipeline(
993
+ old_content,
994
+ new_content,
995
+ old_path,
996
+ None,
997
+ new_filename=new_path,
998
+ _profiler=profiler,
999
+ )
1000
+ if staging_status is not None:
1001
+ result = result.model_copy(update={"staging_status": staging_status})
1002
+ return result
1003
+
1004
+ _skip_exc = (PluginNotFoundError,)
1005
+ _warn_exc = (PluginFuelExhausted, ValueError, RuntimeError)
1006
+
1007
+ parallel = self._config.parallel
1008
+ if not parallel:
1009
+ # ── Sequential (default) ─────────────────────────────────────────
1010
+ results: list[SemanticDiff] = []
1011
+ for idx, item in enumerate(sources):
1012
+ if idx in rust_batch_results:
1013
+ results.append(rust_batch_results[idx])
1014
+ continue
1015
+ # Files the commit BATCH attempted but declined re-run through the normal
1016
+ # differ with the Rust core ON: the batch re-attempt is one cheap native
1017
+ # call and the per-stage finalize routing then serves. (The old
1018
+ # experimental_rust_core=False fallback differ meant "python pipeline";
1019
+ # post-retirement it means the token-fallback kill switch — a comment-only
1020
+ # commit came back as a token ADDITION instead of style-only.)
1021
+ run_differ = self
1022
+ try:
1023
+ results.append(
1024
+ _run_one(
1025
+ run_differ,
1026
+ item,
1027
+ include_source_collection=idx == 0 and not rust_batch_results,
1028
+ )
1029
+ )
1030
+ except RustOnlyGateError:
1031
+ # A genuine token-level fallback under the RUST_ONLY gate is fatal —
1032
+ # propagate it rather than let ``_warn_exc`` silently drop the file.
1033
+ raise
1034
+ except _skip_exc:
1035
+ logger.debug("Skipping %r — no parser available", item[2])
1036
+ except _warn_exc:
1037
+ logger.warning("Failed to diff %r", item[2], exc_info=True)
1038
+ return results
1039
+ else:
1040
+ # ── Parallel ────────────────────────────────────────────────
1041
+ max_workers = (os.cpu_count() or 1) if parallel is True else int(parallel)
1042
+ executor, worker_local = self._parallel_executor_for(max_workers)
1043
+
1044
+ def _worker_differ() -> SemanticDiffer:
1045
+ differ = getattr(worker_local, "differ", None)
1046
+ if differ is None:
1047
+ differ = SemanticDiffer(config=self._config.model_copy())
1048
+ worker_local.differ = differ
1049
+ return differ
1050
+
1051
+
1052
+ def _run_worker(
1053
+ idx: int,
1054
+ item: tuple[str, str, str, str | None, str | None],
1055
+ ) -> SemanticDiff:
1056
+ # Batch-declined files re-run with the Rust core ON (see the
1057
+ # sequential branch note — the finalize tier serves them).
1058
+ return _run_one(
1059
+ _worker_differ(),
1060
+ item,
1061
+ include_source_collection=idx == 0 and not rust_batch_results,
1062
+ )
1063
+
1064
+ results_map: dict[int, SemanticDiff] = dict(rust_batch_results)
1065
+ if executor is not None:
1066
+ future_to_idx = {
1067
+ executor.submit(_run_worker, i, item): i
1068
+ for i, item in enumerate(sources)
1069
+ if i not in rust_batch_results
1070
+ }
1071
+ for future in as_completed(future_to_idx):
1072
+ idx = future_to_idx[future]
1073
+ try:
1074
+ results_map[idx] = future.result()
1075
+ except RustOnlyGateError:
1076
+ # Fatal under the RUST_ONLY gate — propagate, do not drop.
1077
+ raise
1078
+ except _skip_exc:
1079
+ logger.debug("Skipping %r — no parser available", sources[idx][2])
1080
+ except _warn_exc:
1081
+ logger.warning("Failed to diff %r", sources[idx][2], exc_info=True)
1082
+ return [results_map[i] for i in sorted(results_map)]
1083
+
1084
+ def diff_stream(
1085
+ self,
1086
+ source: Source,
1087
+ ) -> Iterator[Change]:
1088
+ """
1089
+ Yield ``Change`` objects one-by-one from a single-file diff.
1090
+
1091
+ The full pipeline runs before the first item is yielded. This is a
1092
+ convenience wrapper over :meth:`diff` for callers that prefer a
1093
+ generator interface (e.g. ``intentumdiff file --stream``).
1094
+ """
1095
+ diff = self.diff(source)
1096
+ yield from diff.changes
1097
+
1098
+ # ── Pipeline ──────────────────────────────────────────────────────────────
1099
+
1100
+ def _run_pipeline(
1101
+ self,
1102
+ old_content: str,
1103
+ new_content: str,
1104
+ filename: str,
1105
+ language_hint: str | None,
1106
+ *,
1107
+ new_filename: str | None = None,
1108
+ parser_plugin_id: str | None = None,
1109
+ _profiler: _PhaseProfiler | None = None,
1110
+ ) -> SemanticDiff:
1111
+ profiler = _profiler or self._new_profiler()
1112
+ result = self._run_stages_1_to_11(
1113
+ old_content,
1114
+ new_content,
1115
+ filename,
1116
+ language_hint,
1117
+ new_filename=new_filename,
1118
+ parser_plugin_id=parser_plugin_id,
1119
+ profiler=profiler,
1120
+ )
1121
+ # _run_stages_1_to_11 always returns a finished SemanticDiff since the python
1122
+ # pipeline retirement (issue #57 payoff, stage 4b).
1123
+ diff = result
1124
+ diff = _attach_content_type_metadata(diff, old_content, new_content)
1125
+ return self._attach_phase_timings(diff, profiler)
1126
+
1127
+ def _run_diff_analyzers_on_routed(
1128
+ self,
1129
+ routed: SemanticDiff,
1130
+ language: str,
1131
+ filename: str,
1132
+ adaptive_fuel: int,
1133
+ ) -> SemanticDiff:
1134
+ for analyzer in self._registry.get_diff_analyzers(language):
1135
+ try:
1136
+ updated_json = analyzer.analyze_diff(
1137
+ routed.model_dump_json(),
1138
+ language,
1139
+ filename,
1140
+ fuel=adaptive_fuel,
1141
+ )
1142
+ routed = SemanticDiff.model_validate_json(updated_json)
1143
+ except Exception as exc:
1144
+ logger.warning(
1145
+ "Diff analyzer %r failed for %r: %s - skipping",
1146
+ getattr(analyzer, "provenance", "?"),
1147
+ filename,
1148
+ exc,
1149
+ )
1150
+ continue
1151
+ return routed
1152
+
1153
+ def _run_stages_1_to_11(
1154
+ self,
1155
+ old_content: str,
1156
+ new_content: str,
1157
+ filename: str,
1158
+ language_hint: str | None,
1159
+ *,
1160
+ new_filename: str | None = None,
1161
+ parser_plugin_id: str | None = None,
1162
+ profiler: _PhaseProfiler | None = None,
1163
+ ) -> SemanticDiff:
1164
+ """Run pipeline stages 1–11 (parse → trivia-strip → Wasm → GumTree → moves).
1165
+
1166
+ Returns ``SemanticDiff`` for early exits (cache hit, style-only, fallback).
1167
+ """
1168
+ # ── 1 + 2. Language detection ────────────────────────────────────────
1169
+ profiler = profiler or self._new_profiler()
1170
+ with profiler.phase("parser_selection"):
1171
+ parser, language = self._registry.detect_parser(
1172
+ filename,
1173
+ new_content,
1174
+ language_hint,
1175
+ plugin_id=parser_plugin_id,
1176
+ phase_recorder=profiler.record if profiler.enabled else None,
1177
+ )
1178
+ logger.debug("Using parser %r for %r (language=%r)", parser.grammar_id, filename, language)
1179
+
1180
+ diagnostics = DiagnosticsRecorder(
1181
+ enabled=self._config.diagnostics,
1182
+ max_events=self._config.diagnostics_max_events,
1183
+ )
1184
+ engine_telemetry: list[dict[str, Any]] = []
1185
+ parse_mode = parser.parser_mode
1186
+ cache_grammar_id = parser.grammar_id
1187
+ diagnostics.summary.update(
1188
+ {
1189
+ "language": language,
1190
+ "old_filename": filename,
1191
+ "new_filename": new_filename if new_filename is not None else filename,
1192
+ "parser_grammar_id": parser.grammar_id,
1193
+ "effective_grammar_id": cache_grammar_id,
1194
+ "parse_mode": parse_mode,
1195
+ }
1196
+ )
1197
+ diagnostics.record(
1198
+ stage="parser",
1199
+ action="select_parser",
1200
+ rule_id="pipeline.parser_selection",
1201
+ reason="parser selected by registry",
1202
+ metadata={
1203
+ "language_hint": language_hint,
1204
+ "parser_plugin_id": parser_plugin_id,
1205
+ "grammar_id": parser.grammar_id,
1206
+ "effective_grammar_id": cache_grammar_id,
1207
+ "parse_mode": parse_mode,
1208
+ },
1209
+ )
1210
+
1211
+ # Adaptive fuel starts at the configured floor; the interpret-cst branch
1212
+ # raises it based on CST node count before calling the plugin.
1213
+ adaptive_fuel: int = self._config.plugin_fuel
1214
+
1215
+ # Save originals for the enricher pass (enrichers see un-preprocessed source)
1216
+ old_raw, new_raw = old_content, new_content
1217
+ file_lifecycle = _infer_file_lifecycle(old_raw, new_raw)
1218
+ diagnostics.summary["file_lifecycle"] = file_lifecycle
1219
+ with profiler.phase("schema_resolution"):
1220
+ schema_resolution = resolve_schema(
1221
+ content=new_content or old_content,
1222
+ filename=new_filename if new_filename is not None else filename,
1223
+ language=language,
1224
+ )
1225
+ if language.lower() == "xml":
1226
+ # User XML dialects (issue #86): marshal the descriptor registry's
1227
+ # declarative coordinate specs into the Rust matcher. Idempotent per
1228
+ # payload; dialects are match-predicated engine-side, so registering
1229
+ # the full set is safe for every xml diff.
1230
+ _user_profiles, _ = load_user_schema_profiles()
1231
+ _dialects = user_xml_dialects_payload(_user_profiles)
1232
+ if _dialects:
1233
+ try_register_user_xml_dialects(_dialects)
1234
+ schema_fingerprint = schema_cache_fingerprint(schema_resolution)
1235
+ cache_schema_suffix = (
1236
+ "" if schema_fingerprint == "schema:none" else f"|{schema_fingerprint}"
1237
+ )
1238
+ schema_metadata = schema_resolution_metadata(schema_resolution)
1239
+ if schema_metadata is not None:
1240
+ diagnostics.summary["schema"] = schema_metadata
1241
+ diagnostics.record(
1242
+ stage="schema",
1243
+ action="resolve_schema",
1244
+ rule_id="analysis.schema_resolution",
1245
+ reason="JSON/YAML schema source resolved for optional keyed-data hints",
1246
+ metadata=schema_metadata,
1247
+ )
1248
+ with profiler.phase("compile_command_resolution"):
1249
+ compile_metadata = compile_commands_metadata(
1250
+ filename=new_filename if new_filename is not None else filename,
1251
+ language=language,
1252
+ )
1253
+ if compile_metadata is not None:
1254
+ diagnostics.summary["compile_commands"] = compile_metadata
1255
+ diagnostics.record(
1256
+ stage="compile_commands",
1257
+ action="resolve_compile_command",
1258
+ rule_id="analysis.compile_commands",
1259
+ reason="C/C++ compile database context resolved",
1260
+ metadata=compile_metadata,
1261
+ )
1262
+ compile_cache_suffix = (
1263
+ "" if compile_metadata is None else f"|compile:{compile_metadata['fingerprint']}"
1264
+ )
1265
+
1266
+ # Cache state — resolved later once we have preprocessed content.
1267
+ _diff_key: str | None = None
1268
+ _guardrail_cache_bypass = guardrails_may_apply(filename, new_filename, self._config)
1269
+ cache_grammar_id = parser.grammar_id
1270
+
1271
+ if (
1272
+ self._config.experimental_rust_core
1273
+ and language in _rust_certified_languages()
1274
+ and not diagnostics.enabled
1275
+ # ``test_matching_engine`` is the dual-run truthiness override.
1276
+ # When set, it forces the matching stage to either Rust or
1277
+ # Python; the certified Python batch path bypasses that stage
1278
+ # entirely, so we must skip it to make the override meaningful.
1279
+ and self._config.test_matching_engine is None
1280
+ ):
1281
+ rust_core_skip_reason: str | None = None
1282
+ if _guardrail_cache_bypass:
1283
+ rust_core_skip_reason = "guardrails may affect final diff"
1284
+ elif self._config.extra_trivia_types:
1285
+ rust_core_skip_reason = "extra trivia types require Python pipeline"
1286
+ else:
1287
+ with profiler.phase("rust_core_certified_surface_check"):
1288
+ if self._registry.get_enrichers(language):
1289
+ rust_core_skip_reason = "enrichers require Python pipeline"
1290
+ elif self._registry.get_diff_analyzers(language):
1291
+ rust_core_skip_reason = "diff analyzers require Python pipeline"
1292
+
1293
+ if rust_core_skip_reason is None:
1294
+ with profiler.phase("rust_core_batch_execution"):
1295
+ rust_batch_attempt = try_rust_core_batch_diff(
1296
+ old_source=old_content,
1297
+ new_source=new_content,
1298
+ old_filename=filename,
1299
+ new_filename=new_filename if new_filename is not None else filename,
1300
+ parser_wasm_path=parser.wasm_path,
1301
+ language=language,
1302
+ parser_plugin_id=parser.plugin_id or parser.grammar_id,
1303
+ config=self._config,
1304
+ file_lifecycle=file_lifecycle,
1305
+ )
1306
+ if rust_batch_attempt.used and rust_batch_attempt.diff is not None:
1307
+ diagnostics.record(
1308
+ stage="rust_core",
1309
+ action="batch_final_diff",
1310
+ rule_id="rust_core.batch_v4",
1311
+ reason="Rust core produced a certified public diff",
1312
+ metadata={
1313
+ "engine": rust_batch_attempt.diff.metadata.get("rust_core", {}).get(
1314
+ "engine"
1315
+ ),
1316
+ "stage": rust_batch_attempt.diff.metadata.get("rust_core", {}).get(
1317
+ "stage"
1318
+ ),
1319
+ },
1320
+ )
1321
+ return _apply_file_lifecycle_to_diff(
1322
+ rust_batch_attempt.diff,
1323
+ file_lifecycle,
1324
+ )
1325
+ # The certified BATCH shortcut declined — fall through to the native
1326
+ # routed finalize path below. This is a Rust→Rust transition (the
1327
+ # routed path finalizes through the Rust core for every certified
1328
+ # language), NOT a Python fallback, so the RUST_ONLY gate does NOT
1329
+ # fire here. It fires only at the genuine token-level fallback sites
1330
+ # (parse errors / finalize declined / rust core disabled), where the
1331
+ # coarse Python token diff — the last non-Rust producer — takes over.
1332
+ logger.debug(
1333
+ "Rust core batch declined for %r: %s; using routed finalize",
1334
+ filename,
1335
+ rust_batch_attempt.fallback_reason or rust_core_skip_reason,
1336
+ )
1337
+ else:
1338
+ logger.debug(
1339
+ "Rust core batch path skipped for %r: %s; using routed finalize",
1340
+ filename,
1341
+ rust_core_skip_reason,
1342
+ )
1343
+
1344
+ if parse_mode == "interpret-cst":
1345
+ trivia_types = list(parser.trivia_node_types) + self._config.extra_trivia_types
1346
+
1347
+ # ── 2.5. Source preprocessing ────────────────────────────────────
1348
+ with profiler.phase("source_preprocessing"):
1349
+ old_content = parser.preprocess_source(old_content)
1350
+ new_content = parser.preprocess_source(new_content)
1351
+
1352
+ # ── 2.6. Diff-level cache check ───────────────────────────────────
1353
+ if self._cache is not None and not diagnostics.enabled and not _guardrail_cache_bypass:
1354
+ _wasm_hash = self._wasm_hash_for(parser)
1355
+ _diff_key = self._cache.diff_key(
1356
+ old_content,
1357
+ new_content,
1358
+ f"{cache_grammar_id}{cache_schema_suffix}{compile_cache_suffix}",
1359
+ _wasm_hash,
1360
+ )
1361
+ _cached = self._cache.get_diff(_diff_key)
1362
+ if _cached is not None:
1363
+ logger.debug("Diff cache hit for %r", filename)
1364
+ return _apply_file_lifecycle_to_diff(
1365
+ SemanticDiff.model_validate_json(_cached),
1366
+ file_lifecycle,
1367
+ )
1368
+
1369
+ old_cst_json = self._parse(
1370
+ old_content,
1371
+ parser,
1372
+ language,
1373
+ filename,
1374
+ profiler=profiler,
1375
+ )
1376
+ new_cst_json = self._parse(
1377
+ new_content,
1378
+ parser,
1379
+ language,
1380
+ filename,
1381
+ profiler=profiler,
1382
+ )
1383
+
1384
+ # ── CST size guard ────────────────────────────────────────────────
1385
+ for _label, _cst in (("old", old_cst_json), ("new", new_cst_json)):
1386
+ if len(_cst) > self._config.max_cst_bytes:
1387
+ raise ValueError(
1388
+ f"CST JSON for the {_label!r} version of {filename!r} is "
1389
+ f"{len(_cst):,} characters, exceeding the limit of "
1390
+ f"{self._config.max_cst_bytes:,} "
1391
+ "(DiffConfig.max_cst_bytes). Increase the limit or switch "
1392
+ "to a full-parse plugin."
1393
+ )
1394
+
1395
+ # ── 5. Trivia stripping ───────────────────────────────────────────
1396
+ with profiler.phase("trivia_stripping"):
1397
+ old_filtered = _strip_trivia_impl(old_cst_json, trivia_types)
1398
+ new_filtered = _strip_trivia_impl(new_cst_json, trivia_types)
1399
+
1400
+ # ── 6. Style-only shortcut ────────────────────────────────────────
1401
+ with profiler.phase("style_hashing"):
1402
+ old_hash = _compute_structural_hash_for_tree(old_filtered)
1403
+ new_hash = _compute_structural_hash_for_tree(new_filtered)
1404
+
1405
+ if old_hash == new_hash:
1406
+ logger.debug("Style-only diff for %r", filename)
1407
+ diagnostics.record(
1408
+ stage="style",
1409
+ action="style_only_shortcut",
1410
+ rule_id="pipeline.style_only_shortcut",
1411
+ reason="filtered CST structural hashes are equal",
1412
+ metadata={"old_hash": old_hash, "new_hash": new_hash},
1413
+ )
1414
+ style_evidence = build_style_only_evidence(
1415
+ old_source=old_content,
1416
+ new_source=new_content,
1417
+ language=language,
1418
+ old_cst_json=old_cst_json,
1419
+ new_cst_json=new_cst_json,
1420
+ )
1421
+ metadata: dict[str, Any] = {}
1422
+ if style_evidence.ignored_style_changes:
1423
+ metadata["ignored_style_changes"] = style_evidence.ignored_style_changes
1424
+ for group in style_evidence.change_groups:
1425
+ diagnostics.record_group(stage="invariance", group=group)
1426
+ if diagnostics.enabled:
1427
+ diagnostics.summary.update(
1428
+ {
1429
+ "early_exit": "style_only",
1430
+ "final_change_count": 0,
1431
+ "final_group_count": len(style_evidence.change_groups),
1432
+ }
1433
+ )
1434
+ metadata["diagnostics"] = diagnostics.snapshot()
1435
+ style_diff = SemanticDiff.style_only(
1436
+ filename,
1437
+ new_filename if new_filename is not None else filename,
1438
+ language,
1439
+ change_groups=style_evidence.change_groups,
1440
+ metadata=metadata,
1441
+ )
1442
+ style_diff = _apply_file_lifecycle_to_diff(style_diff, file_lifecycle)
1443
+ return apply_guardrails_to_diff(
1444
+ style_diff,
1445
+ old_tree=None,
1446
+ new_tree=None,
1447
+ old_source=old_raw,
1448
+ new_source=new_raw,
1449
+ config=self._config,
1450
+ diagnostics=diagnostics,
1451
+ )
1452
+
1453
+ # ── 6.5. Adaptive fuel budget ─────────────────────────────────────
1454
+ old_cst_nodes = _count_cst_nodes(old_filtered)
1455
+ new_cst_nodes = _count_cst_nodes(new_filtered)
1456
+ max_cst_nodes = max(old_cst_nodes, new_cst_nodes)
1457
+ adaptive_fuel = _fuel_budget(
1458
+ self._config.plugin_fuel,
1459
+ 20_000_000 + max_cst_nodes * 200_000,
1460
+ )
1461
+ logger.debug(
1462
+ "[%s] CST: old=%d bytes/%d nodes, new=%d bytes/%d nodes — fuel=%d",
1463
+ filename,
1464
+ len(old_filtered),
1465
+ old_cst_nodes,
1466
+ len(new_filtered),
1467
+ new_cst_nodes,
1468
+ adaptive_fuel,
1469
+ )
1470
+ diagnostics.record(
1471
+ stage="parse",
1472
+ action="interpret_cst",
1473
+ rule_id="pipeline.interpret_cst_parse",
1474
+ reason="host CST parsed, stripped, and sized",
1475
+ metadata={
1476
+ "old_cst_bytes": len(old_filtered),
1477
+ "new_cst_bytes": len(new_filtered),
1478
+ "old_cst_nodes": old_cst_nodes,
1479
+ "new_cst_nodes": new_cst_nodes,
1480
+ "adaptive_fuel": adaptive_fuel,
1481
+ },
1482
+ )
1483
+
1484
+ # ── 7. Plugin dispatch (with parse-tree cache) ───────────────────
1485
+ try:
1486
+ with profiler.phase("wasm_plugin_execution"):
1487
+ if self._cache is not None:
1488
+ old_tree = self._cached_process(
1489
+ parser,
1490
+ old_filtered,
1491
+ language,
1492
+ filename,
1493
+ adaptive_fuel,
1494
+ cache_grammar_id=cache_grammar_id,
1495
+ )
1496
+ new_tree = self._cached_process(
1497
+ parser,
1498
+ new_filtered,
1499
+ language,
1500
+ filename,
1501
+ adaptive_fuel,
1502
+ cache_grammar_id=cache_grammar_id,
1503
+ )
1504
+ else:
1505
+ old_tree = parser.process(
1506
+ old_filtered, language, filename, fuel=adaptive_fuel
1507
+ )
1508
+ new_tree = parser.process(
1509
+ new_filtered, language, filename, fuel=adaptive_fuel
1510
+ )
1511
+ except PluginFuelExhausted as exc:
1512
+ raise PluginFuelExhausted(
1513
+ exc.plugin_id,
1514
+ exc.fuel,
1515
+ f"file={filename!r}, CST nodes up to {max_cst_nodes:,}",
1516
+ ) from exc
1517
+
1518
+ else:
1519
+ # ── 7 (full-parse). Plugin owns parsing — send raw source ─────────
1520
+ # Stages 3-6 are skipped: the plugin handles its own CST construction
1521
+ # and trivia filtering. The style-only shortcut is not available here
1522
+ # because there is no host-side CST to hash.
1523
+ max_source_bytes = max(
1524
+ len(old_content.encode("utf-8")),
1525
+ len(new_content.encode("utf-8")),
1526
+ )
1527
+ adaptive_fuel = _fuel_budget(
1528
+ self._config.plugin_fuel,
1529
+ 50_000_000 + max_source_bytes * 150_000,
1530
+ )
1531
+ logger.debug("full-parse mode for %r via %r", filename, parser.grammar_id)
1532
+
1533
+ # Diff-level cache check for full-parse mode.
1534
+ if self._cache is not None and not diagnostics.enabled and not _guardrail_cache_bypass:
1535
+ _wasm_hash = self._wasm_hash_for(parser)
1536
+ _diff_key = self._cache.diff_key(
1537
+ old_content,
1538
+ new_content,
1539
+ f"{parser.grammar_id}{cache_schema_suffix}{compile_cache_suffix}",
1540
+ _wasm_hash,
1541
+ )
1542
+ _cached = self._cache.get_diff(_diff_key)
1543
+ if _cached is not None:
1544
+ logger.debug("Diff cache hit for %r (full-parse)", filename)
1545
+ return _apply_file_lifecycle_to_diff(
1546
+ SemanticDiff.model_validate_json(_cached),
1547
+ file_lifecycle,
1548
+ )
1549
+
1550
+ with profiler.phase("wasm_plugin_execution"):
1551
+ old_side_absent = file_lifecycle == "added" and old_content == ""
1552
+ new_side_absent = file_lifecycle == "deleted" and new_content == ""
1553
+ if self._cache is not None:
1554
+ old_tree = (
1555
+ _empty_semantic_tree(language)
1556
+ if old_side_absent
1557
+ else self._cached_process(
1558
+ parser, old_content, language, filename, adaptive_fuel
1559
+ )
1560
+ )
1561
+ new_tree = (
1562
+ _empty_semantic_tree(language)
1563
+ if new_side_absent
1564
+ else self._cached_process(
1565
+ parser, new_content, language, filename, adaptive_fuel
1566
+ )
1567
+ )
1568
+ else:
1569
+ old_tree = (
1570
+ _empty_semantic_tree(language)
1571
+ if old_side_absent
1572
+ else parser.process(old_content, language, filename, fuel=adaptive_fuel)
1573
+ )
1574
+ new_tree = (
1575
+ _empty_semantic_tree(language)
1576
+ if new_side_absent
1577
+ else parser.process(new_content, language, filename, fuel=adaptive_fuel)
1578
+ )
1579
+ diagnostics.record(
1580
+ stage="parse",
1581
+ action="full_parse",
1582
+ rule_id="pipeline.full_parse",
1583
+ reason="plugin parsed raw source",
1584
+ metadata={"adaptive_fuel": adaptive_fuel},
1585
+ )
1586
+
1587
+ engine_telemetry.extend(_drain_plugin_telemetry(parser))
1588
+ _record_engine_telemetry(diagnostics, engine_telemetry)
1589
+
1590
+ # ── 7b. Enrichment pass (optional) ─────────────────────────────────────────
1591
+ with profiler.phase("enricher_execution"):
1592
+ enrichers = self._registry.get_enrichers(language)
1593
+ if enrichers:
1594
+ old_tree_json = old_tree.model_dump_json()
1595
+ new_tree_json = new_tree.model_dump_json()
1596
+ for enricher in enrichers:
1597
+ old_tree_json = enricher.enrich(
1598
+ old_tree_json, old_raw, language, filename, fuel=adaptive_fuel
1599
+ )
1600
+ new_tree_json = enricher.enrich(
1601
+ new_tree_json, new_raw, language, filename, fuel=adaptive_fuel
1602
+ )
1603
+ engine_telemetry.extend(_drain_plugin_telemetry(enricher))
1604
+ _record_engine_telemetry(diagnostics, engine_telemetry)
1605
+ try:
1606
+ old_tree = SemanticNode.model_validate_json(old_tree_json)
1607
+ new_tree = SemanticNode.model_validate_json(new_tree_json)
1608
+ except ValidationError as exc:
1609
+ raise PluginOutputError("enricher", str(exc)) from exc
1610
+ # ── 8. Plugin output validation ────────────────────────────────────────
1611
+ with profiler.phase("semantic_node_validation"):
1612
+ old_tree = _enrich_literal_labels(old_tree, old_raw)
1613
+ new_tree = _enrich_literal_labels(new_tree, new_raw)
1614
+ # Cross-language NodeFacts (issue #70): every tree (all parse branches converge here)
1615
+ # gets language-agnostic structural facts derived in the Rust core, so the intent
1616
+ # explainer has signal for ALL languages, not just Python. Idempotent + graceful.
1617
+ old_tree = enrich_node_facts(old_tree)
1618
+ new_tree = enrich_node_facts(new_tree)
1619
+ # Profile-label enrichment runs in the Rust core (issue #57 port).
1620
+ # The Rust core is AUTHORITATIVE (readiness #90 / #82): no Python
1621
+ # engine fallback — if the backend is unavailable the tree passes
1622
+ # through UNCHANGED (graceful, degraded labels) rather than routing
1623
+ # to a Python enricher, so "if Python didn't exist it still works".
1624
+ # Gated by language so non-profile diffs pay no tree-JSON roundtrip.
1625
+ if language.lower() in {"json", "yaml"}:
1626
+ _identity_fields = tuple(schema_resolution.identity_fields)
1627
+ old_tree = try_rust_profile_label_enrichment(
1628
+ old_tree, old_raw, language, identity_fields=_identity_fields
1629
+ ) or old_tree
1630
+ new_tree = try_rust_profile_label_enrichment(
1631
+ new_tree, new_raw, language, identity_fields=_identity_fields
1632
+ ) or new_tree
1633
+ if language.lower() in {"css", "scss", "html", "xml", "mdx"}:
1634
+ old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
1635
+ new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
1636
+ if language.lower() in {"hcl", "puppet"}:
1637
+ old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
1638
+ new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
1639
+ if language.lower() == "sql":
1640
+ old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
1641
+ new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
1642
+ if language.lower() in {"asm", "bash", "delphi"}:
1643
+ old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
1644
+ new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
1645
+ _validate_tree_ids(old_tree, filename)
1646
+ _validate_tree_ids(new_tree, filename)
1647
+ diagnostics.record(
1648
+ stage="parse",
1649
+ action="semantic_tree_ready",
1650
+ rule_id="pipeline.semantic_tree_ready",
1651
+ reason="semantic trees produced and enriched",
1652
+ metadata={
1653
+ "old_nodes": _count_semantic_nodes(old_tree),
1654
+ "new_nodes": _count_semantic_nodes(new_tree),
1655
+ },
1656
+ )
1657
+ # ── 8.5. Token-level fallback for severe parse errors ───────────────
1658
+ # When tree-sitter produced ERROR nodes and fallback is enabled, skip
1659
+ # the GumTree algorithm and return a coarse token-level diff instead.
1660
+ if self._config.fallback_to_token_diff and (
1661
+ _has_error_node(old_tree) or _has_error_node(new_tree)
1662
+ ):
1663
+ logger.warning(
1664
+ "Parse errors detected in %r — falling back to token-level diff",
1665
+ filename,
1666
+ )
1667
+ _raise_rust_only_gate_error("parse errors require Rust token-level fallback")
1668
+ diagnostics.record(
1669
+ stage="fallback",
1670
+ action="token_fallback",
1671
+ rule_id="pipeline.token_fallback",
1672
+ reason="semantic tree contains ERROR nodes",
1673
+ )
1674
+ metadata: dict[str, Any] = {}
1675
+ if diagnostics.enabled:
1676
+ diagnostics.summary.update(
1677
+ {"early_exit": "token_fallback", "final_change_count": None}
1678
+ )
1679
+ metadata["diagnostics"] = diagnostics.snapshot()
1680
+ fallback_diff = _token_fallback_diff(
1681
+ old_raw,
1682
+ new_raw,
1683
+ filename,
1684
+ new_filename if new_filename is not None else filename,
1685
+ language,
1686
+ metadata=metadata,
1687
+ )
1688
+ fallback_diff = _apply_file_lifecycle_to_diff(fallback_diff, file_lifecycle)
1689
+ return apply_guardrails_to_diff(
1690
+ fallback_diff,
1691
+ old_tree=None,
1692
+ new_tree=None,
1693
+ old_source=old_raw,
1694
+ new_source=new_raw,
1695
+ config=self._config,
1696
+ diagnostics=diagnostics,
1697
+ )
1698
+ # ── 9. GumTree matching ────────────────────────────────────────────────
1699
+ # The matching engine is selected by (in priority order):
1700
+ # 1. ``DiffConfig.test_matching_engine`` — test-only hard override used
1701
+ # by the dual-run truthiness matrix to certify both contracts.
1702
+ # 2. The production default: Rust's language-agnostic matcher
1703
+ # (``intentumdiff_rust_core.diff_semantic_tree_json``) when the Rust
1704
+ # core is available and diagnostics are off. This is migration
1705
+ # step 2 of ``docs/ENGINE_BOUNDARY_AUDIT.md``. The four engine
1706
+ # gaps that previously blocked this swap (Python dedent, JS
1707
+ # calc_hash MOVE, YAML block↔flow, C# var↔explicit) are closed —
1708
+ # see ``the retired NOISE_SUPPRESSION_RETUNE doc (git history)`` Phases A–D.
1709
+ # 3. (retired) The python GumTree oracle was deleted with the transitional
1710
+ # layer (issue #57 payoff, stage 4b) — no python matching fallback exists.
1711
+ test_engine = self._config.test_matching_engine
1712
+ rust_attempt: Any = None
1713
+ # Skip Rust matching when either tree is empty (file add/delete
1714
+ # lifecycle). The Rust language-agnostic matcher treats empty-root
1715
+ # ↔ content-root as a structural match, which produces a single
1716
+ # MODIFICATION instead of the correct ADDITION (or DELETION) for
1717
+ # the file-lifecycle case. Letting the Python oracle handle these
1718
+ # edge cases preserves the ADDITION/DELETION shape that downstream
1719
+ # tests and review UIs depend on.
1720
+ either_tree_empty = (
1721
+ not old_tree.children or not new_tree.children
1722
+ )
1723
+ use_rust_matching = (
1724
+ test_engine == "rust"
1725
+ or (
1726
+ test_engine is None
1727
+ and self._config.experimental_rust_core
1728
+ and not diagnostics.enabled
1729
+ and not either_tree_empty
1730
+ )
1731
+ )
1732
+ # ── 9-fin. Rust finalize routing (issue #57) ──────────────────────────
1733
+ # Certified languages skip the python edit-script/refinement/presentation stages
1734
+ # entirely: the Rust core runs the certified batch's refine+finalize from the
1735
+ # semantic trees and returns the finished review. Eligibility is independent of
1736
+ # the matching-engine choice (issue #57 payoff, fallback-tier closure):
1737
+ # - empty-tree lifecycle is handled INSIDE Rust finalize (DELETION+ADDITION
1738
+ # root pair, python parity), so either_tree_empty no longer excludes;
1739
+ # - enriched trees flow through routing (enrichment ran at stage 7b);
1740
+ # - registered diff analyzers run on the ROUTED diff below, mirroring 13.5.
1741
+ # Diagnostics mode routes too (issue #57 payoff, stage 4a): the shell-side
1742
+ # records (parse, trees, guardrails, finalize summary below) still fire; the
1743
+ # per-pass refinement trace is Rust-internal (INTENTUMDIFF_FINALIZE_DEBUG probe)
1744
+ # until #54 formalizes it into the recorder.
1745
+ rust_finalize_eligible = test_engine == "rust" or (
1746
+ test_engine is None and self._config.experimental_rust_core
1747
+ )
1748
+ if rust_finalize_eligible and language in _rust_finalize_languages():
1749
+ from intentumdiff.rust_core import try_rust_finalize_review
1750
+
1751
+ with profiler.phase("rust_finalize_review"):
1752
+ finalize_result = try_rust_finalize_review(
1753
+ old_tree=old_tree,
1754
+ new_tree=new_tree,
1755
+ old_source=old_content,
1756
+ new_source=new_content,
1757
+ language=language,
1758
+ config=self._config,
1759
+ collect_trace=diagnostics.enabled,
1760
+ )
1761
+ if finalize_result is not None:
1762
+ diagnostics.record(
1763
+ stage="finalize",
1764
+ action="rust_finalize_review",
1765
+ rule_id="rust_core.finalize_review_v1",
1766
+ reason="routed per-stage finalize served the review",
1767
+ metadata={
1768
+ "engine_owner": "rust",
1769
+ "change_count": len(finalize_result["changes"]),
1770
+ "is_style_only": finalize_result["is_style_only"],
1771
+ },
1772
+ )
1773
+ # Per-pass trace (issue #54): every probed refine/finalize pass reports
1774
+ # its surviving change count; the recorder mirrors the python pipeline's
1775
+ # per-stage observability on the Rust path.
1776
+ for entry in finalize_result.get("trace") or []:
1777
+ diagnostics.record_count(
1778
+ stage="finalize",
1779
+ action=str(entry.get("pass", "")),
1780
+ rule_id="rust_core.finalize_pass",
1781
+ reason="rust finalize pass completed",
1782
+ count=int(entry.get("changes_after", 0)),
1783
+ )
1784
+ # Semantic invariances (css color / literal equivalence / yaml block↔flow / import
1785
+ # reorder / …) are equivalences that must hold REGARDLESS of routing, but this
1786
+ # short-circuit returns before the stage-12 apply_invariances. Run it here on the
1787
+ # routed changes so canonical-value "zero-change" edits collapse under routing too
1788
+ # (#57: unlocks json/yaml/css/python). No-op when no invariance fires.
1789
+ fin_changes = finalize_result["changes"]
1790
+ fin_is_style_only = finalize_result["is_style_only"]
1791
+ fin_invariant_groups: list[Any] = []
1792
+ fin_invariant_ignored: list[dict[str, Any]] = []
1793
+ if fin_changes:
1794
+ _inv = apply_invariances(
1795
+ fin_changes,
1796
+ old_tree=old_tree,
1797
+ new_tree=new_tree,
1798
+ old_source=old_content,
1799
+ new_source=new_content,
1800
+ language=language,
1801
+ )
1802
+ if len(_inv.changes) != len(fin_changes) or _inv.change_groups:
1803
+ fin_changes = _inv.changes
1804
+ fin_invariant_groups = list(_inv.change_groups)
1805
+ # The invariance's own IGNORED_STYLE groups ARE the style evidence — carry
1806
+ # their rule_id/reason so metadata["ignored_style_changes"] leads with the
1807
+ # specific reason (css.color.canonical_equivalence), not the generic
1808
+ # source-equivalence fallback below.
1809
+ for group in fin_invariant_groups:
1810
+ diagnostics.record_group(stage="invariance", group=group)
1811
+ fin_invariant_ignored = [
1812
+ {"rule_id": g.rule_id, **dict(g.metadata)}
1813
+ for g in fin_invariant_groups
1814
+ if g.kind == ChangeGroupKind.IGNORED_STYLE
1815
+ ]
1816
+ if not fin_changes and old_content != new_content:
1817
+ fin_is_style_only = True
1818
+ # Generic files are line-oriented from the user's point of view
1819
+ # (#57 generic routing, mirroring the stage-12 orchestration): the routed
1820
+ # finalize's parser-token churn is REPLACED wholesale by the Rust text
1821
+ # review's stable line/character spans, then the (Rust-first) markdown
1822
+ # section passes run for .md filenames. Parser-derived groups are dropped
1823
+ # with the churn — carrying them forward invents fake pairings between
1824
+ # unrelated lines.
1825
+ fin_ignored_override: list[dict[str, Any]] | None = None
1826
+ if language.lower() == "generic":
1827
+ generic_presented = normalize_generic_text_for_review(
1828
+ fin_changes,
1829
+ old_content,
1830
+ new_content,
1831
+ )
1832
+ presented_generic = PresentationResult(
1833
+ changes=generic_presented.changes,
1834
+ change_groups=generic_presented.change_groups,
1835
+ ignored_style_changes=[
1836
+ *generic_presented.ignored_style_changes,
1837
+ ],
1838
+ )
1839
+ presented_generic = _markdown_section_move_presentation(
1840
+ presented_generic,
1841
+ old_source=old_content,
1842
+ new_source=new_content,
1843
+ old_filename=filename,
1844
+ new_filename=(
1845
+ new_filename if new_filename is not None else filename
1846
+ ),
1847
+ )
1848
+ presented_generic = _markdown_section_heading_rename_presentation(
1849
+ presented_generic,
1850
+ old_source=old_content,
1851
+ new_source=new_content,
1852
+ old_filename=filename,
1853
+ new_filename=(
1854
+ new_filename if new_filename is not None else filename
1855
+ ),
1856
+ )
1857
+ fin_changes = presented_generic.changes
1858
+ fin_invariant_groups = list(presented_generic.change_groups)
1859
+ fin_ignored_override = list(presented_generic.ignored_style_changes)
1860
+ finalize_result["change_groups"] = []
1861
+ finalize_result["no_surviving_changes"] = False
1862
+ # Mirror the stage-12 style-only resolution: identical sources (or
1863
+ # whitespace-collapsed-equal trees) with no surviving line spans are a
1864
+ # style-only diff, not "no changes". The Rust finalize's flag reflects
1865
+ # its own suppressions, which the replacement just discarded.
1866
+ if not fin_changes:
1867
+
1868
+ def _generic_norm(node: SemanticNode) -> tuple[Any, ...]:
1869
+ label = " ".join(node.label.split())
1870
+ return (
1871
+ node.node_type,
1872
+ label,
1873
+ tuple(_generic_norm(c) for c in node.children),
1874
+ )
1875
+
1876
+ fin_is_style_only = (
1877
+ old_content == new_content
1878
+ or _generic_norm(old_tree) == _generic_norm(new_tree)
1879
+ )
1880
+ # Stage-12 style-only resolution, language-agnostic (markdown #44 exposed
1881
+ # it): ZERO surviving changes with identical (or whitespace-collapsed
1882
+ # tree-equal) sources is a style-only diff for every routed language —
1883
+ # the Rust finalize's flag only reflects its own suppressions.
1884
+ if not fin_changes and not fin_is_style_only:
1885
+
1886
+ def _routed_norm(node: SemanticNode) -> tuple[Any, ...]:
1887
+ label = " ".join(node.label.split())
1888
+ return (
1889
+ node.node_type,
1890
+ label,
1891
+ tuple(_routed_norm(c) for c in node.children),
1892
+ )
1893
+
1894
+ fin_is_style_only = (
1895
+ old_content == new_content
1896
+ or _routed_norm(old_tree) == _routed_norm(new_tree)
1897
+ )
1898
+ metadata_fin: dict[str, Any] = {
1899
+ "engine_owner": "rust",
1900
+ "semantic_contract": "rust_finalize_review_v1",
1901
+ "rust_core": {
1902
+ "engine": "rust_finalize_review_v1",
1903
+ "stage": "per_stage_finalize_routing",
1904
+ "used": True,
1905
+ },
1906
+ }
1907
+ # Context metadata the normal path attaches at stage 11 — the routed
1908
+ # return must carry the same contracts (cpp compile_commands pilot gap).
1909
+ if schema_metadata is not None:
1910
+ metadata_fin["schema"] = schema_metadata
1911
+ if compile_metadata is not None:
1912
+ metadata_fin["compile_commands"] = compile_metadata
1913
+ combined_ignored = (
1914
+ fin_ignored_override
1915
+ if fin_ignored_override is not None
1916
+ else [
1917
+ *fin_invariant_ignored,
1918
+ *finalize_result["ignored_style_changes"],
1919
+ ]
1920
+ )
1921
+ if combined_ignored:
1922
+ metadata_fin["ignored_style_changes"] = combined_ignored
1923
+ if finalize_result["no_surviving_changes"]:
1924
+ metadata_fin["no_surviving_changes"] = True
1925
+ routed_groups = [*finalize_result["change_groups"], *fin_invariant_groups]
1926
+ if not fin_changes and old_content != new_content and not fin_invariant_ignored:
1927
+ # The normal path attaches a source-equivalence suppression group
1928
+ # post-diff when a diff nets to zero changes but the sources differ
1929
+ # (xml attribute reorder -> zero changes must still RECORD the
1930
+ # suppression, not merely be absent). The routed short-circuit returns
1931
+ # before that stage, so mirror it here.
1932
+ style_evidence = build_style_only_evidence(
1933
+ old_source=old_content,
1934
+ new_source=new_content,
1935
+ language=language,
1936
+ )
1937
+ for group in style_evidence.change_groups:
1938
+ diagnostics.record_group(stage="invariance", group=group)
1939
+ routed_groups.extend(style_evidence.change_groups)
1940
+ if style_evidence.ignored_style_changes:
1941
+ metadata_fin.setdefault("ignored_style_changes", []).extend(
1942
+ style_evidence.ignored_style_changes
1943
+ )
1944
+ routed = SemanticDiff(
1945
+ old_filename=filename,
1946
+ new_filename=new_filename if new_filename is not None else filename,
1947
+ language=language,
1948
+ changes=fin_changes,
1949
+ change_groups=routed_groups,
1950
+ has_semantic_changes=bool(fin_changes) and not fin_is_style_only,
1951
+ is_style_only=fin_is_style_only,
1952
+ metadata=metadata_fin,
1953
+ )
1954
+ routed = _apply_file_lifecycle_to_diff(routed, file_lifecycle)
1955
+ routed = apply_guardrails_to_diff(
1956
+ routed,
1957
+ old_tree=old_tree,
1958
+ new_tree=new_tree,
1959
+ old_source=old_content,
1960
+ new_source=new_content,
1961
+ config=self._config,
1962
+ diagnostics=diagnostics,
1963
+ )
1964
+ # Diff-analyzer pass on the routed diff (stage 13.5 mirror; issue #57
1965
+ # payoff): analyzers are diff->diff Wasm transforms over the final
1966
+ # SemanticDiff JSON, so the routed review feeds them identically.
1967
+ with profiler.phase("diff_analyzers"):
1968
+ routed = self._run_diff_analyzers_on_routed(
1969
+ routed, language, filename, adaptive_fuel
1970
+ )
1971
+ return _attach_run_telemetry(routed, engine_telemetry, diagnostics)
1972
+ # The Rust finalize declined (tree_too_large / backend unavailable). The
1973
+ # transitional python pipeline is retired (issue #57 payoff, stage 4b) —
1974
+ # degrade honestly to the coarse token-level diff, same as parse errors.
1975
+ logger.warning(
1976
+ "Rust finalize declined for %r (%s) — token-level fallback",
1977
+ filename,
1978
+ language,
1979
+ )
1980
+ _raise_rust_only_gate_error("rust finalize declined")
1981
+ diagnostics.record(
1982
+ stage="fallback",
1983
+ action="token_fallback",
1984
+ rule_id="pipeline.finalize_declined",
1985
+ reason="rust finalize declined; python pipeline retired (#57)",
1986
+ )
1987
+ fallback_metadata: dict[str, Any] = {
1988
+ "fallback_reason": "rust_finalize_declined",
1989
+ }
1990
+ if diagnostics.enabled:
1991
+ diagnostics.summary.update(
1992
+ {"early_exit": "token_fallback", "final_change_count": None}
1993
+ )
1994
+ fallback_metadata["diagnostics"] = diagnostics.snapshot()
1995
+ fallback_diff = _token_fallback_diff(
1996
+ old_raw,
1997
+ new_raw,
1998
+ filename,
1999
+ new_filename if new_filename is not None else filename,
2000
+ language,
2001
+ metadata=fallback_metadata,
2002
+ )
2003
+ fallback_diff = _apply_file_lifecycle_to_diff(fallback_diff, file_lifecycle)
2004
+ return apply_guardrails_to_diff(
2005
+ fallback_diff,
2006
+ old_tree=None,
2007
+ new_tree=None,
2008
+ old_source=old_raw,
2009
+ new_source=new_raw,
2010
+ config=self._config,
2011
+ diagnostics=diagnostics,
2012
+ )
2013
+ if test_engine == "python":
2014
+ raise NotImplementedError(
2015
+ "test_matching_engine='python' was retired with the transitional "
2016
+ "python pipeline (issue #57); the Rust finalize routing serves every "
2017
+ "certified language."
2018
+ )
2019
+
2020
+ # Rust core disabled (DiffConfig kill switch / INTENTUMDIFF_RUST_CORE=0): the
2021
+ # transitional python pipeline is retired (issue #57 payoff) — semantic
2022
+ # diffing REQUIRES the Rust core; degrade honestly to the token-level diff.
2023
+ logger.warning(
2024
+ "Rust core disabled for %r — token-level fallback (python pipeline retired)",
2025
+ filename,
2026
+ )
2027
+ _raise_rust_only_gate_error("rust core disabled")
2028
+ diagnostics.record(
2029
+ stage="fallback",
2030
+ action="token_fallback",
2031
+ rule_id="pipeline.rust_core_disabled",
2032
+ reason="rust core disabled; python pipeline retired (#57)",
2033
+ )
2034
+ disabled_metadata: dict[str, Any] = {
2035
+ "fallback_reason": "rust_core_disabled",
2036
+ }
2037
+ if diagnostics.enabled:
2038
+ diagnostics.summary.update(
2039
+ {"early_exit": "token_fallback", "final_change_count": None}
2040
+ )
2041
+ disabled_metadata["diagnostics"] = diagnostics.snapshot()
2042
+ disabled_diff = _token_fallback_diff(
2043
+ old_raw,
2044
+ new_raw,
2045
+ filename,
2046
+ new_filename if new_filename is not None else filename,
2047
+ language,
2048
+ metadata=disabled_metadata,
2049
+ )
2050
+ disabled_diff = _apply_file_lifecycle_to_diff(disabled_diff, file_lifecycle)
2051
+ return apply_guardrails_to_diff(
2052
+ disabled_diff,
2053
+ old_tree=None,
2054
+ new_tree=None,
2055
+ old_source=old_raw,
2056
+ new_source=new_raw,
2057
+ config=self._config,
2058
+ diagnostics=diagnostics,
2059
+ )
2060
+
2061
+ def diff_stream_progressive(
2062
+ self,
2063
+ source: Source,
2064
+ ) -> Iterator[ChangeStreamEvent]:
2065
+ """Yield ``ChangeStreamEvent`` objects as each pipeline phase completes.
2066
+
2067
+ Three phases are emitted in order:
2068
+
2069
+ 1. **STRUCTURAL** — one ``action="add"`` event per change after the
2070
+ GumTree edit script and move-promotion pass (stage 11). These are
2071
+ available immediately, before the (potentially expensive) refactoring
2072
+ detection runs.
2073
+
2074
+ 2. **REFINED** — events emitted once refactoring detection (stage 12) has
2075
+ run. Consumed raw changes are replaced by ``action="revise"`` events
2076
+ carrying a ``REFACTORING`` change and ``replaced_ids`` listing the
2077
+ node IDs of the Phase-1 changes they supersede. Removed-but-not-
2078
+ replaced changes are emitted as ``action="remove"``.
2079
+
2080
+ 3. **FINAL** — events emitted after classification (stage 13) and any
2081
+ diff-analyzer plugins (stage 13.5) have run. Emitted only when these
2082
+ passes modify the change list.
2083
+
2084
+ Consumers that only care about the final result can wait until all events
2085
+ are consumed; consumers that want to display partial results can render
2086
+ STRUCTURAL changes immediately and update/remove them as REFINED and
2087
+ FINAL events arrive.
2088
+ """
2089
+ old_content, new_content, filename, language_hint = source.get_content()
2090
+ yield from self._run_pipeline_streaming(old_content, new_content, filename, language_hint)
2091
+
2092
+ def _run_pipeline_streaming(
2093
+ self,
2094
+ old_content: str,
2095
+ new_content: str,
2096
+ filename: str,
2097
+ language_hint: str | None,
2098
+ *,
2099
+ new_filename: str | None = None,
2100
+ ) -> Iterator[ChangeStreamEvent]:
2101
+ """Internal generator — see ``diff_stream_progressive`` for the public API."""
2102
+ result = self._run_stages_1_to_11(
2103
+ old_content, new_content, filename, language_hint, new_filename=new_filename
2104
+ )
2105
+
2106
+ if isinstance(result, SemanticDiff):
2107
+ # Early exit (cache hit / style-only / token fallback): emit all changes
2108
+ # as FINAL so consumers still receive a well-formed event stream.
2109
+ for change in result.changes:
2110
+ yield ChangeStreamEvent(
2111
+ phase=ChangeStreamPhase.FINAL,
2112
+ action="add",
2113
+ change=change,
2114
+ )
2115
+ return
2116
+
2117
+ raise AssertionError(
2118
+ "unreachable: _run_stages_1_to_11 always returns a SemanticDiff since the "
2119
+ "python pipeline retirement (issue #57 payoff, stage 4b)"
2120
+ )
2121
+
2122
+ def _cached_process(
2123
+ self,
2124
+ parser: Any,
2125
+ input_: str,
2126
+ language: str,
2127
+ filename: str,
2128
+ fuel: int,
2129
+ *,
2130
+ cache_grammar_id: str | None = None,
2131
+ ) -> SemanticNode:
2132
+ """
2133
+ Call ``parser.process()`` with a parse-tree cache layer.
2134
+
2135
+ On a cache hit the wasm plugin is skipped entirely and the cached
2136
+ ``SemanticNode`` tree is returned. On a miss the result is stored
2137
+ before returning.
2138
+ """
2139
+ cache = self._cache
2140
+ if cache is None:
2141
+ raise RuntimeError("parse cache store is not configured")
2142
+ wasm_hash = self._wasm_hash_for(parser)
2143
+ grammar_id = cache_grammar_id or parser.grammar_id
2144
+ key = cache.parse_key(input_, grammar_id, wasm_hash)
2145
+ cached = cache.get_parse(key)
2146
+ if cached is not None:
2147
+ logger.debug("Parse cache hit for grammar=%r", grammar_id)
2148
+ return SemanticNode.model_validate_json(cached)
2149
+
2150
+ node = parser.process(input_, language, filename, fuel=fuel)
2151
+ cache.put_parse(key, node.model_dump_json(), grammar_id=grammar_id)
2152
+ return node
2153
+
2154
+ def _wasm_hash_for(self, parser: Any) -> str:
2155
+ """
2156
+ Return a short (16-hex-char) sha256 hash of the parser's Wasm binary.
2157
+
2158
+ Computed once per grammar_id per process and memoised. Falls back to
2159
+ ``"unknown"`` if the path cannot be read (e.g. in-memory test stubs).
2160
+ """
2161
+ grammar_id: str = parser.grammar_id
2162
+ if grammar_id in self._wasm_hash_cache:
2163
+ return self._wasm_hash_cache[grammar_id]
2164
+ try:
2165
+ wasm_bytes = Path(parser.wasm_path).read_bytes()
2166
+ h = hashlib.sha256(wasm_bytes).hexdigest()[:16]
2167
+ except (AttributeError, OSError):
2168
+ h = "unknown"
2169
+ self._wasm_hash_cache[grammar_id] = h
2170
+ return h
2171
+
2172
+ def _parse(
2173
+ self,
2174
+ content: str,
2175
+ parser_adapter: Any,
2176
+ language: str,
2177
+ filename: str,
2178
+ *,
2179
+ profiler: _PhaseProfiler | None = None,
2180
+ ) -> str:
2181
+ """
2182
+ Prepare source for a parser plugin.
2183
+
2184
+ FullParse parsers receive raw source. Legacy ``interpret-cst`` parsers
2185
+ are rejected so public paths cannot host parsing in Python.
2186
+ """
2187
+ if parser_adapter.parser_mode == "full-parse":
2188
+ return content # plugin will parse internally
2189
+
2190
+ raise PluginNotFoundError(
2191
+ "IntentumDiff no longer supports Python-hosted InterpretCst parsers "
2192
+ "on public product paths. Rebuild or update parser "
2193
+ f"{getattr(parser_adapter, 'grammar_id', '<unknown>')!r} as a "
2194
+ "FullParse Rust/Wasm plugin."
2195
+ )