intentumdiff-python 0.0.1__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- intentumdiff/__init__.py +134 -0
- intentumdiff/_differ_gate.py +210 -0
- intentumdiff/_differ_presentation.py +786 -0
- intentumdiff/_differ_runtime.py +320 -0
- intentumdiff/analysis/__init__.py +1 -0
- intentumdiff/analysis/compile_commands.py +151 -0
- intentumdiff/analysis/cross_file.py +71 -0
- intentumdiff/analysis/diagnostics.py +309 -0
- intentumdiff/analysis/guardrail_reports.py +254 -0
- intentumdiff/analysis/guardrails.py +318 -0
- intentumdiff/analysis/invariances.py +1207 -0
- intentumdiff/analysis/keyed_profiles.py +25 -0
- intentumdiff/analysis/resource_profiles.py +14 -0
- intentumdiff/analysis/schema_resolver.py +816 -0
- intentumdiff/analysis/text_review.py +46 -0
- intentumdiff/analysis/user_schemas.py +430 -0
- intentumdiff/assets.py +45 -0
- intentumdiff/cache/__init__.py +37 -0
- intentumdiff/cache/duckdb_store.py +98 -0
- intentumdiff/cache/sqlite_store.py +149 -0
- intentumdiff/cache/store.py +190 -0
- intentumdiff/cli/__init__.py +199 -0
- intentumdiff/cli/_commands.py +2141 -0
- intentumdiff/cli/_parser.py +1061 -0
- intentumdiff/cli/_shared.py +644 -0
- intentumdiff/content_type.py +59 -0
- intentumdiff/core/__init__.py +1 -0
- intentumdiff/core/commit_differ.py +599 -0
- intentumdiff/core/config.py +51 -0
- intentumdiff/core/cst_serializer.py +101 -0
- intentumdiff/core/diffignore.py +87 -0
- intentumdiff/core/index.py +189 -0
- intentumdiff/core/indexer.py +792 -0
- intentumdiff/core/models.py +837 -0
- intentumdiff/differ.py +2195 -0
- intentumdiff/github_app.py +287 -0
- intentumdiff/intentumdiff_rust_core/__init__.py +7 -0
- intentumdiff/intentumdiff_rust_core/ffi.py +8 -0
- intentumdiff/intentumdiff_rust_core/intentumdiff_rust_core.dll +0 -0
- intentumdiff/invariances/__init__.py +2 -0
- intentumdiff/invariances/rules.schema.json +283 -0
- intentumdiff/invariances/rules.yaml +349 -0
- intentumdiff/live_server.py +1314 -0
- intentumdiff/lsp/__init__.py +67 -0
- intentumdiff/lsp/client.py +544 -0
- intentumdiff/lsp/config.py +96 -0
- intentumdiff/lsp/enricher.py +227 -0
- intentumdiff/lsp/exceptions.py +27 -0
- intentumdiff/lsp/launcher.py +273 -0
- intentumdiff/lsp/servers.py +356 -0
- intentumdiff/lsp_server/__init__.py +14 -0
- intentumdiff/lsp_server/_codelens.py +67 -0
- intentumdiff/lsp_server/_diagnostics.py +62 -0
- intentumdiff/lsp_server/_handlers.py +175 -0
- intentumdiff/lsp_server/server.py +205 -0
- intentumdiff/plugins/__init__.py +1 -0
- intentumdiff/plugins/adapter.py +397 -0
- intentumdiff/plugins/builtins.py +323 -0
- intentumdiff/plugins/exceptions.py +93 -0
- intentumdiff/plugins/hub.py +1155 -0
- intentumdiff/plugins/language_metadata.py +250 -0
- intentumdiff/plugins/loader.py +1252 -0
- intentumdiff/plugins/registry.py +1079 -0
- intentumdiff/plugins/registry.schema.json +99 -0
- intentumdiff/plugins/registry_schema.py +280 -0
- intentumdiff/plugins/wit/plugin.wit +351 -0
- intentumdiff/rust_core.py +2370 -0
- intentumdiff/serve/__init__.py +19 -0
- intentumdiff/serve/_app.py +485 -0
- intentumdiff/serve/_static/index.html +326 -0
- intentumdiff/serve/_static/playground.js +567 -0
- intentumdiff/sources/__init__.py +1 -0
- intentumdiff/sources/base.py +28 -0
- intentumdiff/sources/file_source.py +48 -0
- intentumdiff/sources/git_source.py +375 -0
- intentumdiff/sources/live_buffer_source.py +113 -0
- intentumdiff/sources/patch_source.py +126 -0
- intentumdiff/sources/string_source.py +44 -0
- intentumdiff/testing.py +165 -0
- intentumdiff/vcs/__init__.py +37 -0
- intentumdiff/vcs/_safe_path.py +50 -0
- intentumdiff/vcs/base.py +136 -0
- intentumdiff/vcs/git_backend.py +97 -0
- intentumdiff/vcs/git_cli.py +100 -0
- intentumdiff/vcs/hg_backend.py +94 -0
- intentumdiff/vcs/perforce_backend.py +109 -0
- intentumdiff/vcs/svn_backend.py +105 -0
- intentumdiff/wasm/.gitkeep +2 -0
- intentumdiff/wasm/abap_parser.wasm +0 -0
- intentumdiff/wasm/adf_parser.wasm +0 -0
- intentumdiff/wasm/asciidoc_parser.wasm +0 -0
- intentumdiff/wasm/asm_parser.wasm +0 -0
- intentumdiff/wasm/assemblyscript_parser.wasm +0 -0
- intentumdiff/wasm/astro_parser.wasm +0 -0
- intentumdiff/wasm/bash_parser.wasm +0 -0
- intentumdiff/wasm/clojure_parser.wasm +0 -0
- intentumdiff/wasm/cmake_parser.wasm +0 -0
- intentumdiff/wasm/cpp_parser.wasm +0 -0
- intentumdiff/wasm/csharp_parser.wasm +0 -0
- intentumdiff/wasm/css_parser.wasm +0 -0
- intentumdiff/wasm/dart_parser.wasm +0 -0
- intentumdiff/wasm/databricks_parser.wasm +0 -0
- intentumdiff/wasm/dax_parser.wasm +0 -0
- intentumdiff/wasm/delphi_parser.wasm +0 -0
- intentumdiff/wasm/dockerfile_parser.wasm +0 -0
- intentumdiff/wasm/elixir_parser.wasm +0 -0
- intentumdiff/wasm/freebasic_parser.wasm +0 -0
- intentumdiff/wasm/generic_parser.wasm +0 -0
- intentumdiff/wasm/gitignore_parser.wasm +0 -0
- intentumdiff/wasm/go_parser.wasm +0 -0
- intentumdiff/wasm/gomod_parser.wasm +0 -0
- intentumdiff/wasm/graphql_parser.wasm +0 -0
- intentumdiff/wasm/groovy_parser.wasm +0 -0
- intentumdiff/wasm/haskell_parser.wasm +0 -0
- intentumdiff/wasm/html_parser.wasm +0 -0
- intentumdiff/wasm/ini_parser.wasm +0 -0
- intentumdiff/wasm/java_parser.wasm +0 -0
- intentumdiff/wasm/js_ts_parser.wasm +0 -0
- intentumdiff/wasm/json_parser.wasm +0 -0
- intentumdiff/wasm/kotlin_parser.wasm +0 -0
- intentumdiff/wasm/latex_parser.wasm +0 -0
- intentumdiff/wasm/lua_parser.wasm +0 -0
- intentumdiff/wasm/make_parser.wasm +0 -0
- intentumdiff/wasm/markdown_parser.wasm +0 -0
- intentumdiff/wasm/mdx_parser.wasm +0 -0
- intentumdiff/wasm/ocaml_parser.wasm +0 -0
- intentumdiff/wasm/odin_parser.wasm +0 -0
- intentumdiff/wasm/parser_manifest.json +590 -0
- intentumdiff/wasm/perl_parser.wasm +0 -0
- intentumdiff/wasm/php_parser.wasm +0 -0
- intentumdiff/wasm/plsql_parser.wasm +0 -0
- intentumdiff/wasm/po_parser.wasm +0 -0
- intentumdiff/wasm/postscript_parser.wasm +0 -0
- intentumdiff/wasm/powershell_parser.wasm +0 -0
- intentumdiff/wasm/proto_parser.wasm +0 -0
- intentumdiff/wasm/puppet_parser.wasm +0 -0
- intentumdiff/wasm/python_parser.wasm +0 -0
- intentumdiff/wasm/qsharp_parser.wasm +0 -0
- intentumdiff/wasm/r_parser.wasm +0 -0
- intentumdiff/wasm/reasonml_parser.wasm +0 -0
- intentumdiff/wasm/ruby_parser.wasm +0 -0
- intentumdiff/wasm/rust_parser.wasm +0 -0
- intentumdiff/wasm/sas_parser.wasm +0 -0
- intentumdiff/wasm/scala_parser.wasm +0 -0
- intentumdiff/wasm/scss_parser.wasm +0 -0
- intentumdiff/wasm/sql_parser.wasm +0 -0
- intentumdiff/wasm/squirrel_parser.wasm +0 -0
- intentumdiff/wasm/svelte_parser.wasm +0 -0
- intentumdiff/wasm/swift_parser.wasm +0 -0
- intentumdiff/wasm/terraform_parser.wasm +0 -0
- intentumdiff/wasm/toml_parser.wasm +0 -0
- intentumdiff/wasm/tsql_parser.wasm +0 -0
- intentumdiff/wasm/vbnet_parser.wasm +0 -0
- intentumdiff/wasm/vue_parser.wasm +0 -0
- intentumdiff/wasm/wat_parser.wasm +0 -0
- intentumdiff/wasm/xml_parser.wasm +0 -0
- intentumdiff/wasm/yaml_parser.wasm +0 -0
- intentumdiff/wasm/zig_parser.wasm +0 -0
- intentumdiff/watcher.py +259 -0
- intentumdiff_python-0.0.1.dist-info/METADATA +112 -0
- intentumdiff_python-0.0.1.dist-info/RECORD +165 -0
- intentumdiff_python-0.0.1.dist-info/WHEEL +4 -0
- intentumdiff_python-0.0.1.dist-info/entry_points.txt +80 -0
- intentumdiff_python-0.0.1.dist-info/licenses/LICENSE +21 -0
- intentumdiff_python-0.0.1.dist-info/sboms/intentumdiff-rust-core.cyclonedx.json +10968 -0
intentumdiff/differ.py
ADDED
|
@@ -0,0 +1,2195 @@
|
|
|
1
|
+
"""
|
|
2
|
+
intentumdiff.differ
|
|
3
|
+
~~~~~~~~~~~~~~~~~~~~~~~~
|
|
4
|
+
|
|
5
|
+
Pipeline orchestrator — the central coordinator.
|
|
6
|
+
|
|
7
|
+
Pipeline stages
|
|
8
|
+
───────────────
|
|
9
|
+
1. Source retrieval — call ``Source.get_content()``
|
|
10
|
+
2. Language detection — ``PluginRegistry.detect_parser(...)``
|
|
11
|
+
3. Parser dispatch - full-parse Wasm parser consumes raw source
|
|
12
|
+
4. Parser output - parser returns semantic tree JSON
|
|
13
|
+
5. Trivia stripping — ``_strip_trivia_impl`` removes comment / whitespace nodes
|
|
14
|
+
6. Style-only shortcut — compare root structural hashes; short-circuit if equal
|
|
15
|
+
7. Plugin dispatch - call ``ParserAdapter.process(source, ...)`` (Wasm)
|
|
16
|
+
8. ID validation — ``_validate_tree_ids`` catches duplicate IDs from plugins
|
|
17
|
+
9. Rust finalize routing — ``finalize_review_json`` (matching, edit script,
|
|
18
|
+
refinement, presentation, and grouping all run in the Rust core; the
|
|
19
|
+
transitional python stages 9-13 were retired — issue #57 payoff)
|
|
20
|
+
13.5 Diff-analyzer pass — ``DiffAnalyzerAdapter.analyze_diff(...)`` (Wasm, optional)
|
|
21
|
+
|
|
22
|
+
The ``SemanticDiffer`` class is the primary public API object.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import hashlib
|
|
28
|
+
import logging
|
|
29
|
+
import os
|
|
30
|
+
import threading
|
|
31
|
+
import time
|
|
32
|
+
from collections.abc import Iterator
|
|
33
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from typing import Any
|
|
36
|
+
|
|
37
|
+
from pydantic import ValidationError
|
|
38
|
+
|
|
39
|
+
from intentumdiff._differ_gate import (
|
|
40
|
+
RUST_CERTIFIED_LANGUAGES as RUST_CERTIFIED_LANGUAGES,
|
|
41
|
+
)
|
|
42
|
+
from intentumdiff._differ_gate import (
|
|
43
|
+
RUST_FINALIZE_LANGUAGES as RUST_FINALIZE_LANGUAGES,
|
|
44
|
+
)
|
|
45
|
+
from intentumdiff._differ_gate import (
|
|
46
|
+
STRICT_RUST_GATE_ENV as STRICT_RUST_GATE_ENV,
|
|
47
|
+
)
|
|
48
|
+
from intentumdiff._differ_gate import (
|
|
49
|
+
RustOnlyGateError as RustOnlyGateError,
|
|
50
|
+
)
|
|
51
|
+
from intentumdiff._differ_gate import (
|
|
52
|
+
_raise_rust_only_gate_error as _raise_rust_only_gate_error,
|
|
53
|
+
)
|
|
54
|
+
from intentumdiff._differ_gate import (
|
|
55
|
+
_rust_certified_languages as _rust_certified_languages,
|
|
56
|
+
)
|
|
57
|
+
from intentumdiff._differ_gate import (
|
|
58
|
+
_rust_finalize_languages as _rust_finalize_languages,
|
|
59
|
+
)
|
|
60
|
+
from intentumdiff._differ_gate import (
|
|
61
|
+
_strict_rust_only_enabled as _strict_rust_only_enabled,
|
|
62
|
+
)
|
|
63
|
+
from intentumdiff._differ_gate import (
|
|
64
|
+
_validate_git_ref as _validate_git_ref,
|
|
65
|
+
)
|
|
66
|
+
from intentumdiff._differ_presentation import (
|
|
67
|
+
_GENERIC_STRING_LABELS as _GENERIC_STRING_LABELS,
|
|
68
|
+
)
|
|
69
|
+
from intentumdiff._differ_presentation import (
|
|
70
|
+
_NAMED_ENTITY_NODE_TYPES as _NAMED_ENTITY_NODE_TYPES,
|
|
71
|
+
)
|
|
72
|
+
from intentumdiff._differ_presentation import (
|
|
73
|
+
_all_semantic_nodes as _all_semantic_nodes,
|
|
74
|
+
)
|
|
75
|
+
from intentumdiff._differ_presentation import (
|
|
76
|
+
_annotate_text_diffs as _annotate_text_diffs,
|
|
77
|
+
)
|
|
78
|
+
from intentumdiff._differ_presentation import (
|
|
79
|
+
_changes_to_stream_events as _changes_to_stream_events,
|
|
80
|
+
)
|
|
81
|
+
from intentumdiff._differ_presentation import (
|
|
82
|
+
_clean_string_literal_label as _clean_string_literal_label,
|
|
83
|
+
)
|
|
84
|
+
from intentumdiff._differ_presentation import (
|
|
85
|
+
_compute_structural_hash_for_tree as _compute_structural_hash_for_tree,
|
|
86
|
+
)
|
|
87
|
+
from intentumdiff._differ_presentation import (
|
|
88
|
+
_count_cst_nodes as _count_cst_nodes,
|
|
89
|
+
)
|
|
90
|
+
from intentumdiff._differ_presentation import (
|
|
91
|
+
_count_semantic_nodes as _count_semantic_nodes,
|
|
92
|
+
)
|
|
93
|
+
from intentumdiff._differ_presentation import (
|
|
94
|
+
_empty_semantic_tree as _empty_semantic_tree,
|
|
95
|
+
)
|
|
96
|
+
from intentumdiff._differ_presentation import (
|
|
97
|
+
_enrich_literal_labels as _enrich_literal_labels,
|
|
98
|
+
)
|
|
99
|
+
from intentumdiff._differ_presentation import (
|
|
100
|
+
_has_error_node as _has_error_node,
|
|
101
|
+
)
|
|
102
|
+
from intentumdiff._differ_presentation import (
|
|
103
|
+
_is_markdown_filename as _is_markdown_filename,
|
|
104
|
+
)
|
|
105
|
+
from intentumdiff._differ_presentation import (
|
|
106
|
+
_is_named_entity_node as _is_named_entity_node,
|
|
107
|
+
)
|
|
108
|
+
from intentumdiff._differ_presentation import (
|
|
109
|
+
_markdown_section_body_hashes as _markdown_section_body_hashes,
|
|
110
|
+
)
|
|
111
|
+
from intentumdiff._differ_presentation import (
|
|
112
|
+
_markdown_section_heading_rename_presentation as _markdown_section_heading_rename_presentation,
|
|
113
|
+
)
|
|
114
|
+
from intentumdiff._differ_presentation import (
|
|
115
|
+
_markdown_section_move_presentation as _markdown_section_move_presentation,
|
|
116
|
+
)
|
|
117
|
+
from intentumdiff._differ_presentation import (
|
|
118
|
+
_markdown_sections as _markdown_sections,
|
|
119
|
+
)
|
|
120
|
+
from intentumdiff._differ_presentation import (
|
|
121
|
+
_node_to_dict as _node_to_dict,
|
|
122
|
+
)
|
|
123
|
+
from intentumdiff._differ_presentation import (
|
|
124
|
+
_root_structural_hash as _root_structural_hash,
|
|
125
|
+
)
|
|
126
|
+
from intentumdiff._differ_presentation import (
|
|
127
|
+
_semantic_parent_map as _semantic_parent_map,
|
|
128
|
+
)
|
|
129
|
+
from intentumdiff._differ_presentation import (
|
|
130
|
+
_slice_source_text as _slice_source_text,
|
|
131
|
+
)
|
|
132
|
+
from intentumdiff._differ_presentation import (
|
|
133
|
+
_surface_changed_in_place_entities as _surface_changed_in_place_entities,
|
|
134
|
+
)
|
|
135
|
+
from intentumdiff._differ_presentation import (
|
|
136
|
+
_token_fallback_diff as _token_fallback_diff,
|
|
137
|
+
)
|
|
138
|
+
from intentumdiff._differ_presentation import (
|
|
139
|
+
_validate_tree_ids as _validate_tree_ids,
|
|
140
|
+
)
|
|
141
|
+
from intentumdiff._differ_runtime import (
|
|
142
|
+
_ADDED_FILE_STATUSES as _ADDED_FILE_STATUSES,
|
|
143
|
+
)
|
|
144
|
+
from intentumdiff._differ_runtime import (
|
|
145
|
+
_DEFAULT_PLUGIN_FUEL as _DEFAULT_PLUGIN_FUEL,
|
|
146
|
+
)
|
|
147
|
+
from intentumdiff._differ_runtime import (
|
|
148
|
+
_DELETED_FILE_STATUSES as _DELETED_FILE_STATUSES,
|
|
149
|
+
)
|
|
150
|
+
from intentumdiff._differ_runtime import (
|
|
151
|
+
_EXPLICIT_FUEL_EXHAUSTION_TEST_CAP as _EXPLICIT_FUEL_EXHAUSTION_TEST_CAP,
|
|
152
|
+
)
|
|
153
|
+
from intentumdiff._differ_runtime import (
|
|
154
|
+
_FUEL_HOTSPOT_ABSOLUTE as _FUEL_HOTSPOT_ABSOLUTE,
|
|
155
|
+
)
|
|
156
|
+
from intentumdiff._differ_runtime import (
|
|
157
|
+
_FUEL_HOTSPOT_PER_KB as _FUEL_HOTSPOT_PER_KB,
|
|
158
|
+
)
|
|
159
|
+
from intentumdiff._differ_runtime import (
|
|
160
|
+
_FUEL_HOTSPOT_PER_LINE as _FUEL_HOTSPOT_PER_LINE,
|
|
161
|
+
)
|
|
162
|
+
from intentumdiff._differ_runtime import (
|
|
163
|
+
_apply_file_lifecycle_to_diff as _apply_file_lifecycle_to_diff,
|
|
164
|
+
)
|
|
165
|
+
from intentumdiff._differ_runtime import (
|
|
166
|
+
_attach_content_type_metadata as _attach_content_type_metadata,
|
|
167
|
+
)
|
|
168
|
+
from intentumdiff._differ_runtime import (
|
|
169
|
+
_attach_run_telemetry as _attach_run_telemetry,
|
|
170
|
+
)
|
|
171
|
+
from intentumdiff._differ_runtime import (
|
|
172
|
+
_drain_plugin_telemetry as _drain_plugin_telemetry,
|
|
173
|
+
)
|
|
174
|
+
from intentumdiff._differ_runtime import (
|
|
175
|
+
_fuel_budget as _fuel_budget,
|
|
176
|
+
)
|
|
177
|
+
from intentumdiff._differ_runtime import (
|
|
178
|
+
_fuel_hotspot_for_call as _fuel_hotspot_for_call,
|
|
179
|
+
)
|
|
180
|
+
from intentumdiff._differ_runtime import (
|
|
181
|
+
_infer_file_lifecycle as _infer_file_lifecycle,
|
|
182
|
+
)
|
|
183
|
+
from intentumdiff._differ_runtime import (
|
|
184
|
+
_PhaseProfiler as _PhaseProfiler,
|
|
185
|
+
)
|
|
186
|
+
from intentumdiff._differ_runtime import (
|
|
187
|
+
_record_engine_telemetry as _record_engine_telemetry,
|
|
188
|
+
)
|
|
189
|
+
from intentumdiff._differ_runtime import (
|
|
190
|
+
_summarize_engine_telemetry as _summarize_engine_telemetry,
|
|
191
|
+
)
|
|
192
|
+
from intentumdiff.analysis.compile_commands import compile_commands_metadata
|
|
193
|
+
from intentumdiff.analysis.diagnostics import (
|
|
194
|
+
DiagnosticsRecorder,
|
|
195
|
+
)
|
|
196
|
+
from intentumdiff.analysis.guardrails import (
|
|
197
|
+
apply_guardrails_to_diff,
|
|
198
|
+
guardrails_may_apply,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
# All five profile-enrichment families (keyed / path / query / statement /
|
|
202
|
+
# resource) are now Rust-authoritative — no Python enricher on the diff path (#90).
|
|
203
|
+
from intentumdiff.analysis.schema_resolver import (
|
|
204
|
+
resolve_schema,
|
|
205
|
+
schema_cache_fingerprint,
|
|
206
|
+
schema_resolution_metadata,
|
|
207
|
+
)
|
|
208
|
+
from intentumdiff.analysis.text_review import (
|
|
209
|
+
PresentationResult,
|
|
210
|
+
normalize_generic_text_for_review,
|
|
211
|
+
)
|
|
212
|
+
from intentumdiff.analysis.user_schemas import (
|
|
213
|
+
load_user_schema_profiles,
|
|
214
|
+
user_xml_dialects_payload,
|
|
215
|
+
)
|
|
216
|
+
from intentumdiff.cache.store import CacheStore
|
|
217
|
+
from intentumdiff.core.models import (
|
|
218
|
+
Change,
|
|
219
|
+
ChangeGroupKind,
|
|
220
|
+
ChangeStreamEvent,
|
|
221
|
+
ChangeStreamPhase,
|
|
222
|
+
ChangeType,
|
|
223
|
+
DetectionResult,
|
|
224
|
+
DiffConfig,
|
|
225
|
+
LanguageInfoGroup,
|
|
226
|
+
SemanticDiff,
|
|
227
|
+
SemanticNode,
|
|
228
|
+
)
|
|
229
|
+
from intentumdiff.plugins.exceptions import (
|
|
230
|
+
PluginFuelExhausted,
|
|
231
|
+
PluginNotFoundError,
|
|
232
|
+
PluginOutputError,
|
|
233
|
+
)
|
|
234
|
+
from intentumdiff.plugins.loader import _strip_trivia_impl
|
|
235
|
+
from intentumdiff.plugins.registry import PluginRegistry
|
|
236
|
+
from intentumdiff.rust_core import (
|
|
237
|
+
RustCoreCommitJsonAttempt,
|
|
238
|
+
apply_invariances,
|
|
239
|
+
build_style_only_evidence,
|
|
240
|
+
enrich_node_facts,
|
|
241
|
+
try_register_user_xml_dialects,
|
|
242
|
+
try_rust_core_batch_diff,
|
|
243
|
+
try_rust_core_batch_diffs,
|
|
244
|
+
try_rust_core_commit_json,
|
|
245
|
+
try_rust_core_working_tree_commit_json,
|
|
246
|
+
try_rust_profile_label_enrichment,
|
|
247
|
+
)
|
|
248
|
+
from intentumdiff.sources.base import Source
|
|
249
|
+
|
|
250
|
+
logger = logging.getLogger(__name__)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
class SemanticDiffer:
|
|
254
|
+
"""
|
|
255
|
+
High-level semantic differ.
|
|
256
|
+
|
|
257
|
+
Usage::
|
|
258
|
+
|
|
259
|
+
differ = SemanticDiffer()
|
|
260
|
+
diff = differ.diff(GitSource(repo_path, "src/main.py", "HEAD~1", "HEAD"))
|
|
261
|
+
print(diff.has_semantic_changes)
|
|
262
|
+
"""
|
|
263
|
+
|
|
264
|
+
def __init__(
|
|
265
|
+
self,
|
|
266
|
+
config: DiffConfig | None = None,
|
|
267
|
+
registry: PluginRegistry | None = None,
|
|
268
|
+
cache: CacheStore | None = None,
|
|
269
|
+
) -> None:
|
|
270
|
+
# Defensive copy so callers cannot mutate config after construction.
|
|
271
|
+
self._config = config.model_copy() if config is not None else DiffConfig()
|
|
272
|
+
self._registry = registry or PluginRegistry(self._config)
|
|
273
|
+
self._parallel_executor: ThreadPoolExecutor | None = None
|
|
274
|
+
self._parallel_executor_workers: int | None = None
|
|
275
|
+
self._parallel_executor_lock = threading.Lock()
|
|
276
|
+
self._parallel_worker_local = threading.local()
|
|
277
|
+
# Per-grammar wasm binary hash, computed once and memoised.
|
|
278
|
+
self._wasm_hash_cache: dict[str, str] = {}
|
|
279
|
+
|
|
280
|
+
# ── Cache / analytics setup ───────────────────────────────────────
|
|
281
|
+
if cache is not None:
|
|
282
|
+
# Explicitly injected store (e.g. for tests)
|
|
283
|
+
self._cache: CacheStore | None = cache
|
|
284
|
+
self._analytics = None
|
|
285
|
+
elif self._config.cache_path is not None:
|
|
286
|
+
from intentumdiff.cache.sqlite_store import SqliteCacheStore # noqa: PLC0415
|
|
287
|
+
|
|
288
|
+
self._cache = SqliteCacheStore(
|
|
289
|
+
self._config.cache_path / "cache.db",
|
|
290
|
+
ttl_days=self._config.cache_ttl_days,
|
|
291
|
+
max_mb=self._config.cache_max_mb,
|
|
292
|
+
)
|
|
293
|
+
if self._config.analytics_path is not None:
|
|
294
|
+
from intentumdiff.cache.duckdb_store import (
|
|
295
|
+
DuckDBAnalyticsStore, # noqa: PLC0415
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
self._analytics = DuckDBAnalyticsStore(
|
|
299
|
+
self._config.analytics_path / "analytics.duckdb"
|
|
300
|
+
)
|
|
301
|
+
else:
|
|
302
|
+
self._analytics = None
|
|
303
|
+
else:
|
|
304
|
+
self._cache = None
|
|
305
|
+
self._analytics = None
|
|
306
|
+
|
|
307
|
+
# ── Public API ────────────────────────────────────────────────────────────
|
|
308
|
+
|
|
309
|
+
def close(self) -> None:
|
|
310
|
+
"""Release background resources owned by this differ."""
|
|
311
|
+
with self._parallel_executor_lock:
|
|
312
|
+
executor = self._parallel_executor
|
|
313
|
+
self._parallel_executor = None
|
|
314
|
+
self._parallel_executor_workers = None
|
|
315
|
+
self._parallel_worker_local = threading.local()
|
|
316
|
+
if executor is not None:
|
|
317
|
+
executor.shutdown(wait=False, cancel_futures=True)
|
|
318
|
+
|
|
319
|
+
def __del__(self) -> None:
|
|
320
|
+
executor = getattr(self, "_parallel_executor", None)
|
|
321
|
+
if executor is not None:
|
|
322
|
+
executor.shutdown(wait=False, cancel_futures=True)
|
|
323
|
+
|
|
324
|
+
def _new_profiler(self) -> _PhaseProfiler:
|
|
325
|
+
return _PhaseProfiler(bool(self._config.profile_phases))
|
|
326
|
+
|
|
327
|
+
def _parallel_executor_for(
|
|
328
|
+
self,
|
|
329
|
+
max_workers: int,
|
|
330
|
+
) -> tuple[ThreadPoolExecutor, threading.local]:
|
|
331
|
+
with self._parallel_executor_lock:
|
|
332
|
+
if self._parallel_executor is None or self._parallel_executor_workers != max_workers:
|
|
333
|
+
old_executor = self._parallel_executor
|
|
334
|
+
self._parallel_worker_local = threading.local()
|
|
335
|
+
self._parallel_executor = ThreadPoolExecutor(max_workers=max_workers)
|
|
336
|
+
self._parallel_executor_workers = max_workers
|
|
337
|
+
else:
|
|
338
|
+
old_executor = None
|
|
339
|
+
executor = self._parallel_executor
|
|
340
|
+
worker_local = self._parallel_worker_local
|
|
341
|
+
if old_executor is not None:
|
|
342
|
+
old_executor.shutdown(wait=True, cancel_futures=False)
|
|
343
|
+
return executor, worker_local
|
|
344
|
+
|
|
345
|
+
def _attach_phase_timings(
|
|
346
|
+
self,
|
|
347
|
+
diff: SemanticDiff,
|
|
348
|
+
profiler: _PhaseProfiler,
|
|
349
|
+
) -> SemanticDiff:
|
|
350
|
+
if not profiler.enabled:
|
|
351
|
+
return diff
|
|
352
|
+
metadata = dict(diff.metadata)
|
|
353
|
+
metadata["phase_timings"] = profiler.snapshot()
|
|
354
|
+
return diff.model_copy(update={"metadata": metadata})
|
|
355
|
+
|
|
356
|
+
def _attach_commit_batch_phase_timings(
|
|
357
|
+
self,
|
|
358
|
+
diff: SemanticDiff,
|
|
359
|
+
*,
|
|
360
|
+
source_collection_ms: float,
|
|
361
|
+
rust_batch_ms: float,
|
|
362
|
+
rust_commit_merge_ms: float = 0.0,
|
|
363
|
+
rust_adapter_phase_timings: list[dict[str, Any]] | None = None,
|
|
364
|
+
rust_batch_parallel_auto: bool,
|
|
365
|
+
rust_batch_parallel_workers: int | None,
|
|
366
|
+
rust_batch_size: int,
|
|
367
|
+
rust_batch_fallback_count: int,
|
|
368
|
+
) -> SemanticDiff:
|
|
369
|
+
metadata = dict(diff.metadata)
|
|
370
|
+
batch_metadata = dict(metadata.get("rust_core_batch") or {})
|
|
371
|
+
batch_metadata.update(
|
|
372
|
+
{
|
|
373
|
+
"rust_core_batch_parallel_auto": rust_batch_parallel_auto,
|
|
374
|
+
"parallel_workers": rust_batch_parallel_workers,
|
|
375
|
+
"batch_size": rust_batch_size,
|
|
376
|
+
"fallback_count": rust_batch_fallback_count,
|
|
377
|
+
}
|
|
378
|
+
)
|
|
379
|
+
adapter_phases = [
|
|
380
|
+
{
|
|
381
|
+
**phase,
|
|
382
|
+
"shared": True,
|
|
383
|
+
}
|
|
384
|
+
for phase in rust_adapter_phase_timings or []
|
|
385
|
+
if isinstance(phase, dict)
|
|
386
|
+
]
|
|
387
|
+
if adapter_phases:
|
|
388
|
+
batch_metadata["adapter_phase_timings"] = adapter_phases
|
|
389
|
+
metadata["rust_core_batch"] = batch_metadata
|
|
390
|
+
if not self._config.profile_phases:
|
|
391
|
+
return diff.model_copy(update={"metadata": metadata})
|
|
392
|
+
phases: list[dict[str, Any]] = [
|
|
393
|
+
{
|
|
394
|
+
"name": "source_collection",
|
|
395
|
+
"duration_ms": round(source_collection_ms, 3),
|
|
396
|
+
"shared": True,
|
|
397
|
+
}
|
|
398
|
+
]
|
|
399
|
+
if adapter_phases:
|
|
400
|
+
phases.extend(adapter_phases)
|
|
401
|
+
else:
|
|
402
|
+
phases.append(
|
|
403
|
+
{
|
|
404
|
+
"name": "rust_core_batch_execution",
|
|
405
|
+
"duration_ms": round(rust_batch_ms, 3),
|
|
406
|
+
"shared": True,
|
|
407
|
+
}
|
|
408
|
+
)
|
|
409
|
+
metadata["phase_timings"] = {
|
|
410
|
+
"schema_version": 1,
|
|
411
|
+
"total_ms": round(
|
|
412
|
+
source_collection_ms + rust_batch_ms + rust_commit_merge_ms,
|
|
413
|
+
3,
|
|
414
|
+
),
|
|
415
|
+
"phases": phases,
|
|
416
|
+
}
|
|
417
|
+
if rust_commit_merge_ms:
|
|
418
|
+
metadata["phase_timings"]["phases"].append(
|
|
419
|
+
{
|
|
420
|
+
"name": "commit_result_merge",
|
|
421
|
+
"duration_ms": round(rust_commit_merge_ms, 3),
|
|
422
|
+
"shared": True,
|
|
423
|
+
}
|
|
424
|
+
)
|
|
425
|
+
return diff.model_copy(update={"metadata": metadata})
|
|
426
|
+
|
|
427
|
+
def _rust_core_python_batch_skip_reason(self) -> str | None:
|
|
428
|
+
if not self._config.experimental_rust_core:
|
|
429
|
+
return "disabled"
|
|
430
|
+
if self._config.diagnostics:
|
|
431
|
+
return "diagnostics require Python pipeline"
|
|
432
|
+
if self._config.extra_trivia_types:
|
|
433
|
+
return "extra trivia types require Python pipeline"
|
|
434
|
+
if self._config.extra_grammars.get("python"):
|
|
435
|
+
return "custom Python grammar requires Python pipeline"
|
|
436
|
+
allowed = self._config.allowed_plugins
|
|
437
|
+
if allowed is not None and not any(
|
|
438
|
+
plugin_id in allowed
|
|
439
|
+
for plugin_id in (
|
|
440
|
+
"python",
|
|
441
|
+
"python-parser",
|
|
442
|
+
"python_parser",
|
|
443
|
+
"intentumdiff:python:python",
|
|
444
|
+
)
|
|
445
|
+
):
|
|
446
|
+
return "Python parser is not allowed by plugin policy"
|
|
447
|
+
if self._registry.get_enrichers("python"):
|
|
448
|
+
return "enrichers require Python pipeline"
|
|
449
|
+
if self._registry.get_diff_analyzers("python"):
|
|
450
|
+
return "diff analyzers require Python pipeline"
|
|
451
|
+
return None
|
|
452
|
+
|
|
453
|
+
@staticmethod
|
|
454
|
+
def _rust_core_parallel_workers(parallel: int | bool) -> int | None:
|
|
455
|
+
if parallel is True:
|
|
456
|
+
return os.cpu_count() or 1
|
|
457
|
+
if isinstance(parallel, int) and parallel > 0:
|
|
458
|
+
return parallel
|
|
459
|
+
return None
|
|
460
|
+
|
|
461
|
+
@staticmethod
|
|
462
|
+
def _rust_core_batch_parallel_plan(
|
|
463
|
+
file_count: int,
|
|
464
|
+
parallel: int | bool,
|
|
465
|
+
) -> tuple[bool, int | None, bool]:
|
|
466
|
+
"""Return Rust batch parallel settings for the opt-in Rust product lane."""
|
|
467
|
+
if file_count <= 1:
|
|
468
|
+
return False, 1 if file_count == 1 else None, False
|
|
469
|
+
cpu_count = os.cpu_count() or 1
|
|
470
|
+
if parallel is True:
|
|
471
|
+
return True, min(cpu_count, file_count), False
|
|
472
|
+
if isinstance(parallel, int) and not isinstance(parallel, bool) and parallel > 0:
|
|
473
|
+
return True, min(parallel, file_count), False
|
|
474
|
+
return True, min(cpu_count, file_count), True
|
|
475
|
+
|
|
476
|
+
def diff(self, source: Source) -> SemanticDiff:
|
|
477
|
+
"""
|
|
478
|
+
Run the full semantic diff pipeline on a ``Source``.
|
|
479
|
+
|
|
480
|
+
Parameters
|
|
481
|
+
----------
|
|
482
|
+
source:
|
|
483
|
+
Any ``Source`` subclass (``GitSource``, ``FileSource``,
|
|
484
|
+
``StringSource``, ``PatchSource``).
|
|
485
|
+
|
|
486
|
+
Returns
|
|
487
|
+
-------
|
|
488
|
+
SemanticDiff
|
|
489
|
+
Immutable result object.
|
|
490
|
+
"""
|
|
491
|
+
profiler = self._new_profiler()
|
|
492
|
+
with profiler.phase("source_loading"):
|
|
493
|
+
old_content, new_content, filename, language_hint = source.get_content()
|
|
494
|
+
return self._run_pipeline(
|
|
495
|
+
old_content,
|
|
496
|
+
new_content,
|
|
497
|
+
filename,
|
|
498
|
+
language_hint,
|
|
499
|
+
_profiler=profiler,
|
|
500
|
+
)
|
|
501
|
+
|
|
502
|
+
def diff_strings(
|
|
503
|
+
self,
|
|
504
|
+
old: str,
|
|
505
|
+
new: str,
|
|
506
|
+
filename: str,
|
|
507
|
+
language_hint: str | None = None,
|
|
508
|
+
parser_plugin_id: str | None = None,
|
|
509
|
+
) -> SemanticDiff:
|
|
510
|
+
"""Convenience method for in-memory diffs."""
|
|
511
|
+
return self._run_pipeline(
|
|
512
|
+
old,
|
|
513
|
+
new,
|
|
514
|
+
filename,
|
|
515
|
+
language_hint,
|
|
516
|
+
parser_plugin_id=parser_plugin_id,
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
def detect_all(
|
|
520
|
+
self,
|
|
521
|
+
content: str,
|
|
522
|
+
candidates: list[str] | None = None,
|
|
523
|
+
preferred_plugins: dict[str, str] | None = None,
|
|
524
|
+
plugin_id: str | None = None,
|
|
525
|
+
) -> list[DetectionResult]:
|
|
526
|
+
"""Return every parser that claims to handle *content*, ranked by priority.
|
|
527
|
+
|
|
528
|
+
Each registered parser is asked via its own Wasm ``detect-language``
|
|
529
|
+
function — no regex heuristics in Python.
|
|
530
|
+
|
|
531
|
+
Parameters
|
|
532
|
+
----------
|
|
533
|
+
content:
|
|
534
|
+
Source snippet to identify (first 4 KB are inspected).
|
|
535
|
+
candidates:
|
|
536
|
+
Optional shortlist of language IDs. When given, only parsers that
|
|
537
|
+
handle at least one of those IDs are consulted.
|
|
538
|
+
|
|
539
|
+
Returns
|
|
540
|
+
-------
|
|
541
|
+
list[DetectionResult]
|
|
542
|
+
Possibly empty. First element is the most confident match.
|
|
543
|
+
"""
|
|
544
|
+
return self._registry.detect_by_content(
|
|
545
|
+
content,
|
|
546
|
+
candidates,
|
|
547
|
+
preferred_plugins,
|
|
548
|
+
plugin_id,
|
|
549
|
+
)
|
|
550
|
+
|
|
551
|
+
def supported_languages(self) -> list[str]:
|
|
552
|
+
"""Return language IDs reported by registered parser plugins.
|
|
553
|
+
|
|
554
|
+
This is the API used by the playground language picker. The values
|
|
555
|
+
come from each parser plugin's ``language-ids`` export, so plugin
|
|
556
|
+
authors own aliases and displayable IDs in one place.
|
|
557
|
+
"""
|
|
558
|
+
return self._registry.language_ids()
|
|
559
|
+
|
|
560
|
+
def language_info(self) -> list[LanguageInfoGroup]:
|
|
561
|
+
"""Return rich parser-plugin metadata grouped by language ID."""
|
|
562
|
+
return self._registry.language_info()
|
|
563
|
+
|
|
564
|
+
def detect_language(
|
|
565
|
+
self,
|
|
566
|
+
content: str,
|
|
567
|
+
candidates: list[str] | None = None,
|
|
568
|
+
preferred_plugins: dict[str, str] | None = None,
|
|
569
|
+
plugin_id: str | None = None,
|
|
570
|
+
) -> DetectionResult | None:
|
|
571
|
+
"""Return the best-matching language for *content*, or ``None``.
|
|
572
|
+
|
|
573
|
+
Convenience wrapper around :meth:`detect_all` that returns only the
|
|
574
|
+
top result.
|
|
575
|
+
"""
|
|
576
|
+
results = self.detect_all(content, candidates, preferred_plugins, plugin_id)
|
|
577
|
+
return results[0] if results else None
|
|
578
|
+
|
|
579
|
+
def playground_example(
|
|
580
|
+
self,
|
|
581
|
+
language: str,
|
|
582
|
+
plugin_id: str | None = None,
|
|
583
|
+
) -> dict[str, str] | None:
|
|
584
|
+
"""Return the ``{\"old\": ..., \"new\": ...}`` demo pair from the plugin
|
|
585
|
+
that owns *language*, or ``None`` if unavailable."""
|
|
586
|
+
return self._registry.example_for(language, plugin_id)
|
|
587
|
+
|
|
588
|
+
def parse(
|
|
589
|
+
self,
|
|
590
|
+
content: str,
|
|
591
|
+
filename: str,
|
|
592
|
+
language_hint: str | None = None,
|
|
593
|
+
) -> tuple[SemanticNode, str]:
|
|
594
|
+
"""
|
|
595
|
+
Parse *content* as *filename* and return ``(SemanticNode, language)``.
|
|
596
|
+
|
|
597
|
+
This runs pipeline stages 1–7 (language detection, preprocessing,
|
|
598
|
+
tree-sitter parsing, trivia stripping, and Wasm plugin dispatch) and
|
|
599
|
+
respects the parse-tree cache if one is configured.
|
|
600
|
+
|
|
601
|
+
Raises :class:`~intentumdiff.plugins.exceptions.PluginNotFoundError`
|
|
602
|
+
when no registered parser supports *filename*.
|
|
603
|
+
|
|
604
|
+
Parameters
|
|
605
|
+
----------
|
|
606
|
+
content:
|
|
607
|
+
Raw source text of the file.
|
|
608
|
+
filename:
|
|
609
|
+
File name used for language detection and debug messages.
|
|
610
|
+
language_hint:
|
|
611
|
+
Optional language override; if omitted the registry detects it.
|
|
612
|
+
|
|
613
|
+
Returns
|
|
614
|
+
-------
|
|
615
|
+
tuple[SemanticNode, str]
|
|
616
|
+
``(tree, language)`` where *tree* is the fully processed semantic
|
|
617
|
+
tree and *language* is the detected language identifier.
|
|
618
|
+
"""
|
|
619
|
+
parser, language = self._registry.detect_parser(filename, content, language_hint)
|
|
620
|
+
adaptive_fuel = self._config.plugin_fuel
|
|
621
|
+
|
|
622
|
+
if parser.parser_mode == "interpret-cst":
|
|
623
|
+
content = parser.preprocess_source(content)
|
|
624
|
+
cst_json = self._parse(content, parser, language, filename)
|
|
625
|
+
if len(cst_json) > self._config.max_cst_bytes:
|
|
626
|
+
raise ValueError(
|
|
627
|
+
f"CST JSON for {filename!r} is {len(cst_json):,} characters, "
|
|
628
|
+
f"exceeding the limit of {self._config.max_cst_bytes:,} "
|
|
629
|
+
"(DiffConfig.max_cst_bytes)."
|
|
630
|
+
)
|
|
631
|
+
trivia_types = list(parser.trivia_node_types) + self._config.extra_trivia_types
|
|
632
|
+
filtered = _strip_trivia_impl(cst_json, trivia_types)
|
|
633
|
+
cst_nodes = _count_cst_nodes(filtered)
|
|
634
|
+
adaptive_fuel = _fuel_budget(self._config.plugin_fuel, 20_000_000 + cst_nodes * 200_000)
|
|
635
|
+
input_ = filtered
|
|
636
|
+
else:
|
|
637
|
+
input_ = content
|
|
638
|
+
|
|
639
|
+
if self._cache is not None:
|
|
640
|
+
node = self._cached_process(parser, input_, language, filename, adaptive_fuel)
|
|
641
|
+
else:
|
|
642
|
+
node = parser.process(input_, language, filename, fuel=adaptive_fuel)
|
|
643
|
+
|
|
644
|
+
return node, language
|
|
645
|
+
|
|
646
|
+
def render(self, diff: SemanticDiff, output_format: str = "terminal") -> str:
|
|
647
|
+
"""
|
|
648
|
+
Render a ``SemanticDiff`` to a plain string.
|
|
649
|
+
|
|
650
|
+
Parameters
|
|
651
|
+
----------
|
|
652
|
+
diff:
|
|
653
|
+
The diff result to render.
|
|
654
|
+
output_format:
|
|
655
|
+
Output format. Built-in: ``"terminal"`` (default, no ANSI),
|
|
656
|
+
``"terminal-color"`` (ANSI via renderer plugin), ``"json"``.
|
|
657
|
+
Plugin-provided: ``"patch"``, ``"html"``, ``"llm"``, or any
|
|
658
|
+
format name registered by a third-party renderer plugin.
|
|
659
|
+
|
|
660
|
+
Returns
|
|
661
|
+
-------
|
|
662
|
+
str
|
|
663
|
+
The rendered string. ``"terminal"`` output contains no ANSI codes.
|
|
664
|
+
"""
|
|
665
|
+
if output_format == "json":
|
|
666
|
+
return diff.model_dump_json(indent=2)
|
|
667
|
+
|
|
668
|
+
if output_format == "terminal":
|
|
669
|
+
# Inline Python renderer — always available, no ANSI codes.
|
|
670
|
+
from io import StringIO
|
|
671
|
+
|
|
672
|
+
from rich.console import Console as _Console
|
|
673
|
+
|
|
674
|
+
buf = StringIO()
|
|
675
|
+
console = _Console(file=buf, highlight=False, no_color=True)
|
|
676
|
+
console.print(
|
|
677
|
+
f"Semantic diff: {diff.old_filename} -> {diff.new_filename} ({diff.language})"
|
|
678
|
+
)
|
|
679
|
+
for change in diff.changes:
|
|
680
|
+
ct_str = (
|
|
681
|
+
change.change_type.value
|
|
682
|
+
if isinstance(change.change_type, ChangeType)
|
|
683
|
+
else str(change.change_type)
|
|
684
|
+
)
|
|
685
|
+
console.print(f" {ct_str:14} {change.description}")
|
|
686
|
+
return buf.getvalue()
|
|
687
|
+
|
|
688
|
+
try:
|
|
689
|
+
renderer = self._registry.get_renderer(output_format)
|
|
690
|
+
except KeyError as exc:
|
|
691
|
+
raise ValueError(str(exc)) from exc
|
|
692
|
+
diff_json = diff.model_dump_json()
|
|
693
|
+
adaptive_fuel = _fuel_budget(self._config.plugin_fuel, 20_000_000 + len(diff_json) * 100)
|
|
694
|
+
return renderer.render(diff_json, fuel=adaptive_fuel)
|
|
695
|
+
|
|
696
|
+
def _diff_commit_certified_json(
|
|
697
|
+
self,
|
|
698
|
+
repo_path: str | os.PathLike[str] = ".",
|
|
699
|
+
old_ref: str = "HEAD",
|
|
700
|
+
new_ref: str = "",
|
|
701
|
+
) -> RustCoreCommitJsonAttempt:
|
|
702
|
+
"""Return Rust-certified CommitDiff JSON bytes for narrow Python batches."""
|
|
703
|
+
_validate_git_ref(old_ref)
|
|
704
|
+
_validate_git_ref(new_ref)
|
|
705
|
+
rust_batch_skip_reason = self._rust_core_python_batch_skip_reason()
|
|
706
|
+
if rust_batch_skip_reason is not None:
|
|
707
|
+
_raise_rust_only_gate_error(rust_batch_skip_reason)
|
|
708
|
+
return RustCoreCommitJsonAttempt(fallback_reason=rust_batch_skip_reason)
|
|
709
|
+
|
|
710
|
+
if not new_ref:
|
|
711
|
+
rust_batch_parallel, rust_batch_workers, _rust_batch_parallel_auto = (
|
|
712
|
+
True,
|
|
713
|
+
self._rust_core_parallel_workers(self._config.parallel),
|
|
714
|
+
True,
|
|
715
|
+
)
|
|
716
|
+
working_tree_attempt = try_rust_core_working_tree_commit_json(
|
|
717
|
+
repo_path=repo_path,
|
|
718
|
+
old_ref=old_ref,
|
|
719
|
+
config=self._config,
|
|
720
|
+
parallel=rust_batch_parallel,
|
|
721
|
+
max_workers=rust_batch_workers,
|
|
722
|
+
)
|
|
723
|
+
if working_tree_attempt.used:
|
|
724
|
+
return working_tree_attempt
|
|
725
|
+
if working_tree_attempt.fallback_reason and not (
|
|
726
|
+
"does not expose diff_working_tree_python_commit_json"
|
|
727
|
+
in working_tree_attempt.fallback_reason
|
|
728
|
+
or working_tree_attempt.fallback_reason.startswith("unavailable:")
|
|
729
|
+
):
|
|
730
|
+
_raise_rust_only_gate_error(working_tree_attempt.fallback_reason)
|
|
731
|
+
return working_tree_attempt
|
|
732
|
+
|
|
733
|
+
from intentumdiff.plugins.builtins import python_parser_entry
|
|
734
|
+
from intentumdiff.sources.git_source import (
|
|
735
|
+
collect_working_tree_python_sources_fast,
|
|
736
|
+
iter_changed_sources,
|
|
737
|
+
)
|
|
738
|
+
|
|
739
|
+
source_started = time.perf_counter()
|
|
740
|
+
fast_collection = (
|
|
741
|
+
collect_working_tree_python_sources_fast(repo_path, old_ref) if not new_ref else None
|
|
742
|
+
)
|
|
743
|
+
if fast_collection is None:
|
|
744
|
+
sources = list(iter_changed_sources(repo_path, old_ref, new_ref))
|
|
745
|
+
fast_fallback_reason = None
|
|
746
|
+
else:
|
|
747
|
+
sources, fast_fallback_reason = fast_collection
|
|
748
|
+
source_collection_ms = (time.perf_counter() - source_started) * 1000
|
|
749
|
+
if fast_fallback_reason is not None:
|
|
750
|
+
_raise_rust_only_gate_error(fast_fallback_reason)
|
|
751
|
+
return RustCoreCommitJsonAttempt(fallback_reason=fast_fallback_reason)
|
|
752
|
+
if not sources:
|
|
753
|
+
return RustCoreCommitJsonAttempt()
|
|
754
|
+
|
|
755
|
+
parser_wasm_path = python_parser_entry()
|
|
756
|
+
request_files: list[dict[str, Any]] = []
|
|
757
|
+
source_filter_started = time.perf_counter()
|
|
758
|
+
for old_content, new_content, old_path, new_path, staging_status in sources:
|
|
759
|
+
effective_new_path = new_path if new_path is not None else old_path
|
|
760
|
+
file_lifecycle = _infer_file_lifecycle(old_content, new_content, staging_status)
|
|
761
|
+
old_path_lower = old_path.lower()
|
|
762
|
+
effective_new_path_lower = effective_new_path.lower()
|
|
763
|
+
if not (
|
|
764
|
+
old_path_lower.endswith((".py", ".pyi"))
|
|
765
|
+
or effective_new_path_lower.endswith((".py", ".pyi"))
|
|
766
|
+
):
|
|
767
|
+
fallback_attempt = RustCoreCommitJsonAttempt(
|
|
768
|
+
fallback_reason="certified commit JSON requires all changed files to be Python"
|
|
769
|
+
)
|
|
770
|
+
_raise_rust_only_gate_error(fallback_attempt.fallback_reason)
|
|
771
|
+
return fallback_attempt
|
|
772
|
+
# Protected guardrail rules currently apply only to config/profile
|
|
773
|
+
# languages, not Python. Avoid a per-file policy search here; any
|
|
774
|
+
# actual policy-file change is excluded by the Python-only gate.
|
|
775
|
+
request_file: dict[str, Any] = {
|
|
776
|
+
"old_source": old_content,
|
|
777
|
+
"new_source": new_content,
|
|
778
|
+
"old_filename": old_path,
|
|
779
|
+
"new_filename": effective_new_path,
|
|
780
|
+
"language": "python",
|
|
781
|
+
"parser_plugin_id": "python",
|
|
782
|
+
"parser_wasm_path": parser_wasm_path,
|
|
783
|
+
"file_lifecycle": file_lifecycle,
|
|
784
|
+
}
|
|
785
|
+
if staging_status is not None:
|
|
786
|
+
request_file["staging_status"] = staging_status
|
|
787
|
+
request_files.append(request_file)
|
|
788
|
+
source_filter_ms = (time.perf_counter() - source_filter_started) * 1000
|
|
789
|
+
|
|
790
|
+
rust_batch_parallel, rust_batch_workers, _rust_batch_parallel_auto = (
|
|
791
|
+
self._rust_core_batch_parallel_plan(len(request_files), self._config.parallel)
|
|
792
|
+
)
|
|
793
|
+
attempt = try_rust_core_commit_json(
|
|
794
|
+
files=request_files,
|
|
795
|
+
old_ref=old_ref,
|
|
796
|
+
new_ref=new_ref,
|
|
797
|
+
config=self._config,
|
|
798
|
+
parallel=rust_batch_parallel,
|
|
799
|
+
max_workers=rust_batch_workers,
|
|
800
|
+
)
|
|
801
|
+
if not attempt.used:
|
|
802
|
+
_raise_rust_only_gate_error(attempt.fallback_reason)
|
|
803
|
+
return attempt
|
|
804
|
+
adapter_phase_timings = [
|
|
805
|
+
{
|
|
806
|
+
"name": "source_collection",
|
|
807
|
+
"duration_ms": round(source_collection_ms, 3),
|
|
808
|
+
"shared": True,
|
|
809
|
+
},
|
|
810
|
+
{
|
|
811
|
+
"name": "rust_core_commit_source_filtering",
|
|
812
|
+
"duration_ms": round(source_filter_ms, 3),
|
|
813
|
+
"shared": True,
|
|
814
|
+
},
|
|
815
|
+
*attempt.adapter_phase_timings,
|
|
816
|
+
]
|
|
817
|
+
control = dict(attempt.control)
|
|
818
|
+
control["source_collection_ms"] = round(source_collection_ms, 3)
|
|
819
|
+
control["source_filter_ms"] = round(source_filter_ms, 3)
|
|
820
|
+
return RustCoreCommitJsonAttempt(
|
|
821
|
+
control=control,
|
|
822
|
+
commit_diff_json=attempt.commit_diff_json,
|
|
823
|
+
adapter_phase_timings=adapter_phase_timings,
|
|
824
|
+
backend_version=attempt.backend_version,
|
|
825
|
+
)
|
|
826
|
+
|
|
827
|
+
def diff_commit(
|
|
828
|
+
self,
|
|
829
|
+
repo_path: str | os.PathLike[str] = ".",
|
|
830
|
+
old_ref: str = "HEAD",
|
|
831
|
+
new_ref: str = "",
|
|
832
|
+
) -> list[SemanticDiff]:
|
|
833
|
+
_validate_git_ref(old_ref)
|
|
834
|
+
_validate_git_ref(new_ref)
|
|
835
|
+
"""
|
|
836
|
+
Diff every file that changed between two git refs.
|
|
837
|
+
|
|
838
|
+
Files that have no registered parser are silently skipped.
|
|
839
|
+
|
|
840
|
+
Parameters
|
|
841
|
+
----------
|
|
842
|
+
repo_path:
|
|
843
|
+
Path to (any directory inside) the git repository.
|
|
844
|
+
old_ref:
|
|
845
|
+
Old git ref (commit SHA, tag, branch). Defaults to ``"HEAD"``.
|
|
846
|
+
new_ref:
|
|
847
|
+
New git ref. Defaults to ``""`` (working tree — diff against
|
|
848
|
+
current unsaved/uncommitted files).
|
|
849
|
+
|
|
850
|
+
Returns
|
|
851
|
+
-------
|
|
852
|
+
list[SemanticDiff]
|
|
853
|
+
One entry per changed file that could be parsed.
|
|
854
|
+
"""
|
|
855
|
+
from intentumdiff.sources.git_source import iter_changed_sources
|
|
856
|
+
|
|
857
|
+
source_started = time.perf_counter()
|
|
858
|
+
sources = list(iter_changed_sources(repo_path, old_ref, new_ref))
|
|
859
|
+
source_collection_ms = (time.perf_counter() - source_started) * 1000
|
|
860
|
+
rust_batch_results: dict[int, SemanticDiff] = {}
|
|
861
|
+
rust_batch_attempted: set[int] = set()
|
|
862
|
+
rust_batch_skip_reason = self._rust_core_python_batch_skip_reason()
|
|
863
|
+
if rust_batch_skip_reason is None and sources:
|
|
864
|
+
from intentumdiff.plugins.builtins import python_parser_entry
|
|
865
|
+
|
|
866
|
+
parser_wasm_path = python_parser_entry()
|
|
867
|
+
request_files: list[dict[str, Any]] = []
|
|
868
|
+
request_to_source_index: list[int] = []
|
|
869
|
+
rust_batch_source_filter_started = time.perf_counter()
|
|
870
|
+
for idx, item in enumerate(sources):
|
|
871
|
+
old_content, new_content, old_path, new_path, _staging_status = item
|
|
872
|
+
effective_new_path = new_path if new_path is not None else old_path
|
|
873
|
+
file_lifecycle = _infer_file_lifecycle(old_content, new_content, _staging_status)
|
|
874
|
+
old_path_lower = old_path.lower()
|
|
875
|
+
effective_new_path_lower = effective_new_path.lower()
|
|
876
|
+
if not (
|
|
877
|
+
old_path_lower.endswith((".py", ".pyi"))
|
|
878
|
+
or effective_new_path_lower.endswith((".py", ".pyi"))
|
|
879
|
+
):
|
|
880
|
+
continue
|
|
881
|
+
# Protected guardrail rules currently apply only to
|
|
882
|
+
# config/profile languages. Certified Python batches can skip
|
|
883
|
+
# the expensive policy lookup; policy-file changes are not
|
|
884
|
+
# Python files and therefore remain on the fallback path.
|
|
885
|
+
request_to_source_index.append(idx)
|
|
886
|
+
request_files.append(
|
|
887
|
+
{
|
|
888
|
+
"old_source": old_content,
|
|
889
|
+
"new_source": new_content,
|
|
890
|
+
"old_filename": old_path,
|
|
891
|
+
"new_filename": effective_new_path,
|
|
892
|
+
"language": "python",
|
|
893
|
+
"parser_plugin_id": "python",
|
|
894
|
+
"parser_wasm_path": parser_wasm_path,
|
|
895
|
+
"file_lifecycle": file_lifecycle,
|
|
896
|
+
}
|
|
897
|
+
)
|
|
898
|
+
rust_batch_source_filter_ms = (
|
|
899
|
+
time.perf_counter() - rust_batch_source_filter_started
|
|
900
|
+
) * 1000
|
|
901
|
+
if request_files:
|
|
902
|
+
rust_batch_attempted = set(request_to_source_index)
|
|
903
|
+
(
|
|
904
|
+
rust_batch_parallel,
|
|
905
|
+
rust_batch_workers,
|
|
906
|
+
rust_batch_parallel_auto,
|
|
907
|
+
) = self._rust_core_batch_parallel_plan(
|
|
908
|
+
len(request_files),
|
|
909
|
+
self._config.parallel,
|
|
910
|
+
)
|
|
911
|
+
rust_batch_started = time.perf_counter()
|
|
912
|
+
rust_batch = try_rust_core_batch_diffs(
|
|
913
|
+
files=request_files,
|
|
914
|
+
config=self._config,
|
|
915
|
+
parallel=rust_batch_parallel,
|
|
916
|
+
max_workers=rust_batch_workers,
|
|
917
|
+
)
|
|
918
|
+
rust_batch_ms = (time.perf_counter() - rust_batch_started) * 1000
|
|
919
|
+
rust_batch_fallback_count = int(
|
|
920
|
+
rust_batch.batch_metadata.get("fallback_count")
|
|
921
|
+
or len(rust_batch.fallback_reasons)
|
|
922
|
+
)
|
|
923
|
+
# A per-item batch fallback is NOT a Python fallback: the declined
|
|
924
|
+
# files re-run through ``_run_one`` -> ``_run_pipeline`` below, whose
|
|
925
|
+
# per-stage Rust finalize serves them natively. The RUST_ONLY gate
|
|
926
|
+
# therefore does not fire here; it fires inside ``_run_pipeline`` only
|
|
927
|
+
# if an item genuinely reaches a token-level fallback (and the fan-out
|
|
928
|
+
# re-raises that RustOnlyGateError instead of swallowing it).
|
|
929
|
+
rust_merge_started = time.perf_counter()
|
|
930
|
+
rust_merged_results: list[tuple[int, SemanticDiff]] = []
|
|
931
|
+
for request_index, diff in rust_batch.diffs.items():
|
|
932
|
+
if request_index < 0 or request_index >= len(request_to_source_index):
|
|
933
|
+
continue
|
|
934
|
+
source_index = request_to_source_index[request_index]
|
|
935
|
+
_old_content, _new_content, _old_path, _new_path, staging_status = sources[
|
|
936
|
+
source_index
|
|
937
|
+
]
|
|
938
|
+
diff = _apply_file_lifecycle_to_diff(
|
|
939
|
+
diff,
|
|
940
|
+
_infer_file_lifecycle(_old_content, _new_content, staging_status),
|
|
941
|
+
)
|
|
942
|
+
if staging_status is not None:
|
|
943
|
+
diff = diff.model_copy(update={"staging_status": staging_status})
|
|
944
|
+
rust_merged_results.append((source_index, diff))
|
|
945
|
+
rust_commit_merge_ms = (time.perf_counter() - rust_merge_started) * 1000
|
|
946
|
+
for source_index, diff in rust_merged_results:
|
|
947
|
+
rust_batch_results[source_index] = self._attach_commit_batch_phase_timings(
|
|
948
|
+
diff,
|
|
949
|
+
source_collection_ms=source_collection_ms,
|
|
950
|
+
rust_batch_ms=rust_batch_ms,
|
|
951
|
+
rust_commit_merge_ms=rust_commit_merge_ms,
|
|
952
|
+
rust_adapter_phase_timings=[
|
|
953
|
+
{
|
|
954
|
+
"name": "rust_core_commit_source_filtering",
|
|
955
|
+
"duration_ms": round(rust_batch_source_filter_ms, 3),
|
|
956
|
+
"shared": True,
|
|
957
|
+
},
|
|
958
|
+
*rust_batch.adapter_phase_timings,
|
|
959
|
+
],
|
|
960
|
+
rust_batch_parallel_auto=rust_batch_parallel_auto,
|
|
961
|
+
rust_batch_parallel_workers=rust_batch_workers,
|
|
962
|
+
rust_batch_size=len(request_files),
|
|
963
|
+
rust_batch_fallback_count=rust_batch_fallback_count,
|
|
964
|
+
)
|
|
965
|
+
if rust_batch.fallback_reasons or rust_batch.fallback_reason:
|
|
966
|
+
logger.debug(
|
|
967
|
+
"Rust core commit batch used %d/%d files; fallbacks=%s reason=%s",
|
|
968
|
+
len(rust_batch_results),
|
|
969
|
+
len(request_files),
|
|
970
|
+
rust_batch.fallback_reasons,
|
|
971
|
+
rust_batch.fallback_reason,
|
|
972
|
+
)
|
|
973
|
+
elif rust_batch_skip_reason is not None:
|
|
974
|
+
# The commit BATCH shortcut was skipped (guardrails / enrichers / analyzers
|
|
975
|
+
# / diagnostics / trivia / policy / rust-core disabled). Every source now
|
|
976
|
+
# runs through ``_run_one`` -> ``_run_pipeline`` below, which is the native
|
|
977
|
+
# per-stage Rust finalize path; the RUST_ONLY gate fires there, and only if
|
|
978
|
+
# an item genuinely reaches a token-level fallback. A batch skip is not
|
|
979
|
+
# itself a Python fallback, so no gate fires here.
|
|
980
|
+
logger.debug("Rust core commit batch skipped: %s", rust_batch_skip_reason)
|
|
981
|
+
|
|
982
|
+
def _run_one(
|
|
983
|
+
differ: SemanticDiffer,
|
|
984
|
+
item: tuple[str, str, str, str | None, str | None],
|
|
985
|
+
*,
|
|
986
|
+
include_source_collection: bool = False,
|
|
987
|
+
) -> SemanticDiff:
|
|
988
|
+
old_content, new_content, old_path, new_path, staging_status = item
|
|
989
|
+
profiler = differ._new_profiler()
|
|
990
|
+
if include_source_collection:
|
|
991
|
+
profiler.record("source_collection", source_collection_ms, shared=True)
|
|
992
|
+
result = differ._run_pipeline(
|
|
993
|
+
old_content,
|
|
994
|
+
new_content,
|
|
995
|
+
old_path,
|
|
996
|
+
None,
|
|
997
|
+
new_filename=new_path,
|
|
998
|
+
_profiler=profiler,
|
|
999
|
+
)
|
|
1000
|
+
if staging_status is not None:
|
|
1001
|
+
result = result.model_copy(update={"staging_status": staging_status})
|
|
1002
|
+
return result
|
|
1003
|
+
|
|
1004
|
+
_skip_exc = (PluginNotFoundError,)
|
|
1005
|
+
_warn_exc = (PluginFuelExhausted, ValueError, RuntimeError)
|
|
1006
|
+
|
|
1007
|
+
parallel = self._config.parallel
|
|
1008
|
+
if not parallel:
|
|
1009
|
+
# ── Sequential (default) ─────────────────────────────────────────
|
|
1010
|
+
results: list[SemanticDiff] = []
|
|
1011
|
+
for idx, item in enumerate(sources):
|
|
1012
|
+
if idx in rust_batch_results:
|
|
1013
|
+
results.append(rust_batch_results[idx])
|
|
1014
|
+
continue
|
|
1015
|
+
# Files the commit BATCH attempted but declined re-run through the normal
|
|
1016
|
+
# differ with the Rust core ON: the batch re-attempt is one cheap native
|
|
1017
|
+
# call and the per-stage finalize routing then serves. (The old
|
|
1018
|
+
# experimental_rust_core=False fallback differ meant "python pipeline";
|
|
1019
|
+
# post-retirement it means the token-fallback kill switch — a comment-only
|
|
1020
|
+
# commit came back as a token ADDITION instead of style-only.)
|
|
1021
|
+
run_differ = self
|
|
1022
|
+
try:
|
|
1023
|
+
results.append(
|
|
1024
|
+
_run_one(
|
|
1025
|
+
run_differ,
|
|
1026
|
+
item,
|
|
1027
|
+
include_source_collection=idx == 0 and not rust_batch_results,
|
|
1028
|
+
)
|
|
1029
|
+
)
|
|
1030
|
+
except RustOnlyGateError:
|
|
1031
|
+
# A genuine token-level fallback under the RUST_ONLY gate is fatal —
|
|
1032
|
+
# propagate it rather than let ``_warn_exc`` silently drop the file.
|
|
1033
|
+
raise
|
|
1034
|
+
except _skip_exc:
|
|
1035
|
+
logger.debug("Skipping %r — no parser available", item[2])
|
|
1036
|
+
except _warn_exc:
|
|
1037
|
+
logger.warning("Failed to diff %r", item[2], exc_info=True)
|
|
1038
|
+
return results
|
|
1039
|
+
else:
|
|
1040
|
+
# ── Parallel ────────────────────────────────────────────────
|
|
1041
|
+
max_workers = (os.cpu_count() or 1) if parallel is True else int(parallel)
|
|
1042
|
+
executor, worker_local = self._parallel_executor_for(max_workers)
|
|
1043
|
+
|
|
1044
|
+
def _worker_differ() -> SemanticDiffer:
|
|
1045
|
+
differ = getattr(worker_local, "differ", None)
|
|
1046
|
+
if differ is None:
|
|
1047
|
+
differ = SemanticDiffer(config=self._config.model_copy())
|
|
1048
|
+
worker_local.differ = differ
|
|
1049
|
+
return differ
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def _run_worker(
|
|
1053
|
+
idx: int,
|
|
1054
|
+
item: tuple[str, str, str, str | None, str | None],
|
|
1055
|
+
) -> SemanticDiff:
|
|
1056
|
+
# Batch-declined files re-run with the Rust core ON (see the
|
|
1057
|
+
# sequential branch note — the finalize tier serves them).
|
|
1058
|
+
return _run_one(
|
|
1059
|
+
_worker_differ(),
|
|
1060
|
+
item,
|
|
1061
|
+
include_source_collection=idx == 0 and not rust_batch_results,
|
|
1062
|
+
)
|
|
1063
|
+
|
|
1064
|
+
results_map: dict[int, SemanticDiff] = dict(rust_batch_results)
|
|
1065
|
+
if executor is not None:
|
|
1066
|
+
future_to_idx = {
|
|
1067
|
+
executor.submit(_run_worker, i, item): i
|
|
1068
|
+
for i, item in enumerate(sources)
|
|
1069
|
+
if i not in rust_batch_results
|
|
1070
|
+
}
|
|
1071
|
+
for future in as_completed(future_to_idx):
|
|
1072
|
+
idx = future_to_idx[future]
|
|
1073
|
+
try:
|
|
1074
|
+
results_map[idx] = future.result()
|
|
1075
|
+
except RustOnlyGateError:
|
|
1076
|
+
# Fatal under the RUST_ONLY gate — propagate, do not drop.
|
|
1077
|
+
raise
|
|
1078
|
+
except _skip_exc:
|
|
1079
|
+
logger.debug("Skipping %r — no parser available", sources[idx][2])
|
|
1080
|
+
except _warn_exc:
|
|
1081
|
+
logger.warning("Failed to diff %r", sources[idx][2], exc_info=True)
|
|
1082
|
+
return [results_map[i] for i in sorted(results_map)]
|
|
1083
|
+
|
|
1084
|
+
def diff_stream(
|
|
1085
|
+
self,
|
|
1086
|
+
source: Source,
|
|
1087
|
+
) -> Iterator[Change]:
|
|
1088
|
+
"""
|
|
1089
|
+
Yield ``Change`` objects one-by-one from a single-file diff.
|
|
1090
|
+
|
|
1091
|
+
The full pipeline runs before the first item is yielded. This is a
|
|
1092
|
+
convenience wrapper over :meth:`diff` for callers that prefer a
|
|
1093
|
+
generator interface (e.g. ``intentumdiff file --stream``).
|
|
1094
|
+
"""
|
|
1095
|
+
diff = self.diff(source)
|
|
1096
|
+
yield from diff.changes
|
|
1097
|
+
|
|
1098
|
+
# ── Pipeline ──────────────────────────────────────────────────────────────
|
|
1099
|
+
|
|
1100
|
+
def _run_pipeline(
|
|
1101
|
+
self,
|
|
1102
|
+
old_content: str,
|
|
1103
|
+
new_content: str,
|
|
1104
|
+
filename: str,
|
|
1105
|
+
language_hint: str | None,
|
|
1106
|
+
*,
|
|
1107
|
+
new_filename: str | None = None,
|
|
1108
|
+
parser_plugin_id: str | None = None,
|
|
1109
|
+
_profiler: _PhaseProfiler | None = None,
|
|
1110
|
+
) -> SemanticDiff:
|
|
1111
|
+
profiler = _profiler or self._new_profiler()
|
|
1112
|
+
result = self._run_stages_1_to_11(
|
|
1113
|
+
old_content,
|
|
1114
|
+
new_content,
|
|
1115
|
+
filename,
|
|
1116
|
+
language_hint,
|
|
1117
|
+
new_filename=new_filename,
|
|
1118
|
+
parser_plugin_id=parser_plugin_id,
|
|
1119
|
+
profiler=profiler,
|
|
1120
|
+
)
|
|
1121
|
+
# _run_stages_1_to_11 always returns a finished SemanticDiff since the python
|
|
1122
|
+
# pipeline retirement (issue #57 payoff, stage 4b).
|
|
1123
|
+
diff = result
|
|
1124
|
+
diff = _attach_content_type_metadata(diff, old_content, new_content)
|
|
1125
|
+
return self._attach_phase_timings(diff, profiler)
|
|
1126
|
+
|
|
1127
|
+
def _run_diff_analyzers_on_routed(
|
|
1128
|
+
self,
|
|
1129
|
+
routed: SemanticDiff,
|
|
1130
|
+
language: str,
|
|
1131
|
+
filename: str,
|
|
1132
|
+
adaptive_fuel: int,
|
|
1133
|
+
) -> SemanticDiff:
|
|
1134
|
+
for analyzer in self._registry.get_diff_analyzers(language):
|
|
1135
|
+
try:
|
|
1136
|
+
updated_json = analyzer.analyze_diff(
|
|
1137
|
+
routed.model_dump_json(),
|
|
1138
|
+
language,
|
|
1139
|
+
filename,
|
|
1140
|
+
fuel=adaptive_fuel,
|
|
1141
|
+
)
|
|
1142
|
+
routed = SemanticDiff.model_validate_json(updated_json)
|
|
1143
|
+
except Exception as exc:
|
|
1144
|
+
logger.warning(
|
|
1145
|
+
"Diff analyzer %r failed for %r: %s - skipping",
|
|
1146
|
+
getattr(analyzer, "provenance", "?"),
|
|
1147
|
+
filename,
|
|
1148
|
+
exc,
|
|
1149
|
+
)
|
|
1150
|
+
continue
|
|
1151
|
+
return routed
|
|
1152
|
+
|
|
1153
|
+
def _run_stages_1_to_11(
|
|
1154
|
+
self,
|
|
1155
|
+
old_content: str,
|
|
1156
|
+
new_content: str,
|
|
1157
|
+
filename: str,
|
|
1158
|
+
language_hint: str | None,
|
|
1159
|
+
*,
|
|
1160
|
+
new_filename: str | None = None,
|
|
1161
|
+
parser_plugin_id: str | None = None,
|
|
1162
|
+
profiler: _PhaseProfiler | None = None,
|
|
1163
|
+
) -> SemanticDiff:
|
|
1164
|
+
"""Run pipeline stages 1–11 (parse → trivia-strip → Wasm → GumTree → moves).
|
|
1165
|
+
|
|
1166
|
+
Returns ``SemanticDiff`` for early exits (cache hit, style-only, fallback).
|
|
1167
|
+
"""
|
|
1168
|
+
# ── 1 + 2. Language detection ────────────────────────────────────────
|
|
1169
|
+
profiler = profiler or self._new_profiler()
|
|
1170
|
+
with profiler.phase("parser_selection"):
|
|
1171
|
+
parser, language = self._registry.detect_parser(
|
|
1172
|
+
filename,
|
|
1173
|
+
new_content,
|
|
1174
|
+
language_hint,
|
|
1175
|
+
plugin_id=parser_plugin_id,
|
|
1176
|
+
phase_recorder=profiler.record if profiler.enabled else None,
|
|
1177
|
+
)
|
|
1178
|
+
logger.debug("Using parser %r for %r (language=%r)", parser.grammar_id, filename, language)
|
|
1179
|
+
|
|
1180
|
+
diagnostics = DiagnosticsRecorder(
|
|
1181
|
+
enabled=self._config.diagnostics,
|
|
1182
|
+
max_events=self._config.diagnostics_max_events,
|
|
1183
|
+
)
|
|
1184
|
+
engine_telemetry: list[dict[str, Any]] = []
|
|
1185
|
+
parse_mode = parser.parser_mode
|
|
1186
|
+
cache_grammar_id = parser.grammar_id
|
|
1187
|
+
diagnostics.summary.update(
|
|
1188
|
+
{
|
|
1189
|
+
"language": language,
|
|
1190
|
+
"old_filename": filename,
|
|
1191
|
+
"new_filename": new_filename if new_filename is not None else filename,
|
|
1192
|
+
"parser_grammar_id": parser.grammar_id,
|
|
1193
|
+
"effective_grammar_id": cache_grammar_id,
|
|
1194
|
+
"parse_mode": parse_mode,
|
|
1195
|
+
}
|
|
1196
|
+
)
|
|
1197
|
+
diagnostics.record(
|
|
1198
|
+
stage="parser",
|
|
1199
|
+
action="select_parser",
|
|
1200
|
+
rule_id="pipeline.parser_selection",
|
|
1201
|
+
reason="parser selected by registry",
|
|
1202
|
+
metadata={
|
|
1203
|
+
"language_hint": language_hint,
|
|
1204
|
+
"parser_plugin_id": parser_plugin_id,
|
|
1205
|
+
"grammar_id": parser.grammar_id,
|
|
1206
|
+
"effective_grammar_id": cache_grammar_id,
|
|
1207
|
+
"parse_mode": parse_mode,
|
|
1208
|
+
},
|
|
1209
|
+
)
|
|
1210
|
+
|
|
1211
|
+
# Adaptive fuel starts at the configured floor; the interpret-cst branch
|
|
1212
|
+
# raises it based on CST node count before calling the plugin.
|
|
1213
|
+
adaptive_fuel: int = self._config.plugin_fuel
|
|
1214
|
+
|
|
1215
|
+
# Save originals for the enricher pass (enrichers see un-preprocessed source)
|
|
1216
|
+
old_raw, new_raw = old_content, new_content
|
|
1217
|
+
file_lifecycle = _infer_file_lifecycle(old_raw, new_raw)
|
|
1218
|
+
diagnostics.summary["file_lifecycle"] = file_lifecycle
|
|
1219
|
+
with profiler.phase("schema_resolution"):
|
|
1220
|
+
schema_resolution = resolve_schema(
|
|
1221
|
+
content=new_content or old_content,
|
|
1222
|
+
filename=new_filename if new_filename is not None else filename,
|
|
1223
|
+
language=language,
|
|
1224
|
+
)
|
|
1225
|
+
if language.lower() == "xml":
|
|
1226
|
+
# User XML dialects (issue #86): marshal the descriptor registry's
|
|
1227
|
+
# declarative coordinate specs into the Rust matcher. Idempotent per
|
|
1228
|
+
# payload; dialects are match-predicated engine-side, so registering
|
|
1229
|
+
# the full set is safe for every xml diff.
|
|
1230
|
+
_user_profiles, _ = load_user_schema_profiles()
|
|
1231
|
+
_dialects = user_xml_dialects_payload(_user_profiles)
|
|
1232
|
+
if _dialects:
|
|
1233
|
+
try_register_user_xml_dialects(_dialects)
|
|
1234
|
+
schema_fingerprint = schema_cache_fingerprint(schema_resolution)
|
|
1235
|
+
cache_schema_suffix = (
|
|
1236
|
+
"" if schema_fingerprint == "schema:none" else f"|{schema_fingerprint}"
|
|
1237
|
+
)
|
|
1238
|
+
schema_metadata = schema_resolution_metadata(schema_resolution)
|
|
1239
|
+
if schema_metadata is not None:
|
|
1240
|
+
diagnostics.summary["schema"] = schema_metadata
|
|
1241
|
+
diagnostics.record(
|
|
1242
|
+
stage="schema",
|
|
1243
|
+
action="resolve_schema",
|
|
1244
|
+
rule_id="analysis.schema_resolution",
|
|
1245
|
+
reason="JSON/YAML schema source resolved for optional keyed-data hints",
|
|
1246
|
+
metadata=schema_metadata,
|
|
1247
|
+
)
|
|
1248
|
+
with profiler.phase("compile_command_resolution"):
|
|
1249
|
+
compile_metadata = compile_commands_metadata(
|
|
1250
|
+
filename=new_filename if new_filename is not None else filename,
|
|
1251
|
+
language=language,
|
|
1252
|
+
)
|
|
1253
|
+
if compile_metadata is not None:
|
|
1254
|
+
diagnostics.summary["compile_commands"] = compile_metadata
|
|
1255
|
+
diagnostics.record(
|
|
1256
|
+
stage="compile_commands",
|
|
1257
|
+
action="resolve_compile_command",
|
|
1258
|
+
rule_id="analysis.compile_commands",
|
|
1259
|
+
reason="C/C++ compile database context resolved",
|
|
1260
|
+
metadata=compile_metadata,
|
|
1261
|
+
)
|
|
1262
|
+
compile_cache_suffix = (
|
|
1263
|
+
"" if compile_metadata is None else f"|compile:{compile_metadata['fingerprint']}"
|
|
1264
|
+
)
|
|
1265
|
+
|
|
1266
|
+
# Cache state — resolved later once we have preprocessed content.
|
|
1267
|
+
_diff_key: str | None = None
|
|
1268
|
+
_guardrail_cache_bypass = guardrails_may_apply(filename, new_filename, self._config)
|
|
1269
|
+
cache_grammar_id = parser.grammar_id
|
|
1270
|
+
|
|
1271
|
+
if (
|
|
1272
|
+
self._config.experimental_rust_core
|
|
1273
|
+
and language in _rust_certified_languages()
|
|
1274
|
+
and not diagnostics.enabled
|
|
1275
|
+
# ``test_matching_engine`` is the dual-run truthiness override.
|
|
1276
|
+
# When set, it forces the matching stage to either Rust or
|
|
1277
|
+
# Python; the certified Python batch path bypasses that stage
|
|
1278
|
+
# entirely, so we must skip it to make the override meaningful.
|
|
1279
|
+
and self._config.test_matching_engine is None
|
|
1280
|
+
):
|
|
1281
|
+
rust_core_skip_reason: str | None = None
|
|
1282
|
+
if _guardrail_cache_bypass:
|
|
1283
|
+
rust_core_skip_reason = "guardrails may affect final diff"
|
|
1284
|
+
elif self._config.extra_trivia_types:
|
|
1285
|
+
rust_core_skip_reason = "extra trivia types require Python pipeline"
|
|
1286
|
+
else:
|
|
1287
|
+
with profiler.phase("rust_core_certified_surface_check"):
|
|
1288
|
+
if self._registry.get_enrichers(language):
|
|
1289
|
+
rust_core_skip_reason = "enrichers require Python pipeline"
|
|
1290
|
+
elif self._registry.get_diff_analyzers(language):
|
|
1291
|
+
rust_core_skip_reason = "diff analyzers require Python pipeline"
|
|
1292
|
+
|
|
1293
|
+
if rust_core_skip_reason is None:
|
|
1294
|
+
with profiler.phase("rust_core_batch_execution"):
|
|
1295
|
+
rust_batch_attempt = try_rust_core_batch_diff(
|
|
1296
|
+
old_source=old_content,
|
|
1297
|
+
new_source=new_content,
|
|
1298
|
+
old_filename=filename,
|
|
1299
|
+
new_filename=new_filename if new_filename is not None else filename,
|
|
1300
|
+
parser_wasm_path=parser.wasm_path,
|
|
1301
|
+
language=language,
|
|
1302
|
+
parser_plugin_id=parser.plugin_id or parser.grammar_id,
|
|
1303
|
+
config=self._config,
|
|
1304
|
+
file_lifecycle=file_lifecycle,
|
|
1305
|
+
)
|
|
1306
|
+
if rust_batch_attempt.used and rust_batch_attempt.diff is not None:
|
|
1307
|
+
diagnostics.record(
|
|
1308
|
+
stage="rust_core",
|
|
1309
|
+
action="batch_final_diff",
|
|
1310
|
+
rule_id="rust_core.batch_v4",
|
|
1311
|
+
reason="Rust core produced a certified public diff",
|
|
1312
|
+
metadata={
|
|
1313
|
+
"engine": rust_batch_attempt.diff.metadata.get("rust_core", {}).get(
|
|
1314
|
+
"engine"
|
|
1315
|
+
),
|
|
1316
|
+
"stage": rust_batch_attempt.diff.metadata.get("rust_core", {}).get(
|
|
1317
|
+
"stage"
|
|
1318
|
+
),
|
|
1319
|
+
},
|
|
1320
|
+
)
|
|
1321
|
+
return _apply_file_lifecycle_to_diff(
|
|
1322
|
+
rust_batch_attempt.diff,
|
|
1323
|
+
file_lifecycle,
|
|
1324
|
+
)
|
|
1325
|
+
# The certified BATCH shortcut declined — fall through to the native
|
|
1326
|
+
# routed finalize path below. This is a Rust→Rust transition (the
|
|
1327
|
+
# routed path finalizes through the Rust core for every certified
|
|
1328
|
+
# language), NOT a Python fallback, so the RUST_ONLY gate does NOT
|
|
1329
|
+
# fire here. It fires only at the genuine token-level fallback sites
|
|
1330
|
+
# (parse errors / finalize declined / rust core disabled), where the
|
|
1331
|
+
# coarse Python token diff — the last non-Rust producer — takes over.
|
|
1332
|
+
logger.debug(
|
|
1333
|
+
"Rust core batch declined for %r: %s; using routed finalize",
|
|
1334
|
+
filename,
|
|
1335
|
+
rust_batch_attempt.fallback_reason or rust_core_skip_reason,
|
|
1336
|
+
)
|
|
1337
|
+
else:
|
|
1338
|
+
logger.debug(
|
|
1339
|
+
"Rust core batch path skipped for %r: %s; using routed finalize",
|
|
1340
|
+
filename,
|
|
1341
|
+
rust_core_skip_reason,
|
|
1342
|
+
)
|
|
1343
|
+
|
|
1344
|
+
if parse_mode == "interpret-cst":
|
|
1345
|
+
trivia_types = list(parser.trivia_node_types) + self._config.extra_trivia_types
|
|
1346
|
+
|
|
1347
|
+
# ── 2.5. Source preprocessing ────────────────────────────────────
|
|
1348
|
+
with profiler.phase("source_preprocessing"):
|
|
1349
|
+
old_content = parser.preprocess_source(old_content)
|
|
1350
|
+
new_content = parser.preprocess_source(new_content)
|
|
1351
|
+
|
|
1352
|
+
# ── 2.6. Diff-level cache check ───────────────────────────────────
|
|
1353
|
+
if self._cache is not None and not diagnostics.enabled and not _guardrail_cache_bypass:
|
|
1354
|
+
_wasm_hash = self._wasm_hash_for(parser)
|
|
1355
|
+
_diff_key = self._cache.diff_key(
|
|
1356
|
+
old_content,
|
|
1357
|
+
new_content,
|
|
1358
|
+
f"{cache_grammar_id}{cache_schema_suffix}{compile_cache_suffix}",
|
|
1359
|
+
_wasm_hash,
|
|
1360
|
+
)
|
|
1361
|
+
_cached = self._cache.get_diff(_diff_key)
|
|
1362
|
+
if _cached is not None:
|
|
1363
|
+
logger.debug("Diff cache hit for %r", filename)
|
|
1364
|
+
return _apply_file_lifecycle_to_diff(
|
|
1365
|
+
SemanticDiff.model_validate_json(_cached),
|
|
1366
|
+
file_lifecycle,
|
|
1367
|
+
)
|
|
1368
|
+
|
|
1369
|
+
old_cst_json = self._parse(
|
|
1370
|
+
old_content,
|
|
1371
|
+
parser,
|
|
1372
|
+
language,
|
|
1373
|
+
filename,
|
|
1374
|
+
profiler=profiler,
|
|
1375
|
+
)
|
|
1376
|
+
new_cst_json = self._parse(
|
|
1377
|
+
new_content,
|
|
1378
|
+
parser,
|
|
1379
|
+
language,
|
|
1380
|
+
filename,
|
|
1381
|
+
profiler=profiler,
|
|
1382
|
+
)
|
|
1383
|
+
|
|
1384
|
+
# ── CST size guard ────────────────────────────────────────────────
|
|
1385
|
+
for _label, _cst in (("old", old_cst_json), ("new", new_cst_json)):
|
|
1386
|
+
if len(_cst) > self._config.max_cst_bytes:
|
|
1387
|
+
raise ValueError(
|
|
1388
|
+
f"CST JSON for the {_label!r} version of {filename!r} is "
|
|
1389
|
+
f"{len(_cst):,} characters, exceeding the limit of "
|
|
1390
|
+
f"{self._config.max_cst_bytes:,} "
|
|
1391
|
+
"(DiffConfig.max_cst_bytes). Increase the limit or switch "
|
|
1392
|
+
"to a full-parse plugin."
|
|
1393
|
+
)
|
|
1394
|
+
|
|
1395
|
+
# ── 5. Trivia stripping ───────────────────────────────────────────
|
|
1396
|
+
with profiler.phase("trivia_stripping"):
|
|
1397
|
+
old_filtered = _strip_trivia_impl(old_cst_json, trivia_types)
|
|
1398
|
+
new_filtered = _strip_trivia_impl(new_cst_json, trivia_types)
|
|
1399
|
+
|
|
1400
|
+
# ── 6. Style-only shortcut ────────────────────────────────────────
|
|
1401
|
+
with profiler.phase("style_hashing"):
|
|
1402
|
+
old_hash = _compute_structural_hash_for_tree(old_filtered)
|
|
1403
|
+
new_hash = _compute_structural_hash_for_tree(new_filtered)
|
|
1404
|
+
|
|
1405
|
+
if old_hash == new_hash:
|
|
1406
|
+
logger.debug("Style-only diff for %r", filename)
|
|
1407
|
+
diagnostics.record(
|
|
1408
|
+
stage="style",
|
|
1409
|
+
action="style_only_shortcut",
|
|
1410
|
+
rule_id="pipeline.style_only_shortcut",
|
|
1411
|
+
reason="filtered CST structural hashes are equal",
|
|
1412
|
+
metadata={"old_hash": old_hash, "new_hash": new_hash},
|
|
1413
|
+
)
|
|
1414
|
+
style_evidence = build_style_only_evidence(
|
|
1415
|
+
old_source=old_content,
|
|
1416
|
+
new_source=new_content,
|
|
1417
|
+
language=language,
|
|
1418
|
+
old_cst_json=old_cst_json,
|
|
1419
|
+
new_cst_json=new_cst_json,
|
|
1420
|
+
)
|
|
1421
|
+
metadata: dict[str, Any] = {}
|
|
1422
|
+
if style_evidence.ignored_style_changes:
|
|
1423
|
+
metadata["ignored_style_changes"] = style_evidence.ignored_style_changes
|
|
1424
|
+
for group in style_evidence.change_groups:
|
|
1425
|
+
diagnostics.record_group(stage="invariance", group=group)
|
|
1426
|
+
if diagnostics.enabled:
|
|
1427
|
+
diagnostics.summary.update(
|
|
1428
|
+
{
|
|
1429
|
+
"early_exit": "style_only",
|
|
1430
|
+
"final_change_count": 0,
|
|
1431
|
+
"final_group_count": len(style_evidence.change_groups),
|
|
1432
|
+
}
|
|
1433
|
+
)
|
|
1434
|
+
metadata["diagnostics"] = diagnostics.snapshot()
|
|
1435
|
+
style_diff = SemanticDiff.style_only(
|
|
1436
|
+
filename,
|
|
1437
|
+
new_filename if new_filename is not None else filename,
|
|
1438
|
+
language,
|
|
1439
|
+
change_groups=style_evidence.change_groups,
|
|
1440
|
+
metadata=metadata,
|
|
1441
|
+
)
|
|
1442
|
+
style_diff = _apply_file_lifecycle_to_diff(style_diff, file_lifecycle)
|
|
1443
|
+
return apply_guardrails_to_diff(
|
|
1444
|
+
style_diff,
|
|
1445
|
+
old_tree=None,
|
|
1446
|
+
new_tree=None,
|
|
1447
|
+
old_source=old_raw,
|
|
1448
|
+
new_source=new_raw,
|
|
1449
|
+
config=self._config,
|
|
1450
|
+
diagnostics=diagnostics,
|
|
1451
|
+
)
|
|
1452
|
+
|
|
1453
|
+
# ── 6.5. Adaptive fuel budget ─────────────────────────────────────
|
|
1454
|
+
old_cst_nodes = _count_cst_nodes(old_filtered)
|
|
1455
|
+
new_cst_nodes = _count_cst_nodes(new_filtered)
|
|
1456
|
+
max_cst_nodes = max(old_cst_nodes, new_cst_nodes)
|
|
1457
|
+
adaptive_fuel = _fuel_budget(
|
|
1458
|
+
self._config.plugin_fuel,
|
|
1459
|
+
20_000_000 + max_cst_nodes * 200_000,
|
|
1460
|
+
)
|
|
1461
|
+
logger.debug(
|
|
1462
|
+
"[%s] CST: old=%d bytes/%d nodes, new=%d bytes/%d nodes — fuel=%d",
|
|
1463
|
+
filename,
|
|
1464
|
+
len(old_filtered),
|
|
1465
|
+
old_cst_nodes,
|
|
1466
|
+
len(new_filtered),
|
|
1467
|
+
new_cst_nodes,
|
|
1468
|
+
adaptive_fuel,
|
|
1469
|
+
)
|
|
1470
|
+
diagnostics.record(
|
|
1471
|
+
stage="parse",
|
|
1472
|
+
action="interpret_cst",
|
|
1473
|
+
rule_id="pipeline.interpret_cst_parse",
|
|
1474
|
+
reason="host CST parsed, stripped, and sized",
|
|
1475
|
+
metadata={
|
|
1476
|
+
"old_cst_bytes": len(old_filtered),
|
|
1477
|
+
"new_cst_bytes": len(new_filtered),
|
|
1478
|
+
"old_cst_nodes": old_cst_nodes,
|
|
1479
|
+
"new_cst_nodes": new_cst_nodes,
|
|
1480
|
+
"adaptive_fuel": adaptive_fuel,
|
|
1481
|
+
},
|
|
1482
|
+
)
|
|
1483
|
+
|
|
1484
|
+
# ── 7. Plugin dispatch (with parse-tree cache) ───────────────────
|
|
1485
|
+
try:
|
|
1486
|
+
with profiler.phase("wasm_plugin_execution"):
|
|
1487
|
+
if self._cache is not None:
|
|
1488
|
+
old_tree = self._cached_process(
|
|
1489
|
+
parser,
|
|
1490
|
+
old_filtered,
|
|
1491
|
+
language,
|
|
1492
|
+
filename,
|
|
1493
|
+
adaptive_fuel,
|
|
1494
|
+
cache_grammar_id=cache_grammar_id,
|
|
1495
|
+
)
|
|
1496
|
+
new_tree = self._cached_process(
|
|
1497
|
+
parser,
|
|
1498
|
+
new_filtered,
|
|
1499
|
+
language,
|
|
1500
|
+
filename,
|
|
1501
|
+
adaptive_fuel,
|
|
1502
|
+
cache_grammar_id=cache_grammar_id,
|
|
1503
|
+
)
|
|
1504
|
+
else:
|
|
1505
|
+
old_tree = parser.process(
|
|
1506
|
+
old_filtered, language, filename, fuel=adaptive_fuel
|
|
1507
|
+
)
|
|
1508
|
+
new_tree = parser.process(
|
|
1509
|
+
new_filtered, language, filename, fuel=adaptive_fuel
|
|
1510
|
+
)
|
|
1511
|
+
except PluginFuelExhausted as exc:
|
|
1512
|
+
raise PluginFuelExhausted(
|
|
1513
|
+
exc.plugin_id,
|
|
1514
|
+
exc.fuel,
|
|
1515
|
+
f"file={filename!r}, CST nodes up to {max_cst_nodes:,}",
|
|
1516
|
+
) from exc
|
|
1517
|
+
|
|
1518
|
+
else:
|
|
1519
|
+
# ── 7 (full-parse). Plugin owns parsing — send raw source ─────────
|
|
1520
|
+
# Stages 3-6 are skipped: the plugin handles its own CST construction
|
|
1521
|
+
# and trivia filtering. The style-only shortcut is not available here
|
|
1522
|
+
# because there is no host-side CST to hash.
|
|
1523
|
+
max_source_bytes = max(
|
|
1524
|
+
len(old_content.encode("utf-8")),
|
|
1525
|
+
len(new_content.encode("utf-8")),
|
|
1526
|
+
)
|
|
1527
|
+
adaptive_fuel = _fuel_budget(
|
|
1528
|
+
self._config.plugin_fuel,
|
|
1529
|
+
50_000_000 + max_source_bytes * 150_000,
|
|
1530
|
+
)
|
|
1531
|
+
logger.debug("full-parse mode for %r via %r", filename, parser.grammar_id)
|
|
1532
|
+
|
|
1533
|
+
# Diff-level cache check for full-parse mode.
|
|
1534
|
+
if self._cache is not None and not diagnostics.enabled and not _guardrail_cache_bypass:
|
|
1535
|
+
_wasm_hash = self._wasm_hash_for(parser)
|
|
1536
|
+
_diff_key = self._cache.diff_key(
|
|
1537
|
+
old_content,
|
|
1538
|
+
new_content,
|
|
1539
|
+
f"{parser.grammar_id}{cache_schema_suffix}{compile_cache_suffix}",
|
|
1540
|
+
_wasm_hash,
|
|
1541
|
+
)
|
|
1542
|
+
_cached = self._cache.get_diff(_diff_key)
|
|
1543
|
+
if _cached is not None:
|
|
1544
|
+
logger.debug("Diff cache hit for %r (full-parse)", filename)
|
|
1545
|
+
return _apply_file_lifecycle_to_diff(
|
|
1546
|
+
SemanticDiff.model_validate_json(_cached),
|
|
1547
|
+
file_lifecycle,
|
|
1548
|
+
)
|
|
1549
|
+
|
|
1550
|
+
with profiler.phase("wasm_plugin_execution"):
|
|
1551
|
+
old_side_absent = file_lifecycle == "added" and old_content == ""
|
|
1552
|
+
new_side_absent = file_lifecycle == "deleted" and new_content == ""
|
|
1553
|
+
if self._cache is not None:
|
|
1554
|
+
old_tree = (
|
|
1555
|
+
_empty_semantic_tree(language)
|
|
1556
|
+
if old_side_absent
|
|
1557
|
+
else self._cached_process(
|
|
1558
|
+
parser, old_content, language, filename, adaptive_fuel
|
|
1559
|
+
)
|
|
1560
|
+
)
|
|
1561
|
+
new_tree = (
|
|
1562
|
+
_empty_semantic_tree(language)
|
|
1563
|
+
if new_side_absent
|
|
1564
|
+
else self._cached_process(
|
|
1565
|
+
parser, new_content, language, filename, adaptive_fuel
|
|
1566
|
+
)
|
|
1567
|
+
)
|
|
1568
|
+
else:
|
|
1569
|
+
old_tree = (
|
|
1570
|
+
_empty_semantic_tree(language)
|
|
1571
|
+
if old_side_absent
|
|
1572
|
+
else parser.process(old_content, language, filename, fuel=adaptive_fuel)
|
|
1573
|
+
)
|
|
1574
|
+
new_tree = (
|
|
1575
|
+
_empty_semantic_tree(language)
|
|
1576
|
+
if new_side_absent
|
|
1577
|
+
else parser.process(new_content, language, filename, fuel=adaptive_fuel)
|
|
1578
|
+
)
|
|
1579
|
+
diagnostics.record(
|
|
1580
|
+
stage="parse",
|
|
1581
|
+
action="full_parse",
|
|
1582
|
+
rule_id="pipeline.full_parse",
|
|
1583
|
+
reason="plugin parsed raw source",
|
|
1584
|
+
metadata={"adaptive_fuel": adaptive_fuel},
|
|
1585
|
+
)
|
|
1586
|
+
|
|
1587
|
+
engine_telemetry.extend(_drain_plugin_telemetry(parser))
|
|
1588
|
+
_record_engine_telemetry(diagnostics, engine_telemetry)
|
|
1589
|
+
|
|
1590
|
+
# ── 7b. Enrichment pass (optional) ─────────────────────────────────────────
|
|
1591
|
+
with profiler.phase("enricher_execution"):
|
|
1592
|
+
enrichers = self._registry.get_enrichers(language)
|
|
1593
|
+
if enrichers:
|
|
1594
|
+
old_tree_json = old_tree.model_dump_json()
|
|
1595
|
+
new_tree_json = new_tree.model_dump_json()
|
|
1596
|
+
for enricher in enrichers:
|
|
1597
|
+
old_tree_json = enricher.enrich(
|
|
1598
|
+
old_tree_json, old_raw, language, filename, fuel=adaptive_fuel
|
|
1599
|
+
)
|
|
1600
|
+
new_tree_json = enricher.enrich(
|
|
1601
|
+
new_tree_json, new_raw, language, filename, fuel=adaptive_fuel
|
|
1602
|
+
)
|
|
1603
|
+
engine_telemetry.extend(_drain_plugin_telemetry(enricher))
|
|
1604
|
+
_record_engine_telemetry(diagnostics, engine_telemetry)
|
|
1605
|
+
try:
|
|
1606
|
+
old_tree = SemanticNode.model_validate_json(old_tree_json)
|
|
1607
|
+
new_tree = SemanticNode.model_validate_json(new_tree_json)
|
|
1608
|
+
except ValidationError as exc:
|
|
1609
|
+
raise PluginOutputError("enricher", str(exc)) from exc
|
|
1610
|
+
# ── 8. Plugin output validation ────────────────────────────────────────
|
|
1611
|
+
with profiler.phase("semantic_node_validation"):
|
|
1612
|
+
old_tree = _enrich_literal_labels(old_tree, old_raw)
|
|
1613
|
+
new_tree = _enrich_literal_labels(new_tree, new_raw)
|
|
1614
|
+
# Cross-language NodeFacts (issue #70): every tree (all parse branches converge here)
|
|
1615
|
+
# gets language-agnostic structural facts derived in the Rust core, so the intent
|
|
1616
|
+
# explainer has signal for ALL languages, not just Python. Idempotent + graceful.
|
|
1617
|
+
old_tree = enrich_node_facts(old_tree)
|
|
1618
|
+
new_tree = enrich_node_facts(new_tree)
|
|
1619
|
+
# Profile-label enrichment runs in the Rust core (issue #57 port).
|
|
1620
|
+
# The Rust core is AUTHORITATIVE (readiness #90 / #82): no Python
|
|
1621
|
+
# engine fallback — if the backend is unavailable the tree passes
|
|
1622
|
+
# through UNCHANGED (graceful, degraded labels) rather than routing
|
|
1623
|
+
# to a Python enricher, so "if Python didn't exist it still works".
|
|
1624
|
+
# Gated by language so non-profile diffs pay no tree-JSON roundtrip.
|
|
1625
|
+
if language.lower() in {"json", "yaml"}:
|
|
1626
|
+
_identity_fields = tuple(schema_resolution.identity_fields)
|
|
1627
|
+
old_tree = try_rust_profile_label_enrichment(
|
|
1628
|
+
old_tree, old_raw, language, identity_fields=_identity_fields
|
|
1629
|
+
) or old_tree
|
|
1630
|
+
new_tree = try_rust_profile_label_enrichment(
|
|
1631
|
+
new_tree, new_raw, language, identity_fields=_identity_fields
|
|
1632
|
+
) or new_tree
|
|
1633
|
+
if language.lower() in {"css", "scss", "html", "xml", "mdx"}:
|
|
1634
|
+
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
|
|
1635
|
+
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
|
|
1636
|
+
if language.lower() in {"hcl", "puppet"}:
|
|
1637
|
+
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
|
|
1638
|
+
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
|
|
1639
|
+
if language.lower() == "sql":
|
|
1640
|
+
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
|
|
1641
|
+
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
|
|
1642
|
+
if language.lower() in {"asm", "bash", "delphi"}:
|
|
1643
|
+
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
|
|
1644
|
+
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
|
|
1645
|
+
_validate_tree_ids(old_tree, filename)
|
|
1646
|
+
_validate_tree_ids(new_tree, filename)
|
|
1647
|
+
diagnostics.record(
|
|
1648
|
+
stage="parse",
|
|
1649
|
+
action="semantic_tree_ready",
|
|
1650
|
+
rule_id="pipeline.semantic_tree_ready",
|
|
1651
|
+
reason="semantic trees produced and enriched",
|
|
1652
|
+
metadata={
|
|
1653
|
+
"old_nodes": _count_semantic_nodes(old_tree),
|
|
1654
|
+
"new_nodes": _count_semantic_nodes(new_tree),
|
|
1655
|
+
},
|
|
1656
|
+
)
|
|
1657
|
+
# ── 8.5. Token-level fallback for severe parse errors ───────────────
|
|
1658
|
+
# When tree-sitter produced ERROR nodes and fallback is enabled, skip
|
|
1659
|
+
# the GumTree algorithm and return a coarse token-level diff instead.
|
|
1660
|
+
if self._config.fallback_to_token_diff and (
|
|
1661
|
+
_has_error_node(old_tree) or _has_error_node(new_tree)
|
|
1662
|
+
):
|
|
1663
|
+
logger.warning(
|
|
1664
|
+
"Parse errors detected in %r — falling back to token-level diff",
|
|
1665
|
+
filename,
|
|
1666
|
+
)
|
|
1667
|
+
_raise_rust_only_gate_error("parse errors require Rust token-level fallback")
|
|
1668
|
+
diagnostics.record(
|
|
1669
|
+
stage="fallback",
|
|
1670
|
+
action="token_fallback",
|
|
1671
|
+
rule_id="pipeline.token_fallback",
|
|
1672
|
+
reason="semantic tree contains ERROR nodes",
|
|
1673
|
+
)
|
|
1674
|
+
metadata: dict[str, Any] = {}
|
|
1675
|
+
if diagnostics.enabled:
|
|
1676
|
+
diagnostics.summary.update(
|
|
1677
|
+
{"early_exit": "token_fallback", "final_change_count": None}
|
|
1678
|
+
)
|
|
1679
|
+
metadata["diagnostics"] = diagnostics.snapshot()
|
|
1680
|
+
fallback_diff = _token_fallback_diff(
|
|
1681
|
+
old_raw,
|
|
1682
|
+
new_raw,
|
|
1683
|
+
filename,
|
|
1684
|
+
new_filename if new_filename is not None else filename,
|
|
1685
|
+
language,
|
|
1686
|
+
metadata=metadata,
|
|
1687
|
+
)
|
|
1688
|
+
fallback_diff = _apply_file_lifecycle_to_diff(fallback_diff, file_lifecycle)
|
|
1689
|
+
return apply_guardrails_to_diff(
|
|
1690
|
+
fallback_diff,
|
|
1691
|
+
old_tree=None,
|
|
1692
|
+
new_tree=None,
|
|
1693
|
+
old_source=old_raw,
|
|
1694
|
+
new_source=new_raw,
|
|
1695
|
+
config=self._config,
|
|
1696
|
+
diagnostics=diagnostics,
|
|
1697
|
+
)
|
|
1698
|
+
# ── 9. GumTree matching ────────────────────────────────────────────────
|
|
1699
|
+
# The matching engine is selected by (in priority order):
|
|
1700
|
+
# 1. ``DiffConfig.test_matching_engine`` — test-only hard override used
|
|
1701
|
+
# by the dual-run truthiness matrix to certify both contracts.
|
|
1702
|
+
# 2. The production default: Rust's language-agnostic matcher
|
|
1703
|
+
# (``intentumdiff_rust_core.diff_semantic_tree_json``) when the Rust
|
|
1704
|
+
# core is available and diagnostics are off. This is migration
|
|
1705
|
+
# step 2 of ``docs/ENGINE_BOUNDARY_AUDIT.md``. The four engine
|
|
1706
|
+
# gaps that previously blocked this swap (Python dedent, JS
|
|
1707
|
+
# calc_hash MOVE, YAML block↔flow, C# var↔explicit) are closed —
|
|
1708
|
+
# see ``the retired NOISE_SUPPRESSION_RETUNE doc (git history)`` Phases A–D.
|
|
1709
|
+
# 3. (retired) The python GumTree oracle was deleted with the transitional
|
|
1710
|
+
# layer (issue #57 payoff, stage 4b) — no python matching fallback exists.
|
|
1711
|
+
test_engine = self._config.test_matching_engine
|
|
1712
|
+
rust_attempt: Any = None
|
|
1713
|
+
# Skip Rust matching when either tree is empty (file add/delete
|
|
1714
|
+
# lifecycle). The Rust language-agnostic matcher treats empty-root
|
|
1715
|
+
# ↔ content-root as a structural match, which produces a single
|
|
1716
|
+
# MODIFICATION instead of the correct ADDITION (or DELETION) for
|
|
1717
|
+
# the file-lifecycle case. Letting the Python oracle handle these
|
|
1718
|
+
# edge cases preserves the ADDITION/DELETION shape that downstream
|
|
1719
|
+
# tests and review UIs depend on.
|
|
1720
|
+
either_tree_empty = (
|
|
1721
|
+
not old_tree.children or not new_tree.children
|
|
1722
|
+
)
|
|
1723
|
+
use_rust_matching = (
|
|
1724
|
+
test_engine == "rust"
|
|
1725
|
+
or (
|
|
1726
|
+
test_engine is None
|
|
1727
|
+
and self._config.experimental_rust_core
|
|
1728
|
+
and not diagnostics.enabled
|
|
1729
|
+
and not either_tree_empty
|
|
1730
|
+
)
|
|
1731
|
+
)
|
|
1732
|
+
# ── 9-fin. Rust finalize routing (issue #57) ──────────────────────────
|
|
1733
|
+
# Certified languages skip the python edit-script/refinement/presentation stages
|
|
1734
|
+
# entirely: the Rust core runs the certified batch's refine+finalize from the
|
|
1735
|
+
# semantic trees and returns the finished review. Eligibility is independent of
|
|
1736
|
+
# the matching-engine choice (issue #57 payoff, fallback-tier closure):
|
|
1737
|
+
# - empty-tree lifecycle is handled INSIDE Rust finalize (DELETION+ADDITION
|
|
1738
|
+
# root pair, python parity), so either_tree_empty no longer excludes;
|
|
1739
|
+
# - enriched trees flow through routing (enrichment ran at stage 7b);
|
|
1740
|
+
# - registered diff analyzers run on the ROUTED diff below, mirroring 13.5.
|
|
1741
|
+
# Diagnostics mode routes too (issue #57 payoff, stage 4a): the shell-side
|
|
1742
|
+
# records (parse, trees, guardrails, finalize summary below) still fire; the
|
|
1743
|
+
# per-pass refinement trace is Rust-internal (INTENTUMDIFF_FINALIZE_DEBUG probe)
|
|
1744
|
+
# until #54 formalizes it into the recorder.
|
|
1745
|
+
rust_finalize_eligible = test_engine == "rust" or (
|
|
1746
|
+
test_engine is None and self._config.experimental_rust_core
|
|
1747
|
+
)
|
|
1748
|
+
if rust_finalize_eligible and language in _rust_finalize_languages():
|
|
1749
|
+
from intentumdiff.rust_core import try_rust_finalize_review
|
|
1750
|
+
|
|
1751
|
+
with profiler.phase("rust_finalize_review"):
|
|
1752
|
+
finalize_result = try_rust_finalize_review(
|
|
1753
|
+
old_tree=old_tree,
|
|
1754
|
+
new_tree=new_tree,
|
|
1755
|
+
old_source=old_content,
|
|
1756
|
+
new_source=new_content,
|
|
1757
|
+
language=language,
|
|
1758
|
+
config=self._config,
|
|
1759
|
+
collect_trace=diagnostics.enabled,
|
|
1760
|
+
)
|
|
1761
|
+
if finalize_result is not None:
|
|
1762
|
+
diagnostics.record(
|
|
1763
|
+
stage="finalize",
|
|
1764
|
+
action="rust_finalize_review",
|
|
1765
|
+
rule_id="rust_core.finalize_review_v1",
|
|
1766
|
+
reason="routed per-stage finalize served the review",
|
|
1767
|
+
metadata={
|
|
1768
|
+
"engine_owner": "rust",
|
|
1769
|
+
"change_count": len(finalize_result["changes"]),
|
|
1770
|
+
"is_style_only": finalize_result["is_style_only"],
|
|
1771
|
+
},
|
|
1772
|
+
)
|
|
1773
|
+
# Per-pass trace (issue #54): every probed refine/finalize pass reports
|
|
1774
|
+
# its surviving change count; the recorder mirrors the python pipeline's
|
|
1775
|
+
# per-stage observability on the Rust path.
|
|
1776
|
+
for entry in finalize_result.get("trace") or []:
|
|
1777
|
+
diagnostics.record_count(
|
|
1778
|
+
stage="finalize",
|
|
1779
|
+
action=str(entry.get("pass", "")),
|
|
1780
|
+
rule_id="rust_core.finalize_pass",
|
|
1781
|
+
reason="rust finalize pass completed",
|
|
1782
|
+
count=int(entry.get("changes_after", 0)),
|
|
1783
|
+
)
|
|
1784
|
+
# Semantic invariances (css color / literal equivalence / yaml block↔flow / import
|
|
1785
|
+
# reorder / …) are equivalences that must hold REGARDLESS of routing, but this
|
|
1786
|
+
# short-circuit returns before the stage-12 apply_invariances. Run it here on the
|
|
1787
|
+
# routed changes so canonical-value "zero-change" edits collapse under routing too
|
|
1788
|
+
# (#57: unlocks json/yaml/css/python). No-op when no invariance fires.
|
|
1789
|
+
fin_changes = finalize_result["changes"]
|
|
1790
|
+
fin_is_style_only = finalize_result["is_style_only"]
|
|
1791
|
+
fin_invariant_groups: list[Any] = []
|
|
1792
|
+
fin_invariant_ignored: list[dict[str, Any]] = []
|
|
1793
|
+
if fin_changes:
|
|
1794
|
+
_inv = apply_invariances(
|
|
1795
|
+
fin_changes,
|
|
1796
|
+
old_tree=old_tree,
|
|
1797
|
+
new_tree=new_tree,
|
|
1798
|
+
old_source=old_content,
|
|
1799
|
+
new_source=new_content,
|
|
1800
|
+
language=language,
|
|
1801
|
+
)
|
|
1802
|
+
if len(_inv.changes) != len(fin_changes) or _inv.change_groups:
|
|
1803
|
+
fin_changes = _inv.changes
|
|
1804
|
+
fin_invariant_groups = list(_inv.change_groups)
|
|
1805
|
+
# The invariance's own IGNORED_STYLE groups ARE the style evidence — carry
|
|
1806
|
+
# their rule_id/reason so metadata["ignored_style_changes"] leads with the
|
|
1807
|
+
# specific reason (css.color.canonical_equivalence), not the generic
|
|
1808
|
+
# source-equivalence fallback below.
|
|
1809
|
+
for group in fin_invariant_groups:
|
|
1810
|
+
diagnostics.record_group(stage="invariance", group=group)
|
|
1811
|
+
fin_invariant_ignored = [
|
|
1812
|
+
{"rule_id": g.rule_id, **dict(g.metadata)}
|
|
1813
|
+
for g in fin_invariant_groups
|
|
1814
|
+
if g.kind == ChangeGroupKind.IGNORED_STYLE
|
|
1815
|
+
]
|
|
1816
|
+
if not fin_changes and old_content != new_content:
|
|
1817
|
+
fin_is_style_only = True
|
|
1818
|
+
# Generic files are line-oriented from the user's point of view
|
|
1819
|
+
# (#57 generic routing, mirroring the stage-12 orchestration): the routed
|
|
1820
|
+
# finalize's parser-token churn is REPLACED wholesale by the Rust text
|
|
1821
|
+
# review's stable line/character spans, then the (Rust-first) markdown
|
|
1822
|
+
# section passes run for .md filenames. Parser-derived groups are dropped
|
|
1823
|
+
# with the churn — carrying them forward invents fake pairings between
|
|
1824
|
+
# unrelated lines.
|
|
1825
|
+
fin_ignored_override: list[dict[str, Any]] | None = None
|
|
1826
|
+
if language.lower() == "generic":
|
|
1827
|
+
generic_presented = normalize_generic_text_for_review(
|
|
1828
|
+
fin_changes,
|
|
1829
|
+
old_content,
|
|
1830
|
+
new_content,
|
|
1831
|
+
)
|
|
1832
|
+
presented_generic = PresentationResult(
|
|
1833
|
+
changes=generic_presented.changes,
|
|
1834
|
+
change_groups=generic_presented.change_groups,
|
|
1835
|
+
ignored_style_changes=[
|
|
1836
|
+
*generic_presented.ignored_style_changes,
|
|
1837
|
+
],
|
|
1838
|
+
)
|
|
1839
|
+
presented_generic = _markdown_section_move_presentation(
|
|
1840
|
+
presented_generic,
|
|
1841
|
+
old_source=old_content,
|
|
1842
|
+
new_source=new_content,
|
|
1843
|
+
old_filename=filename,
|
|
1844
|
+
new_filename=(
|
|
1845
|
+
new_filename if new_filename is not None else filename
|
|
1846
|
+
),
|
|
1847
|
+
)
|
|
1848
|
+
presented_generic = _markdown_section_heading_rename_presentation(
|
|
1849
|
+
presented_generic,
|
|
1850
|
+
old_source=old_content,
|
|
1851
|
+
new_source=new_content,
|
|
1852
|
+
old_filename=filename,
|
|
1853
|
+
new_filename=(
|
|
1854
|
+
new_filename if new_filename is not None else filename
|
|
1855
|
+
),
|
|
1856
|
+
)
|
|
1857
|
+
fin_changes = presented_generic.changes
|
|
1858
|
+
fin_invariant_groups = list(presented_generic.change_groups)
|
|
1859
|
+
fin_ignored_override = list(presented_generic.ignored_style_changes)
|
|
1860
|
+
finalize_result["change_groups"] = []
|
|
1861
|
+
finalize_result["no_surviving_changes"] = False
|
|
1862
|
+
# Mirror the stage-12 style-only resolution: identical sources (or
|
|
1863
|
+
# whitespace-collapsed-equal trees) with no surviving line spans are a
|
|
1864
|
+
# style-only diff, not "no changes". The Rust finalize's flag reflects
|
|
1865
|
+
# its own suppressions, which the replacement just discarded.
|
|
1866
|
+
if not fin_changes:
|
|
1867
|
+
|
|
1868
|
+
def _generic_norm(node: SemanticNode) -> tuple[Any, ...]:
|
|
1869
|
+
label = " ".join(node.label.split())
|
|
1870
|
+
return (
|
|
1871
|
+
node.node_type,
|
|
1872
|
+
label,
|
|
1873
|
+
tuple(_generic_norm(c) for c in node.children),
|
|
1874
|
+
)
|
|
1875
|
+
|
|
1876
|
+
fin_is_style_only = (
|
|
1877
|
+
old_content == new_content
|
|
1878
|
+
or _generic_norm(old_tree) == _generic_norm(new_tree)
|
|
1879
|
+
)
|
|
1880
|
+
# Stage-12 style-only resolution, language-agnostic (markdown #44 exposed
|
|
1881
|
+
# it): ZERO surviving changes with identical (or whitespace-collapsed
|
|
1882
|
+
# tree-equal) sources is a style-only diff for every routed language —
|
|
1883
|
+
# the Rust finalize's flag only reflects its own suppressions.
|
|
1884
|
+
if not fin_changes and not fin_is_style_only:
|
|
1885
|
+
|
|
1886
|
+
def _routed_norm(node: SemanticNode) -> tuple[Any, ...]:
|
|
1887
|
+
label = " ".join(node.label.split())
|
|
1888
|
+
return (
|
|
1889
|
+
node.node_type,
|
|
1890
|
+
label,
|
|
1891
|
+
tuple(_routed_norm(c) for c in node.children),
|
|
1892
|
+
)
|
|
1893
|
+
|
|
1894
|
+
fin_is_style_only = (
|
|
1895
|
+
old_content == new_content
|
|
1896
|
+
or _routed_norm(old_tree) == _routed_norm(new_tree)
|
|
1897
|
+
)
|
|
1898
|
+
metadata_fin: dict[str, Any] = {
|
|
1899
|
+
"engine_owner": "rust",
|
|
1900
|
+
"semantic_contract": "rust_finalize_review_v1",
|
|
1901
|
+
"rust_core": {
|
|
1902
|
+
"engine": "rust_finalize_review_v1",
|
|
1903
|
+
"stage": "per_stage_finalize_routing",
|
|
1904
|
+
"used": True,
|
|
1905
|
+
},
|
|
1906
|
+
}
|
|
1907
|
+
# Context metadata the normal path attaches at stage 11 — the routed
|
|
1908
|
+
# return must carry the same contracts (cpp compile_commands pilot gap).
|
|
1909
|
+
if schema_metadata is not None:
|
|
1910
|
+
metadata_fin["schema"] = schema_metadata
|
|
1911
|
+
if compile_metadata is not None:
|
|
1912
|
+
metadata_fin["compile_commands"] = compile_metadata
|
|
1913
|
+
combined_ignored = (
|
|
1914
|
+
fin_ignored_override
|
|
1915
|
+
if fin_ignored_override is not None
|
|
1916
|
+
else [
|
|
1917
|
+
*fin_invariant_ignored,
|
|
1918
|
+
*finalize_result["ignored_style_changes"],
|
|
1919
|
+
]
|
|
1920
|
+
)
|
|
1921
|
+
if combined_ignored:
|
|
1922
|
+
metadata_fin["ignored_style_changes"] = combined_ignored
|
|
1923
|
+
if finalize_result["no_surviving_changes"]:
|
|
1924
|
+
metadata_fin["no_surviving_changes"] = True
|
|
1925
|
+
routed_groups = [*finalize_result["change_groups"], *fin_invariant_groups]
|
|
1926
|
+
if not fin_changes and old_content != new_content and not fin_invariant_ignored:
|
|
1927
|
+
# The normal path attaches a source-equivalence suppression group
|
|
1928
|
+
# post-diff when a diff nets to zero changes but the sources differ
|
|
1929
|
+
# (xml attribute reorder -> zero changes must still RECORD the
|
|
1930
|
+
# suppression, not merely be absent). The routed short-circuit returns
|
|
1931
|
+
# before that stage, so mirror it here.
|
|
1932
|
+
style_evidence = build_style_only_evidence(
|
|
1933
|
+
old_source=old_content,
|
|
1934
|
+
new_source=new_content,
|
|
1935
|
+
language=language,
|
|
1936
|
+
)
|
|
1937
|
+
for group in style_evidence.change_groups:
|
|
1938
|
+
diagnostics.record_group(stage="invariance", group=group)
|
|
1939
|
+
routed_groups.extend(style_evidence.change_groups)
|
|
1940
|
+
if style_evidence.ignored_style_changes:
|
|
1941
|
+
metadata_fin.setdefault("ignored_style_changes", []).extend(
|
|
1942
|
+
style_evidence.ignored_style_changes
|
|
1943
|
+
)
|
|
1944
|
+
routed = SemanticDiff(
|
|
1945
|
+
old_filename=filename,
|
|
1946
|
+
new_filename=new_filename if new_filename is not None else filename,
|
|
1947
|
+
language=language,
|
|
1948
|
+
changes=fin_changes,
|
|
1949
|
+
change_groups=routed_groups,
|
|
1950
|
+
has_semantic_changes=bool(fin_changes) and not fin_is_style_only,
|
|
1951
|
+
is_style_only=fin_is_style_only,
|
|
1952
|
+
metadata=metadata_fin,
|
|
1953
|
+
)
|
|
1954
|
+
routed = _apply_file_lifecycle_to_diff(routed, file_lifecycle)
|
|
1955
|
+
routed = apply_guardrails_to_diff(
|
|
1956
|
+
routed,
|
|
1957
|
+
old_tree=old_tree,
|
|
1958
|
+
new_tree=new_tree,
|
|
1959
|
+
old_source=old_content,
|
|
1960
|
+
new_source=new_content,
|
|
1961
|
+
config=self._config,
|
|
1962
|
+
diagnostics=diagnostics,
|
|
1963
|
+
)
|
|
1964
|
+
# Diff-analyzer pass on the routed diff (stage 13.5 mirror; issue #57
|
|
1965
|
+
# payoff): analyzers are diff->diff Wasm transforms over the final
|
|
1966
|
+
# SemanticDiff JSON, so the routed review feeds them identically.
|
|
1967
|
+
with profiler.phase("diff_analyzers"):
|
|
1968
|
+
routed = self._run_diff_analyzers_on_routed(
|
|
1969
|
+
routed, language, filename, adaptive_fuel
|
|
1970
|
+
)
|
|
1971
|
+
return _attach_run_telemetry(routed, engine_telemetry, diagnostics)
|
|
1972
|
+
# The Rust finalize declined (tree_too_large / backend unavailable). The
|
|
1973
|
+
# transitional python pipeline is retired (issue #57 payoff, stage 4b) —
|
|
1974
|
+
# degrade honestly to the coarse token-level diff, same as parse errors.
|
|
1975
|
+
logger.warning(
|
|
1976
|
+
"Rust finalize declined for %r (%s) — token-level fallback",
|
|
1977
|
+
filename,
|
|
1978
|
+
language,
|
|
1979
|
+
)
|
|
1980
|
+
_raise_rust_only_gate_error("rust finalize declined")
|
|
1981
|
+
diagnostics.record(
|
|
1982
|
+
stage="fallback",
|
|
1983
|
+
action="token_fallback",
|
|
1984
|
+
rule_id="pipeline.finalize_declined",
|
|
1985
|
+
reason="rust finalize declined; python pipeline retired (#57)",
|
|
1986
|
+
)
|
|
1987
|
+
fallback_metadata: dict[str, Any] = {
|
|
1988
|
+
"fallback_reason": "rust_finalize_declined",
|
|
1989
|
+
}
|
|
1990
|
+
if diagnostics.enabled:
|
|
1991
|
+
diagnostics.summary.update(
|
|
1992
|
+
{"early_exit": "token_fallback", "final_change_count": None}
|
|
1993
|
+
)
|
|
1994
|
+
fallback_metadata["diagnostics"] = diagnostics.snapshot()
|
|
1995
|
+
fallback_diff = _token_fallback_diff(
|
|
1996
|
+
old_raw,
|
|
1997
|
+
new_raw,
|
|
1998
|
+
filename,
|
|
1999
|
+
new_filename if new_filename is not None else filename,
|
|
2000
|
+
language,
|
|
2001
|
+
metadata=fallback_metadata,
|
|
2002
|
+
)
|
|
2003
|
+
fallback_diff = _apply_file_lifecycle_to_diff(fallback_diff, file_lifecycle)
|
|
2004
|
+
return apply_guardrails_to_diff(
|
|
2005
|
+
fallback_diff,
|
|
2006
|
+
old_tree=None,
|
|
2007
|
+
new_tree=None,
|
|
2008
|
+
old_source=old_raw,
|
|
2009
|
+
new_source=new_raw,
|
|
2010
|
+
config=self._config,
|
|
2011
|
+
diagnostics=diagnostics,
|
|
2012
|
+
)
|
|
2013
|
+
if test_engine == "python":
|
|
2014
|
+
raise NotImplementedError(
|
|
2015
|
+
"test_matching_engine='python' was retired with the transitional "
|
|
2016
|
+
"python pipeline (issue #57); the Rust finalize routing serves every "
|
|
2017
|
+
"certified language."
|
|
2018
|
+
)
|
|
2019
|
+
|
|
2020
|
+
# Rust core disabled (DiffConfig kill switch / INTENTUMDIFF_RUST_CORE=0): the
|
|
2021
|
+
# transitional python pipeline is retired (issue #57 payoff) — semantic
|
|
2022
|
+
# diffing REQUIRES the Rust core; degrade honestly to the token-level diff.
|
|
2023
|
+
logger.warning(
|
|
2024
|
+
"Rust core disabled for %r — token-level fallback (python pipeline retired)",
|
|
2025
|
+
filename,
|
|
2026
|
+
)
|
|
2027
|
+
_raise_rust_only_gate_error("rust core disabled")
|
|
2028
|
+
diagnostics.record(
|
|
2029
|
+
stage="fallback",
|
|
2030
|
+
action="token_fallback",
|
|
2031
|
+
rule_id="pipeline.rust_core_disabled",
|
|
2032
|
+
reason="rust core disabled; python pipeline retired (#57)",
|
|
2033
|
+
)
|
|
2034
|
+
disabled_metadata: dict[str, Any] = {
|
|
2035
|
+
"fallback_reason": "rust_core_disabled",
|
|
2036
|
+
}
|
|
2037
|
+
if diagnostics.enabled:
|
|
2038
|
+
diagnostics.summary.update(
|
|
2039
|
+
{"early_exit": "token_fallback", "final_change_count": None}
|
|
2040
|
+
)
|
|
2041
|
+
disabled_metadata["diagnostics"] = diagnostics.snapshot()
|
|
2042
|
+
disabled_diff = _token_fallback_diff(
|
|
2043
|
+
old_raw,
|
|
2044
|
+
new_raw,
|
|
2045
|
+
filename,
|
|
2046
|
+
new_filename if new_filename is not None else filename,
|
|
2047
|
+
language,
|
|
2048
|
+
metadata=disabled_metadata,
|
|
2049
|
+
)
|
|
2050
|
+
disabled_diff = _apply_file_lifecycle_to_diff(disabled_diff, file_lifecycle)
|
|
2051
|
+
return apply_guardrails_to_diff(
|
|
2052
|
+
disabled_diff,
|
|
2053
|
+
old_tree=None,
|
|
2054
|
+
new_tree=None,
|
|
2055
|
+
old_source=old_raw,
|
|
2056
|
+
new_source=new_raw,
|
|
2057
|
+
config=self._config,
|
|
2058
|
+
diagnostics=diagnostics,
|
|
2059
|
+
)
|
|
2060
|
+
|
|
2061
|
+
def diff_stream_progressive(
|
|
2062
|
+
self,
|
|
2063
|
+
source: Source,
|
|
2064
|
+
) -> Iterator[ChangeStreamEvent]:
|
|
2065
|
+
"""Yield ``ChangeStreamEvent`` objects as each pipeline phase completes.
|
|
2066
|
+
|
|
2067
|
+
Three phases are emitted in order:
|
|
2068
|
+
|
|
2069
|
+
1. **STRUCTURAL** — one ``action="add"`` event per change after the
|
|
2070
|
+
GumTree edit script and move-promotion pass (stage 11). These are
|
|
2071
|
+
available immediately, before the (potentially expensive) refactoring
|
|
2072
|
+
detection runs.
|
|
2073
|
+
|
|
2074
|
+
2. **REFINED** — events emitted once refactoring detection (stage 12) has
|
|
2075
|
+
run. Consumed raw changes are replaced by ``action="revise"`` events
|
|
2076
|
+
carrying a ``REFACTORING`` change and ``replaced_ids`` listing the
|
|
2077
|
+
node IDs of the Phase-1 changes they supersede. Removed-but-not-
|
|
2078
|
+
replaced changes are emitted as ``action="remove"``.
|
|
2079
|
+
|
|
2080
|
+
3. **FINAL** — events emitted after classification (stage 13) and any
|
|
2081
|
+
diff-analyzer plugins (stage 13.5) have run. Emitted only when these
|
|
2082
|
+
passes modify the change list.
|
|
2083
|
+
|
|
2084
|
+
Consumers that only care about the final result can wait until all events
|
|
2085
|
+
are consumed; consumers that want to display partial results can render
|
|
2086
|
+
STRUCTURAL changes immediately and update/remove them as REFINED and
|
|
2087
|
+
FINAL events arrive.
|
|
2088
|
+
"""
|
|
2089
|
+
old_content, new_content, filename, language_hint = source.get_content()
|
|
2090
|
+
yield from self._run_pipeline_streaming(old_content, new_content, filename, language_hint)
|
|
2091
|
+
|
|
2092
|
+
def _run_pipeline_streaming(
|
|
2093
|
+
self,
|
|
2094
|
+
old_content: str,
|
|
2095
|
+
new_content: str,
|
|
2096
|
+
filename: str,
|
|
2097
|
+
language_hint: str | None,
|
|
2098
|
+
*,
|
|
2099
|
+
new_filename: str | None = None,
|
|
2100
|
+
) -> Iterator[ChangeStreamEvent]:
|
|
2101
|
+
"""Internal generator — see ``diff_stream_progressive`` for the public API."""
|
|
2102
|
+
result = self._run_stages_1_to_11(
|
|
2103
|
+
old_content, new_content, filename, language_hint, new_filename=new_filename
|
|
2104
|
+
)
|
|
2105
|
+
|
|
2106
|
+
if isinstance(result, SemanticDiff):
|
|
2107
|
+
# Early exit (cache hit / style-only / token fallback): emit all changes
|
|
2108
|
+
# as FINAL so consumers still receive a well-formed event stream.
|
|
2109
|
+
for change in result.changes:
|
|
2110
|
+
yield ChangeStreamEvent(
|
|
2111
|
+
phase=ChangeStreamPhase.FINAL,
|
|
2112
|
+
action="add",
|
|
2113
|
+
change=change,
|
|
2114
|
+
)
|
|
2115
|
+
return
|
|
2116
|
+
|
|
2117
|
+
raise AssertionError(
|
|
2118
|
+
"unreachable: _run_stages_1_to_11 always returns a SemanticDiff since the "
|
|
2119
|
+
"python pipeline retirement (issue #57 payoff, stage 4b)"
|
|
2120
|
+
)
|
|
2121
|
+
|
|
2122
|
+
def _cached_process(
|
|
2123
|
+
self,
|
|
2124
|
+
parser: Any,
|
|
2125
|
+
input_: str,
|
|
2126
|
+
language: str,
|
|
2127
|
+
filename: str,
|
|
2128
|
+
fuel: int,
|
|
2129
|
+
*,
|
|
2130
|
+
cache_grammar_id: str | None = None,
|
|
2131
|
+
) -> SemanticNode:
|
|
2132
|
+
"""
|
|
2133
|
+
Call ``parser.process()`` with a parse-tree cache layer.
|
|
2134
|
+
|
|
2135
|
+
On a cache hit the wasm plugin is skipped entirely and the cached
|
|
2136
|
+
``SemanticNode`` tree is returned. On a miss the result is stored
|
|
2137
|
+
before returning.
|
|
2138
|
+
"""
|
|
2139
|
+
cache = self._cache
|
|
2140
|
+
if cache is None:
|
|
2141
|
+
raise RuntimeError("parse cache store is not configured")
|
|
2142
|
+
wasm_hash = self._wasm_hash_for(parser)
|
|
2143
|
+
grammar_id = cache_grammar_id or parser.grammar_id
|
|
2144
|
+
key = cache.parse_key(input_, grammar_id, wasm_hash)
|
|
2145
|
+
cached = cache.get_parse(key)
|
|
2146
|
+
if cached is not None:
|
|
2147
|
+
logger.debug("Parse cache hit for grammar=%r", grammar_id)
|
|
2148
|
+
return SemanticNode.model_validate_json(cached)
|
|
2149
|
+
|
|
2150
|
+
node = parser.process(input_, language, filename, fuel=fuel)
|
|
2151
|
+
cache.put_parse(key, node.model_dump_json(), grammar_id=grammar_id)
|
|
2152
|
+
return node
|
|
2153
|
+
|
|
2154
|
+
def _wasm_hash_for(self, parser: Any) -> str:
|
|
2155
|
+
"""
|
|
2156
|
+
Return a short (16-hex-char) sha256 hash of the parser's Wasm binary.
|
|
2157
|
+
|
|
2158
|
+
Computed once per grammar_id per process and memoised. Falls back to
|
|
2159
|
+
``"unknown"`` if the path cannot be read (e.g. in-memory test stubs).
|
|
2160
|
+
"""
|
|
2161
|
+
grammar_id: str = parser.grammar_id
|
|
2162
|
+
if grammar_id in self._wasm_hash_cache:
|
|
2163
|
+
return self._wasm_hash_cache[grammar_id]
|
|
2164
|
+
try:
|
|
2165
|
+
wasm_bytes = Path(parser.wasm_path).read_bytes()
|
|
2166
|
+
h = hashlib.sha256(wasm_bytes).hexdigest()[:16]
|
|
2167
|
+
except (AttributeError, OSError):
|
|
2168
|
+
h = "unknown"
|
|
2169
|
+
self._wasm_hash_cache[grammar_id] = h
|
|
2170
|
+
return h
|
|
2171
|
+
|
|
2172
|
+
def _parse(
|
|
2173
|
+
self,
|
|
2174
|
+
content: str,
|
|
2175
|
+
parser_adapter: Any,
|
|
2176
|
+
language: str,
|
|
2177
|
+
filename: str,
|
|
2178
|
+
*,
|
|
2179
|
+
profiler: _PhaseProfiler | None = None,
|
|
2180
|
+
) -> str:
|
|
2181
|
+
"""
|
|
2182
|
+
Prepare source for a parser plugin.
|
|
2183
|
+
|
|
2184
|
+
FullParse parsers receive raw source. Legacy ``interpret-cst`` parsers
|
|
2185
|
+
are rejected so public paths cannot host parsing in Python.
|
|
2186
|
+
"""
|
|
2187
|
+
if parser_adapter.parser_mode == "full-parse":
|
|
2188
|
+
return content # plugin will parse internally
|
|
2189
|
+
|
|
2190
|
+
raise PluginNotFoundError(
|
|
2191
|
+
"IntentumDiff no longer supports Python-hosted InterpretCst parsers "
|
|
2192
|
+
"on public product paths. Rebuild or update parser "
|
|
2193
|
+
f"{getattr(parser_adapter, 'grammar_id', '<unknown>')!r} as a "
|
|
2194
|
+
"FullParse Rust/Wasm plugin."
|
|
2195
|
+
)
|