intentumdiff-python 0.0.1__py3-none-win_amd64.whl → 0.0.3b1__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- intentumdiff/__init__.py +9 -0
- intentumdiff/__main__.py +12 -0
- intentumdiff/_differ_gate.py +0 -136
- intentumdiff/_differ_presentation.py +17 -735
- intentumdiff/_differ_runtime.py +13 -64
- intentumdiff/analysis/compile_commands.py +26 -141
- intentumdiff/analysis/cross_file.py +23 -25
- intentumdiff/analysis/diagnostics.py +1 -164
- intentumdiff/analysis/guardrail_reports.py +31 -33
- intentumdiff/analysis/guardrails.py +111 -188
- intentumdiff/analysis/schema_resolver.py +72 -360
- intentumdiff/analysis/user_schemas.py +57 -254
- intentumdiff/api.py +21 -0
- intentumdiff/cache/sqlite_store.py +5 -23
- intentumdiff/cli/__init__.py +59 -0
- intentumdiff/cli/_commands.py +243 -137
- intentumdiff/cli/_parser.py +51 -56
- intentumdiff/cli/_shared.py +151 -116
- intentumdiff/content_type.py +13 -30
- intentumdiff/core/commit_differ.py +137 -286
- intentumdiff/core/diffignore.py +28 -34
- intentumdiff/core/index.py +16 -18
- intentumdiff/core/indexer.py +148 -169
- intentumdiff/core/models.py +279 -83
- intentumdiff/differ.py +366 -1000
- intentumdiff/github_app.py +40 -27
- intentumdiff/intentumdiff_rust_core/intentumdiff_rust_core.dll +0 -0
- intentumdiff/live_server.py +197 -34
- intentumdiff/lsp/client.py +49 -43
- intentumdiff/lsp/enricher.py +19 -119
- intentumdiff/lsp/exceptions.py +6 -7
- intentumdiff/lsp/servers.py +10 -11
- intentumdiff/lsp_server/_codelens.py +24 -57
- intentumdiff/lsp_server/_diagnostics.py +23 -51
- intentumdiff/lsp_server/_handlers.py +67 -9
- intentumdiff/lsp_server/server.py +28 -8
- intentumdiff/plugins/adapter.py +5 -0
- intentumdiff/plugins/builtins.py +7 -14
- intentumdiff/plugins/exceptions.py +28 -0
- intentumdiff/plugins/hub.py +39 -43
- intentumdiff/plugins/loader.py +229 -215
- intentumdiff/plugins/registry.py +259 -207
- intentumdiff/plugins/registry.schema.json +156 -15
- intentumdiff/plugins/registry_schema.py +47 -140
- intentumdiff/rust_core.py +623 -1326
- intentumdiff/serve/_app.py +26 -35
- intentumdiff/sources/base.py +24 -0
- intentumdiff/sources/file_source.py +15 -2
- intentumdiff/sources/git_source.py +64 -185
- intentumdiff/sources/live_buffer_source.py +45 -5
- intentumdiff/sources/patch_source.py +36 -109
- intentumdiff/vcs/_safe_path.py +7 -0
- intentumdiff/vcs/base.py +6 -0
- intentumdiff/vcs/git_backend.py +1 -1
- intentumdiff/vcs/git_cli.py +0 -27
- intentumdiff/vcs/hg_backend.py +1 -1
- intentumdiff/vcs/perforce_backend.py +7 -1
- intentumdiff/vcs/svn_backend.py +1 -1
- intentumdiff/wasm/abap_parser.wasm +0 -0
- intentumdiff/wasm/adf_parser.wasm +0 -0
- intentumdiff/wasm/asciidoc_parser.wasm +0 -0
- intentumdiff/wasm/asm_parser.wasm +0 -0
- intentumdiff/wasm/assemblyscript_parser.wasm +0 -0
- intentumdiff/wasm/astro_parser.wasm +0 -0
- intentumdiff/wasm/bash_parser.wasm +0 -0
- intentumdiff/wasm/clojure_parser.wasm +0 -0
- intentumdiff/wasm/cmake_parser.wasm +0 -0
- intentumdiff/wasm/cpp_parser.wasm +0 -0
- intentumdiff/wasm/csharp_parser.wasm +0 -0
- intentumdiff/wasm/css_parser.wasm +0 -0
- intentumdiff/wasm/dart_parser.wasm +0 -0
- intentumdiff/wasm/databricks_parser.wasm +0 -0
- intentumdiff/wasm/dax_parser.wasm +0 -0
- intentumdiff/wasm/delphi_parser.wasm +0 -0
- intentumdiff/wasm/dockerfile_parser.wasm +0 -0
- intentumdiff/wasm/elixir_parser.wasm +0 -0
- intentumdiff/wasm/freebasic_parser.wasm +0 -0
- intentumdiff/wasm/generic_parser.wasm +0 -0
- intentumdiff/wasm/gitignore_parser.wasm +0 -0
- intentumdiff/wasm/go_parser.wasm +0 -0
- intentumdiff/wasm/gomod_parser.wasm +0 -0
- intentumdiff/wasm/graphql_parser.wasm +0 -0
- intentumdiff/wasm/groovy_parser.wasm +0 -0
- intentumdiff/wasm/haskell_parser.wasm +0 -0
- intentumdiff/wasm/html_parser.wasm +0 -0
- intentumdiff/wasm/html_renderer.wasm +0 -0
- intentumdiff/wasm/ini_parser.wasm +0 -0
- intentumdiff/wasm/java_parser.wasm +0 -0
- intentumdiff/wasm/js_ts_parser.wasm +0 -0
- intentumdiff/wasm/json_parser.wasm +0 -0
- intentumdiff/wasm/kotlin_parser.wasm +0 -0
- intentumdiff/wasm/latex_parser.wasm +0 -0
- intentumdiff/wasm/llm_renderer.wasm +0 -0
- intentumdiff/wasm/lua_parser.wasm +0 -0
- intentumdiff/wasm/make_parser.wasm +0 -0
- intentumdiff/wasm/markdown_parser.wasm +0 -0
- intentumdiff/wasm/mdx_parser.wasm +0 -0
- intentumdiff/wasm/ocaml_parser.wasm +0 -0
- intentumdiff/wasm/odin_parser.wasm +0 -0
- intentumdiff/wasm/parser_manifest.json +81 -2
- intentumdiff/wasm/patch_renderer.wasm +0 -0
- intentumdiff/wasm/perl_parser.wasm +0 -0
- intentumdiff/wasm/php_parser.wasm +0 -0
- intentumdiff/wasm/plsql_parser.wasm +0 -0
- intentumdiff/wasm/po_parser.wasm +0 -0
- intentumdiff/wasm/postscript_parser.wasm +0 -0
- intentumdiff/wasm/powershell_parser.wasm +0 -0
- intentumdiff/wasm/proto_parser.wasm +0 -0
- intentumdiff/wasm/puppet_parser.wasm +0 -0
- intentumdiff/wasm/python_parser.wasm +0 -0
- intentumdiff/wasm/qsharp_parser.wasm +0 -0
- intentumdiff/wasm/r_parser.wasm +0 -0
- intentumdiff/wasm/reasonml_parser.wasm +0 -0
- intentumdiff/wasm/ruby_parser.wasm +0 -0
- intentumdiff/wasm/rust_parser.wasm +0 -0
- intentumdiff/wasm/sas_parser.wasm +0 -0
- intentumdiff/wasm/scala_parser.wasm +0 -0
- intentumdiff/wasm/scss_parser.wasm +0 -0
- intentumdiff/wasm/sql_parser.wasm +0 -0
- intentumdiff/wasm/squirrel_parser.wasm +0 -0
- intentumdiff/wasm/svelte_parser.wasm +0 -0
- intentumdiff/wasm/swift_parser.wasm +0 -0
- intentumdiff/wasm/terminal_renderer.wasm +0 -0
- intentumdiff/wasm/terraform_parser.wasm +0 -0
- intentumdiff/wasm/toml_parser.wasm +0 -0
- intentumdiff/wasm/tsql_parser.wasm +0 -0
- intentumdiff/wasm/vbnet_parser.wasm +0 -0
- intentumdiff/wasm/vue_parser.wasm +0 -0
- intentumdiff/wasm/wasm_provenance.json +884 -0
- intentumdiff/wasm/wat_parser.wasm +0 -0
- intentumdiff/wasm/xml_parser.wasm +0 -0
- intentumdiff/wasm/yaml_parser.wasm +0 -0
- intentumdiff/wasm/zig_parser.wasm +0 -0
- intentumdiff/watcher.py +15 -17
- intentumdiff_python-0.0.3b1.dist-info/METADATA +188 -0
- intentumdiff_python-0.0.3b1.dist-info/RECORD +164 -0
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.3b1.dist-info}/WHEEL +1 -1
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.3b1.dist-info}/sboms/intentumdiff-rust-core.cyclonedx.json +905 -379
- intentumdiff/analysis/invariances.py +0 -1207
- intentumdiff/analysis/keyed_profiles.py +0 -25
- intentumdiff/analysis/resource_profiles.py +0 -14
- intentumdiff/analysis/text_review.py +0 -46
- intentumdiff/core/cst_serializer.py +0 -101
- intentumdiff/invariances/__init__.py +0 -2
- intentumdiff/invariances/rules.schema.json +0 -283
- intentumdiff/invariances/rules.yaml +0 -349
- intentumdiff_python-0.0.1.dist-info/METADATA +0 -112
- intentumdiff_python-0.0.1.dist-info/RECORD +0 -165
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.3b1.dist-info}/entry_points.txt +0 -0
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.3b1.dist-info}/licenses/LICENSE +0 -0
intentumdiff/differ.py
CHANGED
|
@@ -25,6 +25,7 @@ The ``SemanticDiffer`` class is the primary public API object.
|
|
|
25
25
|
from __future__ import annotations
|
|
26
26
|
|
|
27
27
|
import hashlib
|
|
28
|
+
import json
|
|
28
29
|
import logging
|
|
29
30
|
import os
|
|
30
31
|
import threading
|
|
@@ -39,9 +40,6 @@ from pydantic import ValidationError
|
|
|
39
40
|
from intentumdiff._differ_gate import (
|
|
40
41
|
RUST_CERTIFIED_LANGUAGES as RUST_CERTIFIED_LANGUAGES,
|
|
41
42
|
)
|
|
42
|
-
from intentumdiff._differ_gate import (
|
|
43
|
-
RUST_FINALIZE_LANGUAGES as RUST_FINALIZE_LANGUAGES,
|
|
44
|
-
)
|
|
45
43
|
from intentumdiff._differ_gate import (
|
|
46
44
|
STRICT_RUST_GATE_ENV as STRICT_RUST_GATE_ENV,
|
|
47
45
|
)
|
|
@@ -54,39 +52,9 @@ from intentumdiff._differ_gate import (
|
|
|
54
52
|
from intentumdiff._differ_gate import (
|
|
55
53
|
_rust_certified_languages as _rust_certified_languages,
|
|
56
54
|
)
|
|
57
|
-
from intentumdiff._differ_gate import (
|
|
58
|
-
_rust_finalize_languages as _rust_finalize_languages,
|
|
59
|
-
)
|
|
60
|
-
from intentumdiff._differ_gate import (
|
|
61
|
-
_strict_rust_only_enabled as _strict_rust_only_enabled,
|
|
62
|
-
)
|
|
63
55
|
from intentumdiff._differ_gate import (
|
|
64
56
|
_validate_git_ref as _validate_git_ref,
|
|
65
57
|
)
|
|
66
|
-
from intentumdiff._differ_presentation import (
|
|
67
|
-
_GENERIC_STRING_LABELS as _GENERIC_STRING_LABELS,
|
|
68
|
-
)
|
|
69
|
-
from intentumdiff._differ_presentation import (
|
|
70
|
-
_NAMED_ENTITY_NODE_TYPES as _NAMED_ENTITY_NODE_TYPES,
|
|
71
|
-
)
|
|
72
|
-
from intentumdiff._differ_presentation import (
|
|
73
|
-
_all_semantic_nodes as _all_semantic_nodes,
|
|
74
|
-
)
|
|
75
|
-
from intentumdiff._differ_presentation import (
|
|
76
|
-
_annotate_text_diffs as _annotate_text_diffs,
|
|
77
|
-
)
|
|
78
|
-
from intentumdiff._differ_presentation import (
|
|
79
|
-
_changes_to_stream_events as _changes_to_stream_events,
|
|
80
|
-
)
|
|
81
|
-
from intentumdiff._differ_presentation import (
|
|
82
|
-
_clean_string_literal_label as _clean_string_literal_label,
|
|
83
|
-
)
|
|
84
|
-
from intentumdiff._differ_presentation import (
|
|
85
|
-
_compute_structural_hash_for_tree as _compute_structural_hash_for_tree,
|
|
86
|
-
)
|
|
87
|
-
from intentumdiff._differ_presentation import (
|
|
88
|
-
_count_cst_nodes as _count_cst_nodes,
|
|
89
|
-
)
|
|
90
58
|
from intentumdiff._differ_presentation import (
|
|
91
59
|
_count_semantic_nodes as _count_semantic_nodes,
|
|
92
60
|
)
|
|
@@ -99,66 +67,15 @@ from intentumdiff._differ_presentation import (
|
|
|
99
67
|
from intentumdiff._differ_presentation import (
|
|
100
68
|
_has_error_node as _has_error_node,
|
|
101
69
|
)
|
|
102
|
-
from intentumdiff._differ_presentation import (
|
|
103
|
-
_is_markdown_filename as _is_markdown_filename,
|
|
104
|
-
)
|
|
105
|
-
from intentumdiff._differ_presentation import (
|
|
106
|
-
_is_named_entity_node as _is_named_entity_node,
|
|
107
|
-
)
|
|
108
|
-
from intentumdiff._differ_presentation import (
|
|
109
|
-
_markdown_section_body_hashes as _markdown_section_body_hashes,
|
|
110
|
-
)
|
|
111
|
-
from intentumdiff._differ_presentation import (
|
|
112
|
-
_markdown_section_heading_rename_presentation as _markdown_section_heading_rename_presentation,
|
|
113
|
-
)
|
|
114
|
-
from intentumdiff._differ_presentation import (
|
|
115
|
-
_markdown_section_move_presentation as _markdown_section_move_presentation,
|
|
116
|
-
)
|
|
117
|
-
from intentumdiff._differ_presentation import (
|
|
118
|
-
_markdown_sections as _markdown_sections,
|
|
119
|
-
)
|
|
120
|
-
from intentumdiff._differ_presentation import (
|
|
121
|
-
_node_to_dict as _node_to_dict,
|
|
122
|
-
)
|
|
123
|
-
from intentumdiff._differ_presentation import (
|
|
124
|
-
_root_structural_hash as _root_structural_hash,
|
|
125
|
-
)
|
|
126
|
-
from intentumdiff._differ_presentation import (
|
|
127
|
-
_semantic_parent_map as _semantic_parent_map,
|
|
128
|
-
)
|
|
129
|
-
from intentumdiff._differ_presentation import (
|
|
130
|
-
_slice_source_text as _slice_source_text,
|
|
131
|
-
)
|
|
132
|
-
from intentumdiff._differ_presentation import (
|
|
133
|
-
_surface_changed_in_place_entities as _surface_changed_in_place_entities,
|
|
134
|
-
)
|
|
135
70
|
from intentumdiff._differ_presentation import (
|
|
136
71
|
_token_fallback_diff as _token_fallback_diff,
|
|
137
72
|
)
|
|
138
73
|
from intentumdiff._differ_presentation import (
|
|
139
74
|
_validate_tree_ids as _validate_tree_ids,
|
|
140
75
|
)
|
|
141
|
-
from intentumdiff._differ_runtime import (
|
|
142
|
-
_ADDED_FILE_STATUSES as _ADDED_FILE_STATUSES,
|
|
143
|
-
)
|
|
144
|
-
from intentumdiff._differ_runtime import (
|
|
145
|
-
_DEFAULT_PLUGIN_FUEL as _DEFAULT_PLUGIN_FUEL,
|
|
146
|
-
)
|
|
147
|
-
from intentumdiff._differ_runtime import (
|
|
148
|
-
_DELETED_FILE_STATUSES as _DELETED_FILE_STATUSES,
|
|
149
|
-
)
|
|
150
|
-
from intentumdiff._differ_runtime import (
|
|
151
|
-
_EXPLICIT_FUEL_EXHAUSTION_TEST_CAP as _EXPLICIT_FUEL_EXHAUSTION_TEST_CAP,
|
|
152
|
-
)
|
|
153
76
|
from intentumdiff._differ_runtime import (
|
|
154
77
|
_FUEL_HOTSPOT_ABSOLUTE as _FUEL_HOTSPOT_ABSOLUTE,
|
|
155
78
|
)
|
|
156
|
-
from intentumdiff._differ_runtime import (
|
|
157
|
-
_FUEL_HOTSPOT_PER_KB as _FUEL_HOTSPOT_PER_KB,
|
|
158
|
-
)
|
|
159
|
-
from intentumdiff._differ_runtime import (
|
|
160
|
-
_FUEL_HOTSPOT_PER_LINE as _FUEL_HOTSPOT_PER_LINE,
|
|
161
|
-
)
|
|
162
79
|
from intentumdiff._differ_runtime import (
|
|
163
80
|
_apply_file_lifecycle_to_diff as _apply_file_lifecycle_to_diff,
|
|
164
81
|
)
|
|
@@ -171,12 +88,6 @@ from intentumdiff._differ_runtime import (
|
|
|
171
88
|
from intentumdiff._differ_runtime import (
|
|
172
89
|
_drain_plugin_telemetry as _drain_plugin_telemetry,
|
|
173
90
|
)
|
|
174
|
-
from intentumdiff._differ_runtime import (
|
|
175
|
-
_fuel_budget as _fuel_budget,
|
|
176
|
-
)
|
|
177
|
-
from intentumdiff._differ_runtime import (
|
|
178
|
-
_fuel_hotspot_for_call as _fuel_hotspot_for_call,
|
|
179
|
-
)
|
|
180
91
|
from intentumdiff._differ_runtime import (
|
|
181
92
|
_infer_file_lifecycle as _infer_file_lifecycle,
|
|
182
93
|
)
|
|
@@ -202,13 +113,8 @@ from intentumdiff.analysis.guardrails import (
|
|
|
202
113
|
# resource) are now Rust-authoritative — no Python enricher on the diff path (#90).
|
|
203
114
|
from intentumdiff.analysis.schema_resolver import (
|
|
204
115
|
resolve_schema,
|
|
205
|
-
schema_cache_fingerprint,
|
|
206
116
|
schema_resolution_metadata,
|
|
207
117
|
)
|
|
208
|
-
from intentumdiff.analysis.text_review import (
|
|
209
|
-
PresentationResult,
|
|
210
|
-
normalize_generic_text_for_review,
|
|
211
|
-
)
|
|
212
118
|
from intentumdiff.analysis.user_schemas import (
|
|
213
119
|
load_user_schema_profiles,
|
|
214
120
|
user_xml_dialects_payload,
|
|
@@ -216,36 +122,31 @@ from intentumdiff.analysis.user_schemas import (
|
|
|
216
122
|
from intentumdiff.cache.store import CacheStore
|
|
217
123
|
from intentumdiff.core.models import (
|
|
218
124
|
Change,
|
|
219
|
-
ChangeGroupKind,
|
|
220
125
|
ChangeStreamEvent,
|
|
221
126
|
ChangeStreamPhase,
|
|
222
|
-
ChangeType,
|
|
223
127
|
DetectionResult,
|
|
224
128
|
DiffConfig,
|
|
225
129
|
LanguageInfoGroup,
|
|
226
130
|
SemanticDiff,
|
|
227
131
|
SemanticNode,
|
|
228
132
|
)
|
|
133
|
+
from intentumdiff.plugins.adapter import ParserAdapter
|
|
229
134
|
from intentumdiff.plugins.exceptions import (
|
|
230
135
|
PluginFuelExhausted,
|
|
231
136
|
PluginNotFoundError,
|
|
232
137
|
PluginOutputError,
|
|
233
138
|
)
|
|
234
|
-
from intentumdiff.plugins.loader import _strip_trivia_impl
|
|
235
139
|
from intentumdiff.plugins.registry import PluginRegistry
|
|
236
140
|
from intentumdiff.rust_core import (
|
|
237
|
-
RustCoreCommitJsonAttempt,
|
|
238
|
-
apply_invariances,
|
|
239
|
-
build_style_only_evidence,
|
|
240
141
|
enrich_node_facts,
|
|
142
|
+
fuel_budget,
|
|
241
143
|
try_register_user_xml_dialects,
|
|
242
144
|
try_rust_core_batch_diff,
|
|
243
145
|
try_rust_core_batch_diffs,
|
|
244
|
-
try_rust_core_commit_json,
|
|
245
|
-
try_rust_core_working_tree_commit_json,
|
|
246
146
|
try_rust_profile_label_enrichment,
|
|
247
147
|
)
|
|
248
148
|
from intentumdiff.sources.base import Source
|
|
149
|
+
from intentumdiff.sources.file_source import FileSource as FileSource # compatibility re-export
|
|
249
150
|
|
|
250
151
|
logger = logging.getLogger(__name__)
|
|
251
152
|
|
|
@@ -274,8 +175,8 @@ class SemanticDiffer:
|
|
|
274
175
|
self._parallel_executor_workers: int | None = None
|
|
275
176
|
self._parallel_executor_lock = threading.Lock()
|
|
276
177
|
self._parallel_worker_local = threading.local()
|
|
277
|
-
#
|
|
278
|
-
self._wasm_hash_cache: dict[str, str] = {}
|
|
178
|
+
# Parser digest identity includes the actual on-disk artifact, never grammar alone.
|
|
179
|
+
self._wasm_hash_cache: dict[tuple[str, int, int, int, int, int], str] = {}
|
|
279
180
|
|
|
280
181
|
# ── Cache / analytics setup ───────────────────────────────────────
|
|
281
182
|
if cache is not None:
|
|
@@ -425,24 +326,22 @@ class SemanticDiffer:
|
|
|
425
326
|
return diff.model_copy(update={"metadata": metadata})
|
|
426
327
|
|
|
427
328
|
def _rust_core_python_batch_skip_reason(self) -> str | None:
|
|
428
|
-
if not self._config.experimental_rust_core:
|
|
429
|
-
return "disabled"
|
|
430
329
|
if self._config.diagnostics:
|
|
431
330
|
return "diagnostics require Python pipeline"
|
|
432
331
|
if self._config.extra_trivia_types:
|
|
433
332
|
return "extra trivia types require Python pipeline"
|
|
434
333
|
if self._config.extra_grammars.get("python"):
|
|
435
334
|
return "custom Python grammar requires Python pipeline"
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
):
|
|
335
|
+
# Core's allowlist rule (#311) on the batch parser's catalogue identity, the one
|
|
336
|
+
# filename selection sees: resolved path, catalogue id + entry names, languages.
|
|
337
|
+
from intentumdiff.plugins.builtins import python_parser_entry
|
|
338
|
+
from intentumdiff.rust_core import parser_allowlist_matches
|
|
339
|
+
|
|
340
|
+
path = str(Path(python_parser_entry()).resolve())
|
|
341
|
+
identity = next(({"plugin_id": path, "aliases": [e.plugin_id, *e.entry_names],
|
|
342
|
+
"languages": list(e.language_guesses)}
|
|
343
|
+
for e in self._registry._catalog() if e.resolved_path == path), {})
|
|
344
|
+
if not parser_allowlist_matches(self._config.allowed_plugins, identity)["matches"]:
|
|
446
345
|
return "Python parser is not allowed by plugin policy"
|
|
447
346
|
if self._registry.get_enrichers("python"):
|
|
448
347
|
return "enrichers require Python pipeline"
|
|
@@ -450,14 +349,6 @@ class SemanticDiffer:
|
|
|
450
349
|
return "diff analyzers require Python pipeline"
|
|
451
350
|
return None
|
|
452
351
|
|
|
453
|
-
@staticmethod
|
|
454
|
-
def _rust_core_parallel_workers(parallel: int | bool) -> int | None:
|
|
455
|
-
if parallel is True:
|
|
456
|
-
return os.cpu_count() or 1
|
|
457
|
-
if isinstance(parallel, int) and parallel > 0:
|
|
458
|
-
return parallel
|
|
459
|
-
return None
|
|
460
|
-
|
|
461
352
|
@staticmethod
|
|
462
353
|
def _rust_core_batch_parallel_plan(
|
|
463
354
|
file_count: int,
|
|
@@ -491,13 +382,29 @@ class SemanticDiffer:
|
|
|
491
382
|
profiler = self._new_profiler()
|
|
492
383
|
with profiler.phase("source_loading"):
|
|
493
384
|
old_content, new_content, filename, language_hint = source.get_content()
|
|
494
|
-
|
|
385
|
+
# Resolve host filesystem identity separately from public display labels.
|
|
386
|
+
# Source adapters validate repo roots/paths; only Rust evaluates the
|
|
387
|
+
# discovered guardrail policies against the semantic changes.
|
|
388
|
+
policy_context = getattr(source, "guardrail_policy_paths", None)
|
|
389
|
+
policy_old_filename, policy_source_filename = (
|
|
390
|
+
policy_context() if callable(policy_context) else (None, None)
|
|
391
|
+
)
|
|
392
|
+
diff = self._run_pipeline(
|
|
495
393
|
old_content,
|
|
496
394
|
new_content,
|
|
497
395
|
filename,
|
|
498
396
|
language_hint,
|
|
397
|
+
_policy_source_filename=policy_source_filename,
|
|
398
|
+
_policy_old_filename=policy_old_filename,
|
|
499
399
|
_profiler=profiler,
|
|
500
400
|
)
|
|
401
|
+
# The pipeline works from the single filename language detection needs. A source
|
|
402
|
+
# comparing two differently named files knows both, so restore them here rather
|
|
403
|
+
# than threading a second name through every construction site.
|
|
404
|
+
names = source.display_names()
|
|
405
|
+
if names is not None:
|
|
406
|
+
diff = diff.model_copy(update={"old_filename": names[0], "new_filename": names[1]})
|
|
407
|
+
return diff
|
|
501
408
|
|
|
502
409
|
def diff_strings(
|
|
503
410
|
self,
|
|
@@ -594,9 +501,9 @@ class SemanticDiffer:
|
|
|
594
501
|
"""
|
|
595
502
|
Parse *content* as *filename* and return ``(SemanticNode, language)``.
|
|
596
503
|
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
504
|
+
A FullParse Rust/Wasm plugin owns parsing; Python handles plugin dispatch
|
|
505
|
+
and the optional parse-tree cache. Legacy Python-hosted CST processing is
|
|
506
|
+
deliberately rejected.
|
|
600
507
|
|
|
601
508
|
Raises :class:`~intentumdiff.plugins.exceptions.PluginNotFoundError`
|
|
602
509
|
when no registered parser supports *filename*.
|
|
@@ -619,22 +526,7 @@ class SemanticDiffer:
|
|
|
619
526
|
parser, language = self._registry.detect_parser(filename, content, language_hint)
|
|
620
527
|
adaptive_fuel = self._config.plugin_fuel
|
|
621
528
|
|
|
622
|
-
|
|
623
|
-
content = parser.preprocess_source(content)
|
|
624
|
-
cst_json = self._parse(content, parser, language, filename)
|
|
625
|
-
if len(cst_json) > self._config.max_cst_bytes:
|
|
626
|
-
raise ValueError(
|
|
627
|
-
f"CST JSON for {filename!r} is {len(cst_json):,} characters, "
|
|
628
|
-
f"exceeding the limit of {self._config.max_cst_bytes:,} "
|
|
629
|
-
"(DiffConfig.max_cst_bytes)."
|
|
630
|
-
)
|
|
631
|
-
trivia_types = list(parser.trivia_node_types) + self._config.extra_trivia_types
|
|
632
|
-
filtered = _strip_trivia_impl(cst_json, trivia_types)
|
|
633
|
-
cst_nodes = _count_cst_nodes(filtered)
|
|
634
|
-
adaptive_fuel = _fuel_budget(self._config.plugin_fuel, 20_000_000 + cst_nodes * 200_000)
|
|
635
|
-
input_ = filtered
|
|
636
|
-
else:
|
|
637
|
-
input_ = content
|
|
529
|
+
input_ = self._parse(content, parser, language, filename)
|
|
638
530
|
|
|
639
531
|
if self._cache is not None:
|
|
640
532
|
node = self._cached_process(parser, input_, language, filename, adaptive_fuel)
|
|
@@ -666,164 +558,20 @@ class SemanticDiffer:
|
|
|
666
558
|
return diff.model_dump_json(indent=2)
|
|
667
559
|
|
|
668
560
|
if output_format == "terminal":
|
|
669
|
-
#
|
|
670
|
-
|
|
561
|
+
# Core's terminal presentation, exactly as the CLI renders it: source text such as
|
|
562
|
+
# `[/].py` or `[red]` stays literal instead of becoming Rich markup (#306).
|
|
563
|
+
from intentumdiff.rust_core import render_cli_review
|
|
671
564
|
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
buf = StringIO()
|
|
675
|
-
console = _Console(file=buf, highlight=False, no_color=True)
|
|
676
|
-
console.print(
|
|
677
|
-
f"Semantic diff: {diff.old_filename} -> {diff.new_filename} ({diff.language})"
|
|
678
|
-
)
|
|
679
|
-
for change in diff.changes:
|
|
680
|
-
ct_str = (
|
|
681
|
-
change.change_type.value
|
|
682
|
-
if isinstance(change.change_type, ChangeType)
|
|
683
|
-
else str(change.change_type)
|
|
684
|
-
)
|
|
685
|
-
console.print(f" {ct_str:14} {change.description}")
|
|
686
|
-
return buf.getvalue()
|
|
565
|
+
return render_cli_review(diff, width=80, color=False)
|
|
687
566
|
|
|
688
567
|
try:
|
|
689
568
|
renderer = self._registry.get_renderer(output_format)
|
|
690
569
|
except KeyError as exc:
|
|
691
570
|
raise ValueError(str(exc)) from exc
|
|
692
571
|
diff_json = diff.model_dump_json()
|
|
693
|
-
adaptive_fuel =
|
|
572
|
+
adaptive_fuel = fuel_budget(self._config.plugin_fuel, len(diff_json.encode("utf-8")))
|
|
694
573
|
return renderer.render(diff_json, fuel=adaptive_fuel)
|
|
695
574
|
|
|
696
|
-
def _diff_commit_certified_json(
|
|
697
|
-
self,
|
|
698
|
-
repo_path: str | os.PathLike[str] = ".",
|
|
699
|
-
old_ref: str = "HEAD",
|
|
700
|
-
new_ref: str = "",
|
|
701
|
-
) -> RustCoreCommitJsonAttempt:
|
|
702
|
-
"""Return Rust-certified CommitDiff JSON bytes for narrow Python batches."""
|
|
703
|
-
_validate_git_ref(old_ref)
|
|
704
|
-
_validate_git_ref(new_ref)
|
|
705
|
-
rust_batch_skip_reason = self._rust_core_python_batch_skip_reason()
|
|
706
|
-
if rust_batch_skip_reason is not None:
|
|
707
|
-
_raise_rust_only_gate_error(rust_batch_skip_reason)
|
|
708
|
-
return RustCoreCommitJsonAttempt(fallback_reason=rust_batch_skip_reason)
|
|
709
|
-
|
|
710
|
-
if not new_ref:
|
|
711
|
-
rust_batch_parallel, rust_batch_workers, _rust_batch_parallel_auto = (
|
|
712
|
-
True,
|
|
713
|
-
self._rust_core_parallel_workers(self._config.parallel),
|
|
714
|
-
True,
|
|
715
|
-
)
|
|
716
|
-
working_tree_attempt = try_rust_core_working_tree_commit_json(
|
|
717
|
-
repo_path=repo_path,
|
|
718
|
-
old_ref=old_ref,
|
|
719
|
-
config=self._config,
|
|
720
|
-
parallel=rust_batch_parallel,
|
|
721
|
-
max_workers=rust_batch_workers,
|
|
722
|
-
)
|
|
723
|
-
if working_tree_attempt.used:
|
|
724
|
-
return working_tree_attempt
|
|
725
|
-
if working_tree_attempt.fallback_reason and not (
|
|
726
|
-
"does not expose diff_working_tree_python_commit_json"
|
|
727
|
-
in working_tree_attempt.fallback_reason
|
|
728
|
-
or working_tree_attempt.fallback_reason.startswith("unavailable:")
|
|
729
|
-
):
|
|
730
|
-
_raise_rust_only_gate_error(working_tree_attempt.fallback_reason)
|
|
731
|
-
return working_tree_attempt
|
|
732
|
-
|
|
733
|
-
from intentumdiff.plugins.builtins import python_parser_entry
|
|
734
|
-
from intentumdiff.sources.git_source import (
|
|
735
|
-
collect_working_tree_python_sources_fast,
|
|
736
|
-
iter_changed_sources,
|
|
737
|
-
)
|
|
738
|
-
|
|
739
|
-
source_started = time.perf_counter()
|
|
740
|
-
fast_collection = (
|
|
741
|
-
collect_working_tree_python_sources_fast(repo_path, old_ref) if not new_ref else None
|
|
742
|
-
)
|
|
743
|
-
if fast_collection is None:
|
|
744
|
-
sources = list(iter_changed_sources(repo_path, old_ref, new_ref))
|
|
745
|
-
fast_fallback_reason = None
|
|
746
|
-
else:
|
|
747
|
-
sources, fast_fallback_reason = fast_collection
|
|
748
|
-
source_collection_ms = (time.perf_counter() - source_started) * 1000
|
|
749
|
-
if fast_fallback_reason is not None:
|
|
750
|
-
_raise_rust_only_gate_error(fast_fallback_reason)
|
|
751
|
-
return RustCoreCommitJsonAttempt(fallback_reason=fast_fallback_reason)
|
|
752
|
-
if not sources:
|
|
753
|
-
return RustCoreCommitJsonAttempt()
|
|
754
|
-
|
|
755
|
-
parser_wasm_path = python_parser_entry()
|
|
756
|
-
request_files: list[dict[str, Any]] = []
|
|
757
|
-
source_filter_started = time.perf_counter()
|
|
758
|
-
for old_content, new_content, old_path, new_path, staging_status in sources:
|
|
759
|
-
effective_new_path = new_path if new_path is not None else old_path
|
|
760
|
-
file_lifecycle = _infer_file_lifecycle(old_content, new_content, staging_status)
|
|
761
|
-
old_path_lower = old_path.lower()
|
|
762
|
-
effective_new_path_lower = effective_new_path.lower()
|
|
763
|
-
if not (
|
|
764
|
-
old_path_lower.endswith((".py", ".pyi"))
|
|
765
|
-
or effective_new_path_lower.endswith((".py", ".pyi"))
|
|
766
|
-
):
|
|
767
|
-
fallback_attempt = RustCoreCommitJsonAttempt(
|
|
768
|
-
fallback_reason="certified commit JSON requires all changed files to be Python"
|
|
769
|
-
)
|
|
770
|
-
_raise_rust_only_gate_error(fallback_attempt.fallback_reason)
|
|
771
|
-
return fallback_attempt
|
|
772
|
-
# Protected guardrail rules currently apply only to config/profile
|
|
773
|
-
# languages, not Python. Avoid a per-file policy search here; any
|
|
774
|
-
# actual policy-file change is excluded by the Python-only gate.
|
|
775
|
-
request_file: dict[str, Any] = {
|
|
776
|
-
"old_source": old_content,
|
|
777
|
-
"new_source": new_content,
|
|
778
|
-
"old_filename": old_path,
|
|
779
|
-
"new_filename": effective_new_path,
|
|
780
|
-
"language": "python",
|
|
781
|
-
"parser_plugin_id": "python",
|
|
782
|
-
"parser_wasm_path": parser_wasm_path,
|
|
783
|
-
"file_lifecycle": file_lifecycle,
|
|
784
|
-
}
|
|
785
|
-
if staging_status is not None:
|
|
786
|
-
request_file["staging_status"] = staging_status
|
|
787
|
-
request_files.append(request_file)
|
|
788
|
-
source_filter_ms = (time.perf_counter() - source_filter_started) * 1000
|
|
789
|
-
|
|
790
|
-
rust_batch_parallel, rust_batch_workers, _rust_batch_parallel_auto = (
|
|
791
|
-
self._rust_core_batch_parallel_plan(len(request_files), self._config.parallel)
|
|
792
|
-
)
|
|
793
|
-
attempt = try_rust_core_commit_json(
|
|
794
|
-
files=request_files,
|
|
795
|
-
old_ref=old_ref,
|
|
796
|
-
new_ref=new_ref,
|
|
797
|
-
config=self._config,
|
|
798
|
-
parallel=rust_batch_parallel,
|
|
799
|
-
max_workers=rust_batch_workers,
|
|
800
|
-
)
|
|
801
|
-
if not attempt.used:
|
|
802
|
-
_raise_rust_only_gate_error(attempt.fallback_reason)
|
|
803
|
-
return attempt
|
|
804
|
-
adapter_phase_timings = [
|
|
805
|
-
{
|
|
806
|
-
"name": "source_collection",
|
|
807
|
-
"duration_ms": round(source_collection_ms, 3),
|
|
808
|
-
"shared": True,
|
|
809
|
-
},
|
|
810
|
-
{
|
|
811
|
-
"name": "rust_core_commit_source_filtering",
|
|
812
|
-
"duration_ms": round(source_filter_ms, 3),
|
|
813
|
-
"shared": True,
|
|
814
|
-
},
|
|
815
|
-
*attempt.adapter_phase_timings,
|
|
816
|
-
]
|
|
817
|
-
control = dict(attempt.control)
|
|
818
|
-
control["source_collection_ms"] = round(source_collection_ms, 3)
|
|
819
|
-
control["source_filter_ms"] = round(source_filter_ms, 3)
|
|
820
|
-
return RustCoreCommitJsonAttempt(
|
|
821
|
-
control=control,
|
|
822
|
-
commit_diff_json=attempt.commit_diff_json,
|
|
823
|
-
adapter_phase_timings=adapter_phase_timings,
|
|
824
|
-
backend_version=attempt.backend_version,
|
|
825
|
-
)
|
|
826
|
-
|
|
827
575
|
def diff_commit(
|
|
828
576
|
self,
|
|
829
577
|
repo_path: str | os.PathLike[str] = ".",
|
|
@@ -835,7 +583,10 @@ class SemanticDiffer:
|
|
|
835
583
|
"""
|
|
836
584
|
Diff every file that changed between two git refs.
|
|
837
585
|
|
|
838
|
-
|
|
586
|
+
A changed text file with no registered parser is a failed review.
|
|
587
|
+
Missing-parser and native runtime failures abort the entire review with
|
|
588
|
+
the path and original cause. Binary/ignored files are excluded during
|
|
589
|
+
source collection, before the semantic engine is called.
|
|
839
590
|
|
|
840
591
|
Parameters
|
|
841
592
|
----------
|
|
@@ -850,15 +601,17 @@ class SemanticDiffer:
|
|
|
850
601
|
Returns
|
|
851
602
|
-------
|
|
852
603
|
list[SemanticDiff]
|
|
853
|
-
One entry per changed file
|
|
604
|
+
One entry per reviewed changed text file; failures raise rather than disappear.
|
|
854
605
|
"""
|
|
855
606
|
from intentumdiff.sources.git_source import iter_changed_sources
|
|
607
|
+
from intentumdiff.vcs.git_cli import resolve_repo_root
|
|
608
|
+
from intentumdiff.vcs._safe_path import safe_policy_path
|
|
856
609
|
|
|
857
610
|
source_started = time.perf_counter()
|
|
858
|
-
|
|
611
|
+
policy_root = resolve_repo_root(repo_path)
|
|
612
|
+
sources = list(iter_changed_sources(policy_root, old_ref, new_ref))
|
|
859
613
|
source_collection_ms = (time.perf_counter() - source_started) * 1000
|
|
860
614
|
rust_batch_results: dict[int, SemanticDiff] = {}
|
|
861
|
-
rust_batch_attempted: set[int] = set()
|
|
862
615
|
rust_batch_skip_reason = self._rust_core_python_batch_skip_reason()
|
|
863
616
|
if rust_batch_skip_reason is None and sources:
|
|
864
617
|
from intentumdiff.plugins.builtins import python_parser_entry
|
|
@@ -878,10 +631,15 @@ class SemanticDiffer:
|
|
|
878
631
|
or effective_new_path_lower.endswith((".py", ".pyi"))
|
|
879
632
|
):
|
|
880
633
|
continue
|
|
881
|
-
#
|
|
882
|
-
#
|
|
883
|
-
#
|
|
884
|
-
|
|
634
|
+
# A discovered or explicit policy must be validated even when
|
|
635
|
+
# its file is unchanged/ignored. Rust decides whether it applies;
|
|
636
|
+
# policy-relevant sources keep the normal physical-path route.
|
|
637
|
+
if guardrails_may_apply(
|
|
638
|
+
safe_policy_path(policy_root, old_path),
|
|
639
|
+
safe_policy_path(policy_root, effective_new_path),
|
|
640
|
+
self._config,
|
|
641
|
+
):
|
|
642
|
+
continue
|
|
885
643
|
request_to_source_index.append(idx)
|
|
886
644
|
request_files.append(
|
|
887
645
|
{
|
|
@@ -899,7 +657,6 @@ class SemanticDiffer:
|
|
|
899
657
|
time.perf_counter() - rust_batch_source_filter_started
|
|
900
658
|
) * 1000
|
|
901
659
|
if request_files:
|
|
902
|
-
rust_batch_attempted = set(request_to_source_index)
|
|
903
660
|
(
|
|
904
661
|
rust_batch_parallel,
|
|
905
662
|
rust_batch_workers,
|
|
@@ -915,6 +672,15 @@ class SemanticDiffer:
|
|
|
915
672
|
parallel=rust_batch_parallel,
|
|
916
673
|
max_workers=rust_batch_workers,
|
|
917
674
|
)
|
|
675
|
+
# A corrupted Rust batch is an integrity failure, never an
|
|
676
|
+
# invitation to reroute files through a different finalizer.
|
|
677
|
+
# The single-file entrypoint already enforces this contract;
|
|
678
|
+
# public commit reviews must enforce it before merging results.
|
|
679
|
+
if rust_batch.invalid_output:
|
|
680
|
+
raise RuntimeError(
|
|
681
|
+
"Malformed Rust commit batch result: "
|
|
682
|
+
f"{rust_batch.fallback_reason or 'invalid native output'}"
|
|
683
|
+
)
|
|
918
684
|
rust_batch_ms = (time.perf_counter() - rust_batch_started) * 1000
|
|
919
685
|
rust_batch_fallback_count = int(
|
|
920
686
|
rust_batch.batch_metadata.get("fallback_count")
|
|
@@ -996,13 +762,15 @@ class SemanticDiffer:
|
|
|
996
762
|
None,
|
|
997
763
|
new_filename=new_path,
|
|
998
764
|
_profiler=profiler,
|
|
765
|
+
_policy_old_filename=safe_policy_path(policy_root, old_path),
|
|
766
|
+
_policy_source_filename=safe_policy_path(policy_root, new_path or old_path),
|
|
999
767
|
)
|
|
1000
768
|
if staging_status is not None:
|
|
1001
769
|
result = result.model_copy(update={"staging_status": staging_status})
|
|
1002
770
|
return result
|
|
1003
771
|
|
|
1004
|
-
|
|
1005
|
-
_warn_exc = (PluginFuelExhausted, ValueError, RuntimeError)
|
|
772
|
+
# Parser absence is not a certified decline: it must fail the whole commit.
|
|
773
|
+
_warn_exc = (PluginNotFoundError, PluginFuelExhausted, ValueError, RuntimeError)
|
|
1006
774
|
|
|
1007
775
|
parallel = self._config.parallel
|
|
1008
776
|
if not parallel:
|
|
@@ -1014,10 +782,7 @@ class SemanticDiffer:
|
|
|
1014
782
|
continue
|
|
1015
783
|
# Files the commit BATCH attempted but declined re-run through the normal
|
|
1016
784
|
# differ with the Rust core ON: the batch re-attempt is one cheap native
|
|
1017
|
-
# call and the per-stage finalize routing then serves.
|
|
1018
|
-
# experimental_rust_core=False fallback differ meant "python pipeline";
|
|
1019
|
-
# post-retirement it means the token-fallback kill switch — a comment-only
|
|
1020
|
-
# commit came back as a token ADDITION instead of style-only.)
|
|
785
|
+
# call and the per-stage finalize routing then serves.
|
|
1021
786
|
run_differ = self
|
|
1022
787
|
try:
|
|
1023
788
|
results.append(
|
|
@@ -1031,10 +796,12 @@ class SemanticDiffer:
|
|
|
1031
796
|
# A genuine token-level fallback under the RUST_ONLY gate is fatal —
|
|
1032
797
|
# propagate it rather than let ``_warn_exc`` silently drop the file.
|
|
1033
798
|
raise
|
|
1034
|
-
except
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
799
|
+
except _warn_exc as exc:
|
|
800
|
+
# A runtime failure cannot be represented in list[SemanticDiff].
|
|
801
|
+
# Preserve the failed file identity; a partial review is unsafe.
|
|
802
|
+
raise RuntimeError(
|
|
803
|
+
f"Commit diff failed for {item[2]!r}: {exc}"
|
|
804
|
+
) from exc
|
|
1038
805
|
return results
|
|
1039
806
|
else:
|
|
1040
807
|
# ── Parallel ────────────────────────────────────────────────
|
|
@@ -1075,10 +842,12 @@ class SemanticDiffer:
|
|
|
1075
842
|
except RustOnlyGateError:
|
|
1076
843
|
# Fatal under the RUST_ONLY gate — propagate, do not drop.
|
|
1077
844
|
raise
|
|
1078
|
-
except
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
845
|
+
except _warn_exc as exc:
|
|
846
|
+
# Match the sequential path: failures must not disappear
|
|
847
|
+
# when one parallel worker succeeds.
|
|
848
|
+
raise RuntimeError(
|
|
849
|
+
f"Commit diff failed for {sources[idx][2]!r}: {exc}"
|
|
850
|
+
) from exc
|
|
1082
851
|
return [results_map[i] for i in sorted(results_map)]
|
|
1083
852
|
|
|
1084
853
|
def diff_stream(
|
|
@@ -1106,6 +875,8 @@ class SemanticDiffer:
|
|
|
1106
875
|
*,
|
|
1107
876
|
new_filename: str | None = None,
|
|
1108
877
|
parser_plugin_id: str | None = None,
|
|
878
|
+
_policy_source_filename: str | None = None,
|
|
879
|
+
_policy_old_filename: str | None = None,
|
|
1109
880
|
_profiler: _PhaseProfiler | None = None,
|
|
1110
881
|
) -> SemanticDiff:
|
|
1111
882
|
profiler = _profiler or self._new_profiler()
|
|
@@ -1116,6 +887,8 @@ class SemanticDiffer:
|
|
|
1116
887
|
language_hint,
|
|
1117
888
|
new_filename=new_filename,
|
|
1118
889
|
parser_plugin_id=parser_plugin_id,
|
|
890
|
+
policy_source_filename=_policy_source_filename,
|
|
891
|
+
policy_old_filename=_policy_old_filename,
|
|
1119
892
|
profiler=profiler,
|
|
1120
893
|
)
|
|
1121
894
|
# _run_stages_1_to_11 always returns a finished SemanticDiff since the python
|
|
@@ -1139,15 +912,21 @@ class SemanticDiffer:
|
|
|
1139
912
|
filename,
|
|
1140
913
|
fuel=adaptive_fuel,
|
|
1141
914
|
)
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
exc,
|
|
915
|
+
# A Wasm analyzer is a diff-to-diff transform, never permission
|
|
916
|
+
# to replace a Rust review with Pydantic's empty DTO defaults.
|
|
917
|
+
from intentumdiff.rust_core import _require_complete_batch_diff_dto
|
|
918
|
+
|
|
919
|
+
routed = SemanticDiff.model_validate(
|
|
920
|
+
_require_complete_batch_diff_dto(json.loads(updated_json))
|
|
1149
921
|
)
|
|
1150
|
-
|
|
922
|
+
except Exception as exc:
|
|
923
|
+
# Semantic analyzers are configured diff-to-diff processing.
|
|
924
|
+
# Omitting a failed one would return an incomplete review as
|
|
925
|
+
# successful, including on fuel exhaustion or malformed DTOs.
|
|
926
|
+
provenance = getattr(analyzer, "provenance", "?")
|
|
927
|
+
raise RuntimeError(
|
|
928
|
+
f"Diff analyzer {provenance!r} failed for {filename!r}: {exc}"
|
|
929
|
+
) from exc
|
|
1151
930
|
return routed
|
|
1152
931
|
|
|
1153
932
|
def _run_stages_1_to_11(
|
|
@@ -1159,11 +938,13 @@ class SemanticDiffer:
|
|
|
1159
938
|
*,
|
|
1160
939
|
new_filename: str | None = None,
|
|
1161
940
|
parser_plugin_id: str | None = None,
|
|
941
|
+
policy_source_filename: str | None = None,
|
|
942
|
+
policy_old_filename: str | None = None,
|
|
1162
943
|
profiler: _PhaseProfiler | None = None,
|
|
1163
944
|
) -> SemanticDiff:
|
|
1164
945
|
"""Run pipeline stages 1–11 (parse → trivia-strip → Wasm → GumTree → moves).
|
|
1165
946
|
|
|
1166
|
-
Returns
|
|
947
|
+
Returns Rust-owned results or explicitly marked source fallbacks.
|
|
1167
948
|
"""
|
|
1168
949
|
# ── 1 + 2. Language detection ────────────────────────────────────────
|
|
1169
950
|
profiler = profiler or self._new_profiler()
|
|
@@ -1174,6 +955,7 @@ class SemanticDiffer:
|
|
|
1174
955
|
language_hint,
|
|
1175
956
|
plugin_id=parser_plugin_id,
|
|
1176
957
|
phase_recorder=profiler.record if profiler.enabled else None,
|
|
958
|
+
old_content=old_content,
|
|
1177
959
|
)
|
|
1178
960
|
logger.debug("Using parser %r for %r (language=%r)", parser.grammar_id, filename, language)
|
|
1179
961
|
|
|
@@ -1183,6 +965,10 @@ class SemanticDiffer:
|
|
|
1183
965
|
)
|
|
1184
966
|
engine_telemetry: list[dict[str, Any]] = []
|
|
1185
967
|
parse_mode = parser.parser_mode
|
|
968
|
+
# Retired InterpretCst execution cannot survive in a thin Python binding.
|
|
969
|
+
# `_parse` checks the capability and raises with an actionable upgrade path.
|
|
970
|
+
if parse_mode != "full-parse":
|
|
971
|
+
self._parse(new_content, parser, language, filename)
|
|
1186
972
|
cache_grammar_id = parser.grammar_id
|
|
1187
973
|
diagnostics.summary.update(
|
|
1188
974
|
{
|
|
@@ -1208,8 +994,7 @@ class SemanticDiffer:
|
|
|
1208
994
|
},
|
|
1209
995
|
)
|
|
1210
996
|
|
|
1211
|
-
#
|
|
1212
|
-
# raises it based on CST node count before calling the plugin.
|
|
997
|
+
# The configured plugin fuel is adapted for full-parse source size below.
|
|
1213
998
|
adaptive_fuel: int = self._config.plugin_fuel
|
|
1214
999
|
|
|
1215
1000
|
# Save originals for the enricher pass (enrichers see un-preprocessed source)
|
|
@@ -1229,12 +1014,8 @@ class SemanticDiffer:
|
|
|
1229
1014
|
# the full set is safe for every xml diff.
|
|
1230
1015
|
_user_profiles, _ = load_user_schema_profiles()
|
|
1231
1016
|
_dialects = user_xml_dialects_payload(_user_profiles)
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
schema_fingerprint = schema_cache_fingerprint(schema_resolution)
|
|
1235
|
-
cache_schema_suffix = (
|
|
1236
|
-
"" if schema_fingerprint == "schema:none" else f"|{schema_fingerprint}"
|
|
1237
|
-
)
|
|
1017
|
+
# Empty configuration must clear previously registered native dialects.
|
|
1018
|
+
try_register_user_xml_dialects(_dialects)
|
|
1238
1019
|
schema_metadata = schema_resolution_metadata(schema_resolution)
|
|
1239
1020
|
if schema_metadata is not None:
|
|
1240
1021
|
diagnostics.summary["schema"] = schema_metadata
|
|
@@ -1259,18 +1040,18 @@ class SemanticDiffer:
|
|
|
1259
1040
|
reason="C/C++ compile database context resolved",
|
|
1260
1041
|
metadata=compile_metadata,
|
|
1261
1042
|
)
|
|
1262
|
-
compile_cache_suffix = (
|
|
1263
|
-
"" if compile_metadata is None else f"|compile:{compile_metadata['fingerprint']}"
|
|
1264
|
-
)
|
|
1265
1043
|
|
|
1266
|
-
#
|
|
1267
|
-
|
|
1268
|
-
_guardrail_cache_bypass = guardrails_may_apply(
|
|
1044
|
+
# Both physical sides may carry protection. Display aliases must not
|
|
1045
|
+
# hide the source policy on moves out of protected directories (#262).
|
|
1046
|
+
_guardrail_cache_bypass = guardrails_may_apply(
|
|
1047
|
+
policy_old_filename or filename,
|
|
1048
|
+
policy_source_filename or new_filename,
|
|
1049
|
+
self._config,
|
|
1050
|
+
)
|
|
1269
1051
|
cache_grammar_id = parser.grammar_id
|
|
1270
1052
|
|
|
1271
1053
|
if (
|
|
1272
|
-
|
|
1273
|
-
and language in _rust_certified_languages()
|
|
1054
|
+
language in _rust_certified_languages()
|
|
1274
1055
|
and not diagnostics.enabled
|
|
1275
1056
|
# ``test_matching_engine`` is the dual-run truthiness override.
|
|
1276
1057
|
# When set, it forces the matching stage to either Rust or
|
|
@@ -1322,13 +1103,22 @@ class SemanticDiffer:
|
|
|
1322
1103
|
rust_batch_attempt.diff,
|
|
1323
1104
|
file_lifecycle,
|
|
1324
1105
|
)
|
|
1106
|
+
# An invalid claimed-complete Rust result is a broken engine/DTO
|
|
1107
|
+
# contract, NOT an ordinary batch decline. Routing it through the
|
|
1108
|
+
# alternate Rust finalizer can silently publish different semantic
|
|
1109
|
+
# changes (observed in the 74-language corpus, #205).
|
|
1110
|
+
if rust_batch_attempt.invalid_output:
|
|
1111
|
+
raise RuntimeError(
|
|
1112
|
+
f"Malformed Rust batch result for {filename!r}: "
|
|
1113
|
+
f"{rust_batch_attempt.fallback_reason}"
|
|
1114
|
+
)
|
|
1325
1115
|
# The certified BATCH shortcut declined — fall through to the native
|
|
1326
1116
|
# routed finalize path below. This is a Rust→Rust transition (the
|
|
1327
1117
|
# routed path finalizes through the Rust core for every certified
|
|
1328
1118
|
# language), NOT a Python fallback, so the RUST_ONLY gate does NOT
|
|
1329
|
-
# fire here.
|
|
1330
|
-
#
|
|
1331
|
-
#
|
|
1119
|
+
# fire here. Parse-error source comparison also runs in Rust;
|
|
1120
|
+
# explicit core disabling and unexpected finalize declines still
|
|
1121
|
+
# fail the strict gate.
|
|
1332
1122
|
logger.debug(
|
|
1333
1123
|
"Rust core batch declined for %r: %s; using routed finalize",
|
|
1334
1124
|
filename,
|
|
@@ -1341,248 +1131,56 @@ class SemanticDiffer:
|
|
|
1341
1131
|
rust_core_skip_reason,
|
|
1342
1132
|
)
|
|
1343
1133
|
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
)
|
|
1368
|
-
|
|
1369
|
-
old_cst_json = self._parse(
|
|
1370
|
-
old_content,
|
|
1371
|
-
parser,
|
|
1372
|
-
language,
|
|
1373
|
-
filename,
|
|
1374
|
-
profiler=profiler,
|
|
1375
|
-
)
|
|
1376
|
-
new_cst_json = self._parse(
|
|
1377
|
-
new_content,
|
|
1378
|
-
parser,
|
|
1379
|
-
language,
|
|
1380
|
-
filename,
|
|
1381
|
-
profiler=profiler,
|
|
1382
|
-
)
|
|
1383
|
-
|
|
1384
|
-
# ── CST size guard ────────────────────────────────────────────────
|
|
1385
|
-
for _label, _cst in (("old", old_cst_json), ("new", new_cst_json)):
|
|
1386
|
-
if len(_cst) > self._config.max_cst_bytes:
|
|
1387
|
-
raise ValueError(
|
|
1388
|
-
f"CST JSON for the {_label!r} version of {filename!r} is "
|
|
1389
|
-
f"{len(_cst):,} characters, exceeding the limit of "
|
|
1390
|
-
f"{self._config.max_cst_bytes:,} "
|
|
1391
|
-
"(DiffConfig.max_cst_bytes). Increase the limit or switch "
|
|
1392
|
-
"to a full-parse plugin."
|
|
1134
|
+
# ── 7 (full-parse). Plugin owns parsing — send raw source ─────────
|
|
1135
|
+
# Stages 3-6 are skipped: the plugin handles its own CST construction
|
|
1136
|
+
# and trivia filtering. The style-only shortcut is not available here
|
|
1137
|
+
# because there is no host-side CST to hash.
|
|
1138
|
+
adaptive_fuel = fuel_budget( # core's pair policy scales by both sides' bytes (#308)
|
|
1139
|
+
self._config.plugin_fuel,
|
|
1140
|
+
len(old_content.encode("utf-8")) + len(new_content.encode("utf-8")),
|
|
1141
|
+
)
|
|
1142
|
+
logger.debug("full-parse mode for %r via %r", filename, parser.grammar_id)
|
|
1143
|
+
|
|
1144
|
+
# Full-diff cache reads are disabled for this release candidate (#187).
|
|
1145
|
+
# Historical entries omit the Rust engine/configuration identity and
|
|
1146
|
+
# can silently replay stale style-only results after semantic fixes.
|
|
1147
|
+
# Parser-tree caching remains available and is keyed by actual Wasm bytes.
|
|
1148
|
+
with profiler.phase("wasm_plugin_execution"):
|
|
1149
|
+
old_side_absent = file_lifecycle == "added" and old_content == ""
|
|
1150
|
+
new_side_absent = file_lifecycle == "deleted" and new_content == ""
|
|
1151
|
+
if self._cache is not None:
|
|
1152
|
+
old_tree = (
|
|
1153
|
+
_empty_semantic_tree(language)
|
|
1154
|
+
if old_side_absent
|
|
1155
|
+
else self._cached_process(
|
|
1156
|
+
parser, old_content, language, filename, adaptive_fuel
|
|
1393
1157
|
)
|
|
1394
|
-
|
|
1395
|
-
# ── 5. Trivia stripping ───────────────────────────────────────────
|
|
1396
|
-
with profiler.phase("trivia_stripping"):
|
|
1397
|
-
old_filtered = _strip_trivia_impl(old_cst_json, trivia_types)
|
|
1398
|
-
new_filtered = _strip_trivia_impl(new_cst_json, trivia_types)
|
|
1399
|
-
|
|
1400
|
-
# ── 6. Style-only shortcut ────────────────────────────────────────
|
|
1401
|
-
with profiler.phase("style_hashing"):
|
|
1402
|
-
old_hash = _compute_structural_hash_for_tree(old_filtered)
|
|
1403
|
-
new_hash = _compute_structural_hash_for_tree(new_filtered)
|
|
1404
|
-
|
|
1405
|
-
if old_hash == new_hash:
|
|
1406
|
-
logger.debug("Style-only diff for %r", filename)
|
|
1407
|
-
diagnostics.record(
|
|
1408
|
-
stage="style",
|
|
1409
|
-
action="style_only_shortcut",
|
|
1410
|
-
rule_id="pipeline.style_only_shortcut",
|
|
1411
|
-
reason="filtered CST structural hashes are equal",
|
|
1412
|
-
metadata={"old_hash": old_hash, "new_hash": new_hash},
|
|
1413
|
-
)
|
|
1414
|
-
style_evidence = build_style_only_evidence(
|
|
1415
|
-
old_source=old_content,
|
|
1416
|
-
new_source=new_content,
|
|
1417
|
-
language=language,
|
|
1418
|
-
old_cst_json=old_cst_json,
|
|
1419
|
-
new_cst_json=new_cst_json,
|
|
1420
1158
|
)
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
if diagnostics.enabled:
|
|
1427
|
-
diagnostics.summary.update(
|
|
1428
|
-
{
|
|
1429
|
-
"early_exit": "style_only",
|
|
1430
|
-
"final_change_count": 0,
|
|
1431
|
-
"final_group_count": len(style_evidence.change_groups),
|
|
1432
|
-
}
|
|
1159
|
+
new_tree = (
|
|
1160
|
+
_empty_semantic_tree(language)
|
|
1161
|
+
if new_side_absent
|
|
1162
|
+
else self._cached_process(
|
|
1163
|
+
parser, new_content, language, filename, adaptive_fuel
|
|
1433
1164
|
)
|
|
1434
|
-
metadata["diagnostics"] = diagnostics.snapshot()
|
|
1435
|
-
style_diff = SemanticDiff.style_only(
|
|
1436
|
-
filename,
|
|
1437
|
-
new_filename if new_filename is not None else filename,
|
|
1438
|
-
language,
|
|
1439
|
-
change_groups=style_evidence.change_groups,
|
|
1440
|
-
metadata=metadata,
|
|
1441
1165
|
)
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
old_source=old_raw,
|
|
1448
|
-
new_source=new_raw,
|
|
1449
|
-
config=self._config,
|
|
1450
|
-
diagnostics=diagnostics,
|
|
1166
|
+
else:
|
|
1167
|
+
old_tree = (
|
|
1168
|
+
_empty_semantic_tree(language)
|
|
1169
|
+
if old_side_absent
|
|
1170
|
+
else parser.process(old_content, language, filename, fuel=adaptive_fuel)
|
|
1451
1171
|
)
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
max_cst_nodes = max(old_cst_nodes, new_cst_nodes)
|
|
1457
|
-
adaptive_fuel = _fuel_budget(
|
|
1458
|
-
self._config.plugin_fuel,
|
|
1459
|
-
20_000_000 + max_cst_nodes * 200_000,
|
|
1460
|
-
)
|
|
1461
|
-
logger.debug(
|
|
1462
|
-
"[%s] CST: old=%d bytes/%d nodes, new=%d bytes/%d nodes — fuel=%d",
|
|
1463
|
-
filename,
|
|
1464
|
-
len(old_filtered),
|
|
1465
|
-
old_cst_nodes,
|
|
1466
|
-
len(new_filtered),
|
|
1467
|
-
new_cst_nodes,
|
|
1468
|
-
adaptive_fuel,
|
|
1469
|
-
)
|
|
1470
|
-
diagnostics.record(
|
|
1471
|
-
stage="parse",
|
|
1472
|
-
action="interpret_cst",
|
|
1473
|
-
rule_id="pipeline.interpret_cst_parse",
|
|
1474
|
-
reason="host CST parsed, stripped, and sized",
|
|
1475
|
-
metadata={
|
|
1476
|
-
"old_cst_bytes": len(old_filtered),
|
|
1477
|
-
"new_cst_bytes": len(new_filtered),
|
|
1478
|
-
"old_cst_nodes": old_cst_nodes,
|
|
1479
|
-
"new_cst_nodes": new_cst_nodes,
|
|
1480
|
-
"adaptive_fuel": adaptive_fuel,
|
|
1481
|
-
},
|
|
1482
|
-
)
|
|
1483
|
-
|
|
1484
|
-
# ── 7. Plugin dispatch (with parse-tree cache) ───────────────────
|
|
1485
|
-
try:
|
|
1486
|
-
with profiler.phase("wasm_plugin_execution"):
|
|
1487
|
-
if self._cache is not None:
|
|
1488
|
-
old_tree = self._cached_process(
|
|
1489
|
-
parser,
|
|
1490
|
-
old_filtered,
|
|
1491
|
-
language,
|
|
1492
|
-
filename,
|
|
1493
|
-
adaptive_fuel,
|
|
1494
|
-
cache_grammar_id=cache_grammar_id,
|
|
1495
|
-
)
|
|
1496
|
-
new_tree = self._cached_process(
|
|
1497
|
-
parser,
|
|
1498
|
-
new_filtered,
|
|
1499
|
-
language,
|
|
1500
|
-
filename,
|
|
1501
|
-
adaptive_fuel,
|
|
1502
|
-
cache_grammar_id=cache_grammar_id,
|
|
1503
|
-
)
|
|
1504
|
-
else:
|
|
1505
|
-
old_tree = parser.process(
|
|
1506
|
-
old_filtered, language, filename, fuel=adaptive_fuel
|
|
1507
|
-
)
|
|
1508
|
-
new_tree = parser.process(
|
|
1509
|
-
new_filtered, language, filename, fuel=adaptive_fuel
|
|
1510
|
-
)
|
|
1511
|
-
except PluginFuelExhausted as exc:
|
|
1512
|
-
raise PluginFuelExhausted(
|
|
1513
|
-
exc.plugin_id,
|
|
1514
|
-
exc.fuel,
|
|
1515
|
-
f"file={filename!r}, CST nodes up to {max_cst_nodes:,}",
|
|
1516
|
-
) from exc
|
|
1517
|
-
|
|
1518
|
-
else:
|
|
1519
|
-
# ── 7 (full-parse). Plugin owns parsing — send raw source ─────────
|
|
1520
|
-
# Stages 3-6 are skipped: the plugin handles its own CST construction
|
|
1521
|
-
# and trivia filtering. The style-only shortcut is not available here
|
|
1522
|
-
# because there is no host-side CST to hash.
|
|
1523
|
-
max_source_bytes = max(
|
|
1524
|
-
len(old_content.encode("utf-8")),
|
|
1525
|
-
len(new_content.encode("utf-8")),
|
|
1526
|
-
)
|
|
1527
|
-
adaptive_fuel = _fuel_budget(
|
|
1528
|
-
self._config.plugin_fuel,
|
|
1529
|
-
50_000_000 + max_source_bytes * 150_000,
|
|
1530
|
-
)
|
|
1531
|
-
logger.debug("full-parse mode for %r via %r", filename, parser.grammar_id)
|
|
1532
|
-
|
|
1533
|
-
# Diff-level cache check for full-parse mode.
|
|
1534
|
-
if self._cache is not None and not diagnostics.enabled and not _guardrail_cache_bypass:
|
|
1535
|
-
_wasm_hash = self._wasm_hash_for(parser)
|
|
1536
|
-
_diff_key = self._cache.diff_key(
|
|
1537
|
-
old_content,
|
|
1538
|
-
new_content,
|
|
1539
|
-
f"{parser.grammar_id}{cache_schema_suffix}{compile_cache_suffix}",
|
|
1540
|
-
_wasm_hash,
|
|
1172
|
+
new_tree = (
|
|
1173
|
+
_empty_semantic_tree(language)
|
|
1174
|
+
if new_side_absent
|
|
1175
|
+
else parser.process(new_content, language, filename, fuel=adaptive_fuel)
|
|
1541
1176
|
)
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
with profiler.phase("wasm_plugin_execution"):
|
|
1551
|
-
old_side_absent = file_lifecycle == "added" and old_content == ""
|
|
1552
|
-
new_side_absent = file_lifecycle == "deleted" and new_content == ""
|
|
1553
|
-
if self._cache is not None:
|
|
1554
|
-
old_tree = (
|
|
1555
|
-
_empty_semantic_tree(language)
|
|
1556
|
-
if old_side_absent
|
|
1557
|
-
else self._cached_process(
|
|
1558
|
-
parser, old_content, language, filename, adaptive_fuel
|
|
1559
|
-
)
|
|
1560
|
-
)
|
|
1561
|
-
new_tree = (
|
|
1562
|
-
_empty_semantic_tree(language)
|
|
1563
|
-
if new_side_absent
|
|
1564
|
-
else self._cached_process(
|
|
1565
|
-
parser, new_content, language, filename, adaptive_fuel
|
|
1566
|
-
)
|
|
1567
|
-
)
|
|
1568
|
-
else:
|
|
1569
|
-
old_tree = (
|
|
1570
|
-
_empty_semantic_tree(language)
|
|
1571
|
-
if old_side_absent
|
|
1572
|
-
else parser.process(old_content, language, filename, fuel=adaptive_fuel)
|
|
1573
|
-
)
|
|
1574
|
-
new_tree = (
|
|
1575
|
-
_empty_semantic_tree(language)
|
|
1576
|
-
if new_side_absent
|
|
1577
|
-
else parser.process(new_content, language, filename, fuel=adaptive_fuel)
|
|
1578
|
-
)
|
|
1579
|
-
diagnostics.record(
|
|
1580
|
-
stage="parse",
|
|
1581
|
-
action="full_parse",
|
|
1582
|
-
rule_id="pipeline.full_parse",
|
|
1583
|
-
reason="plugin parsed raw source",
|
|
1584
|
-
metadata={"adaptive_fuel": adaptive_fuel},
|
|
1585
|
-
)
|
|
1177
|
+
diagnostics.record(
|
|
1178
|
+
stage="parse",
|
|
1179
|
+
action="full_parse",
|
|
1180
|
+
rule_id="pipeline.full_parse",
|
|
1181
|
+
reason="plugin parsed raw source",
|
|
1182
|
+
metadata={"adaptive_fuel": adaptive_fuel},
|
|
1183
|
+
)
|
|
1586
1184
|
|
|
1587
1185
|
engine_telemetry.extend(_drain_plugin_telemetry(parser))
|
|
1588
1186
|
_record_engine_telemetry(diagnostics, engine_telemetry)
|
|
@@ -1611,37 +1209,24 @@ class SemanticDiffer:
|
|
|
1611
1209
|
with profiler.phase("semantic_node_validation"):
|
|
1612
1210
|
old_tree = _enrich_literal_labels(old_tree, old_raw)
|
|
1613
1211
|
new_tree = _enrich_literal_labels(new_tree, new_raw)
|
|
1614
|
-
# Cross-language NodeFacts (issue #70):
|
|
1615
|
-
#
|
|
1616
|
-
#
|
|
1212
|
+
# Cross-language NodeFacts (issue #70): Rust derives structural facts for
|
|
1213
|
+
# every language. Missing or malformed native output is a contract failure,
|
|
1214
|
+
# not permission to silently publish a semantically impoverished tree.
|
|
1617
1215
|
old_tree = enrich_node_facts(old_tree)
|
|
1618
1216
|
new_tree = enrich_node_facts(new_tree)
|
|
1619
1217
|
# Profile-label enrichment runs in the Rust core (issue #57 port).
|
|
1620
|
-
#
|
|
1621
|
-
#
|
|
1622
|
-
#
|
|
1623
|
-
#
|
|
1624
|
-
#
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
old_tree =
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
new_tree =
|
|
1631
|
-
|
|
1632
|
-
) or new_tree
|
|
1633
|
-
if language.lower() in {"css", "scss", "html", "xml", "mdx"}:
|
|
1634
|
-
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
|
|
1635
|
-
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
|
|
1636
|
-
if language.lower() in {"hcl", "puppet"}:
|
|
1637
|
-
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
|
|
1638
|
-
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
|
|
1639
|
-
if language.lower() == "sql":
|
|
1640
|
-
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
|
|
1641
|
-
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
|
|
1642
|
-
if language.lower() in {"asm", "bash", "delphi"}:
|
|
1643
|
-
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language) or old_tree
|
|
1644
|
-
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language) or new_tree
|
|
1218
|
+
# Rust owns profile enrichment. Missing ABI handlers are fatal:
|
|
1219
|
+
# accepting un-enriched trees silently changes identity matching
|
|
1220
|
+
# and can suppress a meaningful diff. Never substitute Python logic.
|
|
1221
|
+
# Every language is sent: core decides which profile family (if any)
|
|
1222
|
+
# applies and returns other trees unchanged.
|
|
1223
|
+
_identity_fields = tuple(sorted(schema_resolution.identity_fields))
|
|
1224
|
+
old_tree = try_rust_profile_label_enrichment(
|
|
1225
|
+
old_tree, old_raw, language, identity_fields=_identity_fields
|
|
1226
|
+
)
|
|
1227
|
+
new_tree = try_rust_profile_label_enrichment(
|
|
1228
|
+
new_tree, new_raw, language, identity_fields=_identity_fields
|
|
1229
|
+
)
|
|
1645
1230
|
_validate_tree_ids(old_tree, filename)
|
|
1646
1231
|
_validate_tree_ids(new_tree, filename)
|
|
1647
1232
|
diagnostics.record(
|
|
@@ -1654,17 +1239,16 @@ class SemanticDiffer:
|
|
|
1654
1239
|
"new_nodes": _count_semantic_nodes(new_tree),
|
|
1655
1240
|
},
|
|
1656
1241
|
)
|
|
1657
|
-
# ── 8.5.
|
|
1242
|
+
# ── 8.5. Rust source fallback for severe parse errors ───────────────
|
|
1658
1243
|
# When tree-sitter produced ERROR nodes and fallback is enabled, skip
|
|
1659
|
-
#
|
|
1244
|
+
# semantic matching and preserve the changed source range in Rust.
|
|
1660
1245
|
if self._config.fallback_to_token_diff and (
|
|
1661
1246
|
_has_error_node(old_tree) or _has_error_node(new_tree)
|
|
1662
1247
|
):
|
|
1663
1248
|
logger.warning(
|
|
1664
|
-
"Parse errors detected in %r
|
|
1249
|
+
"Parse errors detected in %r - using Rust source fallback",
|
|
1665
1250
|
filename,
|
|
1666
1251
|
)
|
|
1667
|
-
_raise_rust_only_gate_error("parse errors require Rust token-level fallback")
|
|
1668
1252
|
diagnostics.record(
|
|
1669
1253
|
stage="fallback",
|
|
1670
1254
|
action="token_fallback",
|
|
@@ -1693,368 +1277,151 @@ class SemanticDiffer:
|
|
|
1693
1277
|
old_source=old_raw,
|
|
1694
1278
|
new_source=new_raw,
|
|
1695
1279
|
config=self._config,
|
|
1280
|
+
policy_filename=policy_source_filename,
|
|
1281
|
+
policy_old_filename=policy_old_filename,
|
|
1696
1282
|
diagnostics=diagnostics,
|
|
1697
1283
|
)
|
|
1698
|
-
# ── 9.
|
|
1699
|
-
#
|
|
1700
|
-
#
|
|
1701
|
-
#
|
|
1702
|
-
# 2. The production default: Rust's language-agnostic matcher
|
|
1703
|
-
# (``intentumdiff_rust_core.diff_semantic_tree_json``) when the Rust
|
|
1704
|
-
# core is available and diagnostics are off. This is migration
|
|
1705
|
-
# step 2 of ``docs/ENGINE_BOUNDARY_AUDIT.md``. The four engine
|
|
1706
|
-
# gaps that previously blocked this swap (Python dedent, JS
|
|
1707
|
-
# calc_hash MOVE, YAML block↔flow, C# var↔explicit) are closed —
|
|
1708
|
-
# see ``the retired NOISE_SUPPRESSION_RETUNE doc (git history)`` Phases A–D.
|
|
1709
|
-
# 3. (retired) The python GumTree oracle was deleted with the transitional
|
|
1710
|
-
# layer (issue #57 payoff, stage 4b) — no python matching fallback exists.
|
|
1284
|
+
# ── 9. Matching + finalize routing ─────────────────────────────────────
|
|
1285
|
+
# Rust owns matching and finalization; there is no Python matching path.
|
|
1286
|
+
# ``DiffConfig.test_matching_engine`` is a test-only override of the routing
|
|
1287
|
+
# decision below ("python" is retired and raises).
|
|
1711
1288
|
test_engine = self._config.test_matching_engine
|
|
1712
|
-
rust_attempt: Any = None
|
|
1713
|
-
# Skip Rust matching when either tree is empty (file add/delete
|
|
1714
|
-
# lifecycle). The Rust language-agnostic matcher treats empty-root
|
|
1715
|
-
# ↔ content-root as a structural match, which produces a single
|
|
1716
|
-
# MODIFICATION instead of the correct ADDITION (or DELETION) for
|
|
1717
|
-
# the file-lifecycle case. Letting the Python oracle handle these
|
|
1718
|
-
# edge cases preserves the ADDITION/DELETION shape that downstream
|
|
1719
|
-
# tests and review UIs depend on.
|
|
1720
|
-
either_tree_empty = (
|
|
1721
|
-
not old_tree.children or not new_tree.children
|
|
1722
|
-
)
|
|
1723
|
-
use_rust_matching = (
|
|
1724
|
-
test_engine == "rust"
|
|
1725
|
-
or (
|
|
1726
|
-
test_engine is None
|
|
1727
|
-
and self._config.experimental_rust_core
|
|
1728
|
-
and not diagnostics.enabled
|
|
1729
|
-
and not either_tree_empty
|
|
1730
|
-
)
|
|
1731
|
-
)
|
|
1732
1289
|
# ── 9-fin. Rust finalize routing (issue #57) ──────────────────────────
|
|
1733
|
-
#
|
|
1734
|
-
#
|
|
1735
|
-
#
|
|
1736
|
-
# the
|
|
1290
|
+
# Every parser language (first-party, dbt, third-party plugins) is finalized
|
|
1291
|
+
# by core: there is no Python language allowlist in front of it. The Rust
|
|
1292
|
+
# core runs the certified batch's refine+finalize from the semantic trees and
|
|
1293
|
+
# returns the finished review:
|
|
1737
1294
|
# - empty-tree lifecycle is handled INSIDE Rust finalize (DELETION+ADDITION
|
|
1738
|
-
# root pair
|
|
1295
|
+
# root pair);
|
|
1739
1296
|
# - enriched trees flow through routing (enrichment ran at stage 7b);
|
|
1740
1297
|
# - registered diff analyzers run on the ROUTED diff below, mirroring 13.5.
|
|
1741
1298
|
# Diagnostics mode routes too (issue #57 payoff, stage 4a): the shell-side
|
|
1742
1299
|
# records (parse, trees, guardrails, finalize summary below) still fire; the
|
|
1743
1300
|
# per-pass refinement trace is Rust-internal (INTENTUMDIFF_FINALIZE_DEBUG probe)
|
|
1744
1301
|
# until #54 formalizes it into the recorder.
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
with profiler.phase("rust_finalize_review"):
|
|
1752
|
-
finalize_result = try_rust_finalize_review(
|
|
1753
|
-
old_tree=old_tree,
|
|
1754
|
-
new_tree=new_tree,
|
|
1755
|
-
old_source=old_content,
|
|
1756
|
-
new_source=new_content,
|
|
1757
|
-
language=language,
|
|
1758
|
-
config=self._config,
|
|
1759
|
-
collect_trace=diagnostics.enabled,
|
|
1760
|
-
)
|
|
1761
|
-
if finalize_result is not None:
|
|
1762
|
-
diagnostics.record(
|
|
1763
|
-
stage="finalize",
|
|
1764
|
-
action="rust_finalize_review",
|
|
1765
|
-
rule_id="rust_core.finalize_review_v1",
|
|
1766
|
-
reason="routed per-stage finalize served the review",
|
|
1767
|
-
metadata={
|
|
1768
|
-
"engine_owner": "rust",
|
|
1769
|
-
"change_count": len(finalize_result["changes"]),
|
|
1770
|
-
"is_style_only": finalize_result["is_style_only"],
|
|
1771
|
-
},
|
|
1772
|
-
)
|
|
1773
|
-
# Per-pass trace (issue #54): every probed refine/finalize pass reports
|
|
1774
|
-
# its surviving change count; the recorder mirrors the python pipeline's
|
|
1775
|
-
# per-stage observability on the Rust path.
|
|
1776
|
-
for entry in finalize_result.get("trace") or []:
|
|
1777
|
-
diagnostics.record_count(
|
|
1778
|
-
stage="finalize",
|
|
1779
|
-
action=str(entry.get("pass", "")),
|
|
1780
|
-
rule_id="rust_core.finalize_pass",
|
|
1781
|
-
reason="rust finalize pass completed",
|
|
1782
|
-
count=int(entry.get("changes_after", 0)),
|
|
1783
|
-
)
|
|
1784
|
-
# Semantic invariances (css color / literal equivalence / yaml block↔flow / import
|
|
1785
|
-
# reorder / …) are equivalences that must hold REGARDLESS of routing, but this
|
|
1786
|
-
# short-circuit returns before the stage-12 apply_invariances. Run it here on the
|
|
1787
|
-
# routed changes so canonical-value "zero-change" edits collapse under routing too
|
|
1788
|
-
# (#57: unlocks json/yaml/css/python). No-op when no invariance fires.
|
|
1789
|
-
fin_changes = finalize_result["changes"]
|
|
1790
|
-
fin_is_style_only = finalize_result["is_style_only"]
|
|
1791
|
-
fin_invariant_groups: list[Any] = []
|
|
1792
|
-
fin_invariant_ignored: list[dict[str, Any]] = []
|
|
1793
|
-
if fin_changes:
|
|
1794
|
-
_inv = apply_invariances(
|
|
1795
|
-
fin_changes,
|
|
1796
|
-
old_tree=old_tree,
|
|
1797
|
-
new_tree=new_tree,
|
|
1798
|
-
old_source=old_content,
|
|
1799
|
-
new_source=new_content,
|
|
1800
|
-
language=language,
|
|
1801
|
-
)
|
|
1802
|
-
if len(_inv.changes) != len(fin_changes) or _inv.change_groups:
|
|
1803
|
-
fin_changes = _inv.changes
|
|
1804
|
-
fin_invariant_groups = list(_inv.change_groups)
|
|
1805
|
-
# The invariance's own IGNORED_STYLE groups ARE the style evidence — carry
|
|
1806
|
-
# their rule_id/reason so metadata["ignored_style_changes"] leads with the
|
|
1807
|
-
# specific reason (css.color.canonical_equivalence), not the generic
|
|
1808
|
-
# source-equivalence fallback below.
|
|
1809
|
-
for group in fin_invariant_groups:
|
|
1810
|
-
diagnostics.record_group(stage="invariance", group=group)
|
|
1811
|
-
fin_invariant_ignored = [
|
|
1812
|
-
{"rule_id": g.rule_id, **dict(g.metadata)}
|
|
1813
|
-
for g in fin_invariant_groups
|
|
1814
|
-
if g.kind == ChangeGroupKind.IGNORED_STYLE
|
|
1815
|
-
]
|
|
1816
|
-
if not fin_changes and old_content != new_content:
|
|
1817
|
-
fin_is_style_only = True
|
|
1818
|
-
# Generic files are line-oriented from the user's point of view
|
|
1819
|
-
# (#57 generic routing, mirroring the stage-12 orchestration): the routed
|
|
1820
|
-
# finalize's parser-token churn is REPLACED wholesale by the Rust text
|
|
1821
|
-
# review's stable line/character spans, then the (Rust-first) markdown
|
|
1822
|
-
# section passes run for .md filenames. Parser-derived groups are dropped
|
|
1823
|
-
# with the churn — carrying them forward invents fake pairings between
|
|
1824
|
-
# unrelated lines.
|
|
1825
|
-
fin_ignored_override: list[dict[str, Any]] | None = None
|
|
1826
|
-
if language.lower() == "generic":
|
|
1827
|
-
generic_presented = normalize_generic_text_for_review(
|
|
1828
|
-
fin_changes,
|
|
1829
|
-
old_content,
|
|
1830
|
-
new_content,
|
|
1831
|
-
)
|
|
1832
|
-
presented_generic = PresentationResult(
|
|
1833
|
-
changes=generic_presented.changes,
|
|
1834
|
-
change_groups=generic_presented.change_groups,
|
|
1835
|
-
ignored_style_changes=[
|
|
1836
|
-
*generic_presented.ignored_style_changes,
|
|
1837
|
-
],
|
|
1838
|
-
)
|
|
1839
|
-
presented_generic = _markdown_section_move_presentation(
|
|
1840
|
-
presented_generic,
|
|
1841
|
-
old_source=old_content,
|
|
1842
|
-
new_source=new_content,
|
|
1843
|
-
old_filename=filename,
|
|
1844
|
-
new_filename=(
|
|
1845
|
-
new_filename if new_filename is not None else filename
|
|
1846
|
-
),
|
|
1847
|
-
)
|
|
1848
|
-
presented_generic = _markdown_section_heading_rename_presentation(
|
|
1849
|
-
presented_generic,
|
|
1850
|
-
old_source=old_content,
|
|
1851
|
-
new_source=new_content,
|
|
1852
|
-
old_filename=filename,
|
|
1853
|
-
new_filename=(
|
|
1854
|
-
new_filename if new_filename is not None else filename
|
|
1855
|
-
),
|
|
1856
|
-
)
|
|
1857
|
-
fin_changes = presented_generic.changes
|
|
1858
|
-
fin_invariant_groups = list(presented_generic.change_groups)
|
|
1859
|
-
fin_ignored_override = list(presented_generic.ignored_style_changes)
|
|
1860
|
-
finalize_result["change_groups"] = []
|
|
1861
|
-
finalize_result["no_surviving_changes"] = False
|
|
1862
|
-
# Mirror the stage-12 style-only resolution: identical sources (or
|
|
1863
|
-
# whitespace-collapsed-equal trees) with no surviving line spans are a
|
|
1864
|
-
# style-only diff, not "no changes". The Rust finalize's flag reflects
|
|
1865
|
-
# its own suppressions, which the replacement just discarded.
|
|
1866
|
-
if not fin_changes:
|
|
1867
|
-
|
|
1868
|
-
def _generic_norm(node: SemanticNode) -> tuple[Any, ...]:
|
|
1869
|
-
label = " ".join(node.label.split())
|
|
1870
|
-
return (
|
|
1871
|
-
node.node_type,
|
|
1872
|
-
label,
|
|
1873
|
-
tuple(_generic_norm(c) for c in node.children),
|
|
1874
|
-
)
|
|
1875
|
-
|
|
1876
|
-
fin_is_style_only = (
|
|
1877
|
-
old_content == new_content
|
|
1878
|
-
or _generic_norm(old_tree) == _generic_norm(new_tree)
|
|
1879
|
-
)
|
|
1880
|
-
# Stage-12 style-only resolution, language-agnostic (markdown #44 exposed
|
|
1881
|
-
# it): ZERO surviving changes with identical (or whitespace-collapsed
|
|
1882
|
-
# tree-equal) sources is a style-only diff for every routed language —
|
|
1883
|
-
# the Rust finalize's flag only reflects its own suppressions.
|
|
1884
|
-
if not fin_changes and not fin_is_style_only:
|
|
1885
|
-
|
|
1886
|
-
def _routed_norm(node: SemanticNode) -> tuple[Any, ...]:
|
|
1887
|
-
label = " ".join(node.label.split())
|
|
1888
|
-
return (
|
|
1889
|
-
node.node_type,
|
|
1890
|
-
label,
|
|
1891
|
-
tuple(_routed_norm(c) for c in node.children),
|
|
1892
|
-
)
|
|
1893
|
-
|
|
1894
|
-
fin_is_style_only = (
|
|
1895
|
-
old_content == new_content
|
|
1896
|
-
or _routed_norm(old_tree) == _routed_norm(new_tree)
|
|
1897
|
-
)
|
|
1898
|
-
metadata_fin: dict[str, Any] = {
|
|
1899
|
-
"engine_owner": "rust",
|
|
1900
|
-
"semantic_contract": "rust_finalize_review_v1",
|
|
1901
|
-
"rust_core": {
|
|
1902
|
-
"engine": "rust_finalize_review_v1",
|
|
1903
|
-
"stage": "per_stage_finalize_routing",
|
|
1904
|
-
"used": True,
|
|
1905
|
-
},
|
|
1906
|
-
}
|
|
1907
|
-
# Context metadata the normal path attaches at stage 11 — the routed
|
|
1908
|
-
# return must carry the same contracts (cpp compile_commands pilot gap).
|
|
1909
|
-
if schema_metadata is not None:
|
|
1910
|
-
metadata_fin["schema"] = schema_metadata
|
|
1911
|
-
if compile_metadata is not None:
|
|
1912
|
-
metadata_fin["compile_commands"] = compile_metadata
|
|
1913
|
-
combined_ignored = (
|
|
1914
|
-
fin_ignored_override
|
|
1915
|
-
if fin_ignored_override is not None
|
|
1916
|
-
else [
|
|
1917
|
-
*fin_invariant_ignored,
|
|
1918
|
-
*finalize_result["ignored_style_changes"],
|
|
1919
|
-
]
|
|
1920
|
-
)
|
|
1921
|
-
if combined_ignored:
|
|
1922
|
-
metadata_fin["ignored_style_changes"] = combined_ignored
|
|
1923
|
-
if finalize_result["no_surviving_changes"]:
|
|
1924
|
-
metadata_fin["no_surviving_changes"] = True
|
|
1925
|
-
routed_groups = [*finalize_result["change_groups"], *fin_invariant_groups]
|
|
1926
|
-
if not fin_changes and old_content != new_content and not fin_invariant_ignored:
|
|
1927
|
-
# The normal path attaches a source-equivalence suppression group
|
|
1928
|
-
# post-diff when a diff nets to zero changes but the sources differ
|
|
1929
|
-
# (xml attribute reorder -> zero changes must still RECORD the
|
|
1930
|
-
# suppression, not merely be absent). The routed short-circuit returns
|
|
1931
|
-
# before that stage, so mirror it here.
|
|
1932
|
-
style_evidence = build_style_only_evidence(
|
|
1933
|
-
old_source=old_content,
|
|
1934
|
-
new_source=new_content,
|
|
1935
|
-
language=language,
|
|
1936
|
-
)
|
|
1937
|
-
for group in style_evidence.change_groups:
|
|
1938
|
-
diagnostics.record_group(stage="invariance", group=group)
|
|
1939
|
-
routed_groups.extend(style_evidence.change_groups)
|
|
1940
|
-
if style_evidence.ignored_style_changes:
|
|
1941
|
-
metadata_fin.setdefault("ignored_style_changes", []).extend(
|
|
1942
|
-
style_evidence.ignored_style_changes
|
|
1943
|
-
)
|
|
1944
|
-
routed = SemanticDiff(
|
|
1945
|
-
old_filename=filename,
|
|
1946
|
-
new_filename=new_filename if new_filename is not None else filename,
|
|
1947
|
-
language=language,
|
|
1948
|
-
changes=fin_changes,
|
|
1949
|
-
change_groups=routed_groups,
|
|
1950
|
-
has_semantic_changes=bool(fin_changes) and not fin_is_style_only,
|
|
1951
|
-
is_style_only=fin_is_style_only,
|
|
1952
|
-
metadata=metadata_fin,
|
|
1953
|
-
)
|
|
1954
|
-
routed = _apply_file_lifecycle_to_diff(routed, file_lifecycle)
|
|
1955
|
-
routed = apply_guardrails_to_diff(
|
|
1956
|
-
routed,
|
|
1957
|
-
old_tree=old_tree,
|
|
1958
|
-
new_tree=new_tree,
|
|
1959
|
-
old_source=old_content,
|
|
1960
|
-
new_source=new_content,
|
|
1961
|
-
config=self._config,
|
|
1962
|
-
diagnostics=diagnostics,
|
|
1963
|
-
)
|
|
1964
|
-
# Diff-analyzer pass on the routed diff (stage 13.5 mirror; issue #57
|
|
1965
|
-
# payoff): analyzers are diff->diff Wasm transforms over the final
|
|
1966
|
-
# SemanticDiff JSON, so the routed review feeds them identically.
|
|
1967
|
-
with profiler.phase("diff_analyzers"):
|
|
1968
|
-
routed = self._run_diff_analyzers_on_routed(
|
|
1969
|
-
routed, language, filename, adaptive_fuel
|
|
1970
|
-
)
|
|
1971
|
-
return _attach_run_telemetry(routed, engine_telemetry, diagnostics)
|
|
1972
|
-
# The Rust finalize declined (tree_too_large / backend unavailable). The
|
|
1973
|
-
# transitional python pipeline is retired (issue #57 payoff, stage 4b) —
|
|
1974
|
-
# degrade honestly to the coarse token-level diff, same as parse errors.
|
|
1975
|
-
logger.warning(
|
|
1976
|
-
"Rust finalize declined for %r (%s) — token-level fallback",
|
|
1977
|
-
filename,
|
|
1978
|
-
language,
|
|
1302
|
+
if test_engine == "python":
|
|
1303
|
+
raise NotImplementedError(
|
|
1304
|
+
"test_matching_engine='python' was retired with the transitional "
|
|
1305
|
+
"python pipeline (issue #57); the Rust finalize routing serves every "
|
|
1306
|
+
"certified language."
|
|
1979
1307
|
)
|
|
1980
|
-
|
|
1308
|
+
from intentumdiff.rust_core import try_rust_finalize_review
|
|
1309
|
+
|
|
1310
|
+
with profiler.phase("rust_finalize_review"):
|
|
1311
|
+
finalize_result = try_rust_finalize_review(
|
|
1312
|
+
old_tree=old_tree,
|
|
1313
|
+
new_tree=new_tree,
|
|
1314
|
+
old_source=old_content,
|
|
1315
|
+
new_source=new_content,
|
|
1316
|
+
language=language,
|
|
1317
|
+
config=self._config,
|
|
1318
|
+
collect_trace=diagnostics.enabled,
|
|
1319
|
+
)
|
|
1320
|
+
if finalize_result is not None:
|
|
1321
|
+
if finalize_result.get("fallback_diff") is not None:
|
|
1322
|
+
fallback_diff = finalize_result["fallback_diff"].model_copy(update={
|
|
1323
|
+
"old_filename": filename,
|
|
1324
|
+
"new_filename": new_filename if new_filename is not None else filename})
|
|
1325
|
+
fallback_diff = _apply_file_lifecycle_to_diff(fallback_diff, file_lifecycle)
|
|
1326
|
+
return apply_guardrails_to_diff(fallback_diff, old_tree=None, new_tree=None,
|
|
1327
|
+
old_source=old_raw, new_source=new_raw, config=self._config,
|
|
1328
|
+
policy_filename=policy_source_filename,
|
|
1329
|
+
policy_old_filename=policy_old_filename, diagnostics=diagnostics)
|
|
1981
1330
|
diagnostics.record(
|
|
1982
|
-
stage="
|
|
1983
|
-
action="
|
|
1984
|
-
rule_id="
|
|
1985
|
-
reason="
|
|
1331
|
+
stage="finalize",
|
|
1332
|
+
action="rust_finalize_review",
|
|
1333
|
+
rule_id="rust_core.finalize_review_v1",
|
|
1334
|
+
reason="routed per-stage finalize served the review",
|
|
1335
|
+
metadata={
|
|
1336
|
+
"engine_owner": "rust",
|
|
1337
|
+
"change_count": len(finalize_result["changes"]),
|
|
1338
|
+
"is_style_only": finalize_result["is_style_only"],
|
|
1339
|
+
},
|
|
1986
1340
|
)
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
diagnostics.
|
|
1992
|
-
|
|
1341
|
+
# Per-pass trace (issue #54): every probed refine/finalize pass reports
|
|
1342
|
+
# its surviving change count; the recorder mirrors the python pipeline's
|
|
1343
|
+
# per-stage observability on the Rust path.
|
|
1344
|
+
for entry in finalize_result.get("trace") or []:
|
|
1345
|
+
diagnostics.record_count(
|
|
1346
|
+
stage="finalize",
|
|
1347
|
+
action=str(entry.get("pass", "")),
|
|
1348
|
+
rule_id="rust_core.finalize_pass",
|
|
1349
|
+
reason="rust finalize pass completed",
|
|
1350
|
+
count=int(entry.get("changes_after", 0)),
|
|
1993
1351
|
)
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
1352
|
+
from intentumdiff.rust_core import complete_routed_review
|
|
1353
|
+
|
|
1354
|
+
routed = complete_routed_review(
|
|
1355
|
+
finalized=finalize_result, old_tree=old_tree, new_tree=new_tree,
|
|
1356
|
+
old_source=old_content, new_source=new_content,
|
|
1357
|
+
old_filename=filename,
|
|
1358
|
+
new_filename=new_filename if new_filename is not None else filename,
|
|
1359
|
+
language=language, schema_metadata=schema_metadata,
|
|
1360
|
+
compile_metadata=compile_metadata,
|
|
2002
1361
|
)
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
1362
|
+
for group in routed.change_groups:
|
|
1363
|
+
diagnostics.record_group(stage="invariance", group=group)
|
|
1364
|
+
routed = _apply_file_lifecycle_to_diff(routed, file_lifecycle)
|
|
1365
|
+
routed = apply_guardrails_to_diff(
|
|
1366
|
+
routed,
|
|
1367
|
+
old_tree=old_tree,
|
|
1368
|
+
new_tree=new_tree,
|
|
1369
|
+
old_source=old_content,
|
|
1370
|
+
new_source=new_content,
|
|
2010
1371
|
config=self._config,
|
|
1372
|
+
policy_filename=policy_source_filename,
|
|
1373
|
+
policy_old_filename=policy_old_filename,
|
|
2011
1374
|
diagnostics=diagnostics,
|
|
2012
1375
|
)
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
#
|
|
2022
|
-
#
|
|
1376
|
+
# Diff-analyzer pass on the routed diff (stage 13.5 mirror; issue #57
|
|
1377
|
+
# payoff): analyzers are diff->diff Wasm transforms over the final
|
|
1378
|
+
# SemanticDiff JSON, so the routed review feeds them identically.
|
|
1379
|
+
with profiler.phase("diff_analyzers"):
|
|
1380
|
+
routed = self._run_diff_analyzers_on_routed(
|
|
1381
|
+
routed, language, filename, adaptive_fuel
|
|
1382
|
+
)
|
|
1383
|
+
return _attach_run_telemetry(routed, engine_telemetry, diagnostics)
|
|
1384
|
+
# The Rust finalize declined (tree_too_large / backend unavailable). The
|
|
1385
|
+
# transitional python pipeline is retired (issue #57 payoff, stage 4b) —
|
|
1386
|
+
# degrade honestly to the coarse token-level diff, same as parse errors.
|
|
2023
1387
|
logger.warning(
|
|
2024
|
-
"Rust
|
|
1388
|
+
"Rust finalize declined for %r (%s) — Rust source fallback",
|
|
2025
1389
|
filename,
|
|
1390
|
+
language,
|
|
2026
1391
|
)
|
|
2027
|
-
_raise_rust_only_gate_error("rust
|
|
1392
|
+
_raise_rust_only_gate_error("rust finalize declined")
|
|
2028
1393
|
diagnostics.record(
|
|
2029
1394
|
stage="fallback",
|
|
2030
1395
|
action="token_fallback",
|
|
2031
|
-
rule_id="pipeline.
|
|
2032
|
-
reason="rust
|
|
1396
|
+
rule_id="pipeline.finalize_declined",
|
|
1397
|
+
reason="rust finalize declined; python pipeline retired (#57)",
|
|
2033
1398
|
)
|
|
2034
|
-
|
|
2035
|
-
"fallback_reason": "
|
|
1399
|
+
fallback_metadata: dict[str, Any] = {
|
|
1400
|
+
"fallback_reason": "rust_finalize_declined",
|
|
2036
1401
|
}
|
|
2037
1402
|
if diagnostics.enabled:
|
|
2038
1403
|
diagnostics.summary.update(
|
|
2039
1404
|
{"early_exit": "token_fallback", "final_change_count": None}
|
|
2040
1405
|
)
|
|
2041
|
-
|
|
2042
|
-
|
|
1406
|
+
fallback_metadata["diagnostics"] = diagnostics.snapshot()
|
|
1407
|
+
fallback_diff = _token_fallback_diff(
|
|
2043
1408
|
old_raw,
|
|
2044
1409
|
new_raw,
|
|
2045
1410
|
filename,
|
|
2046
1411
|
new_filename if new_filename is not None else filename,
|
|
2047
1412
|
language,
|
|
2048
|
-
metadata=
|
|
1413
|
+
metadata=fallback_metadata,
|
|
2049
1414
|
)
|
|
2050
|
-
|
|
1415
|
+
fallback_diff = _apply_file_lifecycle_to_diff(fallback_diff, file_lifecycle)
|
|
2051
1416
|
return apply_guardrails_to_diff(
|
|
2052
|
-
|
|
1417
|
+
fallback_diff,
|
|
2053
1418
|
old_tree=None,
|
|
2054
1419
|
new_tree=None,
|
|
2055
1420
|
old_source=old_raw,
|
|
2056
1421
|
new_source=new_raw,
|
|
2057
1422
|
config=self._config,
|
|
1423
|
+
policy_filename=policy_source_filename,
|
|
1424
|
+
policy_old_filename=policy_old_filename,
|
|
2058
1425
|
diagnostics=diagnostics,
|
|
2059
1426
|
)
|
|
2060
1427
|
|
|
@@ -2062,62 +1429,22 @@ class SemanticDiffer:
|
|
|
2062
1429
|
self,
|
|
2063
1430
|
source: Source,
|
|
2064
1431
|
) -> Iterator[ChangeStreamEvent]:
|
|
2065
|
-
"""
|
|
2066
|
-
|
|
2067
|
-
Three phases are emitted in order:
|
|
2068
|
-
|
|
2069
|
-
1. **STRUCTURAL** — one ``action="add"`` event per change after the
|
|
2070
|
-
GumTree edit script and move-promotion pass (stage 11). These are
|
|
2071
|
-
available immediately, before the (potentially expensive) refactoring
|
|
2072
|
-
detection runs.
|
|
1432
|
+
"""Stream finalized Rust-backed changes as FINAL add events.
|
|
2073
1433
|
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
replaced changes are emitted as ``action="remove"``.
|
|
2079
|
-
|
|
2080
|
-
3. **FINAL** — events emitted after classification (stage 13) and any
|
|
2081
|
-
diff-analyzer plugins (stage 13.5) have run. Emitted only when these
|
|
2082
|
-
passes modify the change list.
|
|
2083
|
-
|
|
2084
|
-
Consumers that only care about the final result can wait until all events
|
|
2085
|
-
are consumed; consumers that want to display partial results can render
|
|
2086
|
-
STRUCTURAL changes immediately and update/remove them as REFINED and
|
|
2087
|
-
FINAL events arrive.
|
|
1434
|
+
The Python pipeline was retired in 0.0.2. Rust returns a complete diff,
|
|
1435
|
+
not intermediate structural/refinement snapshots. This API preserves its
|
|
1436
|
+
event-shaped transport contract, but it does not manufacture progressive
|
|
1437
|
+
semantic phases or reconcile changes using Python object identity.
|
|
2088
1438
|
"""
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
*,
|
|
2099
|
-
new_filename: str | None = None,
|
|
2100
|
-
) -> Iterator[ChangeStreamEvent]:
|
|
2101
|
-
"""Internal generator — see ``diff_stream_progressive`` for the public API."""
|
|
2102
|
-
result = self._run_stages_1_to_11(
|
|
2103
|
-
old_content, new_content, filename, language_hint, new_filename=new_filename
|
|
2104
|
-
)
|
|
2105
|
-
|
|
2106
|
-
if isinstance(result, SemanticDiff):
|
|
2107
|
-
# Early exit (cache hit / style-only / token fallback): emit all changes
|
|
2108
|
-
# as FINAL so consumers still receive a well-formed event stream.
|
|
2109
|
-
for change in result.changes:
|
|
2110
|
-
yield ChangeStreamEvent(
|
|
2111
|
-
phase=ChangeStreamPhase.FINAL,
|
|
2112
|
-
action="add",
|
|
2113
|
-
change=change,
|
|
2114
|
-
)
|
|
2115
|
-
return
|
|
2116
|
-
|
|
2117
|
-
raise AssertionError(
|
|
2118
|
-
"unreachable: _run_stages_1_to_11 always returns a SemanticDiff since the "
|
|
2119
|
-
"python pipeline retirement (issue #57 payoff, stage 4b)"
|
|
2120
|
-
)
|
|
1439
|
+
# Reuse the public path so FileSource policy discovery retains both
|
|
1440
|
+
# physical paths even when the source exposes a display alias.
|
|
1441
|
+
result = self.diff(source)
|
|
1442
|
+
for change in result.changes:
|
|
1443
|
+
yield ChangeStreamEvent(
|
|
1444
|
+
phase=ChangeStreamPhase.FINAL,
|
|
1445
|
+
action="add",
|
|
1446
|
+
change=change,
|
|
1447
|
+
)
|
|
2121
1448
|
|
|
2122
1449
|
def _cached_process(
|
|
2123
1450
|
self,
|
|
@@ -2141,33 +1468,72 @@ class SemanticDiffer:
|
|
|
2141
1468
|
raise RuntimeError("parse cache store is not configured")
|
|
2142
1469
|
wasm_hash = self._wasm_hash_for(parser)
|
|
2143
1470
|
grammar_id = cache_grammar_id or parser.grammar_id
|
|
2144
|
-
key
|
|
1471
|
+
# Rust owns the canonical key hashing. The host must supply every
|
|
1472
|
+
# argument that can affect the Wasm parser output, including language
|
|
1473
|
+
# and filename for a multi-language/shared-grammar component (#188).
|
|
1474
|
+
parser_context = json.dumps(
|
|
1475
|
+
{"grammar_id": grammar_id, "language": language, "filename": filename},
|
|
1476
|
+
sort_keys=True, separators=(",", ":"),
|
|
1477
|
+
)
|
|
1478
|
+
key = cache.parse_key(input_, parser_context, wasm_hash)
|
|
2145
1479
|
cached = cache.get_parse(key)
|
|
2146
1480
|
if cached is not None:
|
|
2147
1481
|
logger.debug("Parse cache hit for grammar=%r", grammar_id)
|
|
2148
1482
|
return SemanticNode.model_validate_json(cached)
|
|
2149
1483
|
|
|
2150
1484
|
node = parser.process(input_, language, filename, fuel=fuel)
|
|
1485
|
+
# A trap may reload the Wasm module during process(); never write its
|
|
1486
|
+
# output under a key computed for a different compiled component.
|
|
1487
|
+
if self._wasm_hash_for(parser) != wasm_hash:
|
|
1488
|
+
raise RuntimeError("parser Wasm identity changed during parse")
|
|
2151
1489
|
cache.put_parse(key, node.model_dump_json(), grammar_id=grammar_id)
|
|
2152
1490
|
return node
|
|
2153
1491
|
|
|
2154
1492
|
def _wasm_hash_for(self, parser: Any) -> str:
|
|
2155
|
-
"""
|
|
2156
|
-
Return a short (16-hex-char) sha256 hash of the parser's Wasm binary.
|
|
1493
|
+
"""Fingerprint the actual parser Wasm bytes before permitting any cache hit.
|
|
2157
1494
|
|
|
2158
|
-
|
|
2159
|
-
|
|
1495
|
+
A missing/unreadable component cannot share an ``"unknown"`` cache key:
|
|
1496
|
+
doing so could replay a stale style-only diff without invoking Rust.
|
|
1497
|
+
File identity (not grammar ID) scopes the memoized digest, so different
|
|
1498
|
+
component revisions cannot share previously certified parse/diff results.
|
|
2160
1499
|
"""
|
|
2161
|
-
grammar_id: str = parser.grammar_id
|
|
2162
|
-
if grammar_id in self._wasm_hash_cache:
|
|
2163
|
-
return self._wasm_hash_cache[grammar_id]
|
|
2164
1500
|
try:
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
except (AttributeError, OSError):
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
1501
|
+
path = Path(parser.wasm_path).resolve(strict=True)
|
|
1502
|
+
stat_before = path.stat()
|
|
1503
|
+
except (AttributeError, TypeError, OSError) as exc:
|
|
1504
|
+
raise RuntimeError(
|
|
1505
|
+
f"cannot fingerprint parser Wasm at {getattr(parser, 'wasm_path', None)!r}"
|
|
1506
|
+
) from exc
|
|
1507
|
+
|
|
1508
|
+
def _identity(stat: Any) -> tuple[int, int, int, int, int]:
|
|
1509
|
+
return (stat.st_dev, stat.st_ino, stat.st_size,
|
|
1510
|
+
stat.st_mtime_ns, stat.st_ctime_ns)
|
|
1511
|
+
|
|
1512
|
+
identity = (str(path), *_identity(stat_before))
|
|
1513
|
+
digest = self._wasm_hash_cache.get(identity)
|
|
1514
|
+
if digest is None:
|
|
1515
|
+
try:
|
|
1516
|
+
wasm_bytes = path.read_bytes()
|
|
1517
|
+
stat_after = path.stat()
|
|
1518
|
+
except OSError as exc:
|
|
1519
|
+
raise RuntimeError(f"cannot fingerprint parser Wasm at {str(path)!r}") from exc
|
|
1520
|
+
if _identity(stat_before) != _identity(stat_after):
|
|
1521
|
+
raise RuntimeError(f"parser Wasm changed while hashing: {path}")
|
|
1522
|
+
digest = hashlib.sha256(wasm_bytes).hexdigest()
|
|
1523
|
+
self._wasm_hash_cache[identity] = digest
|
|
1524
|
+
|
|
1525
|
+
# The path can be replaced while a LoadedPlugin still executes the
|
|
1526
|
+
# previous compiled component. File digest alone is not a safe cache
|
|
1527
|
+
# identity: bind it to the *instantiated* Wasmtime bytes (#196).
|
|
1528
|
+
compiled = getattr(parser, "compiled_wasm_sha256", None)
|
|
1529
|
+
if isinstance(parser, ParserAdapter) and (
|
|
1530
|
+
not isinstance(compiled, str) or len(compiled) != 64
|
|
1531
|
+
or any(c not in "0123456789abcdef" for c in compiled)
|
|
1532
|
+
):
|
|
1533
|
+
raise RuntimeError(f"cannot verify compiled parser Wasm identity: {path}")
|
|
1534
|
+
if isinstance(compiled, str) and compiled != digest:
|
|
1535
|
+
raise RuntimeError(f"compiled parser Wasm differs from backing artifact: {path}")
|
|
1536
|
+
return digest
|
|
2171
1537
|
|
|
2172
1538
|
def _parse(
|
|
2173
1539
|
self,
|