intentumdiff-python 0.0.1__py3-none-win_amd64.whl → 0.0.2b2__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- intentumdiff/__init__.py +2 -0
- intentumdiff/__main__.py +12 -0
- intentumdiff/_differ_presentation.py +35 -682
- intentumdiff/_differ_runtime.py +13 -47
- intentumdiff/analysis/compile_commands.py +26 -141
- intentumdiff/analysis/cross_file.py +2 -4
- intentumdiff/analysis/guardrail_reports.py +7 -5
- intentumdiff/analysis/guardrails.py +110 -122
- intentumdiff/analysis/invariances.py +4 -1003
- intentumdiff/analysis/schema_resolver.py +49 -273
- intentumdiff/analysis/text_review.py +4 -3
- intentumdiff/analysis/user_schemas.py +55 -237
- intentumdiff/api.py +21 -0
- intentumdiff/cache/sqlite_store.py +5 -23
- intentumdiff/cli/__init__.py +59 -0
- intentumdiff/cli/_commands.py +174 -67
- intentumdiff/cli/_parser.py +11 -1
- intentumdiff/cli/_shared.py +96 -112
- intentumdiff/content_type.py +13 -30
- intentumdiff/core/commit_differ.py +105 -220
- intentumdiff/core/diffignore.py +28 -34
- intentumdiff/core/index.py +16 -18
- intentumdiff/core/indexer.py +148 -168
- intentumdiff/core/models.py +53 -14
- intentumdiff/differ.py +278 -605
- intentumdiff/github_app.py +35 -19
- intentumdiff/intentumdiff_rust_core/intentumdiff_rust_core.dll +0 -0
- intentumdiff/live_server.py +180 -17
- intentumdiff/lsp/client.py +7 -1
- intentumdiff/lsp/enricher.py +19 -119
- intentumdiff/lsp/exceptions.py +6 -7
- intentumdiff/lsp_server/_codelens.py +24 -57
- intentumdiff/lsp_server/_diagnostics.py +23 -51
- intentumdiff/lsp_server/_handlers.py +66 -8
- intentumdiff/lsp_server/server.py +28 -7
- intentumdiff/plugins/adapter.py +5 -0
- intentumdiff/plugins/builtins.py +7 -14
- intentumdiff/plugins/exceptions.py +4 -0
- intentumdiff/plugins/hub.py +1 -2
- intentumdiff/plugins/loader.py +229 -215
- intentumdiff/plugins/registry.py +249 -197
- intentumdiff/rust_core.py +744 -784
- intentumdiff/serve/_app.py +3 -10
- intentumdiff/sources/base.py +24 -0
- intentumdiff/sources/file_source.py +15 -2
- intentumdiff/sources/git_source.py +120 -58
- intentumdiff/sources/live_buffer_source.py +45 -5
- intentumdiff/sources/patch_source.py +36 -109
- intentumdiff/vcs/_safe_path.py +7 -0
- intentumdiff/vcs/base.py +6 -0
- intentumdiff/vcs/git_backend.py +1 -1
- intentumdiff/vcs/git_cli.py +38 -18
- intentumdiff/vcs/hg_backend.py +1 -1
- intentumdiff/vcs/perforce_backend.py +7 -1
- intentumdiff/vcs/svn_backend.py +1 -1
- intentumdiff/wasm/abap_parser.wasm +0 -0
- intentumdiff/wasm/adf_parser.wasm +0 -0
- intentumdiff/wasm/asciidoc_parser.wasm +0 -0
- intentumdiff/wasm/asm_parser.wasm +0 -0
- intentumdiff/wasm/assemblyscript_parser.wasm +0 -0
- intentumdiff/wasm/astro_parser.wasm +0 -0
- intentumdiff/wasm/bash_parser.wasm +0 -0
- intentumdiff/wasm/clojure_parser.wasm +0 -0
- intentumdiff/wasm/cmake_parser.wasm +0 -0
- intentumdiff/wasm/cpp_parser.wasm +0 -0
- intentumdiff/wasm/csharp_parser.wasm +0 -0
- intentumdiff/wasm/css_parser.wasm +0 -0
- intentumdiff/wasm/dart_parser.wasm +0 -0
- intentumdiff/wasm/databricks_parser.wasm +0 -0
- intentumdiff/wasm/dax_parser.wasm +0 -0
- intentumdiff/wasm/delphi_parser.wasm +0 -0
- intentumdiff/wasm/dockerfile_parser.wasm +0 -0
- intentumdiff/wasm/elixir_parser.wasm +0 -0
- intentumdiff/wasm/freebasic_parser.wasm +0 -0
- intentumdiff/wasm/generic_parser.wasm +0 -0
- intentumdiff/wasm/gitignore_parser.wasm +0 -0
- intentumdiff/wasm/go_parser.wasm +0 -0
- intentumdiff/wasm/gomod_parser.wasm +0 -0
- intentumdiff/wasm/graphql_parser.wasm +0 -0
- intentumdiff/wasm/groovy_parser.wasm +0 -0
- intentumdiff/wasm/haskell_parser.wasm +0 -0
- intentumdiff/wasm/html_parser.wasm +0 -0
- intentumdiff/wasm/html_renderer.wasm +0 -0
- intentumdiff/wasm/ini_parser.wasm +0 -0
- intentumdiff/wasm/java_parser.wasm +0 -0
- intentumdiff/wasm/js_ts_parser.wasm +0 -0
- intentumdiff/wasm/json_parser.wasm +0 -0
- intentumdiff/wasm/kotlin_parser.wasm +0 -0
- intentumdiff/wasm/latex_parser.wasm +0 -0
- intentumdiff/wasm/llm_renderer.wasm +0 -0
- intentumdiff/wasm/lua_parser.wasm +0 -0
- intentumdiff/wasm/make_parser.wasm +0 -0
- intentumdiff/wasm/markdown_parser.wasm +0 -0
- intentumdiff/wasm/mdx_parser.wasm +0 -0
- intentumdiff/wasm/ocaml_parser.wasm +0 -0
- intentumdiff/wasm/odin_parser.wasm +0 -0
- intentumdiff/wasm/parser_manifest.json +81 -2
- intentumdiff/wasm/patch_renderer.wasm +0 -0
- intentumdiff/wasm/perl_parser.wasm +0 -0
- intentumdiff/wasm/php_parser.wasm +0 -0
- intentumdiff/wasm/plsql_parser.wasm +0 -0
- intentumdiff/wasm/po_parser.wasm +0 -0
- intentumdiff/wasm/postscript_parser.wasm +0 -0
- intentumdiff/wasm/powershell_parser.wasm +0 -0
- intentumdiff/wasm/proto_parser.wasm +0 -0
- intentumdiff/wasm/puppet_parser.wasm +0 -0
- intentumdiff/wasm/python_parser.wasm +0 -0
- intentumdiff/wasm/qsharp_parser.wasm +0 -0
- intentumdiff/wasm/r_parser.wasm +0 -0
- intentumdiff/wasm/reasonml_parser.wasm +0 -0
- intentumdiff/wasm/ruby_parser.wasm +0 -0
- intentumdiff/wasm/rust_parser.wasm +0 -0
- intentumdiff/wasm/sas_parser.wasm +0 -0
- intentumdiff/wasm/scala_parser.wasm +0 -0
- intentumdiff/wasm/scss_parser.wasm +0 -0
- intentumdiff/wasm/sql_parser.wasm +0 -0
- intentumdiff/wasm/squirrel_parser.wasm +0 -0
- intentumdiff/wasm/svelte_parser.wasm +0 -0
- intentumdiff/wasm/swift_parser.wasm +0 -0
- intentumdiff/wasm/terminal_renderer.wasm +0 -0
- intentumdiff/wasm/terraform_parser.wasm +0 -0
- intentumdiff/wasm/toml_parser.wasm +0 -0
- intentumdiff/wasm/tsql_parser.wasm +0 -0
- intentumdiff/wasm/vbnet_parser.wasm +0 -0
- intentumdiff/wasm/vue_parser.wasm +0 -0
- intentumdiff/wasm/wasm_provenance.json +884 -0
- intentumdiff/wasm/wat_parser.wasm +0 -0
- intentumdiff/wasm/xml_parser.wasm +0 -0
- intentumdiff/wasm/yaml_parser.wasm +0 -0
- intentumdiff/wasm/zig_parser.wasm +0 -0
- intentumdiff_python-0.0.2b2.dist-info/METADATA +156 -0
- intentumdiff_python-0.0.2b2.dist-info/RECORD +171 -0
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/WHEEL +1 -1
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/sboms/intentumdiff-rust-core.cyclonedx.json +905 -379
- intentumdiff/core/cst_serializer.py +0 -101
- intentumdiff_python-0.0.1.dist-info/METADATA +0 -112
- intentumdiff_python-0.0.1.dist-info/RECORD +0 -165
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/entry_points.txt +0 -0
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/licenses/LICENSE +0 -0
intentumdiff/differ.py
CHANGED
|
@@ -25,6 +25,7 @@ The ``SemanticDiffer`` class is the primary public API object.
|
|
|
25
25
|
from __future__ import annotations
|
|
26
26
|
|
|
27
27
|
import hashlib
|
|
28
|
+
import json
|
|
28
29
|
import logging
|
|
29
30
|
import os
|
|
30
31
|
import threading
|
|
@@ -63,27 +64,6 @@ from intentumdiff._differ_gate import (
|
|
|
63
64
|
from intentumdiff._differ_gate import (
|
|
64
65
|
_validate_git_ref as _validate_git_ref,
|
|
65
66
|
)
|
|
66
|
-
from intentumdiff._differ_presentation import (
|
|
67
|
-
_GENERIC_STRING_LABELS as _GENERIC_STRING_LABELS,
|
|
68
|
-
)
|
|
69
|
-
from intentumdiff._differ_presentation import (
|
|
70
|
-
_NAMED_ENTITY_NODE_TYPES as _NAMED_ENTITY_NODE_TYPES,
|
|
71
|
-
)
|
|
72
|
-
from intentumdiff._differ_presentation import (
|
|
73
|
-
_all_semantic_nodes as _all_semantic_nodes,
|
|
74
|
-
)
|
|
75
|
-
from intentumdiff._differ_presentation import (
|
|
76
|
-
_annotate_text_diffs as _annotate_text_diffs,
|
|
77
|
-
)
|
|
78
|
-
from intentumdiff._differ_presentation import (
|
|
79
|
-
_changes_to_stream_events as _changes_to_stream_events,
|
|
80
|
-
)
|
|
81
|
-
from intentumdiff._differ_presentation import (
|
|
82
|
-
_clean_string_literal_label as _clean_string_literal_label,
|
|
83
|
-
)
|
|
84
|
-
from intentumdiff._differ_presentation import (
|
|
85
|
-
_compute_structural_hash_for_tree as _compute_structural_hash_for_tree,
|
|
86
|
-
)
|
|
87
67
|
from intentumdiff._differ_presentation import (
|
|
88
68
|
_count_cst_nodes as _count_cst_nodes,
|
|
89
69
|
)
|
|
@@ -99,54 +79,27 @@ from intentumdiff._differ_presentation import (
|
|
|
99
79
|
from intentumdiff._differ_presentation import (
|
|
100
80
|
_has_error_node as _has_error_node,
|
|
101
81
|
)
|
|
102
|
-
from intentumdiff._differ_presentation import (
|
|
103
|
-
_is_markdown_filename as _is_markdown_filename,
|
|
104
|
-
)
|
|
105
|
-
from intentumdiff._differ_presentation import (
|
|
106
|
-
_is_named_entity_node as _is_named_entity_node,
|
|
107
|
-
)
|
|
108
|
-
from intentumdiff._differ_presentation import (
|
|
109
|
-
_markdown_section_body_hashes as _markdown_section_body_hashes,
|
|
110
|
-
)
|
|
111
82
|
from intentumdiff._differ_presentation import (
|
|
112
83
|
_markdown_section_heading_rename_presentation as _markdown_section_heading_rename_presentation,
|
|
113
84
|
)
|
|
114
85
|
from intentumdiff._differ_presentation import (
|
|
115
86
|
_markdown_section_move_presentation as _markdown_section_move_presentation,
|
|
116
87
|
)
|
|
117
|
-
from intentumdiff._differ_presentation import (
|
|
118
|
-
_markdown_sections as _markdown_sections,
|
|
119
|
-
)
|
|
120
88
|
from intentumdiff._differ_presentation import (
|
|
121
89
|
_node_to_dict as _node_to_dict,
|
|
122
90
|
)
|
|
123
91
|
from intentumdiff._differ_presentation import (
|
|
124
92
|
_root_structural_hash as _root_structural_hash,
|
|
125
93
|
)
|
|
126
|
-
from intentumdiff._differ_presentation import (
|
|
127
|
-
_semantic_parent_map as _semantic_parent_map,
|
|
128
|
-
)
|
|
129
|
-
from intentumdiff._differ_presentation import (
|
|
130
|
-
_slice_source_text as _slice_source_text,
|
|
131
|
-
)
|
|
132
|
-
from intentumdiff._differ_presentation import (
|
|
133
|
-
_surface_changed_in_place_entities as _surface_changed_in_place_entities,
|
|
134
|
-
)
|
|
135
94
|
from intentumdiff._differ_presentation import (
|
|
136
95
|
_token_fallback_diff as _token_fallback_diff,
|
|
137
96
|
)
|
|
138
97
|
from intentumdiff._differ_presentation import (
|
|
139
98
|
_validate_tree_ids as _validate_tree_ids,
|
|
140
99
|
)
|
|
141
|
-
from intentumdiff._differ_runtime import (
|
|
142
|
-
_ADDED_FILE_STATUSES as _ADDED_FILE_STATUSES,
|
|
143
|
-
)
|
|
144
100
|
from intentumdiff._differ_runtime import (
|
|
145
101
|
_DEFAULT_PLUGIN_FUEL as _DEFAULT_PLUGIN_FUEL,
|
|
146
102
|
)
|
|
147
|
-
from intentumdiff._differ_runtime import (
|
|
148
|
-
_DELETED_FILE_STATUSES as _DELETED_FILE_STATUSES,
|
|
149
|
-
)
|
|
150
103
|
from intentumdiff._differ_runtime import (
|
|
151
104
|
_EXPLICIT_FUEL_EXHAUSTION_TEST_CAP as _EXPLICIT_FUEL_EXHAUSTION_TEST_CAP,
|
|
152
105
|
)
|
|
@@ -205,10 +158,6 @@ from intentumdiff.analysis.schema_resolver import (
|
|
|
205
158
|
schema_cache_fingerprint,
|
|
206
159
|
schema_resolution_metadata,
|
|
207
160
|
)
|
|
208
|
-
from intentumdiff.analysis.text_review import (
|
|
209
|
-
PresentationResult,
|
|
210
|
-
normalize_generic_text_for_review,
|
|
211
|
-
)
|
|
212
161
|
from intentumdiff.analysis.user_schemas import (
|
|
213
162
|
load_user_schema_profiles,
|
|
214
163
|
user_xml_dialects_payload,
|
|
@@ -226,6 +175,7 @@ from intentumdiff.core.models import (
|
|
|
226
175
|
SemanticDiff,
|
|
227
176
|
SemanticNode,
|
|
228
177
|
)
|
|
178
|
+
from intentumdiff.plugins.adapter import ParserAdapter
|
|
229
179
|
from intentumdiff.plugins.exceptions import (
|
|
230
180
|
PluginFuelExhausted,
|
|
231
181
|
PluginNotFoundError,
|
|
@@ -235,8 +185,6 @@ from intentumdiff.plugins.loader import _strip_trivia_impl
|
|
|
235
185
|
from intentumdiff.plugins.registry import PluginRegistry
|
|
236
186
|
from intentumdiff.rust_core import (
|
|
237
187
|
RustCoreCommitJsonAttempt,
|
|
238
|
-
apply_invariances,
|
|
239
|
-
build_style_only_evidence,
|
|
240
188
|
enrich_node_facts,
|
|
241
189
|
try_register_user_xml_dialects,
|
|
242
190
|
try_rust_core_batch_diff,
|
|
@@ -246,6 +194,7 @@ from intentumdiff.rust_core import (
|
|
|
246
194
|
try_rust_profile_label_enrichment,
|
|
247
195
|
)
|
|
248
196
|
from intentumdiff.sources.base import Source
|
|
197
|
+
from intentumdiff.sources.file_source import FileSource as FileSource # compatibility re-export
|
|
249
198
|
|
|
250
199
|
logger = logging.getLogger(__name__)
|
|
251
200
|
|
|
@@ -274,8 +223,8 @@ class SemanticDiffer:
|
|
|
274
223
|
self._parallel_executor_workers: int | None = None
|
|
275
224
|
self._parallel_executor_lock = threading.Lock()
|
|
276
225
|
self._parallel_worker_local = threading.local()
|
|
277
|
-
#
|
|
278
|
-
self._wasm_hash_cache: dict[str, str] = {}
|
|
226
|
+
# Parser digest identity includes the actual on-disk artifact, never grammar alone.
|
|
227
|
+
self._wasm_hash_cache: dict[tuple[str, int, int, int, int, int], str] = {}
|
|
279
228
|
|
|
280
229
|
# ── Cache / analytics setup ───────────────────────────────────────
|
|
281
230
|
if cache is not None:
|
|
@@ -491,13 +440,29 @@ class SemanticDiffer:
|
|
|
491
440
|
profiler = self._new_profiler()
|
|
492
441
|
with profiler.phase("source_loading"):
|
|
493
442
|
old_content, new_content, filename, language_hint = source.get_content()
|
|
494
|
-
|
|
443
|
+
# Resolve host filesystem identity separately from public display labels.
|
|
444
|
+
# Source adapters validate repo roots/paths; only Rust evaluates the
|
|
445
|
+
# discovered guardrail policies against the semantic changes.
|
|
446
|
+
policy_context = getattr(source, "guardrail_policy_paths", None)
|
|
447
|
+
policy_old_filename, policy_source_filename = (
|
|
448
|
+
policy_context() if callable(policy_context) else (None, None)
|
|
449
|
+
)
|
|
450
|
+
diff = self._run_pipeline(
|
|
495
451
|
old_content,
|
|
496
452
|
new_content,
|
|
497
453
|
filename,
|
|
498
454
|
language_hint,
|
|
455
|
+
_policy_source_filename=policy_source_filename,
|
|
456
|
+
_policy_old_filename=policy_old_filename,
|
|
499
457
|
_profiler=profiler,
|
|
500
458
|
)
|
|
459
|
+
# The pipeline works from the single filename language detection needs. A source
|
|
460
|
+
# comparing two differently named files knows both, so restore them here rather
|
|
461
|
+
# than threading a second name through every construction site.
|
|
462
|
+
names = source.display_names()
|
|
463
|
+
if names is not None:
|
|
464
|
+
diff = diff.model_copy(update={"old_filename": names[0], "new_filename": names[1]})
|
|
465
|
+
return diff
|
|
501
466
|
|
|
502
467
|
def diff_strings(
|
|
503
468
|
self,
|
|
@@ -594,9 +559,9 @@ class SemanticDiffer:
|
|
|
594
559
|
"""
|
|
595
560
|
Parse *content* as *filename* and return ``(SemanticNode, language)``.
|
|
596
561
|
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
562
|
+
A FullParse Rust/Wasm plugin owns parsing; Python handles plugin dispatch
|
|
563
|
+
and the optional parse-tree cache. Legacy Python-hosted CST processing is
|
|
564
|
+
deliberately rejected.
|
|
600
565
|
|
|
601
566
|
Raises :class:`~intentumdiff.plugins.exceptions.PluginNotFoundError`
|
|
602
567
|
when no registered parser supports *filename*.
|
|
@@ -619,22 +584,7 @@ class SemanticDiffer:
|
|
|
619
584
|
parser, language = self._registry.detect_parser(filename, content, language_hint)
|
|
620
585
|
adaptive_fuel = self._config.plugin_fuel
|
|
621
586
|
|
|
622
|
-
|
|
623
|
-
content = parser.preprocess_source(content)
|
|
624
|
-
cst_json = self._parse(content, parser, language, filename)
|
|
625
|
-
if len(cst_json) > self._config.max_cst_bytes:
|
|
626
|
-
raise ValueError(
|
|
627
|
-
f"CST JSON for {filename!r} is {len(cst_json):,} characters, "
|
|
628
|
-
f"exceeding the limit of {self._config.max_cst_bytes:,} "
|
|
629
|
-
"(DiffConfig.max_cst_bytes)."
|
|
630
|
-
)
|
|
631
|
-
trivia_types = list(parser.trivia_node_types) + self._config.extra_trivia_types
|
|
632
|
-
filtered = _strip_trivia_impl(cst_json, trivia_types)
|
|
633
|
-
cst_nodes = _count_cst_nodes(filtered)
|
|
634
|
-
adaptive_fuel = _fuel_budget(self._config.plugin_fuel, 20_000_000 + cst_nodes * 200_000)
|
|
635
|
-
input_ = filtered
|
|
636
|
-
else:
|
|
637
|
-
input_ = content
|
|
587
|
+
input_ = self._parse(content, parser, language, filename)
|
|
638
588
|
|
|
639
589
|
if self._cache is not None:
|
|
640
590
|
node = self._cached_process(parser, input_, language, filename, adaptive_fuel)
|
|
@@ -835,7 +785,10 @@ class SemanticDiffer:
|
|
|
835
785
|
"""
|
|
836
786
|
Diff every file that changed between two git refs.
|
|
837
787
|
|
|
838
|
-
|
|
788
|
+
A changed text file with no registered parser is a failed review.
|
|
789
|
+
Missing-parser and native runtime failures abort the entire review with
|
|
790
|
+
the path and original cause. Binary/ignored files are excluded during
|
|
791
|
+
source collection, before the semantic engine is called.
|
|
839
792
|
|
|
840
793
|
Parameters
|
|
841
794
|
----------
|
|
@@ -850,12 +803,15 @@ class SemanticDiffer:
|
|
|
850
803
|
Returns
|
|
851
804
|
-------
|
|
852
805
|
list[SemanticDiff]
|
|
853
|
-
One entry per changed file
|
|
806
|
+
One entry per reviewed changed text file; failures raise rather than disappear.
|
|
854
807
|
"""
|
|
855
808
|
from intentumdiff.sources.git_source import iter_changed_sources
|
|
809
|
+
from intentumdiff.vcs.git_cli import resolve_repo_root
|
|
810
|
+
from intentumdiff.vcs._safe_path import safe_policy_path
|
|
856
811
|
|
|
857
812
|
source_started = time.perf_counter()
|
|
858
|
-
|
|
813
|
+
policy_root = resolve_repo_root(repo_path)
|
|
814
|
+
sources = list(iter_changed_sources(policy_root, old_ref, new_ref))
|
|
859
815
|
source_collection_ms = (time.perf_counter() - source_started) * 1000
|
|
860
816
|
rust_batch_results: dict[int, SemanticDiff] = {}
|
|
861
817
|
rust_batch_attempted: set[int] = set()
|
|
@@ -878,10 +834,15 @@ class SemanticDiffer:
|
|
|
878
834
|
or effective_new_path_lower.endswith((".py", ".pyi"))
|
|
879
835
|
):
|
|
880
836
|
continue
|
|
881
|
-
#
|
|
882
|
-
#
|
|
883
|
-
#
|
|
884
|
-
|
|
837
|
+
# A discovered or explicit policy must be validated even when
|
|
838
|
+
# its file is unchanged/ignored. Rust decides whether it applies;
|
|
839
|
+
# policy-relevant sources keep the normal physical-path route.
|
|
840
|
+
if guardrails_may_apply(
|
|
841
|
+
safe_policy_path(policy_root, old_path),
|
|
842
|
+
safe_policy_path(policy_root, effective_new_path),
|
|
843
|
+
self._config,
|
|
844
|
+
):
|
|
845
|
+
continue
|
|
885
846
|
request_to_source_index.append(idx)
|
|
886
847
|
request_files.append(
|
|
887
848
|
{
|
|
@@ -915,6 +876,15 @@ class SemanticDiffer:
|
|
|
915
876
|
parallel=rust_batch_parallel,
|
|
916
877
|
max_workers=rust_batch_workers,
|
|
917
878
|
)
|
|
879
|
+
# A corrupted Rust batch is an integrity failure, never an
|
|
880
|
+
# invitation to reroute files through a different finalizer.
|
|
881
|
+
# The single-file entrypoint already enforces this contract;
|
|
882
|
+
# public commit reviews must enforce it before merging results.
|
|
883
|
+
if rust_batch.invalid_output:
|
|
884
|
+
raise RuntimeError(
|
|
885
|
+
"Malformed Rust commit batch result: "
|
|
886
|
+
f"{rust_batch.fallback_reason or 'invalid native output'}"
|
|
887
|
+
)
|
|
918
888
|
rust_batch_ms = (time.perf_counter() - rust_batch_started) * 1000
|
|
919
889
|
rust_batch_fallback_count = int(
|
|
920
890
|
rust_batch.batch_metadata.get("fallback_count")
|
|
@@ -996,13 +966,15 @@ class SemanticDiffer:
|
|
|
996
966
|
None,
|
|
997
967
|
new_filename=new_path,
|
|
998
968
|
_profiler=profiler,
|
|
969
|
+
_policy_old_filename=safe_policy_path(policy_root, old_path),
|
|
970
|
+
_policy_source_filename=safe_policy_path(policy_root, new_path or old_path),
|
|
999
971
|
)
|
|
1000
972
|
if staging_status is not None:
|
|
1001
973
|
result = result.model_copy(update={"staging_status": staging_status})
|
|
1002
974
|
return result
|
|
1003
975
|
|
|
1004
|
-
|
|
1005
|
-
_warn_exc = (PluginFuelExhausted, ValueError, RuntimeError)
|
|
976
|
+
# Parser absence is not a certified decline: it must fail the whole commit.
|
|
977
|
+
_warn_exc = (PluginNotFoundError, PluginFuelExhausted, ValueError, RuntimeError)
|
|
1006
978
|
|
|
1007
979
|
parallel = self._config.parallel
|
|
1008
980
|
if not parallel:
|
|
@@ -1031,10 +1003,12 @@ class SemanticDiffer:
|
|
|
1031
1003
|
# A genuine token-level fallback under the RUST_ONLY gate is fatal —
|
|
1032
1004
|
# propagate it rather than let ``_warn_exc`` silently drop the file.
|
|
1033
1005
|
raise
|
|
1034
|
-
except
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1006
|
+
except _warn_exc as exc:
|
|
1007
|
+
# A runtime failure cannot be represented in list[SemanticDiff].
|
|
1008
|
+
# Preserve the failed file identity; a partial review is unsafe.
|
|
1009
|
+
raise RuntimeError(
|
|
1010
|
+
f"Commit diff failed for {item[2]!r}: {exc}"
|
|
1011
|
+
) from exc
|
|
1038
1012
|
return results
|
|
1039
1013
|
else:
|
|
1040
1014
|
# ── Parallel ────────────────────────────────────────────────
|
|
@@ -1075,10 +1049,12 @@ class SemanticDiffer:
|
|
|
1075
1049
|
except RustOnlyGateError:
|
|
1076
1050
|
# Fatal under the RUST_ONLY gate — propagate, do not drop.
|
|
1077
1051
|
raise
|
|
1078
|
-
except
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1052
|
+
except _warn_exc as exc:
|
|
1053
|
+
# Match the sequential path: failures must not disappear
|
|
1054
|
+
# when one parallel worker succeeds.
|
|
1055
|
+
raise RuntimeError(
|
|
1056
|
+
f"Commit diff failed for {sources[idx][2]!r}: {exc}"
|
|
1057
|
+
) from exc
|
|
1082
1058
|
return [results_map[i] for i in sorted(results_map)]
|
|
1083
1059
|
|
|
1084
1060
|
def diff_stream(
|
|
@@ -1106,6 +1082,8 @@ class SemanticDiffer:
|
|
|
1106
1082
|
*,
|
|
1107
1083
|
new_filename: str | None = None,
|
|
1108
1084
|
parser_plugin_id: str | None = None,
|
|
1085
|
+
_policy_source_filename: str | None = None,
|
|
1086
|
+
_policy_old_filename: str | None = None,
|
|
1109
1087
|
_profiler: _PhaseProfiler | None = None,
|
|
1110
1088
|
) -> SemanticDiff:
|
|
1111
1089
|
profiler = _profiler or self._new_profiler()
|
|
@@ -1116,6 +1094,8 @@ class SemanticDiffer:
|
|
|
1116
1094
|
language_hint,
|
|
1117
1095
|
new_filename=new_filename,
|
|
1118
1096
|
parser_plugin_id=parser_plugin_id,
|
|
1097
|
+
policy_source_filename=_policy_source_filename,
|
|
1098
|
+
policy_old_filename=_policy_old_filename,
|
|
1119
1099
|
profiler=profiler,
|
|
1120
1100
|
)
|
|
1121
1101
|
# _run_stages_1_to_11 always returns a finished SemanticDiff since the python
|
|
@@ -1139,15 +1119,21 @@ class SemanticDiffer:
|
|
|
1139
1119
|
filename,
|
|
1140
1120
|
fuel=adaptive_fuel,
|
|
1141
1121
|
)
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
exc,
|
|
1122
|
+
# A Wasm analyzer is a diff-to-diff transform, never permission
|
|
1123
|
+
# to replace a Rust review with Pydantic's empty DTO defaults.
|
|
1124
|
+
from intentumdiff.rust_core import _require_complete_batch_diff_dto
|
|
1125
|
+
|
|
1126
|
+
routed = SemanticDiff.model_validate(
|
|
1127
|
+
_require_complete_batch_diff_dto(json.loads(updated_json))
|
|
1149
1128
|
)
|
|
1150
|
-
|
|
1129
|
+
except Exception as exc:
|
|
1130
|
+
# Semantic analyzers are configured diff-to-diff processing.
|
|
1131
|
+
# Omitting a failed one would return an incomplete review as
|
|
1132
|
+
# successful, including on fuel exhaustion or malformed DTOs.
|
|
1133
|
+
provenance = getattr(analyzer, "provenance", "?")
|
|
1134
|
+
raise RuntimeError(
|
|
1135
|
+
f"Diff analyzer {provenance!r} failed for {filename!r}: {exc}"
|
|
1136
|
+
) from exc
|
|
1151
1137
|
return routed
|
|
1152
1138
|
|
|
1153
1139
|
def _run_stages_1_to_11(
|
|
@@ -1159,11 +1145,13 @@ class SemanticDiffer:
|
|
|
1159
1145
|
*,
|
|
1160
1146
|
new_filename: str | None = None,
|
|
1161
1147
|
parser_plugin_id: str | None = None,
|
|
1148
|
+
policy_source_filename: str | None = None,
|
|
1149
|
+
policy_old_filename: str | None = None,
|
|
1162
1150
|
profiler: _PhaseProfiler | None = None,
|
|
1163
1151
|
) -> SemanticDiff:
|
|
1164
1152
|
"""Run pipeline stages 1–11 (parse → trivia-strip → Wasm → GumTree → moves).
|
|
1165
1153
|
|
|
1166
|
-
Returns
|
|
1154
|
+
Returns Rust-owned results or explicitly marked source fallbacks.
|
|
1167
1155
|
"""
|
|
1168
1156
|
# ── 1 + 2. Language detection ────────────────────────────────────────
|
|
1169
1157
|
profiler = profiler or self._new_profiler()
|
|
@@ -1174,6 +1162,7 @@ class SemanticDiffer:
|
|
|
1174
1162
|
language_hint,
|
|
1175
1163
|
plugin_id=parser_plugin_id,
|
|
1176
1164
|
phase_recorder=profiler.record if profiler.enabled else None,
|
|
1165
|
+
old_content=old_content,
|
|
1177
1166
|
)
|
|
1178
1167
|
logger.debug("Using parser %r for %r (language=%r)", parser.grammar_id, filename, language)
|
|
1179
1168
|
|
|
@@ -1183,6 +1172,10 @@ class SemanticDiffer:
|
|
|
1183
1172
|
)
|
|
1184
1173
|
engine_telemetry: list[dict[str, Any]] = []
|
|
1185
1174
|
parse_mode = parser.parser_mode
|
|
1175
|
+
# Retired InterpretCst execution cannot survive in a thin Python binding.
|
|
1176
|
+
# `_parse` checks the capability and raises with an actionable upgrade path.
|
|
1177
|
+
if parse_mode != "full-parse":
|
|
1178
|
+
self._parse(new_content, parser, language, filename)
|
|
1186
1179
|
cache_grammar_id = parser.grammar_id
|
|
1187
1180
|
diagnostics.summary.update(
|
|
1188
1181
|
{
|
|
@@ -1208,8 +1201,7 @@ class SemanticDiffer:
|
|
|
1208
1201
|
},
|
|
1209
1202
|
)
|
|
1210
1203
|
|
|
1211
|
-
#
|
|
1212
|
-
# raises it based on CST node count before calling the plugin.
|
|
1204
|
+
# The configured plugin fuel is adapted for full-parse source size below.
|
|
1213
1205
|
adaptive_fuel: int = self._config.plugin_fuel
|
|
1214
1206
|
|
|
1215
1207
|
# Save originals for the enricher pass (enrichers see un-preprocessed source)
|
|
@@ -1229,8 +1221,8 @@ class SemanticDiffer:
|
|
|
1229
1221
|
# the full set is safe for every xml diff.
|
|
1230
1222
|
_user_profiles, _ = load_user_schema_profiles()
|
|
1231
1223
|
_dialects = user_xml_dialects_payload(_user_profiles)
|
|
1232
|
-
|
|
1233
|
-
|
|
1224
|
+
# Empty configuration must clear previously registered native dialects.
|
|
1225
|
+
try_register_user_xml_dialects(_dialects)
|
|
1234
1226
|
schema_fingerprint = schema_cache_fingerprint(schema_resolution)
|
|
1235
1227
|
cache_schema_suffix = (
|
|
1236
1228
|
"" if schema_fingerprint == "schema:none" else f"|{schema_fingerprint}"
|
|
@@ -1263,9 +1255,13 @@ class SemanticDiffer:
|
|
|
1263
1255
|
"" if compile_metadata is None else f"|compile:{compile_metadata['fingerprint']}"
|
|
1264
1256
|
)
|
|
1265
1257
|
|
|
1266
|
-
#
|
|
1267
|
-
|
|
1268
|
-
_guardrail_cache_bypass = guardrails_may_apply(
|
|
1258
|
+
# Both physical sides may carry protection. Display aliases must not
|
|
1259
|
+
# hide the source policy on moves out of protected directories (#262).
|
|
1260
|
+
_guardrail_cache_bypass = guardrails_may_apply(
|
|
1261
|
+
policy_old_filename or filename,
|
|
1262
|
+
policy_source_filename or new_filename,
|
|
1263
|
+
self._config,
|
|
1264
|
+
)
|
|
1269
1265
|
cache_grammar_id = parser.grammar_id
|
|
1270
1266
|
|
|
1271
1267
|
if (
|
|
@@ -1322,13 +1318,22 @@ class SemanticDiffer:
|
|
|
1322
1318
|
rust_batch_attempt.diff,
|
|
1323
1319
|
file_lifecycle,
|
|
1324
1320
|
)
|
|
1321
|
+
# An invalid claimed-complete Rust result is a broken engine/DTO
|
|
1322
|
+
# contract, NOT an ordinary batch decline. Routing it through the
|
|
1323
|
+
# alternate Rust finalizer can silently publish different semantic
|
|
1324
|
+
# changes (observed in the 74-language corpus, #205).
|
|
1325
|
+
if rust_batch_attempt.invalid_output:
|
|
1326
|
+
raise RuntimeError(
|
|
1327
|
+
f"Malformed Rust batch result for {filename!r}: "
|
|
1328
|
+
f"{rust_batch_attempt.fallback_reason}"
|
|
1329
|
+
)
|
|
1325
1330
|
# The certified BATCH shortcut declined — fall through to the native
|
|
1326
1331
|
# routed finalize path below. This is a Rust→Rust transition (the
|
|
1327
1332
|
# routed path finalizes through the Rust core for every certified
|
|
1328
1333
|
# language), NOT a Python fallback, so the RUST_ONLY gate does NOT
|
|
1329
|
-
# fire here.
|
|
1330
|
-
#
|
|
1331
|
-
#
|
|
1334
|
+
# fire here. Parse-error source comparison also runs in Rust;
|
|
1335
|
+
# explicit core disabling and unexpected finalize declines still
|
|
1336
|
+
# fail the strict gate.
|
|
1332
1337
|
logger.debug(
|
|
1333
1338
|
"Rust core batch declined for %r: %s; using routed finalize",
|
|
1334
1339
|
filename,
|
|
@@ -1341,248 +1346,60 @@ class SemanticDiffer:
|
|
|
1341
1346
|
rust_core_skip_reason,
|
|
1342
1347
|
)
|
|
1343
1348
|
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
)
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
parser,
|
|
1372
|
-
language,
|
|
1373
|
-
filename,
|
|
1374
|
-
profiler=profiler,
|
|
1375
|
-
)
|
|
1376
|
-
new_cst_json = self._parse(
|
|
1377
|
-
new_content,
|
|
1378
|
-
parser,
|
|
1379
|
-
language,
|
|
1380
|
-
filename,
|
|
1381
|
-
profiler=profiler,
|
|
1382
|
-
)
|
|
1383
|
-
|
|
1384
|
-
# ── CST size guard ────────────────────────────────────────────────
|
|
1385
|
-
for _label, _cst in (("old", old_cst_json), ("new", new_cst_json)):
|
|
1386
|
-
if len(_cst) > self._config.max_cst_bytes:
|
|
1387
|
-
raise ValueError(
|
|
1388
|
-
f"CST JSON for the {_label!r} version of {filename!r} is "
|
|
1389
|
-
f"{len(_cst):,} characters, exceeding the limit of "
|
|
1390
|
-
f"{self._config.max_cst_bytes:,} "
|
|
1391
|
-
"(DiffConfig.max_cst_bytes). Increase the limit or switch "
|
|
1392
|
-
"to a full-parse plugin."
|
|
1349
|
+
# ── 7 (full-parse). Plugin owns parsing — send raw source ─────────
|
|
1350
|
+
# Stages 3-6 are skipped: the plugin handles its own CST construction
|
|
1351
|
+
# and trivia filtering. The style-only shortcut is not available here
|
|
1352
|
+
# because there is no host-side CST to hash.
|
|
1353
|
+
max_source_bytes = max(
|
|
1354
|
+
len(old_content.encode("utf-8")),
|
|
1355
|
+
len(new_content.encode("utf-8")),
|
|
1356
|
+
)
|
|
1357
|
+
adaptive_fuel = _fuel_budget(
|
|
1358
|
+
self._config.plugin_fuel,
|
|
1359
|
+
50_000_000 + max_source_bytes * 150_000,
|
|
1360
|
+
)
|
|
1361
|
+
logger.debug("full-parse mode for %r via %r", filename, parser.grammar_id)
|
|
1362
|
+
|
|
1363
|
+
# Full-diff cache reads are disabled for this release candidate (#187).
|
|
1364
|
+
# Historical entries omit the Rust engine/configuration identity and
|
|
1365
|
+
# can silently replay stale style-only results after semantic fixes.
|
|
1366
|
+
# Parser-tree caching remains available and is keyed by actual Wasm bytes.
|
|
1367
|
+
with profiler.phase("wasm_plugin_execution"):
|
|
1368
|
+
old_side_absent = file_lifecycle == "added" and old_content == ""
|
|
1369
|
+
new_side_absent = file_lifecycle == "deleted" and new_content == ""
|
|
1370
|
+
if self._cache is not None:
|
|
1371
|
+
old_tree = (
|
|
1372
|
+
_empty_semantic_tree(language)
|
|
1373
|
+
if old_side_absent
|
|
1374
|
+
else self._cached_process(
|
|
1375
|
+
parser, old_content, language, filename, adaptive_fuel
|
|
1393
1376
|
)
|
|
1394
|
-
|
|
1395
|
-
# ── 5. Trivia stripping ───────────────────────────────────────────
|
|
1396
|
-
with profiler.phase("trivia_stripping"):
|
|
1397
|
-
old_filtered = _strip_trivia_impl(old_cst_json, trivia_types)
|
|
1398
|
-
new_filtered = _strip_trivia_impl(new_cst_json, trivia_types)
|
|
1399
|
-
|
|
1400
|
-
# ── 6. Style-only shortcut ────────────────────────────────────────
|
|
1401
|
-
with profiler.phase("style_hashing"):
|
|
1402
|
-
old_hash = _compute_structural_hash_for_tree(old_filtered)
|
|
1403
|
-
new_hash = _compute_structural_hash_for_tree(new_filtered)
|
|
1404
|
-
|
|
1405
|
-
if old_hash == new_hash:
|
|
1406
|
-
logger.debug("Style-only diff for %r", filename)
|
|
1407
|
-
diagnostics.record(
|
|
1408
|
-
stage="style",
|
|
1409
|
-
action="style_only_shortcut",
|
|
1410
|
-
rule_id="pipeline.style_only_shortcut",
|
|
1411
|
-
reason="filtered CST structural hashes are equal",
|
|
1412
|
-
metadata={"old_hash": old_hash, "new_hash": new_hash},
|
|
1413
1377
|
)
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
new_cst_json=new_cst_json,
|
|
1420
|
-
)
|
|
1421
|
-
metadata: dict[str, Any] = {}
|
|
1422
|
-
if style_evidence.ignored_style_changes:
|
|
1423
|
-
metadata["ignored_style_changes"] = style_evidence.ignored_style_changes
|
|
1424
|
-
for group in style_evidence.change_groups:
|
|
1425
|
-
diagnostics.record_group(stage="invariance", group=group)
|
|
1426
|
-
if diagnostics.enabled:
|
|
1427
|
-
diagnostics.summary.update(
|
|
1428
|
-
{
|
|
1429
|
-
"early_exit": "style_only",
|
|
1430
|
-
"final_change_count": 0,
|
|
1431
|
-
"final_group_count": len(style_evidence.change_groups),
|
|
1432
|
-
}
|
|
1378
|
+
new_tree = (
|
|
1379
|
+
_empty_semantic_tree(language)
|
|
1380
|
+
if new_side_absent
|
|
1381
|
+
else self._cached_process(
|
|
1382
|
+
parser, new_content, language, filename, adaptive_fuel
|
|
1433
1383
|
)
|
|
1434
|
-
metadata["diagnostics"] = diagnostics.snapshot()
|
|
1435
|
-
style_diff = SemanticDiff.style_only(
|
|
1436
|
-
filename,
|
|
1437
|
-
new_filename if new_filename is not None else filename,
|
|
1438
|
-
language,
|
|
1439
|
-
change_groups=style_evidence.change_groups,
|
|
1440
|
-
metadata=metadata,
|
|
1441
1384
|
)
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
old_source=old_raw,
|
|
1448
|
-
new_source=new_raw,
|
|
1449
|
-
config=self._config,
|
|
1450
|
-
diagnostics=diagnostics,
|
|
1385
|
+
else:
|
|
1386
|
+
old_tree = (
|
|
1387
|
+
_empty_semantic_tree(language)
|
|
1388
|
+
if old_side_absent
|
|
1389
|
+
else parser.process(old_content, language, filename, fuel=adaptive_fuel)
|
|
1451
1390
|
)
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
max_cst_nodes = max(old_cst_nodes, new_cst_nodes)
|
|
1457
|
-
adaptive_fuel = _fuel_budget(
|
|
1458
|
-
self._config.plugin_fuel,
|
|
1459
|
-
20_000_000 + max_cst_nodes * 200_000,
|
|
1460
|
-
)
|
|
1461
|
-
logger.debug(
|
|
1462
|
-
"[%s] CST: old=%d bytes/%d nodes, new=%d bytes/%d nodes — fuel=%d",
|
|
1463
|
-
filename,
|
|
1464
|
-
len(old_filtered),
|
|
1465
|
-
old_cst_nodes,
|
|
1466
|
-
len(new_filtered),
|
|
1467
|
-
new_cst_nodes,
|
|
1468
|
-
adaptive_fuel,
|
|
1469
|
-
)
|
|
1470
|
-
diagnostics.record(
|
|
1471
|
-
stage="parse",
|
|
1472
|
-
action="interpret_cst",
|
|
1473
|
-
rule_id="pipeline.interpret_cst_parse",
|
|
1474
|
-
reason="host CST parsed, stripped, and sized",
|
|
1475
|
-
metadata={
|
|
1476
|
-
"old_cst_bytes": len(old_filtered),
|
|
1477
|
-
"new_cst_bytes": len(new_filtered),
|
|
1478
|
-
"old_cst_nodes": old_cst_nodes,
|
|
1479
|
-
"new_cst_nodes": new_cst_nodes,
|
|
1480
|
-
"adaptive_fuel": adaptive_fuel,
|
|
1481
|
-
},
|
|
1482
|
-
)
|
|
1483
|
-
|
|
1484
|
-
# ── 7. Plugin dispatch (with parse-tree cache) ───────────────────
|
|
1485
|
-
try:
|
|
1486
|
-
with profiler.phase("wasm_plugin_execution"):
|
|
1487
|
-
if self._cache is not None:
|
|
1488
|
-
old_tree = self._cached_process(
|
|
1489
|
-
parser,
|
|
1490
|
-
old_filtered,
|
|
1491
|
-
language,
|
|
1492
|
-
filename,
|
|
1493
|
-
adaptive_fuel,
|
|
1494
|
-
cache_grammar_id=cache_grammar_id,
|
|
1495
|
-
)
|
|
1496
|
-
new_tree = self._cached_process(
|
|
1497
|
-
parser,
|
|
1498
|
-
new_filtered,
|
|
1499
|
-
language,
|
|
1500
|
-
filename,
|
|
1501
|
-
adaptive_fuel,
|
|
1502
|
-
cache_grammar_id=cache_grammar_id,
|
|
1503
|
-
)
|
|
1504
|
-
else:
|
|
1505
|
-
old_tree = parser.process(
|
|
1506
|
-
old_filtered, language, filename, fuel=adaptive_fuel
|
|
1507
|
-
)
|
|
1508
|
-
new_tree = parser.process(
|
|
1509
|
-
new_filtered, language, filename, fuel=adaptive_fuel
|
|
1510
|
-
)
|
|
1511
|
-
except PluginFuelExhausted as exc:
|
|
1512
|
-
raise PluginFuelExhausted(
|
|
1513
|
-
exc.plugin_id,
|
|
1514
|
-
exc.fuel,
|
|
1515
|
-
f"file={filename!r}, CST nodes up to {max_cst_nodes:,}",
|
|
1516
|
-
) from exc
|
|
1517
|
-
|
|
1518
|
-
else:
|
|
1519
|
-
# ── 7 (full-parse). Plugin owns parsing — send raw source ─────────
|
|
1520
|
-
# Stages 3-6 are skipped: the plugin handles its own CST construction
|
|
1521
|
-
# and trivia filtering. The style-only shortcut is not available here
|
|
1522
|
-
# because there is no host-side CST to hash.
|
|
1523
|
-
max_source_bytes = max(
|
|
1524
|
-
len(old_content.encode("utf-8")),
|
|
1525
|
-
len(new_content.encode("utf-8")),
|
|
1526
|
-
)
|
|
1527
|
-
adaptive_fuel = _fuel_budget(
|
|
1528
|
-
self._config.plugin_fuel,
|
|
1529
|
-
50_000_000 + max_source_bytes * 150_000,
|
|
1530
|
-
)
|
|
1531
|
-
logger.debug("full-parse mode for %r via %r", filename, parser.grammar_id)
|
|
1532
|
-
|
|
1533
|
-
# Diff-level cache check for full-parse mode.
|
|
1534
|
-
if self._cache is not None and not diagnostics.enabled and not _guardrail_cache_bypass:
|
|
1535
|
-
_wasm_hash = self._wasm_hash_for(parser)
|
|
1536
|
-
_diff_key = self._cache.diff_key(
|
|
1537
|
-
old_content,
|
|
1538
|
-
new_content,
|
|
1539
|
-
f"{parser.grammar_id}{cache_schema_suffix}{compile_cache_suffix}",
|
|
1540
|
-
_wasm_hash,
|
|
1391
|
+
new_tree = (
|
|
1392
|
+
_empty_semantic_tree(language)
|
|
1393
|
+
if new_side_absent
|
|
1394
|
+
else parser.process(new_content, language, filename, fuel=adaptive_fuel)
|
|
1541
1395
|
)
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
with profiler.phase("wasm_plugin_execution"):
|
|
1551
|
-
old_side_absent = file_lifecycle == "added" and old_content == ""
|
|
1552
|
-
new_side_absent = file_lifecycle == "deleted" and new_content == ""
|
|
1553
|
-
if self._cache is not None:
|
|
1554
|
-
old_tree = (
|
|
1555
|
-
_empty_semantic_tree(language)
|
|
1556
|
-
if old_side_absent
|
|
1557
|
-
else self._cached_process(
|
|
1558
|
-
parser, old_content, language, filename, adaptive_fuel
|
|
1559
|
-
)
|
|
1560
|
-
)
|
|
1561
|
-
new_tree = (
|
|
1562
|
-
_empty_semantic_tree(language)
|
|
1563
|
-
if new_side_absent
|
|
1564
|
-
else self._cached_process(
|
|
1565
|
-
parser, new_content, language, filename, adaptive_fuel
|
|
1566
|
-
)
|
|
1567
|
-
)
|
|
1568
|
-
else:
|
|
1569
|
-
old_tree = (
|
|
1570
|
-
_empty_semantic_tree(language)
|
|
1571
|
-
if old_side_absent
|
|
1572
|
-
else parser.process(old_content, language, filename, fuel=adaptive_fuel)
|
|
1573
|
-
)
|
|
1574
|
-
new_tree = (
|
|
1575
|
-
_empty_semantic_tree(language)
|
|
1576
|
-
if new_side_absent
|
|
1577
|
-
else parser.process(new_content, language, filename, fuel=adaptive_fuel)
|
|
1578
|
-
)
|
|
1579
|
-
diagnostics.record(
|
|
1580
|
-
stage="parse",
|
|
1581
|
-
action="full_parse",
|
|
1582
|
-
rule_id="pipeline.full_parse",
|
|
1583
|
-
reason="plugin parsed raw source",
|
|
1584
|
-
metadata={"adaptive_fuel": adaptive_fuel},
|
|
1585
|
-
)
|
|
1396
|
+
diagnostics.record(
|
|
1397
|
+
stage="parse",
|
|
1398
|
+
action="full_parse",
|
|
1399
|
+
rule_id="pipeline.full_parse",
|
|
1400
|
+
reason="plugin parsed raw source",
|
|
1401
|
+
metadata={"adaptive_fuel": adaptive_fuel},
|
|
1402
|
+
)
|
|
1586
1403
|
|
|
1587
1404
|
engine_telemetry.extend(_drain_plugin_telemetry(parser))
|
|
1588
1405
|
_record_engine_telemetry(diagnostics, engine_telemetry)
|
|
@@ -1611,37 +1428,36 @@ class SemanticDiffer:
|
|
|
1611
1428
|
with profiler.phase("semantic_node_validation"):
|
|
1612
1429
|
old_tree = _enrich_literal_labels(old_tree, old_raw)
|
|
1613
1430
|
new_tree = _enrich_literal_labels(new_tree, new_raw)
|
|
1614
|
-
# Cross-language NodeFacts (issue #70):
|
|
1615
|
-
#
|
|
1616
|
-
#
|
|
1431
|
+
# Cross-language NodeFacts (issue #70): Rust derives structural facts for
|
|
1432
|
+
# every language. Missing or malformed native output is a contract failure,
|
|
1433
|
+
# not permission to silently publish a semantically impoverished tree.
|
|
1617
1434
|
old_tree = enrich_node_facts(old_tree)
|
|
1618
1435
|
new_tree = enrich_node_facts(new_tree)
|
|
1619
1436
|
# Profile-label enrichment runs in the Rust core (issue #57 port).
|
|
1620
|
-
#
|
|
1621
|
-
#
|
|
1622
|
-
#
|
|
1623
|
-
# to a Python enricher, so "if Python didn't exist it still works".
|
|
1437
|
+
# Rust owns profile enrichment. Missing ABI handlers are fatal:
|
|
1438
|
+
# accepting un-enriched trees silently changes identity matching
|
|
1439
|
+
# and can suppress a meaningful diff. Never substitute Python logic.
|
|
1624
1440
|
# Gated by language so non-profile diffs pay no tree-JSON roundtrip.
|
|
1625
1441
|
if language.lower() in {"json", "yaml"}:
|
|
1626
1442
|
_identity_fields = tuple(schema_resolution.identity_fields)
|
|
1627
1443
|
old_tree = try_rust_profile_label_enrichment(
|
|
1628
1444
|
old_tree, old_raw, language, identity_fields=_identity_fields
|
|
1629
|
-
)
|
|
1445
|
+
)
|
|
1630
1446
|
new_tree = try_rust_profile_label_enrichment(
|
|
1631
1447
|
new_tree, new_raw, language, identity_fields=_identity_fields
|
|
1632
|
-
)
|
|
1448
|
+
)
|
|
1633
1449
|
if language.lower() in {"css", "scss", "html", "xml", "mdx"}:
|
|
1634
|
-
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language)
|
|
1635
|
-
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language)
|
|
1450
|
+
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language)
|
|
1451
|
+
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language)
|
|
1636
1452
|
if language.lower() in {"hcl", "puppet"}:
|
|
1637
|
-
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language)
|
|
1638
|
-
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language)
|
|
1453
|
+
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language)
|
|
1454
|
+
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language)
|
|
1639
1455
|
if language.lower() == "sql":
|
|
1640
|
-
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language)
|
|
1641
|
-
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language)
|
|
1456
|
+
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language)
|
|
1457
|
+
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language)
|
|
1642
1458
|
if language.lower() in {"asm", "bash", "delphi"}:
|
|
1643
|
-
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language)
|
|
1644
|
-
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language)
|
|
1459
|
+
old_tree = try_rust_profile_label_enrichment(old_tree, old_raw, language)
|
|
1460
|
+
new_tree = try_rust_profile_label_enrichment(new_tree, new_raw, language)
|
|
1645
1461
|
_validate_tree_ids(old_tree, filename)
|
|
1646
1462
|
_validate_tree_ids(new_tree, filename)
|
|
1647
1463
|
diagnostics.record(
|
|
@@ -1654,17 +1470,16 @@ class SemanticDiffer:
|
|
|
1654
1470
|
"new_nodes": _count_semantic_nodes(new_tree),
|
|
1655
1471
|
},
|
|
1656
1472
|
)
|
|
1657
|
-
# ── 8.5.
|
|
1473
|
+
# ── 8.5. Rust source fallback for severe parse errors ───────────────
|
|
1658
1474
|
# When tree-sitter produced ERROR nodes and fallback is enabled, skip
|
|
1659
|
-
#
|
|
1475
|
+
# semantic matching and preserve the changed source range in Rust.
|
|
1660
1476
|
if self._config.fallback_to_token_diff and (
|
|
1661
1477
|
_has_error_node(old_tree) or _has_error_node(new_tree)
|
|
1662
1478
|
):
|
|
1663
1479
|
logger.warning(
|
|
1664
|
-
"Parse errors detected in %r
|
|
1480
|
+
"Parse errors detected in %r - using Rust source fallback",
|
|
1665
1481
|
filename,
|
|
1666
1482
|
)
|
|
1667
|
-
_raise_rust_only_gate_error("parse errors require Rust token-level fallback")
|
|
1668
1483
|
diagnostics.record(
|
|
1669
1484
|
stage="fallback",
|
|
1670
1485
|
action="token_fallback",
|
|
@@ -1693,6 +1508,8 @@ class SemanticDiffer:
|
|
|
1693
1508
|
old_source=old_raw,
|
|
1694
1509
|
new_source=new_raw,
|
|
1695
1510
|
config=self._config,
|
|
1511
|
+
policy_filename=policy_source_filename,
|
|
1512
|
+
policy_old_filename=policy_old_filename,
|
|
1696
1513
|
diagnostics=diagnostics,
|
|
1697
1514
|
)
|
|
1698
1515
|
# ── 9. GumTree matching ────────────────────────────────────────────────
|
|
@@ -1759,6 +1576,15 @@ class SemanticDiffer:
|
|
|
1759
1576
|
collect_trace=diagnostics.enabled,
|
|
1760
1577
|
)
|
|
1761
1578
|
if finalize_result is not None:
|
|
1579
|
+
if finalize_result.get("fallback_diff") is not None:
|
|
1580
|
+
fallback_diff = finalize_result["fallback_diff"].model_copy(update={
|
|
1581
|
+
"old_filename": filename,
|
|
1582
|
+
"new_filename": new_filename if new_filename is not None else filename})
|
|
1583
|
+
fallback_diff = _apply_file_lifecycle_to_diff(fallback_diff, file_lifecycle)
|
|
1584
|
+
return apply_guardrails_to_diff(fallback_diff, old_tree=None, new_tree=None,
|
|
1585
|
+
old_source=old_raw, new_source=new_raw, config=self._config,
|
|
1586
|
+
policy_filename=policy_source_filename,
|
|
1587
|
+
policy_old_filename=policy_old_filename, diagnostics=diagnostics)
|
|
1762
1588
|
diagnostics.record(
|
|
1763
1589
|
stage="finalize",
|
|
1764
1590
|
action="rust_finalize_review",
|
|
@@ -1781,176 +1607,18 @@ class SemanticDiffer:
|
|
|
1781
1607
|
reason="rust finalize pass completed",
|
|
1782
1608
|
count=int(entry.get("changes_after", 0)),
|
|
1783
1609
|
)
|
|
1784
|
-
|
|
1785
|
-
# reorder / …) are equivalences that must hold REGARDLESS of routing, but this
|
|
1786
|
-
# short-circuit returns before the stage-12 apply_invariances. Run it here on the
|
|
1787
|
-
# routed changes so canonical-value "zero-change" edits collapse under routing too
|
|
1788
|
-
# (#57: unlocks json/yaml/css/python). No-op when no invariance fires.
|
|
1789
|
-
fin_changes = finalize_result["changes"]
|
|
1790
|
-
fin_is_style_only = finalize_result["is_style_only"]
|
|
1791
|
-
fin_invariant_groups: list[Any] = []
|
|
1792
|
-
fin_invariant_ignored: list[dict[str, Any]] = []
|
|
1793
|
-
if fin_changes:
|
|
1794
|
-
_inv = apply_invariances(
|
|
1795
|
-
fin_changes,
|
|
1796
|
-
old_tree=old_tree,
|
|
1797
|
-
new_tree=new_tree,
|
|
1798
|
-
old_source=old_content,
|
|
1799
|
-
new_source=new_content,
|
|
1800
|
-
language=language,
|
|
1801
|
-
)
|
|
1802
|
-
if len(_inv.changes) != len(fin_changes) or _inv.change_groups:
|
|
1803
|
-
fin_changes = _inv.changes
|
|
1804
|
-
fin_invariant_groups = list(_inv.change_groups)
|
|
1805
|
-
# The invariance's own IGNORED_STYLE groups ARE the style evidence — carry
|
|
1806
|
-
# their rule_id/reason so metadata["ignored_style_changes"] leads with the
|
|
1807
|
-
# specific reason (css.color.canonical_equivalence), not the generic
|
|
1808
|
-
# source-equivalence fallback below.
|
|
1809
|
-
for group in fin_invariant_groups:
|
|
1810
|
-
diagnostics.record_group(stage="invariance", group=group)
|
|
1811
|
-
fin_invariant_ignored = [
|
|
1812
|
-
{"rule_id": g.rule_id, **dict(g.metadata)}
|
|
1813
|
-
for g in fin_invariant_groups
|
|
1814
|
-
if g.kind == ChangeGroupKind.IGNORED_STYLE
|
|
1815
|
-
]
|
|
1816
|
-
if not fin_changes and old_content != new_content:
|
|
1817
|
-
fin_is_style_only = True
|
|
1818
|
-
# Generic files are line-oriented from the user's point of view
|
|
1819
|
-
# (#57 generic routing, mirroring the stage-12 orchestration): the routed
|
|
1820
|
-
# finalize's parser-token churn is REPLACED wholesale by the Rust text
|
|
1821
|
-
# review's stable line/character spans, then the (Rust-first) markdown
|
|
1822
|
-
# section passes run for .md filenames. Parser-derived groups are dropped
|
|
1823
|
-
# with the churn — carrying them forward invents fake pairings between
|
|
1824
|
-
# unrelated lines.
|
|
1825
|
-
fin_ignored_override: list[dict[str, Any]] | None = None
|
|
1826
|
-
if language.lower() == "generic":
|
|
1827
|
-
generic_presented = normalize_generic_text_for_review(
|
|
1828
|
-
fin_changes,
|
|
1829
|
-
old_content,
|
|
1830
|
-
new_content,
|
|
1831
|
-
)
|
|
1832
|
-
presented_generic = PresentationResult(
|
|
1833
|
-
changes=generic_presented.changes,
|
|
1834
|
-
change_groups=generic_presented.change_groups,
|
|
1835
|
-
ignored_style_changes=[
|
|
1836
|
-
*generic_presented.ignored_style_changes,
|
|
1837
|
-
],
|
|
1838
|
-
)
|
|
1839
|
-
presented_generic = _markdown_section_move_presentation(
|
|
1840
|
-
presented_generic,
|
|
1841
|
-
old_source=old_content,
|
|
1842
|
-
new_source=new_content,
|
|
1843
|
-
old_filename=filename,
|
|
1844
|
-
new_filename=(
|
|
1845
|
-
new_filename if new_filename is not None else filename
|
|
1846
|
-
),
|
|
1847
|
-
)
|
|
1848
|
-
presented_generic = _markdown_section_heading_rename_presentation(
|
|
1849
|
-
presented_generic,
|
|
1850
|
-
old_source=old_content,
|
|
1851
|
-
new_source=new_content,
|
|
1852
|
-
old_filename=filename,
|
|
1853
|
-
new_filename=(
|
|
1854
|
-
new_filename if new_filename is not None else filename
|
|
1855
|
-
),
|
|
1856
|
-
)
|
|
1857
|
-
fin_changes = presented_generic.changes
|
|
1858
|
-
fin_invariant_groups = list(presented_generic.change_groups)
|
|
1859
|
-
fin_ignored_override = list(presented_generic.ignored_style_changes)
|
|
1860
|
-
finalize_result["change_groups"] = []
|
|
1861
|
-
finalize_result["no_surviving_changes"] = False
|
|
1862
|
-
# Mirror the stage-12 style-only resolution: identical sources (or
|
|
1863
|
-
# whitespace-collapsed-equal trees) with no surviving line spans are a
|
|
1864
|
-
# style-only diff, not "no changes". The Rust finalize's flag reflects
|
|
1865
|
-
# its own suppressions, which the replacement just discarded.
|
|
1866
|
-
if not fin_changes:
|
|
1867
|
-
|
|
1868
|
-
def _generic_norm(node: SemanticNode) -> tuple[Any, ...]:
|
|
1869
|
-
label = " ".join(node.label.split())
|
|
1870
|
-
return (
|
|
1871
|
-
node.node_type,
|
|
1872
|
-
label,
|
|
1873
|
-
tuple(_generic_norm(c) for c in node.children),
|
|
1874
|
-
)
|
|
1875
|
-
|
|
1876
|
-
fin_is_style_only = (
|
|
1877
|
-
old_content == new_content
|
|
1878
|
-
or _generic_norm(old_tree) == _generic_norm(new_tree)
|
|
1879
|
-
)
|
|
1880
|
-
# Stage-12 style-only resolution, language-agnostic (markdown #44 exposed
|
|
1881
|
-
# it): ZERO surviving changes with identical (or whitespace-collapsed
|
|
1882
|
-
# tree-equal) sources is a style-only diff for every routed language —
|
|
1883
|
-
# the Rust finalize's flag only reflects its own suppressions.
|
|
1884
|
-
if not fin_changes and not fin_is_style_only:
|
|
1885
|
-
|
|
1886
|
-
def _routed_norm(node: SemanticNode) -> tuple[Any, ...]:
|
|
1887
|
-
label = " ".join(node.label.split())
|
|
1888
|
-
return (
|
|
1889
|
-
node.node_type,
|
|
1890
|
-
label,
|
|
1891
|
-
tuple(_routed_norm(c) for c in node.children),
|
|
1892
|
-
)
|
|
1610
|
+
from intentumdiff.rust_core import complete_routed_review
|
|
1893
1611
|
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
)
|
|
1898
|
-
metadata_fin: dict[str, Any] = {
|
|
1899
|
-
"engine_owner": "rust",
|
|
1900
|
-
"semantic_contract": "rust_finalize_review_v1",
|
|
1901
|
-
"rust_core": {
|
|
1902
|
-
"engine": "rust_finalize_review_v1",
|
|
1903
|
-
"stage": "per_stage_finalize_routing",
|
|
1904
|
-
"used": True,
|
|
1905
|
-
},
|
|
1906
|
-
}
|
|
1907
|
-
# Context metadata the normal path attaches at stage 11 — the routed
|
|
1908
|
-
# return must carry the same contracts (cpp compile_commands pilot gap).
|
|
1909
|
-
if schema_metadata is not None:
|
|
1910
|
-
metadata_fin["schema"] = schema_metadata
|
|
1911
|
-
if compile_metadata is not None:
|
|
1912
|
-
metadata_fin["compile_commands"] = compile_metadata
|
|
1913
|
-
combined_ignored = (
|
|
1914
|
-
fin_ignored_override
|
|
1915
|
-
if fin_ignored_override is not None
|
|
1916
|
-
else [
|
|
1917
|
-
*fin_invariant_ignored,
|
|
1918
|
-
*finalize_result["ignored_style_changes"],
|
|
1919
|
-
]
|
|
1920
|
-
)
|
|
1921
|
-
if combined_ignored:
|
|
1922
|
-
metadata_fin["ignored_style_changes"] = combined_ignored
|
|
1923
|
-
if finalize_result["no_surviving_changes"]:
|
|
1924
|
-
metadata_fin["no_surviving_changes"] = True
|
|
1925
|
-
routed_groups = [*finalize_result["change_groups"], *fin_invariant_groups]
|
|
1926
|
-
if not fin_changes and old_content != new_content and not fin_invariant_ignored:
|
|
1927
|
-
# The normal path attaches a source-equivalence suppression group
|
|
1928
|
-
# post-diff when a diff nets to zero changes but the sources differ
|
|
1929
|
-
# (xml attribute reorder -> zero changes must still RECORD the
|
|
1930
|
-
# suppression, not merely be absent). The routed short-circuit returns
|
|
1931
|
-
# before that stage, so mirror it here.
|
|
1932
|
-
style_evidence = build_style_only_evidence(
|
|
1933
|
-
old_source=old_content,
|
|
1934
|
-
new_source=new_content,
|
|
1935
|
-
language=language,
|
|
1936
|
-
)
|
|
1937
|
-
for group in style_evidence.change_groups:
|
|
1938
|
-
diagnostics.record_group(stage="invariance", group=group)
|
|
1939
|
-
routed_groups.extend(style_evidence.change_groups)
|
|
1940
|
-
if style_evidence.ignored_style_changes:
|
|
1941
|
-
metadata_fin.setdefault("ignored_style_changes", []).extend(
|
|
1942
|
-
style_evidence.ignored_style_changes
|
|
1943
|
-
)
|
|
1944
|
-
routed = SemanticDiff(
|
|
1612
|
+
routed = complete_routed_review(
|
|
1613
|
+
finalized=finalize_result, old_tree=old_tree, new_tree=new_tree,
|
|
1614
|
+
old_source=old_content, new_source=new_content,
|
|
1945
1615
|
old_filename=filename,
|
|
1946
1616
|
new_filename=new_filename if new_filename is not None else filename,
|
|
1947
|
-
language=language,
|
|
1948
|
-
|
|
1949
|
-
change_groups=routed_groups,
|
|
1950
|
-
has_semantic_changes=bool(fin_changes) and not fin_is_style_only,
|
|
1951
|
-
is_style_only=fin_is_style_only,
|
|
1952
|
-
metadata=metadata_fin,
|
|
1617
|
+
language=language, schema_metadata=schema_metadata,
|
|
1618
|
+
compile_metadata=compile_metadata,
|
|
1953
1619
|
)
|
|
1620
|
+
for group in routed.change_groups:
|
|
1621
|
+
diagnostics.record_group(stage="invariance", group=group)
|
|
1954
1622
|
routed = _apply_file_lifecycle_to_diff(routed, file_lifecycle)
|
|
1955
1623
|
routed = apply_guardrails_to_diff(
|
|
1956
1624
|
routed,
|
|
@@ -1959,6 +1627,8 @@ class SemanticDiffer:
|
|
|
1959
1627
|
old_source=old_content,
|
|
1960
1628
|
new_source=new_content,
|
|
1961
1629
|
config=self._config,
|
|
1630
|
+
policy_filename=policy_source_filename,
|
|
1631
|
+
policy_old_filename=policy_old_filename,
|
|
1962
1632
|
diagnostics=diagnostics,
|
|
1963
1633
|
)
|
|
1964
1634
|
# Diff-analyzer pass on the routed diff (stage 13.5 mirror; issue #57
|
|
@@ -1973,7 +1643,7 @@ class SemanticDiffer:
|
|
|
1973
1643
|
# transitional python pipeline is retired (issue #57 payoff, stage 4b) —
|
|
1974
1644
|
# degrade honestly to the coarse token-level diff, same as parse errors.
|
|
1975
1645
|
logger.warning(
|
|
1976
|
-
"Rust finalize declined for %r (%s) —
|
|
1646
|
+
"Rust finalize declined for %r (%s) — Rust source fallback",
|
|
1977
1647
|
filename,
|
|
1978
1648
|
language,
|
|
1979
1649
|
)
|
|
@@ -2008,6 +1678,8 @@ class SemanticDiffer:
|
|
|
2008
1678
|
old_source=old_raw,
|
|
2009
1679
|
new_source=new_raw,
|
|
2010
1680
|
config=self._config,
|
|
1681
|
+
policy_filename=policy_source_filename,
|
|
1682
|
+
policy_old_filename=policy_old_filename,
|
|
2011
1683
|
diagnostics=diagnostics,
|
|
2012
1684
|
)
|
|
2013
1685
|
if test_engine == "python":
|
|
@@ -2021,7 +1693,7 @@ class SemanticDiffer:
|
|
|
2021
1693
|
# transitional python pipeline is retired (issue #57 payoff) — semantic
|
|
2022
1694
|
# diffing REQUIRES the Rust core; degrade honestly to the token-level diff.
|
|
2023
1695
|
logger.warning(
|
|
2024
|
-
"Rust core disabled for %r —
|
|
1696
|
+
"Rust core disabled for %r — Rust source fallback (python pipeline retired)",
|
|
2025
1697
|
filename,
|
|
2026
1698
|
)
|
|
2027
1699
|
_raise_rust_only_gate_error("rust core disabled")
|
|
@@ -2055,6 +1727,8 @@ class SemanticDiffer:
|
|
|
2055
1727
|
old_source=old_raw,
|
|
2056
1728
|
new_source=new_raw,
|
|
2057
1729
|
config=self._config,
|
|
1730
|
+
policy_filename=policy_source_filename,
|
|
1731
|
+
policy_old_filename=policy_old_filename,
|
|
2058
1732
|
diagnostics=diagnostics,
|
|
2059
1733
|
)
|
|
2060
1734
|
|
|
@@ -2062,62 +1736,22 @@ class SemanticDiffer:
|
|
|
2062
1736
|
self,
|
|
2063
1737
|
source: Source,
|
|
2064
1738
|
) -> Iterator[ChangeStreamEvent]:
|
|
2065
|
-
"""
|
|
2066
|
-
|
|
2067
|
-
Three phases are emitted in order:
|
|
2068
|
-
|
|
2069
|
-
1. **STRUCTURAL** — one ``action="add"`` event per change after the
|
|
2070
|
-
GumTree edit script and move-promotion pass (stage 11). These are
|
|
2071
|
-
available immediately, before the (potentially expensive) refactoring
|
|
2072
|
-
detection runs.
|
|
2073
|
-
|
|
2074
|
-
2. **REFINED** — events emitted once refactoring detection (stage 12) has
|
|
2075
|
-
run. Consumed raw changes are replaced by ``action="revise"`` events
|
|
2076
|
-
carrying a ``REFACTORING`` change and ``replaced_ids`` listing the
|
|
2077
|
-
node IDs of the Phase-1 changes they supersede. Removed-but-not-
|
|
2078
|
-
replaced changes are emitted as ``action="remove"``.
|
|
2079
|
-
|
|
2080
|
-
3. **FINAL** — events emitted after classification (stage 13) and any
|
|
2081
|
-
diff-analyzer plugins (stage 13.5) have run. Emitted only when these
|
|
2082
|
-
passes modify the change list.
|
|
1739
|
+
"""Stream finalized Rust-backed changes as FINAL add events.
|
|
2083
1740
|
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
1741
|
+
The Python pipeline was retired in 0.0.2. Rust returns a complete diff,
|
|
1742
|
+
not intermediate structural/refinement snapshots. This API preserves its
|
|
1743
|
+
event-shaped transport contract, but it does not manufacture progressive
|
|
1744
|
+
semantic phases or reconcile changes using Python object identity.
|
|
2088
1745
|
"""
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
*,
|
|
2099
|
-
new_filename: str | None = None,
|
|
2100
|
-
) -> Iterator[ChangeStreamEvent]:
|
|
2101
|
-
"""Internal generator — see ``diff_stream_progressive`` for the public API."""
|
|
2102
|
-
result = self._run_stages_1_to_11(
|
|
2103
|
-
old_content, new_content, filename, language_hint, new_filename=new_filename
|
|
2104
|
-
)
|
|
2105
|
-
|
|
2106
|
-
if isinstance(result, SemanticDiff):
|
|
2107
|
-
# Early exit (cache hit / style-only / token fallback): emit all changes
|
|
2108
|
-
# as FINAL so consumers still receive a well-formed event stream.
|
|
2109
|
-
for change in result.changes:
|
|
2110
|
-
yield ChangeStreamEvent(
|
|
2111
|
-
phase=ChangeStreamPhase.FINAL,
|
|
2112
|
-
action="add",
|
|
2113
|
-
change=change,
|
|
2114
|
-
)
|
|
2115
|
-
return
|
|
2116
|
-
|
|
2117
|
-
raise AssertionError(
|
|
2118
|
-
"unreachable: _run_stages_1_to_11 always returns a SemanticDiff since the "
|
|
2119
|
-
"python pipeline retirement (issue #57 payoff, stage 4b)"
|
|
2120
|
-
)
|
|
1746
|
+
# Reuse the public path so FileSource policy discovery retains both
|
|
1747
|
+
# physical paths even when the source exposes a display alias.
|
|
1748
|
+
result = self.diff(source)
|
|
1749
|
+
for change in result.changes:
|
|
1750
|
+
yield ChangeStreamEvent(
|
|
1751
|
+
phase=ChangeStreamPhase.FINAL,
|
|
1752
|
+
action="add",
|
|
1753
|
+
change=change,
|
|
1754
|
+
)
|
|
2121
1755
|
|
|
2122
1756
|
def _cached_process(
|
|
2123
1757
|
self,
|
|
@@ -2141,33 +1775,72 @@ class SemanticDiffer:
|
|
|
2141
1775
|
raise RuntimeError("parse cache store is not configured")
|
|
2142
1776
|
wasm_hash = self._wasm_hash_for(parser)
|
|
2143
1777
|
grammar_id = cache_grammar_id or parser.grammar_id
|
|
2144
|
-
key
|
|
1778
|
+
# Rust owns the canonical key hashing. The host must supply every
|
|
1779
|
+
# argument that can affect the Wasm parser output, including language
|
|
1780
|
+
# and filename for a multi-language/shared-grammar component (#188).
|
|
1781
|
+
parser_context = json.dumps(
|
|
1782
|
+
{"grammar_id": grammar_id, "language": language, "filename": filename},
|
|
1783
|
+
sort_keys=True, separators=(",", ":"),
|
|
1784
|
+
)
|
|
1785
|
+
key = cache.parse_key(input_, parser_context, wasm_hash)
|
|
2145
1786
|
cached = cache.get_parse(key)
|
|
2146
1787
|
if cached is not None:
|
|
2147
1788
|
logger.debug("Parse cache hit for grammar=%r", grammar_id)
|
|
2148
1789
|
return SemanticNode.model_validate_json(cached)
|
|
2149
1790
|
|
|
2150
1791
|
node = parser.process(input_, language, filename, fuel=fuel)
|
|
1792
|
+
# A trap may reload the Wasm module during process(); never write its
|
|
1793
|
+
# output under a key computed for a different compiled component.
|
|
1794
|
+
if self._wasm_hash_for(parser) != wasm_hash:
|
|
1795
|
+
raise RuntimeError("parser Wasm identity changed during parse")
|
|
2151
1796
|
cache.put_parse(key, node.model_dump_json(), grammar_id=grammar_id)
|
|
2152
1797
|
return node
|
|
2153
1798
|
|
|
2154
1799
|
def _wasm_hash_for(self, parser: Any) -> str:
|
|
2155
|
-
"""
|
|
2156
|
-
Return a short (16-hex-char) sha256 hash of the parser's Wasm binary.
|
|
1800
|
+
"""Fingerprint the actual parser Wasm bytes before permitting any cache hit.
|
|
2157
1801
|
|
|
2158
|
-
|
|
2159
|
-
|
|
1802
|
+
A missing/unreadable component cannot share an ``"unknown"`` cache key:
|
|
1803
|
+
doing so could replay a stale style-only diff without invoking Rust.
|
|
1804
|
+
File identity (not grammar ID) scopes the memoized digest, so different
|
|
1805
|
+
component revisions cannot share previously certified parse/diff results.
|
|
2160
1806
|
"""
|
|
2161
|
-
grammar_id: str = parser.grammar_id
|
|
2162
|
-
if grammar_id in self._wasm_hash_cache:
|
|
2163
|
-
return self._wasm_hash_cache[grammar_id]
|
|
2164
1807
|
try:
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
except (AttributeError, OSError):
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
1808
|
+
path = Path(parser.wasm_path).resolve(strict=True)
|
|
1809
|
+
stat_before = path.stat()
|
|
1810
|
+
except (AttributeError, TypeError, OSError) as exc:
|
|
1811
|
+
raise RuntimeError(
|
|
1812
|
+
f"cannot fingerprint parser Wasm at {getattr(parser, 'wasm_path', None)!r}"
|
|
1813
|
+
) from exc
|
|
1814
|
+
|
|
1815
|
+
def _identity(stat: Any) -> tuple[int, int, int, int, int]:
|
|
1816
|
+
return (stat.st_dev, stat.st_ino, stat.st_size,
|
|
1817
|
+
stat.st_mtime_ns, stat.st_ctime_ns)
|
|
1818
|
+
|
|
1819
|
+
identity = (str(path), *_identity(stat_before))
|
|
1820
|
+
digest = self._wasm_hash_cache.get(identity)
|
|
1821
|
+
if digest is None:
|
|
1822
|
+
try:
|
|
1823
|
+
wasm_bytes = path.read_bytes()
|
|
1824
|
+
stat_after = path.stat()
|
|
1825
|
+
except OSError as exc:
|
|
1826
|
+
raise RuntimeError(f"cannot fingerprint parser Wasm at {str(path)!r}") from exc
|
|
1827
|
+
if _identity(stat_before) != _identity(stat_after):
|
|
1828
|
+
raise RuntimeError(f"parser Wasm changed while hashing: {path}")
|
|
1829
|
+
digest = hashlib.sha256(wasm_bytes).hexdigest()
|
|
1830
|
+
self._wasm_hash_cache[identity] = digest
|
|
1831
|
+
|
|
1832
|
+
# The path can be replaced while a LoadedPlugin still executes the
|
|
1833
|
+
# previous compiled component. File digest alone is not a safe cache
|
|
1834
|
+
# identity: bind it to the *instantiated* Wasmtime bytes (#196).
|
|
1835
|
+
compiled = getattr(parser, "compiled_wasm_sha256", None)
|
|
1836
|
+
if isinstance(parser, ParserAdapter) and (
|
|
1837
|
+
not isinstance(compiled, str) or len(compiled) != 64
|
|
1838
|
+
or any(c not in "0123456789abcdef" for c in compiled)
|
|
1839
|
+
):
|
|
1840
|
+
raise RuntimeError(f"cannot verify compiled parser Wasm identity: {path}")
|
|
1841
|
+
if isinstance(compiled, str) and compiled != digest:
|
|
1842
|
+
raise RuntimeError(f"compiled parser Wasm differs from backing artifact: {path}")
|
|
1843
|
+
return digest
|
|
2171
1844
|
|
|
2172
1845
|
def _parse(
|
|
2173
1846
|
self,
|