intentumdiff-python 0.0.1__py3-none-win_amd64.whl → 0.0.2b2__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- intentumdiff/__init__.py +2 -0
- intentumdiff/__main__.py +12 -0
- intentumdiff/_differ_presentation.py +35 -682
- intentumdiff/_differ_runtime.py +13 -47
- intentumdiff/analysis/compile_commands.py +26 -141
- intentumdiff/analysis/cross_file.py +2 -4
- intentumdiff/analysis/guardrail_reports.py +7 -5
- intentumdiff/analysis/guardrails.py +110 -122
- intentumdiff/analysis/invariances.py +4 -1003
- intentumdiff/analysis/schema_resolver.py +49 -273
- intentumdiff/analysis/text_review.py +4 -3
- intentumdiff/analysis/user_schemas.py +55 -237
- intentumdiff/api.py +21 -0
- intentumdiff/cache/sqlite_store.py +5 -23
- intentumdiff/cli/__init__.py +59 -0
- intentumdiff/cli/_commands.py +174 -67
- intentumdiff/cli/_parser.py +11 -1
- intentumdiff/cli/_shared.py +96 -112
- intentumdiff/content_type.py +13 -30
- intentumdiff/core/commit_differ.py +105 -220
- intentumdiff/core/diffignore.py +28 -34
- intentumdiff/core/index.py +16 -18
- intentumdiff/core/indexer.py +148 -168
- intentumdiff/core/models.py +53 -14
- intentumdiff/differ.py +278 -605
- intentumdiff/github_app.py +35 -19
- intentumdiff/intentumdiff_rust_core/intentumdiff_rust_core.dll +0 -0
- intentumdiff/live_server.py +180 -17
- intentumdiff/lsp/client.py +7 -1
- intentumdiff/lsp/enricher.py +19 -119
- intentumdiff/lsp/exceptions.py +6 -7
- intentumdiff/lsp_server/_codelens.py +24 -57
- intentumdiff/lsp_server/_diagnostics.py +23 -51
- intentumdiff/lsp_server/_handlers.py +66 -8
- intentumdiff/lsp_server/server.py +28 -7
- intentumdiff/plugins/adapter.py +5 -0
- intentumdiff/plugins/builtins.py +7 -14
- intentumdiff/plugins/exceptions.py +4 -0
- intentumdiff/plugins/hub.py +1 -2
- intentumdiff/plugins/loader.py +229 -215
- intentumdiff/plugins/registry.py +249 -197
- intentumdiff/rust_core.py +744 -784
- intentumdiff/serve/_app.py +3 -10
- intentumdiff/sources/base.py +24 -0
- intentumdiff/sources/file_source.py +15 -2
- intentumdiff/sources/git_source.py +120 -58
- intentumdiff/sources/live_buffer_source.py +45 -5
- intentumdiff/sources/patch_source.py +36 -109
- intentumdiff/vcs/_safe_path.py +7 -0
- intentumdiff/vcs/base.py +6 -0
- intentumdiff/vcs/git_backend.py +1 -1
- intentumdiff/vcs/git_cli.py +38 -18
- intentumdiff/vcs/hg_backend.py +1 -1
- intentumdiff/vcs/perforce_backend.py +7 -1
- intentumdiff/vcs/svn_backend.py +1 -1
- intentumdiff/wasm/abap_parser.wasm +0 -0
- intentumdiff/wasm/adf_parser.wasm +0 -0
- intentumdiff/wasm/asciidoc_parser.wasm +0 -0
- intentumdiff/wasm/asm_parser.wasm +0 -0
- intentumdiff/wasm/assemblyscript_parser.wasm +0 -0
- intentumdiff/wasm/astro_parser.wasm +0 -0
- intentumdiff/wasm/bash_parser.wasm +0 -0
- intentumdiff/wasm/clojure_parser.wasm +0 -0
- intentumdiff/wasm/cmake_parser.wasm +0 -0
- intentumdiff/wasm/cpp_parser.wasm +0 -0
- intentumdiff/wasm/csharp_parser.wasm +0 -0
- intentumdiff/wasm/css_parser.wasm +0 -0
- intentumdiff/wasm/dart_parser.wasm +0 -0
- intentumdiff/wasm/databricks_parser.wasm +0 -0
- intentumdiff/wasm/dax_parser.wasm +0 -0
- intentumdiff/wasm/delphi_parser.wasm +0 -0
- intentumdiff/wasm/dockerfile_parser.wasm +0 -0
- intentumdiff/wasm/elixir_parser.wasm +0 -0
- intentumdiff/wasm/freebasic_parser.wasm +0 -0
- intentumdiff/wasm/generic_parser.wasm +0 -0
- intentumdiff/wasm/gitignore_parser.wasm +0 -0
- intentumdiff/wasm/go_parser.wasm +0 -0
- intentumdiff/wasm/gomod_parser.wasm +0 -0
- intentumdiff/wasm/graphql_parser.wasm +0 -0
- intentumdiff/wasm/groovy_parser.wasm +0 -0
- intentumdiff/wasm/haskell_parser.wasm +0 -0
- intentumdiff/wasm/html_parser.wasm +0 -0
- intentumdiff/wasm/html_renderer.wasm +0 -0
- intentumdiff/wasm/ini_parser.wasm +0 -0
- intentumdiff/wasm/java_parser.wasm +0 -0
- intentumdiff/wasm/js_ts_parser.wasm +0 -0
- intentumdiff/wasm/json_parser.wasm +0 -0
- intentumdiff/wasm/kotlin_parser.wasm +0 -0
- intentumdiff/wasm/latex_parser.wasm +0 -0
- intentumdiff/wasm/llm_renderer.wasm +0 -0
- intentumdiff/wasm/lua_parser.wasm +0 -0
- intentumdiff/wasm/make_parser.wasm +0 -0
- intentumdiff/wasm/markdown_parser.wasm +0 -0
- intentumdiff/wasm/mdx_parser.wasm +0 -0
- intentumdiff/wasm/ocaml_parser.wasm +0 -0
- intentumdiff/wasm/odin_parser.wasm +0 -0
- intentumdiff/wasm/parser_manifest.json +81 -2
- intentumdiff/wasm/patch_renderer.wasm +0 -0
- intentumdiff/wasm/perl_parser.wasm +0 -0
- intentumdiff/wasm/php_parser.wasm +0 -0
- intentumdiff/wasm/plsql_parser.wasm +0 -0
- intentumdiff/wasm/po_parser.wasm +0 -0
- intentumdiff/wasm/postscript_parser.wasm +0 -0
- intentumdiff/wasm/powershell_parser.wasm +0 -0
- intentumdiff/wasm/proto_parser.wasm +0 -0
- intentumdiff/wasm/puppet_parser.wasm +0 -0
- intentumdiff/wasm/python_parser.wasm +0 -0
- intentumdiff/wasm/qsharp_parser.wasm +0 -0
- intentumdiff/wasm/r_parser.wasm +0 -0
- intentumdiff/wasm/reasonml_parser.wasm +0 -0
- intentumdiff/wasm/ruby_parser.wasm +0 -0
- intentumdiff/wasm/rust_parser.wasm +0 -0
- intentumdiff/wasm/sas_parser.wasm +0 -0
- intentumdiff/wasm/scala_parser.wasm +0 -0
- intentumdiff/wasm/scss_parser.wasm +0 -0
- intentumdiff/wasm/sql_parser.wasm +0 -0
- intentumdiff/wasm/squirrel_parser.wasm +0 -0
- intentumdiff/wasm/svelte_parser.wasm +0 -0
- intentumdiff/wasm/swift_parser.wasm +0 -0
- intentumdiff/wasm/terminal_renderer.wasm +0 -0
- intentumdiff/wasm/terraform_parser.wasm +0 -0
- intentumdiff/wasm/toml_parser.wasm +0 -0
- intentumdiff/wasm/tsql_parser.wasm +0 -0
- intentumdiff/wasm/vbnet_parser.wasm +0 -0
- intentumdiff/wasm/vue_parser.wasm +0 -0
- intentumdiff/wasm/wasm_provenance.json +884 -0
- intentumdiff/wasm/wat_parser.wasm +0 -0
- intentumdiff/wasm/xml_parser.wasm +0 -0
- intentumdiff/wasm/yaml_parser.wasm +0 -0
- intentumdiff/wasm/zig_parser.wasm +0 -0
- intentumdiff_python-0.0.2b2.dist-info/METADATA +156 -0
- intentumdiff_python-0.0.2b2.dist-info/RECORD +171 -0
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/WHEEL +1 -1
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/sboms/intentumdiff-rust-core.cyclonedx.json +905 -379
- intentumdiff/core/cst_serializer.py +0 -101
- intentumdiff_python-0.0.1.dist-info/METADATA +0 -112
- intentumdiff_python-0.0.1.dist-info/RECORD +0 -165
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/entry_points.txt +0 -0
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/licenses/LICENSE +0 -0
intentumdiff/core/index.py
CHANGED
|
@@ -17,9 +17,7 @@ repository snapshot (or any other collection of files). It provides:
|
|
|
17
17
|
The heavy lifting (symbol-table + reference-table extraction) is delegated to
|
|
18
18
|
the Rust core (``index-engine-lib``, via the native ``build_symbol_table_json``
|
|
19
19
|
/ ``build_reference_table_json`` entrypoints). Rust is authoritative — the
|
|
20
|
-
former pure-Python extraction mirror was deleted (#91);
|
|
21
|
-
tables are simply empty ("if Python didn't exist, the engine still lives in
|
|
22
|
-
Rust"). Cross-file diffing lives in ``intentumdiff.analysis.cross_file``, also a
|
|
20
|
+
former pure-Python extraction mirror was deleted (#91); engine failures raise instead of manufacturing empty tables. Cross-file diffing lives in ``intentumdiff.analysis.cross_file``, also a
|
|
23
21
|
thin wrapper over the same Rust core.
|
|
24
22
|
"""
|
|
25
23
|
|
|
@@ -81,24 +79,23 @@ class SemanticIndex:
|
|
|
81
79
|
Rust-authoritative (#91): extraction runs in the native
|
|
82
80
|
``build_symbol_table_json`` / ``build_reference_table_json`` entrypoints
|
|
83
81
|
(the same code the index-engine Wasm plugin wraps). The former Python
|
|
84
|
-
extraction mirror was deleted.
|
|
85
|
-
|
|
86
|
-
engine still lives in Rust".
|
|
82
|
+
extraction mirror was deleted. Required engine failures raise. Both tables are published atomically
|
|
83
|
+
only after successful engine calls and DTO validation.
|
|
87
84
|
"""
|
|
88
85
|
from intentumdiff.rust_core import (
|
|
89
86
|
try_rust_build_reference_table,
|
|
90
87
|
try_rust_build_symbol_table,
|
|
91
88
|
)
|
|
92
89
|
|
|
93
|
-
self._symbols = {}
|
|
94
|
-
self._references = {}
|
|
95
90
|
files_json = self.to_files_json()
|
|
96
91
|
symbol_json = try_rust_build_symbol_table(files_json)
|
|
97
|
-
if symbol_json is not None:
|
|
98
|
-
self.load_symbol_table_json(symbol_json)
|
|
99
92
|
reference_json = try_rust_build_reference_table(files_json)
|
|
100
|
-
|
|
101
|
-
|
|
93
|
+
# Validate DTOs on a temporary index before publishing either table.
|
|
94
|
+
candidate = SemanticIndex()
|
|
95
|
+
candidate.load_symbol_table_json(symbol_json)
|
|
96
|
+
candidate.load_reference_table_json(reference_json)
|
|
97
|
+
self._symbols = candidate._symbols
|
|
98
|
+
self._references = candidate._references
|
|
102
99
|
self._built = True
|
|
103
100
|
return self
|
|
104
101
|
|
|
@@ -144,12 +141,13 @@ class SemanticIndex:
|
|
|
144
141
|
refs = self._references.get(name, [])
|
|
145
142
|
if not resolve:
|
|
146
143
|
return refs
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
144
|
+
from intentumdiff.rust_core import _c_abi_call
|
|
145
|
+
result = _c_abi_call(
|
|
146
|
+
"resolve_references",
|
|
147
|
+
json.dumps([definition.model_dump(mode="json") for definition in self._symbols.get(name, [])]),
|
|
148
|
+
json.dumps([reference.model_dump(mode="json") for reference in refs]),
|
|
149
|
+
)
|
|
150
|
+
return [ReferenceUsage.model_validate(reference) for reference in result]
|
|
153
151
|
|
|
154
152
|
# ------------------------------------------------------------------
|
|
155
153
|
# Serialisation helpers (feed the native index-engine primitives)
|
intentumdiff/core/indexer.py
CHANGED
|
@@ -30,17 +30,11 @@ Pass a no-op lambda to suppress events for a specific call::
|
|
|
30
30
|
|
|
31
31
|
Caching
|
|
32
32
|
-------
|
|
33
|
-
When the
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
the Wasm call entirely.
|
|
39
|
-
|
|
40
|
-
* **Symbol-index cache** — after :meth:`Indexer.index_repo` finishes, the
|
|
41
|
-
resulting symbol and reference tables are stored in SQLite keyed by
|
|
42
|
-
``(repo_root, commit_sha)``. A subsequent call for the *same* commit returns
|
|
43
|
-
the cached index immediately (unless *force=True*).
|
|
33
|
+
When ``DiffConfig.cache_path`` is set, the verified parser-level cache remains
|
|
34
|
+
available through ``SemanticDiffer``. The historical repository symbol-index
|
|
35
|
+
cache and LSP hover-map cache are bypassed in 0.0.2: their keys do not capture
|
|
36
|
+
Rust/parser identity, indexing completeness, or LSP workspace/server state.
|
|
37
|
+
Returning a stale index is worse than recomputing it. See Python #189-192.
|
|
44
38
|
"""
|
|
45
39
|
|
|
46
40
|
from __future__ import annotations
|
|
@@ -53,6 +47,7 @@ from pathlib import Path
|
|
|
53
47
|
from typing import TYPE_CHECKING, Any, Callable
|
|
54
48
|
|
|
55
49
|
from intentumdiff.core.diffignore import load_diffignore
|
|
50
|
+
from intentumdiff.rust_core import attach_lsp_type_map
|
|
56
51
|
|
|
57
52
|
if TYPE_CHECKING:
|
|
58
53
|
from intentumdiff.core.index import SemanticIndex
|
|
@@ -164,41 +159,62 @@ def _make_index_key(repo_root: str, commit_sha: str) -> str:
|
|
|
164
159
|
return hashlib.sha256(encoded).hexdigest()
|
|
165
160
|
|
|
166
161
|
|
|
167
|
-
def _parse_catfile_output(
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
162
|
+
def _parse_catfile_output(buf: bytes) -> dict[str, str | None]:
|
|
163
|
+
"""Decode complete Git batch responses; None marks an intentionally skipped binary.
|
|
164
|
+
|
|
165
|
+
A missing object, bad header, or truncated content is not a binary file.
|
|
166
|
+
Such failures must abort indexing instead of silently dropping source files.
|
|
167
|
+
"""
|
|
168
|
+
blobs: dict[str, str | None] = {}
|
|
173
169
|
i = 0
|
|
174
170
|
n = len(buf)
|
|
175
171
|
while i < n:
|
|
176
172
|
nl = buf.find(b"\n", i)
|
|
177
173
|
if nl == -1:
|
|
178
|
-
|
|
179
|
-
|
|
174
|
+
raise RuntimeError("Incomplete git cat-file batch response header")
|
|
175
|
+
try:
|
|
176
|
+
header = buf[i:nl].decode("ascii", errors="strict")
|
|
177
|
+
except UnicodeDecodeError as exc:
|
|
178
|
+
raise RuntimeError("Invalid git cat-file batch response header") from exc
|
|
180
179
|
i = nl + 1
|
|
181
180
|
parts = header.split()
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
181
|
+
if len(parts) == 2 and parts[1] == "missing":
|
|
182
|
+
raise RuntimeError(f"Git blob {parts[0]} is missing")
|
|
183
|
+
if len(parts) != 3 or parts[1] != "blob":
|
|
184
|
+
raise RuntimeError(f"Unexpected git cat-file batch response: {header!r}")
|
|
185
|
+
sha = parts[0]
|
|
186
|
+
if sha in blobs:
|
|
187
|
+
raise RuntimeError(f"Duplicate git cat-file batch response for {sha}")
|
|
185
188
|
try:
|
|
186
189
|
size = int(parts[2])
|
|
187
|
-
except ValueError:
|
|
188
|
-
|
|
190
|
+
except ValueError as exc:
|
|
191
|
+
raise RuntimeError(f"Invalid git cat-file batch size for {sha}") from exc
|
|
192
|
+
if size < 0 or i + size >= n or buf[i + size] != 0x0A:
|
|
193
|
+
raise RuntimeError(f"Truncated git cat-file batch payload for {sha}")
|
|
189
194
|
content_bytes = buf[i : i + size]
|
|
190
|
-
i += size + 1 #
|
|
191
|
-
if b"\x00" in content_bytes[:8192]:
|
|
195
|
+
i += size + 1 # consume the required trailing newline
|
|
196
|
+
if b"\x00" in content_bytes[:8192]:
|
|
197
|
+
blobs[sha] = None # binary: intentionally excluded from text indexing
|
|
192
198
|
continue
|
|
193
|
-
|
|
194
|
-
|
|
199
|
+
|
|
200
|
+
# Some recognized binaries (for example PDFs) contain no NUL bytes.
|
|
201
|
+
# Use the Rust-owned content classifier on the original blob before
|
|
202
|
+
# decoding; otherwise their binary payload aborts the entire Git index
|
|
203
|
+
# as "invalid UTF-8" (#253). An opaque octet-stream is *not* proof
|
|
204
|
+
# that a source file is safely skippable: still decode it strictly.
|
|
205
|
+
from intentumdiff.content_type import detect_content_type # noqa: PLC0415
|
|
206
|
+
|
|
207
|
+
content_type = detect_content_type(content_bytes)
|
|
208
|
+
if not content_type["is_text"] and content_type["mime"] != "application/octet-stream":
|
|
209
|
+
blobs[sha] = None
|
|
195
210
|
continue
|
|
196
211
|
try:
|
|
197
|
-
|
|
198
|
-
except UnicodeDecodeError:
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
212
|
+
blobs[sha] = content_bytes.decode("utf-8", errors="strict")
|
|
213
|
+
except UnicodeDecodeError as exc:
|
|
214
|
+
raise RuntimeError(
|
|
215
|
+
f"Invalid UTF-8 in Git blob {sha}: refusing lossy symbol indexing"
|
|
216
|
+
) from exc
|
|
217
|
+
return blobs
|
|
202
218
|
|
|
203
219
|
|
|
204
220
|
def _collect_blob_files(
|
|
@@ -206,61 +222,77 @@ def _collect_blob_files(
|
|
|
206
222
|
ref: str,
|
|
207
223
|
diffignore: Any,
|
|
208
224
|
) -> "tuple[list[tuple[str, str]], list[str]]":
|
|
209
|
-
"""Return
|
|
225
|
+
"""Return every non-ignored text path at *ref*, including shared-content blobs.
|
|
210
226
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
Returns ``(files, ignored_files)`` where *ignored_files* are the paths excluded
|
|
216
|
-
by *diffignore*.
|
|
227
|
+
Git stores identical file contents under one blob SHA. Batch-read each unique
|
|
228
|
+
SHA once, then expand that content against the original ls-tree path list;
|
|
229
|
+
collapsing SHA -> one path silently drops files from semantic indexing.
|
|
217
230
|
"""
|
|
218
231
|
from intentumdiff.vcs.git_cli import run_git_bytes # noqa: PLC0415
|
|
219
232
|
|
|
220
233
|
ignored_files: list[str] = []
|
|
221
|
-
|
|
222
|
-
#
|
|
223
|
-
|
|
234
|
+
paths_in_tree_order: list[tuple[str, str]] = []
|
|
235
|
+
# Git's -z output always terminates every record with NUL. Validating
|
|
236
|
+
# the later cat-file blobs cannot discover paths lost from this inventory.
|
|
237
|
+
tree_bytes = run_git_bytes(repo_root, ["ls-tree", "-r", "-z", ref])
|
|
238
|
+
if tree_bytes and not tree_bytes.endswith(b"\x00"):
|
|
239
|
+
raise RuntimeError("Incomplete git ls-tree -z output: missing NUL delimiter")
|
|
240
|
+
for entry in tree_bytes.split(b"\x00"):
|
|
224
241
|
if not entry:
|
|
225
242
|
continue
|
|
226
|
-
meta,
|
|
243
|
+
meta, separator, raw_path = entry.partition(b"\t")
|
|
227
244
|
parts = meta.split()
|
|
228
|
-
if
|
|
245
|
+
if separator != b"\t" or not raw_path or len(parts) != 3:
|
|
246
|
+
raise RuntimeError(f"Malformed git ls-tree entry: {entry[:128]!r}")
|
|
247
|
+
if parts[1] == b"commit":
|
|
248
|
+
# Gitlinks are submodules, not readable blobs at this tree.
|
|
229
249
|
continue
|
|
230
|
-
|
|
250
|
+
if parts[1] != b"blob":
|
|
251
|
+
raise RuntimeError(f"Unexpected git ls-tree object type: {parts[1]!r}")
|
|
252
|
+
# Git may store arbitrary filename bytes on Unix. Replacing undecodable
|
|
253
|
+
# bytes invents a different path and can collapse distinct files (#238).
|
|
254
|
+
try:
|
|
255
|
+
path = raw_path.decode("utf-8", errors="strict")
|
|
256
|
+
except UnicodeDecodeError as exc:
|
|
257
|
+
raise RuntimeError(
|
|
258
|
+
"Invalid UTF-8 in git ls-tree path: refusing lossy indexing"
|
|
259
|
+
) from exc
|
|
260
|
+
try:
|
|
261
|
+
sha = parts[2].decode("ascii", errors="strict")
|
|
262
|
+
except UnicodeDecodeError as exc:
|
|
263
|
+
raise RuntimeError(
|
|
264
|
+
f"Invalid ASCII object ID in git ls-tree for {path!r}"
|
|
265
|
+
) from exc
|
|
231
266
|
if diffignore and diffignore.is_ignored(path):
|
|
232
267
|
ignored_files.append(path)
|
|
233
268
|
continue
|
|
234
|
-
|
|
269
|
+
paths_in_tree_order.append((path, sha))
|
|
235
270
|
|
|
236
|
-
if not
|
|
271
|
+
if not paths_in_tree_order:
|
|
237
272
|
return [], ignored_files
|
|
238
273
|
|
|
239
|
-
|
|
274
|
+
# The caller needs all paths, but cat-file needs only one request per blob.
|
|
275
|
+
shas_input = "\n".join(dict.fromkeys(sha for _, sha in paths_in_tree_order)).encode()
|
|
240
276
|
out = run_git_bytes(repo_root, ["cat-file", "--batch"], input_bytes=shas_input)
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
"""Return a copy of *node* tree with ``type_info`` set from *type_map*.
|
|
249
|
-
|
|
250
|
-
``SemanticNode`` is frozen so we must rebuild changed nodes from the leaves
|
|
251
|
-
up. Only nodes whose id appears in *type_map* are rebuilt.
|
|
252
|
-
"""
|
|
253
|
-
from intentumdiff.core.models import SemanticNode # noqa: PLC0415
|
|
254
|
-
|
|
255
|
-
new_children = [_attach_type_info(c, type_map) for c in node.children]
|
|
256
|
-
if node.id in type_map or new_children != list(node.children):
|
|
257
|
-
return node.model_copy(
|
|
258
|
-
update={
|
|
259
|
-
"children": new_children,
|
|
260
|
-
**({"type_info": type_map[node.id]} if node.id in type_map else {}),
|
|
261
|
-
}
|
|
277
|
+
decoded_by_sha = _parse_catfile_output(out)
|
|
278
|
+
requested = set(sha for _, sha in paths_in_tree_order)
|
|
279
|
+
missing = requested.difference(decoded_by_sha)
|
|
280
|
+
if missing:
|
|
281
|
+
raise RuntimeError(
|
|
282
|
+
"Incomplete git cat-file batch response: missing blob(s) "
|
|
283
|
+
+ ", ".join(sorted(missing)[:5])
|
|
262
284
|
)
|
|
263
|
-
|
|
285
|
+
unexpected = set(decoded_by_sha).difference(requested)
|
|
286
|
+
if unexpected:
|
|
287
|
+
raise RuntimeError(
|
|
288
|
+
"Unexpected git cat-file batch blob(s): " + ", ".join(sorted(unexpected)[:5])
|
|
289
|
+
)
|
|
290
|
+
files: list[tuple[str, str]] = []
|
|
291
|
+
for path, sha in paths_in_tree_order:
|
|
292
|
+
content = decoded_by_sha[sha]
|
|
293
|
+
if content is not None: # only known binary blobs may be omitted
|
|
294
|
+
files.append((path, content))
|
|
295
|
+
return files, ignored_files
|
|
264
296
|
|
|
265
297
|
|
|
266
298
|
# ---------------------------------------------------------------------------
|
|
@@ -363,8 +395,8 @@ class Indexer:
|
|
|
363
395
|
Progress callback for this call. Falls back to the callback
|
|
364
396
|
passed to ``__init__`` when ``None``.
|
|
365
397
|
force:
|
|
366
|
-
|
|
367
|
-
|
|
398
|
+
Retained for API compatibility. Indexing always recomputes the
|
|
399
|
+
symbol index in 0.0.2 (the unproven cache is disabled).
|
|
368
400
|
|
|
369
401
|
Returns
|
|
370
402
|
-------
|
|
@@ -372,41 +404,13 @@ class Indexer:
|
|
|
372
404
|
Fully built index. Check :attr:`IndexResult.from_cache` to
|
|
373
405
|
determine whether the symbol table was loaded from cache.
|
|
374
406
|
"""
|
|
375
|
-
from intentumdiff.vcs.git_cli import
|
|
376
|
-
resolve_repo_root,
|
|
377
|
-
run_git_bytes,
|
|
378
|
-
)
|
|
407
|
+
from intentumdiff.vcs.git_cli import resolve_repo_root # noqa: PLC0415
|
|
379
408
|
|
|
380
409
|
repo_root = resolve_repo_root(repo_path)
|
|
381
|
-
commit_sha = (
|
|
382
|
-
run_git_bytes(repo_root, ["rev-parse", "--verify", ref]).decode().strip()
|
|
383
|
-
)
|
|
384
410
|
callback = on_progress or self._on_progress
|
|
385
411
|
|
|
386
|
-
#
|
|
387
|
-
|
|
388
|
-
index_key = _make_index_key(repo_root, commit_sha)
|
|
389
|
-
cached = self._differ._cache.get_symbol_index(index_key)
|
|
390
|
-
if cached is not None:
|
|
391
|
-
logger.debug(
|
|
392
|
-
"Symbol index cache hit for %s @ %s",
|
|
393
|
-
repo_root,
|
|
394
|
-
commit_sha[:8],
|
|
395
|
-
)
|
|
396
|
-
from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
|
|
397
|
-
|
|
398
|
-
sem_index = SemanticIndex()
|
|
399
|
-
symbols_json, refs_json = cached
|
|
400
|
-
sem_index.load_symbol_table_json(symbols_json)
|
|
401
|
-
sem_index.load_reference_table_json(refs_json)
|
|
402
|
-
return IndexResult(
|
|
403
|
-
files_indexed=0,
|
|
404
|
-
files_skipped=0,
|
|
405
|
-
errors=[],
|
|
406
|
-
semantic_index=sem_index,
|
|
407
|
-
from_cache=True,
|
|
408
|
-
)
|
|
409
|
-
|
|
412
|
+
# Always re-index: historical symbol-cache entries lack Rust/parser
|
|
413
|
+
# identity and failed-file provenance (#189, #190).
|
|
410
414
|
# ── Collect all text files in commit tree ─────────────────────────
|
|
411
415
|
diffignore = load_diffignore(repo_root)
|
|
412
416
|
files, ignored_files = _collect_blob_files(repo_root, ref, diffignore)
|
|
@@ -415,13 +419,7 @@ class Indexer:
|
|
|
415
419
|
result.files_ignored = len(ignored_files)
|
|
416
420
|
result.ignored_files = ignored_files
|
|
417
421
|
|
|
418
|
-
#
|
|
419
|
-
if self._differ._cache is not None:
|
|
420
|
-
index_key = _make_index_key(repo_root, commit_sha)
|
|
421
|
-
self._store_symbol_index(
|
|
422
|
-
result.semantic_index, index_key, result.files_indexed
|
|
423
|
-
)
|
|
424
|
-
|
|
422
|
+
# Do not persist an index whose provenance/completeness cannot be verified.
|
|
425
423
|
return result
|
|
426
424
|
|
|
427
425
|
def index_directory(
|
|
@@ -448,6 +446,8 @@ class Indexer:
|
|
|
448
446
|
directory = Path(directory)
|
|
449
447
|
callback = on_progress or self._on_progress
|
|
450
448
|
|
|
449
|
+
from intentumdiff.content_type import detect_content_type # noqa: PLC0415
|
|
450
|
+
|
|
451
451
|
diffignore = load_diffignore(directory)
|
|
452
452
|
ignored_files: list[str] = []
|
|
453
453
|
files: list[tuple[str, str]] = []
|
|
@@ -459,9 +459,25 @@ class Indexer:
|
|
|
459
459
|
ignored_files.append(rel_posix)
|
|
460
460
|
continue
|
|
461
461
|
try:
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
462
|
+
raw = path.read_bytes()
|
|
463
|
+
# Rust owns content-type classification. A confirmed binary asset
|
|
464
|
+
# is not a source file and must not be UTF-8-decoded (#250).
|
|
465
|
+
content_type = detect_content_type(raw)
|
|
466
|
+
if not content_type["is_text"]:
|
|
467
|
+
if content_type["mime"] == "application/octet-stream" and b"\x00" not in raw:
|
|
468
|
+
# Opaque bytes without a recognized format are not proof
|
|
469
|
+
# that a text source is absent. Reject invalid UTF-8 rather
|
|
470
|
+
# than allowing corrupt source to disappear from the index.
|
|
471
|
+
raw.decode("utf-8", errors="strict")
|
|
472
|
+
continue
|
|
473
|
+
content = raw.decode("utf-8", errors="strict")
|
|
474
|
+
except (OSError, UnicodeError) as exc:
|
|
475
|
+
# Host-I/O and decoding failures are not intentional ignores or unsupported
|
|
476
|
+
# parser. Returning a successful partial directory index would
|
|
477
|
+
# conceal source files from downstream semantic analysis (#203).
|
|
478
|
+
raise RuntimeError(
|
|
479
|
+
f"Failed to read directory index source {rel_posix!r}: {exc}"
|
|
480
|
+
) from exc
|
|
465
481
|
files.append((rel_posix, content))
|
|
466
482
|
|
|
467
483
|
result = self._index_files(files, on_progress=callback)
|
|
@@ -589,44 +605,13 @@ class Indexer:
|
|
|
589
605
|
for fast local servers, lower for shared or rate-limited servers.
|
|
590
606
|
"""
|
|
591
607
|
import asyncio
|
|
592
|
-
from intentumdiff.vcs.git_cli import
|
|
593
|
-
resolve_repo_root,
|
|
594
|
-
run_git_bytes,
|
|
595
|
-
)
|
|
608
|
+
from intentumdiff.vcs.git_cli import resolve_repo_root # noqa: PLC0415
|
|
596
609
|
|
|
597
610
|
repo_root = resolve_repo_root(repo_path)
|
|
598
|
-
commit_sha = (
|
|
599
|
-
run_git_bytes(repo_root, ["rev-parse", "--verify", ref]).decode().strip()
|
|
600
|
-
)
|
|
601
611
|
callback = on_progress or self._on_progress
|
|
602
612
|
|
|
603
|
-
#
|
|
604
|
-
#
|
|
605
|
-
lsp_index_key = _make_index_key(repo_root, "lsp:" + commit_sha)
|
|
606
|
-
|
|
607
|
-
# ── Fast path: cached LSP symbol index ────────────────────────────
|
|
608
|
-
if not force and self._differ._cache is not None:
|
|
609
|
-
cached = self._differ._cache.get_symbol_index(lsp_index_key)
|
|
610
|
-
if cached is not None:
|
|
611
|
-
logger.debug(
|
|
612
|
-
"LSP symbol index cache hit for %s @ %s",
|
|
613
|
-
repo_root,
|
|
614
|
-
commit_sha[:8],
|
|
615
|
-
)
|
|
616
|
-
from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
|
|
617
|
-
|
|
618
|
-
sem_index = SemanticIndex()
|
|
619
|
-
symbols_json, refs_json = cached
|
|
620
|
-
sem_index.load_symbol_table_json(symbols_json)
|
|
621
|
-
sem_index.load_reference_table_json(refs_json)
|
|
622
|
-
return IndexResult(
|
|
623
|
-
files_indexed=0,
|
|
624
|
-
files_skipped=0,
|
|
625
|
-
errors=[],
|
|
626
|
-
semantic_index=sem_index,
|
|
627
|
-
from_cache=True,
|
|
628
|
-
)
|
|
629
|
-
|
|
613
|
+
# Always re-index: LSP type state, parser identity and failed-file
|
|
614
|
+
# provenance are missing from historical cache keys (#189, #190).
|
|
630
615
|
# Collect files (parallel blob reads; same filtering as index_repo)
|
|
631
616
|
diffignore = load_diffignore(repo_root)
|
|
632
617
|
files, ignored_files = await asyncio.to_thread(
|
|
@@ -642,9 +627,7 @@ class Indexer:
|
|
|
642
627
|
result.files_ignored = len(ignored_files)
|
|
643
628
|
result.ignored_files = ignored_files
|
|
644
629
|
|
|
645
|
-
|
|
646
|
-
self._store_symbol_index(result.semantic_index, lsp_index_key, result.files_indexed)
|
|
647
|
-
|
|
630
|
+
# An LSP-enriched index is not reusable without server/workspace identity.
|
|
648
631
|
return result
|
|
649
632
|
|
|
650
633
|
async def _index_files_lsp(
|
|
@@ -684,7 +667,6 @@ class Indexer:
|
|
|
684
667
|
# Error: any other exception
|
|
685
668
|
|
|
686
669
|
sem = asyncio.Semaphore(max(1, file_concurrency))
|
|
687
|
-
cache = self._differ._cache
|
|
688
670
|
differ = self._differ # captured for executor
|
|
689
671
|
|
|
690
672
|
# Use a single-worker executor so Wasm parse calls are serialised.
|
|
@@ -725,22 +707,20 @@ class Indexer:
|
|
|
725
707
|
enricher = enrichers.get(language)
|
|
726
708
|
enriched_count = 0
|
|
727
709
|
if enricher is not None:
|
|
728
|
-
#
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
enriched_count = len(type_map)
|
|
743
|
-
node = _attach_type_info(node, type_map)
|
|
710
|
+
# Hover-map caching lacks file/workspace/server provenance
|
|
711
|
+
# and cannot certify a type result (#191).
|
|
712
|
+
try:
|
|
713
|
+
type_map = await enricher.enrich(filename, content, node)
|
|
714
|
+
if not isinstance(type_map, dict):
|
|
715
|
+
raise TypeError("LSP enricher returned an invalid type-map DTO")
|
|
716
|
+
if type_map:
|
|
717
|
+
# Rust owns semantic-tree mutation and validation.
|
|
718
|
+
node = attach_lsp_type_map(node, type_map)
|
|
719
|
+
enriched_count = len(type_map)
|
|
720
|
+
except Exception as exc: # noqa: BLE001
|
|
721
|
+
# Do not turn a requested enrichment failure into an
|
|
722
|
+
# apparently successful bare semantic index (#192).
|
|
723
|
+
return filename, "ERROR", exc, 0
|
|
744
724
|
return filename, language, node, enriched_count
|
|
745
725
|
|
|
746
726
|
tasks = [asyncio.create_task(_process_one(f, c)) for f, c in files]
|
intentumdiff/core/models.py
CHANGED
|
@@ -78,8 +78,8 @@ class DetectionResult(BaseModel, frozen=True):
|
|
|
78
78
|
ge=0.0,
|
|
79
79
|
le=1.0,
|
|
80
80
|
description=(
|
|
81
|
-
"
|
|
82
|
-
"1.0 for the
|
|
81
|
+
"Reciprocal rank from Rust content detection order, rounded to three decimals. "
|
|
82
|
+
"1.0 for the first match; this is not a measured probability."
|
|
83
83
|
),
|
|
84
84
|
)
|
|
85
85
|
|
|
@@ -123,14 +123,23 @@ class LanguageInfoGroup(BaseModel, frozen=True):
|
|
|
123
123
|
|
|
124
124
|
|
|
125
125
|
class NodeFacts(BaseModel, frozen=True):
|
|
126
|
+
# Unknown Rust-authored facts must fail validation rather than disappear
|
|
127
|
+
# through Pydantic's extra=ignore (0.0.2 thin-wrapper contract, #226).
|
|
128
|
+
model_config = ConfigDict(extra="forbid")
|
|
129
|
+
|
|
126
130
|
"""
|
|
127
|
-
Privacy-safe structural facts about
|
|
131
|
+
Privacy-safe structural facts about an entity or keyed/resource node,
|
|
128
132
|
computed by the Rust parser from the full CST before pruning. Only counts,
|
|
129
133
|
enums, and flags — never source text, identifiers, or literals — so they can
|
|
130
134
|
describe a change's shape ("no parameters", "returns nothing", "no-op body")
|
|
131
135
|
without revealing code. The host carries this through unchanged.
|
|
132
136
|
"""
|
|
133
137
|
|
|
138
|
+
#: Which derivation passes produced these facts, e.g. ``"cst,enrich(derived=14,added=5)"``.
|
|
139
|
+
#: Populated only when ``INTENTUMDIFF_TRACE_FACTS=1``; ``None`` in normal operation.
|
|
140
|
+
#: Diagnostic provenance, never source — safe to paste into a bug report.
|
|
141
|
+
facts_trace: str | None = None
|
|
142
|
+
|
|
134
143
|
param_count: int | None = None
|
|
135
144
|
#: ``"none"`` | ``"value"`` | ``"literal"`` (constant) | ``"unknown"``.
|
|
136
145
|
returns: str | None = None
|
|
@@ -148,6 +157,20 @@ class NodeFacts(BaseModel, frozen=True):
|
|
|
148
157
|
has_error_handling: bool | None = None
|
|
149
158
|
#: The body raises/throws an exception. Flag only.
|
|
150
159
|
throws: bool | None = None
|
|
160
|
+
#: Cross-language structural facts, derived from the pruned canonical tree (#9) rather
|
|
161
|
+
#: than from any one grammar — so they are available for EVERY language, not just the
|
|
162
|
+
#: ones with a bespoke fact extractor.
|
|
163
|
+
#:
|
|
164
|
+
#: These are counts and flags only, in keeping with the rest of NodeFacts: no source, no
|
|
165
|
+
#: identifiers, no literals.
|
|
166
|
+
#:
|
|
167
|
+
#: `has_guard_clause` is deliberately tri-state. The pruned tree can prove a guard is
|
|
168
|
+
#: present, but cannot prove one is absent — statement order survives pruning, operators
|
|
169
|
+
#: do not. So `None` means "not determinable here", NOT "no guard". Omitting beats
|
|
170
|
+
#: claiming; see uast.rs for the reasoning.
|
|
171
|
+
early_exit_count: int | None = None
|
|
172
|
+
negated_condition_count: int | None = None
|
|
173
|
+
has_guard_clause: bool | None = None
|
|
151
174
|
#: The body mutates state — an augmented assignment or an assignment to an
|
|
152
175
|
#: attribute/subscript (``self.x = …``, ``a[i] = …``). Flag only, no target name.
|
|
153
176
|
mutates: bool | None = None
|
|
@@ -195,6 +218,20 @@ class NodeFacts(BaseModel, frozen=True):
|
|
|
195
218
|
call_count: int | None = None
|
|
196
219
|
recursive: bool | None = None
|
|
197
220
|
|
|
221
|
+
# Rust-owned keyed/collection/resource NodeFacts (#179, Core #203,
|
|
222
|
+
# Python #226). These are DTO fields only: the Python host must preserve
|
|
223
|
+
# Rust's privacy-safe counts, flags and shape categories without deriving
|
|
224
|
+
# them or silently dropping them through Pydantic's extra=ignore.
|
|
225
|
+
shape: str | None = None
|
|
226
|
+
child_count: int | None = None
|
|
227
|
+
value_kind: str | None = None
|
|
228
|
+
item_count: int | None = None
|
|
229
|
+
nesting: int | None = None
|
|
230
|
+
is_leaf: bool | None = None
|
|
231
|
+
attribute_count: int | None = None
|
|
232
|
+
nested_block_count: int | None = None
|
|
233
|
+
has_nested_block: bool | None = None
|
|
234
|
+
|
|
198
235
|
|
|
199
236
|
class SemanticNode(BaseModel, frozen=True):
|
|
200
237
|
"""
|
|
@@ -401,14 +438,12 @@ class ChangeGroup(BaseModel, frozen=True):
|
|
|
401
438
|
class ChangeStreamEvent(BaseModel, frozen=True):
|
|
402
439
|
"""A single event emitted by ``SemanticDiffer.diff_stream_progressive``.
|
|
403
440
|
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
441
|
+
The 0.0.2 Python adapter emits FINAL add events only, after Rust has
|
|
442
|
+
produced a complete diff. STRUCTURAL, REFINED, revise and remove remain in
|
|
443
|
+
the DTO schema for compatibility but are not synthesized by Python.
|
|
407
444
|
|
|
408
|
-
``replaced_ids``
|
|
409
|
-
|
|
410
|
-
consumer should look up prior events by these IDs and replace them. When
|
|
411
|
-
``action="remove"`` there is no replacement — the change was absorbed.
|
|
445
|
+
``replaced_ids`` carries identifiers supplied by a backend for revising or
|
|
446
|
+
removing earlier events; the 0.0.2 adapter leaves it empty.
|
|
412
447
|
"""
|
|
413
448
|
|
|
414
449
|
phase: ChangeStreamPhase
|
|
@@ -570,6 +605,11 @@ class DiffConfig(BaseModel):
|
|
|
570
605
|
|
|
571
606
|
Unlike result models, this is NOT frozen — callers may update it freely
|
|
572
607
|
before passing to ``SemanticDiffer``.
|
|
608
|
+
|
|
609
|
+
Compatibility-only fields in 0.0.2: ``ignore_style``, ``detect_refactorings``,
|
|
610
|
+
``approx_move_threshold``, ``leaf_text_diff`` and ``extra_trivia_types``.
|
|
611
|
+
Rust accepts their existing defaults but rejects unsupported overrides.
|
|
612
|
+
Configurable semantics are tracked in Core issues #220–#224.
|
|
573
613
|
"""
|
|
574
614
|
|
|
575
615
|
# GumTree tuning
|
|
@@ -697,11 +737,10 @@ class DiffConfig(BaseModel):
|
|
|
697
737
|
``SemanticDiff`` will have ``is_fallback=True``."""
|
|
698
738
|
|
|
699
739
|
stream_analysis: bool = False
|
|
700
|
-
"""
|
|
740
|
+
"""Legacy stream-analysis configuration flag.
|
|
701
741
|
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
rather than waiting for the entire pipeline to finish."""
|
|
742
|
+
In 0.0.2, ``diff_stream_progressive`` only adapts a completed Rust diff to
|
|
743
|
+
FINAL add events. This flag does not enable partial Rust phase snapshots."""
|
|
705
744
|
|
|
706
745
|
|
|
707
746
|
# ---------------------------------------------------------------------------
|