intentumdiff-python 0.0.1__py3-none-win_amd64.whl → 0.0.2b2__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. intentumdiff/__init__.py +2 -0
  2. intentumdiff/__main__.py +12 -0
  3. intentumdiff/_differ_presentation.py +35 -682
  4. intentumdiff/_differ_runtime.py +13 -47
  5. intentumdiff/analysis/compile_commands.py +26 -141
  6. intentumdiff/analysis/cross_file.py +2 -4
  7. intentumdiff/analysis/guardrail_reports.py +7 -5
  8. intentumdiff/analysis/guardrails.py +110 -122
  9. intentumdiff/analysis/invariances.py +4 -1003
  10. intentumdiff/analysis/schema_resolver.py +49 -273
  11. intentumdiff/analysis/text_review.py +4 -3
  12. intentumdiff/analysis/user_schemas.py +55 -237
  13. intentumdiff/api.py +21 -0
  14. intentumdiff/cache/sqlite_store.py +5 -23
  15. intentumdiff/cli/__init__.py +59 -0
  16. intentumdiff/cli/_commands.py +174 -67
  17. intentumdiff/cli/_parser.py +11 -1
  18. intentumdiff/cli/_shared.py +96 -112
  19. intentumdiff/content_type.py +13 -30
  20. intentumdiff/core/commit_differ.py +105 -220
  21. intentumdiff/core/diffignore.py +28 -34
  22. intentumdiff/core/index.py +16 -18
  23. intentumdiff/core/indexer.py +148 -168
  24. intentumdiff/core/models.py +53 -14
  25. intentumdiff/differ.py +278 -605
  26. intentumdiff/github_app.py +35 -19
  27. intentumdiff/intentumdiff_rust_core/intentumdiff_rust_core.dll +0 -0
  28. intentumdiff/live_server.py +180 -17
  29. intentumdiff/lsp/client.py +7 -1
  30. intentumdiff/lsp/enricher.py +19 -119
  31. intentumdiff/lsp/exceptions.py +6 -7
  32. intentumdiff/lsp_server/_codelens.py +24 -57
  33. intentumdiff/lsp_server/_diagnostics.py +23 -51
  34. intentumdiff/lsp_server/_handlers.py +66 -8
  35. intentumdiff/lsp_server/server.py +28 -7
  36. intentumdiff/plugins/adapter.py +5 -0
  37. intentumdiff/plugins/builtins.py +7 -14
  38. intentumdiff/plugins/exceptions.py +4 -0
  39. intentumdiff/plugins/hub.py +1 -2
  40. intentumdiff/plugins/loader.py +229 -215
  41. intentumdiff/plugins/registry.py +249 -197
  42. intentumdiff/rust_core.py +744 -784
  43. intentumdiff/serve/_app.py +3 -10
  44. intentumdiff/sources/base.py +24 -0
  45. intentumdiff/sources/file_source.py +15 -2
  46. intentumdiff/sources/git_source.py +120 -58
  47. intentumdiff/sources/live_buffer_source.py +45 -5
  48. intentumdiff/sources/patch_source.py +36 -109
  49. intentumdiff/vcs/_safe_path.py +7 -0
  50. intentumdiff/vcs/base.py +6 -0
  51. intentumdiff/vcs/git_backend.py +1 -1
  52. intentumdiff/vcs/git_cli.py +38 -18
  53. intentumdiff/vcs/hg_backend.py +1 -1
  54. intentumdiff/vcs/perforce_backend.py +7 -1
  55. intentumdiff/vcs/svn_backend.py +1 -1
  56. intentumdiff/wasm/abap_parser.wasm +0 -0
  57. intentumdiff/wasm/adf_parser.wasm +0 -0
  58. intentumdiff/wasm/asciidoc_parser.wasm +0 -0
  59. intentumdiff/wasm/asm_parser.wasm +0 -0
  60. intentumdiff/wasm/assemblyscript_parser.wasm +0 -0
  61. intentumdiff/wasm/astro_parser.wasm +0 -0
  62. intentumdiff/wasm/bash_parser.wasm +0 -0
  63. intentumdiff/wasm/clojure_parser.wasm +0 -0
  64. intentumdiff/wasm/cmake_parser.wasm +0 -0
  65. intentumdiff/wasm/cpp_parser.wasm +0 -0
  66. intentumdiff/wasm/csharp_parser.wasm +0 -0
  67. intentumdiff/wasm/css_parser.wasm +0 -0
  68. intentumdiff/wasm/dart_parser.wasm +0 -0
  69. intentumdiff/wasm/databricks_parser.wasm +0 -0
  70. intentumdiff/wasm/dax_parser.wasm +0 -0
  71. intentumdiff/wasm/delphi_parser.wasm +0 -0
  72. intentumdiff/wasm/dockerfile_parser.wasm +0 -0
  73. intentumdiff/wasm/elixir_parser.wasm +0 -0
  74. intentumdiff/wasm/freebasic_parser.wasm +0 -0
  75. intentumdiff/wasm/generic_parser.wasm +0 -0
  76. intentumdiff/wasm/gitignore_parser.wasm +0 -0
  77. intentumdiff/wasm/go_parser.wasm +0 -0
  78. intentumdiff/wasm/gomod_parser.wasm +0 -0
  79. intentumdiff/wasm/graphql_parser.wasm +0 -0
  80. intentumdiff/wasm/groovy_parser.wasm +0 -0
  81. intentumdiff/wasm/haskell_parser.wasm +0 -0
  82. intentumdiff/wasm/html_parser.wasm +0 -0
  83. intentumdiff/wasm/html_renderer.wasm +0 -0
  84. intentumdiff/wasm/ini_parser.wasm +0 -0
  85. intentumdiff/wasm/java_parser.wasm +0 -0
  86. intentumdiff/wasm/js_ts_parser.wasm +0 -0
  87. intentumdiff/wasm/json_parser.wasm +0 -0
  88. intentumdiff/wasm/kotlin_parser.wasm +0 -0
  89. intentumdiff/wasm/latex_parser.wasm +0 -0
  90. intentumdiff/wasm/llm_renderer.wasm +0 -0
  91. intentumdiff/wasm/lua_parser.wasm +0 -0
  92. intentumdiff/wasm/make_parser.wasm +0 -0
  93. intentumdiff/wasm/markdown_parser.wasm +0 -0
  94. intentumdiff/wasm/mdx_parser.wasm +0 -0
  95. intentumdiff/wasm/ocaml_parser.wasm +0 -0
  96. intentumdiff/wasm/odin_parser.wasm +0 -0
  97. intentumdiff/wasm/parser_manifest.json +81 -2
  98. intentumdiff/wasm/patch_renderer.wasm +0 -0
  99. intentumdiff/wasm/perl_parser.wasm +0 -0
  100. intentumdiff/wasm/php_parser.wasm +0 -0
  101. intentumdiff/wasm/plsql_parser.wasm +0 -0
  102. intentumdiff/wasm/po_parser.wasm +0 -0
  103. intentumdiff/wasm/postscript_parser.wasm +0 -0
  104. intentumdiff/wasm/powershell_parser.wasm +0 -0
  105. intentumdiff/wasm/proto_parser.wasm +0 -0
  106. intentumdiff/wasm/puppet_parser.wasm +0 -0
  107. intentumdiff/wasm/python_parser.wasm +0 -0
  108. intentumdiff/wasm/qsharp_parser.wasm +0 -0
  109. intentumdiff/wasm/r_parser.wasm +0 -0
  110. intentumdiff/wasm/reasonml_parser.wasm +0 -0
  111. intentumdiff/wasm/ruby_parser.wasm +0 -0
  112. intentumdiff/wasm/rust_parser.wasm +0 -0
  113. intentumdiff/wasm/sas_parser.wasm +0 -0
  114. intentumdiff/wasm/scala_parser.wasm +0 -0
  115. intentumdiff/wasm/scss_parser.wasm +0 -0
  116. intentumdiff/wasm/sql_parser.wasm +0 -0
  117. intentumdiff/wasm/squirrel_parser.wasm +0 -0
  118. intentumdiff/wasm/svelte_parser.wasm +0 -0
  119. intentumdiff/wasm/swift_parser.wasm +0 -0
  120. intentumdiff/wasm/terminal_renderer.wasm +0 -0
  121. intentumdiff/wasm/terraform_parser.wasm +0 -0
  122. intentumdiff/wasm/toml_parser.wasm +0 -0
  123. intentumdiff/wasm/tsql_parser.wasm +0 -0
  124. intentumdiff/wasm/vbnet_parser.wasm +0 -0
  125. intentumdiff/wasm/vue_parser.wasm +0 -0
  126. intentumdiff/wasm/wasm_provenance.json +884 -0
  127. intentumdiff/wasm/wat_parser.wasm +0 -0
  128. intentumdiff/wasm/xml_parser.wasm +0 -0
  129. intentumdiff/wasm/yaml_parser.wasm +0 -0
  130. intentumdiff/wasm/zig_parser.wasm +0 -0
  131. intentumdiff_python-0.0.2b2.dist-info/METADATA +156 -0
  132. intentumdiff_python-0.0.2b2.dist-info/RECORD +171 -0
  133. {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/WHEEL +1 -1
  134. {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/sboms/intentumdiff-rust-core.cyclonedx.json +905 -379
  135. intentumdiff/core/cst_serializer.py +0 -101
  136. intentumdiff_python-0.0.1.dist-info/METADATA +0 -112
  137. intentumdiff_python-0.0.1.dist-info/RECORD +0 -165
  138. {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/entry_points.txt +0 -0
  139. {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.2b2.dist-info}/licenses/LICENSE +0 -0
@@ -17,9 +17,7 @@ repository snapshot (or any other collection of files). It provides:
17
17
  The heavy lifting (symbol-table + reference-table extraction) is delegated to
18
18
  the Rust core (``index-engine-lib``, via the native ``build_symbol_table_json``
19
19
  / ``build_reference_table_json`` entrypoints). Rust is authoritative — the
20
- former pure-Python extraction mirror was deleted (#91); without the core the
21
- tables are simply empty ("if Python didn't exist, the engine still lives in
22
- Rust"). Cross-file diffing lives in ``intentumdiff.analysis.cross_file``, also a
20
+ former pure-Python extraction mirror was deleted (#91); engine failures raise instead of manufacturing empty tables. Cross-file diffing lives in ``intentumdiff.analysis.cross_file``, also a
23
21
  thin wrapper over the same Rust core.
24
22
  """
25
23
 
@@ -81,24 +79,23 @@ class SemanticIndex:
81
79
  Rust-authoritative (#91): extraction runs in the native
82
80
  ``build_symbol_table_json`` / ``build_reference_table_json`` entrypoints
83
81
  (the same code the index-engine Wasm plugin wraps). The former Python
84
- extraction mirror was deleted. When the core is unavailable the tables
85
- stay empty rather than falling back — "if Python didn't exist, the
86
- engine still lives in Rust".
82
+ extraction mirror was deleted. Required engine failures raise. Both tables are published atomically
83
+ only after successful engine calls and DTO validation.
87
84
  """
88
85
  from intentumdiff.rust_core import (
89
86
  try_rust_build_reference_table,
90
87
  try_rust_build_symbol_table,
91
88
  )
92
89
 
93
- self._symbols = {}
94
- self._references = {}
95
90
  files_json = self.to_files_json()
96
91
  symbol_json = try_rust_build_symbol_table(files_json)
97
- if symbol_json is not None:
98
- self.load_symbol_table_json(symbol_json)
99
92
  reference_json = try_rust_build_reference_table(files_json)
100
- if reference_json is not None:
101
- self.load_reference_table_json(reference_json)
93
+ # Validate DTOs on a temporary index before publishing either table.
94
+ candidate = SemanticIndex()
95
+ candidate.load_symbol_table_json(symbol_json)
96
+ candidate.load_reference_table_json(reference_json)
97
+ self._symbols = candidate._symbols
98
+ self._references = candidate._references
102
99
  self._built = True
103
100
  return self
104
101
 
@@ -144,12 +141,13 @@ class SemanticIndex:
144
141
  refs = self._references.get(name, [])
145
142
  if not resolve:
146
143
  return refs
147
- # Attach resolved definition when the match is unambiguous.
148
- defs = self.find_definition(name)
149
- resolved = defs[0] if len(defs) == 1 else None
150
- return [
151
- ref.model_copy(update={"resolved_definition": resolved}) for ref in refs
152
- ]
144
+ from intentumdiff.rust_core import _c_abi_call
145
+ result = _c_abi_call(
146
+ "resolve_references",
147
+ json.dumps([definition.model_dump(mode="json") for definition in self._symbols.get(name, [])]),
148
+ json.dumps([reference.model_dump(mode="json") for reference in refs]),
149
+ )
150
+ return [ReferenceUsage.model_validate(reference) for reference in result]
153
151
 
154
152
  # ------------------------------------------------------------------
155
153
  # Serialisation helpers (feed the native index-engine primitives)
@@ -30,17 +30,11 @@ Pass a no-op lambda to suppress events for a specific call::
30
30
 
31
31
  Caching
32
32
  -------
33
- When the :class:`~intentumdiff.differ.SemanticDiffer` was created with a
34
- ``DiffConfig.cache_path``, two levels of caching apply automatically:
35
-
36
- * **Parse-tree cache** — each file's :class:`~intentumdiff.core.models.SemanticNode`
37
- is cached keyed by content hash. Re-indexing the same file content skips
38
- the Wasm call entirely.
39
-
40
- * **Symbol-index cache** — after :meth:`Indexer.index_repo` finishes, the
41
- resulting symbol and reference tables are stored in SQLite keyed by
42
- ``(repo_root, commit_sha)``. A subsequent call for the *same* commit returns
43
- the cached index immediately (unless *force=True*).
33
+ When ``DiffConfig.cache_path`` is set, the verified parser-level cache remains
34
+ available through ``SemanticDiffer``. The historical repository symbol-index
35
+ cache and LSP hover-map cache are bypassed in 0.0.2: their keys do not capture
36
+ Rust/parser identity, indexing completeness, or LSP workspace/server state.
37
+ Returning a stale index is worse than recomputing it. See Python #189-192.
44
38
  """
45
39
 
46
40
  from __future__ import annotations
@@ -53,6 +47,7 @@ from pathlib import Path
53
47
  from typing import TYPE_CHECKING, Any, Callable
54
48
 
55
49
  from intentumdiff.core.diffignore import load_diffignore
50
+ from intentumdiff.rust_core import attach_lsp_type_map
56
51
 
57
52
  if TYPE_CHECKING:
58
53
  from intentumdiff.core.index import SemanticIndex
@@ -164,41 +159,62 @@ def _make_index_key(repo_root: str, commit_sha: str) -> str:
164
159
  return hashlib.sha256(encoded).hexdigest()
165
160
 
166
161
 
167
- def _parse_catfile_output(
168
- buf: bytes,
169
- sha_to_path: "dict[str, str]",
170
- ) -> "list[tuple[str, str]]":
171
- """Parse ``git cat-file --batch`` stdout into ``(path, content)`` pairs."""
172
- files: list[tuple[str, str]] = []
162
+ def _parse_catfile_output(buf: bytes) -> dict[str, str | None]:
163
+ """Decode complete Git batch responses; None marks an intentionally skipped binary.
164
+
165
+ A missing object, bad header, or truncated content is not a binary file.
166
+ Such failures must abort indexing instead of silently dropping source files.
167
+ """
168
+ blobs: dict[str, str | None] = {}
173
169
  i = 0
174
170
  n = len(buf)
175
171
  while i < n:
176
172
  nl = buf.find(b"\n", i)
177
173
  if nl == -1:
178
- break
179
- header = buf[i:nl].decode("ascii", errors="replace")
174
+ raise RuntimeError("Incomplete git cat-file batch response header")
175
+ try:
176
+ header = buf[i:nl].decode("ascii", errors="strict")
177
+ except UnicodeDecodeError as exc:
178
+ raise RuntimeError("Invalid git cat-file batch response header") from exc
180
179
  i = nl + 1
181
180
  parts = header.split()
182
- # Expected: "<sha> blob <size>"; skip "missing" / non-blob entries.
183
- if len(parts) < 3 or parts[1] != "blob":
184
- continue
181
+ if len(parts) == 2 and parts[1] == "missing":
182
+ raise RuntimeError(f"Git blob {parts[0]} is missing")
183
+ if len(parts) != 3 or parts[1] != "blob":
184
+ raise RuntimeError(f"Unexpected git cat-file batch response: {header!r}")
185
+ sha = parts[0]
186
+ if sha in blobs:
187
+ raise RuntimeError(f"Duplicate git cat-file batch response for {sha}")
185
188
  try:
186
189
  size = int(parts[2])
187
- except ValueError:
188
- continue
190
+ except ValueError as exc:
191
+ raise RuntimeError(f"Invalid git cat-file batch size for {sha}") from exc
192
+ if size < 0 or i + size >= n or buf[i + size] != 0x0A:
193
+ raise RuntimeError(f"Truncated git cat-file batch payload for {sha}")
189
194
  content_bytes = buf[i : i + size]
190
- i += size + 1 # skip trailing \n
191
- if b"\x00" in content_bytes[:8192]: # binary heuristic
195
+ i += size + 1 # consume the required trailing newline
196
+ if b"\x00" in content_bytes[:8192]:
197
+ blobs[sha] = None # binary: intentionally excluded from text indexing
192
198
  continue
193
- path = sha_to_path.get(parts[0], "")
194
- if not path:
199
+
200
+ # Some recognized binaries (for example PDFs) contain no NUL bytes.
201
+ # Use the Rust-owned content classifier on the original blob before
202
+ # decoding; otherwise their binary payload aborts the entire Git index
203
+ # as "invalid UTF-8" (#253). An opaque octet-stream is *not* proof
204
+ # that a source file is safely skippable: still decode it strictly.
205
+ from intentumdiff.content_type import detect_content_type # noqa: PLC0415
206
+
207
+ content_type = detect_content_type(content_bytes)
208
+ if not content_type["is_text"] and content_type["mime"] != "application/octet-stream":
209
+ blobs[sha] = None
195
210
  continue
196
211
  try:
197
- content = content_bytes.decode("utf-8", errors="strict")
198
- except UnicodeDecodeError:
199
- content = content_bytes.decode("utf-8", errors="replace")
200
- files.append((path, content))
201
- return files
212
+ blobs[sha] = content_bytes.decode("utf-8", errors="strict")
213
+ except UnicodeDecodeError as exc:
214
+ raise RuntimeError(
215
+ f"Invalid UTF-8 in Git blob {sha}: refusing lossy symbol indexing"
216
+ ) from exc
217
+ return blobs
202
218
 
203
219
 
204
220
  def _collect_blob_files(
@@ -206,61 +222,77 @@ def _collect_blob_files(
206
222
  ref: str,
207
223
  diffignore: Any,
208
224
  ) -> "tuple[list[tuple[str, str]], list[str]]":
209
- """Return all text files at *ref* as ``(relative_path, decoded_content)``.
225
+ """Return every non-ignored text path at *ref*, including shared-content blobs.
210
226
 
211
- Shells to the git CLI (#98, A2.4.3): ``git ls-tree -r`` lists every blob and a
212
- single ``git cat-file --batch`` reads them through git's optimised pack reader.
213
- Binary blobs (NUL-byte heuristic in :func:`_parse_catfile_output`) are skipped.
214
-
215
- Returns ``(files, ignored_files)`` where *ignored_files* are the paths excluded
216
- by *diffignore*.
227
+ Git stores identical file contents under one blob SHA. Batch-read each unique
228
+ SHA once, then expand that content against the original ls-tree path list;
229
+ collapsing SHA -> one path silently drops files from semantic indexing.
217
230
  """
218
231
  from intentumdiff.vcs.git_cli import run_git_bytes # noqa: PLC0415
219
232
 
220
233
  ignored_files: list[str] = []
221
- sha_to_path: dict[str, str] = {}
222
- # ls-tree -r -z: "<mode> <type> <sha>\t<path>\0" per entry.
223
- for entry in run_git_bytes(repo_root, ["ls-tree", "-r", "-z", ref]).split(b"\0"):
234
+ paths_in_tree_order: list[tuple[str, str]] = []
235
+ # Git's -z output always terminates every record with NUL. Validating
236
+ # the later cat-file blobs cannot discover paths lost from this inventory.
237
+ tree_bytes = run_git_bytes(repo_root, ["ls-tree", "-r", "-z", ref])
238
+ if tree_bytes and not tree_bytes.endswith(b"\x00"):
239
+ raise RuntimeError("Incomplete git ls-tree -z output: missing NUL delimiter")
240
+ for entry in tree_bytes.split(b"\x00"):
224
241
  if not entry:
225
242
  continue
226
- meta, _tab, raw_path = entry.partition(b"\t")
243
+ meta, separator, raw_path = entry.partition(b"\t")
227
244
  parts = meta.split()
228
- if len(parts) < 3 or parts[1] != b"blob":
245
+ if separator != b"\t" or not raw_path or len(parts) != 3:
246
+ raise RuntimeError(f"Malformed git ls-tree entry: {entry[:128]!r}")
247
+ if parts[1] == b"commit":
248
+ # Gitlinks are submodules, not readable blobs at this tree.
229
249
  continue
230
- path = raw_path.decode("utf-8", errors="replace")
250
+ if parts[1] != b"blob":
251
+ raise RuntimeError(f"Unexpected git ls-tree object type: {parts[1]!r}")
252
+ # Git may store arbitrary filename bytes on Unix. Replacing undecodable
253
+ # bytes invents a different path and can collapse distinct files (#238).
254
+ try:
255
+ path = raw_path.decode("utf-8", errors="strict")
256
+ except UnicodeDecodeError as exc:
257
+ raise RuntimeError(
258
+ "Invalid UTF-8 in git ls-tree path: refusing lossy indexing"
259
+ ) from exc
260
+ try:
261
+ sha = parts[2].decode("ascii", errors="strict")
262
+ except UnicodeDecodeError as exc:
263
+ raise RuntimeError(
264
+ f"Invalid ASCII object ID in git ls-tree for {path!r}"
265
+ ) from exc
231
266
  if diffignore and diffignore.is_ignored(path):
232
267
  ignored_files.append(path)
233
268
  continue
234
- sha_to_path[parts[2].decode("ascii", errors="replace")] = path
269
+ paths_in_tree_order.append((path, sha))
235
270
 
236
- if not sha_to_path:
271
+ if not paths_in_tree_order:
237
272
  return [], ignored_files
238
273
 
239
- shas_input = "\n".join(sha_to_path).encode()
274
+ # The caller needs all paths, but cat-file needs only one request per blob.
275
+ shas_input = "\n".join(dict.fromkeys(sha for _, sha in paths_in_tree_order)).encode()
240
276
  out = run_git_bytes(repo_root, ["cat-file", "--batch"], input_bytes=shas_input)
241
- return _parse_catfile_output(out, sha_to_path), ignored_files
242
-
243
-
244
- def _attach_type_info(
245
- node: "SemanticNode",
246
- type_map: dict[str, str],
247
- ) -> "SemanticNode":
248
- """Return a copy of *node* tree with ``type_info`` set from *type_map*.
249
-
250
- ``SemanticNode`` is frozen so we must rebuild changed nodes from the leaves
251
- up. Only nodes whose id appears in *type_map* are rebuilt.
252
- """
253
- from intentumdiff.core.models import SemanticNode # noqa: PLC0415
254
-
255
- new_children = [_attach_type_info(c, type_map) for c in node.children]
256
- if node.id in type_map or new_children != list(node.children):
257
- return node.model_copy(
258
- update={
259
- "children": new_children,
260
- **({"type_info": type_map[node.id]} if node.id in type_map else {}),
261
- }
277
+ decoded_by_sha = _parse_catfile_output(out)
278
+ requested = set(sha for _, sha in paths_in_tree_order)
279
+ missing = requested.difference(decoded_by_sha)
280
+ if missing:
281
+ raise RuntimeError(
282
+ "Incomplete git cat-file batch response: missing blob(s) "
283
+ + ", ".join(sorted(missing)[:5])
262
284
  )
263
- return node
285
+ unexpected = set(decoded_by_sha).difference(requested)
286
+ if unexpected:
287
+ raise RuntimeError(
288
+ "Unexpected git cat-file batch blob(s): " + ", ".join(sorted(unexpected)[:5])
289
+ )
290
+ files: list[tuple[str, str]] = []
291
+ for path, sha in paths_in_tree_order:
292
+ content = decoded_by_sha[sha]
293
+ if content is not None: # only known binary blobs may be omitted
294
+ files.append((path, content))
295
+ return files, ignored_files
264
296
 
265
297
 
266
298
  # ---------------------------------------------------------------------------
@@ -363,8 +395,8 @@ class Indexer:
363
395
  Progress callback for this call. Falls back to the callback
364
396
  passed to ``__init__`` when ``None``.
365
397
  force:
366
- Re-index even if a cached symbol index already exists for this
367
- commit. Useful after a plugin or wasm binary update.
398
+ Retained for API compatibility. Indexing always recomputes the
399
+ symbol index in 0.0.2 (the unproven cache is disabled).
368
400
 
369
401
  Returns
370
402
  -------
@@ -372,41 +404,13 @@ class Indexer:
372
404
  Fully built index. Check :attr:`IndexResult.from_cache` to
373
405
  determine whether the symbol table was loaded from cache.
374
406
  """
375
- from intentumdiff.vcs.git_cli import ( # noqa: PLC0415
376
- resolve_repo_root,
377
- run_git_bytes,
378
- )
407
+ from intentumdiff.vcs.git_cli import resolve_repo_root # noqa: PLC0415
379
408
 
380
409
  repo_root = resolve_repo_root(repo_path)
381
- commit_sha = (
382
- run_git_bytes(repo_root, ["rev-parse", "--verify", ref]).decode().strip()
383
- )
384
410
  callback = on_progress or self._on_progress
385
411
 
386
- # ── Fast path: cached symbol index ────────────────────────────────
387
- if not force and self._differ._cache is not None:
388
- index_key = _make_index_key(repo_root, commit_sha)
389
- cached = self._differ._cache.get_symbol_index(index_key)
390
- if cached is not None:
391
- logger.debug(
392
- "Symbol index cache hit for %s @ %s",
393
- repo_root,
394
- commit_sha[:8],
395
- )
396
- from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
397
-
398
- sem_index = SemanticIndex()
399
- symbols_json, refs_json = cached
400
- sem_index.load_symbol_table_json(symbols_json)
401
- sem_index.load_reference_table_json(refs_json)
402
- return IndexResult(
403
- files_indexed=0,
404
- files_skipped=0,
405
- errors=[],
406
- semantic_index=sem_index,
407
- from_cache=True,
408
- )
409
-
412
+ # Always re-index: historical symbol-cache entries lack Rust/parser
413
+ # identity and failed-file provenance (#189, #190).
410
414
  # ── Collect all text files in commit tree ─────────────────────────
411
415
  diffignore = load_diffignore(repo_root)
412
416
  files, ignored_files = _collect_blob_files(repo_root, ref, diffignore)
@@ -415,13 +419,7 @@ class Indexer:
415
419
  result.files_ignored = len(ignored_files)
416
420
  result.ignored_files = ignored_files
417
421
 
418
- # ── Persist symbol index ───────────────────────────────────────────
419
- if self._differ._cache is not None:
420
- index_key = _make_index_key(repo_root, commit_sha)
421
- self._store_symbol_index(
422
- result.semantic_index, index_key, result.files_indexed
423
- )
424
-
422
+ # Do not persist an index whose provenance/completeness cannot be verified.
425
423
  return result
426
424
 
427
425
  def index_directory(
@@ -448,6 +446,8 @@ class Indexer:
448
446
  directory = Path(directory)
449
447
  callback = on_progress or self._on_progress
450
448
 
449
+ from intentumdiff.content_type import detect_content_type # noqa: PLC0415
450
+
451
451
  diffignore = load_diffignore(directory)
452
452
  ignored_files: list[str] = []
453
453
  files: list[tuple[str, str]] = []
@@ -459,9 +459,25 @@ class Indexer:
459
459
  ignored_files.append(rel_posix)
460
460
  continue
461
461
  try:
462
- content = path.read_text(encoding="utf-8", errors="replace")
463
- except OSError:
464
- continue
462
+ raw = path.read_bytes()
463
+ # Rust owns content-type classification. A confirmed binary asset
464
+ # is not a source file and must not be UTF-8-decoded (#250).
465
+ content_type = detect_content_type(raw)
466
+ if not content_type["is_text"]:
467
+ if content_type["mime"] == "application/octet-stream" and b"\x00" not in raw:
468
+ # Opaque bytes without a recognized format are not proof
469
+ # that a text source is absent. Reject invalid UTF-8 rather
470
+ # than allowing corrupt source to disappear from the index.
471
+ raw.decode("utf-8", errors="strict")
472
+ continue
473
+ content = raw.decode("utf-8", errors="strict")
474
+ except (OSError, UnicodeError) as exc:
475
+ # Host-I/O and decoding failures are not intentional ignores or unsupported
476
+ # parser. Returning a successful partial directory index would
477
+ # conceal source files from downstream semantic analysis (#203).
478
+ raise RuntimeError(
479
+ f"Failed to read directory index source {rel_posix!r}: {exc}"
480
+ ) from exc
465
481
  files.append((rel_posix, content))
466
482
 
467
483
  result = self._index_files(files, on_progress=callback)
@@ -589,44 +605,13 @@ class Indexer:
589
605
  for fast local servers, lower for shared or rate-limited servers.
590
606
  """
591
607
  import asyncio
592
- from intentumdiff.vcs.git_cli import ( # noqa: PLC0415
593
- resolve_repo_root,
594
- run_git_bytes,
595
- )
608
+ from intentumdiff.vcs.git_cli import resolve_repo_root # noqa: PLC0415
596
609
 
597
610
  repo_root = resolve_repo_root(repo_path)
598
- commit_sha = (
599
- run_git_bytes(repo_root, ["rev-parse", "--verify", ref]).decode().strip()
600
- )
601
611
  callback = on_progress or self._on_progress
602
612
 
603
- # LSP-enriched indexes are stored under a separate key so they don't
604
- # collide with plain (non-enriched) index_repo cache entries.
605
- lsp_index_key = _make_index_key(repo_root, "lsp:" + commit_sha)
606
-
607
- # ── Fast path: cached LSP symbol index ────────────────────────────
608
- if not force and self._differ._cache is not None:
609
- cached = self._differ._cache.get_symbol_index(lsp_index_key)
610
- if cached is not None:
611
- logger.debug(
612
- "LSP symbol index cache hit for %s @ %s",
613
- repo_root,
614
- commit_sha[:8],
615
- )
616
- from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
617
-
618
- sem_index = SemanticIndex()
619
- symbols_json, refs_json = cached
620
- sem_index.load_symbol_table_json(symbols_json)
621
- sem_index.load_reference_table_json(refs_json)
622
- return IndexResult(
623
- files_indexed=0,
624
- files_skipped=0,
625
- errors=[],
626
- semantic_index=sem_index,
627
- from_cache=True,
628
- )
629
-
613
+ # Always re-index: LSP type state, parser identity and failed-file
614
+ # provenance are missing from historical cache keys (#189, #190).
630
615
  # Collect files (parallel blob reads; same filtering as index_repo)
631
616
  diffignore = load_diffignore(repo_root)
632
617
  files, ignored_files = await asyncio.to_thread(
@@ -642,9 +627,7 @@ class Indexer:
642
627
  result.files_ignored = len(ignored_files)
643
628
  result.ignored_files = ignored_files
644
629
 
645
- if self._differ._cache is not None:
646
- self._store_symbol_index(result.semantic_index, lsp_index_key, result.files_indexed)
647
-
630
+ # An LSP-enriched index is not reusable without server/workspace identity.
648
631
  return result
649
632
 
650
633
  async def _index_files_lsp(
@@ -684,7 +667,6 @@ class Indexer:
684
667
  # Error: any other exception
685
668
 
686
669
  sem = asyncio.Semaphore(max(1, file_concurrency))
687
- cache = self._differ._cache
688
670
  differ = self._differ # captured for executor
689
671
 
690
672
  # Use a single-worker executor so Wasm parse calls are serialised.
@@ -725,22 +707,20 @@ class Indexer:
725
707
  enricher = enrichers.get(language)
726
708
  enriched_count = 0
727
709
  if enricher is not None:
728
- # Check hover-map cache first.
729
- type_map: dict[str, str] | None = None
730
- hm_key: str | None = None
731
- if cache is not None:
732
- hm_key = cache.hover_map_key(content, language)
733
- type_map = cache.get_hover_map(hm_key)
734
- if type_map is None:
735
- try:
736
- type_map = await enricher.enrich(filename, content, node)
737
- except Exception:
738
- type_map = None
739
- if type_map and hm_key is not None and cache is not None:
740
- cache.put_hover_map(hm_key, type_map)
741
- if type_map:
742
- enriched_count = len(type_map)
743
- node = _attach_type_info(node, type_map)
710
+ # Hover-map caching lacks file/workspace/server provenance
711
+ # and cannot certify a type result (#191).
712
+ try:
713
+ type_map = await enricher.enrich(filename, content, node)
714
+ if not isinstance(type_map, dict):
715
+ raise TypeError("LSP enricher returned an invalid type-map DTO")
716
+ if type_map:
717
+ # Rust owns semantic-tree mutation and validation.
718
+ node = attach_lsp_type_map(node, type_map)
719
+ enriched_count = len(type_map)
720
+ except Exception as exc: # noqa: BLE001
721
+ # Do not turn a requested enrichment failure into an
722
+ # apparently successful bare semantic index (#192).
723
+ return filename, "ERROR", exc, 0
744
724
  return filename, language, node, enriched_count
745
725
 
746
726
  tasks = [asyncio.create_task(_process_one(f, c)) for f, c in files]
@@ -78,8 +78,8 @@ class DetectionResult(BaseModel, frozen=True):
78
78
  ge=0.0,
79
79
  le=1.0,
80
80
  description=(
81
- "Relative confidence derived from parser priority order. "
82
- "1.0 for the highest-priority match, lower for subsequent matches."
81
+ "Reciprocal rank from Rust content detection order, rounded to three decimals. "
82
+ "1.0 for the first match; this is not a measured probability."
83
83
  ),
84
84
  )
85
85
 
@@ -123,14 +123,23 @@ class LanguageInfoGroup(BaseModel, frozen=True):
123
123
 
124
124
 
125
125
  class NodeFacts(BaseModel, frozen=True):
126
+ # Unknown Rust-authored facts must fail validation rather than disappear
127
+ # through Pydantic's extra=ignore (0.0.2 thin-wrapper contract, #226).
128
+ model_config = ConfigDict(extra="forbid")
129
+
126
130
  """
127
- Privacy-safe structural facts about a definition node (function/method/class),
131
+ Privacy-safe structural facts about an entity or keyed/resource node,
128
132
  computed by the Rust parser from the full CST before pruning. Only counts,
129
133
  enums, and flags — never source text, identifiers, or literals — so they can
130
134
  describe a change's shape ("no parameters", "returns nothing", "no-op body")
131
135
  without revealing code. The host carries this through unchanged.
132
136
  """
133
137
 
138
+ #: Which derivation passes produced these facts, e.g. ``"cst,enrich(derived=14,added=5)"``.
139
+ #: Populated only when ``INTENTUMDIFF_TRACE_FACTS=1``; ``None`` in normal operation.
140
+ #: Diagnostic provenance, never source — safe to paste into a bug report.
141
+ facts_trace: str | None = None
142
+
134
143
  param_count: int | None = None
135
144
  #: ``"none"`` | ``"value"`` | ``"literal"`` (constant) | ``"unknown"``.
136
145
  returns: str | None = None
@@ -148,6 +157,20 @@ class NodeFacts(BaseModel, frozen=True):
148
157
  has_error_handling: bool | None = None
149
158
  #: The body raises/throws an exception. Flag only.
150
159
  throws: bool | None = None
160
+ #: Cross-language structural facts, derived from the pruned canonical tree (#9) rather
161
+ #: than from any one grammar — so they are available for EVERY language, not just the
162
+ #: ones with a bespoke fact extractor.
163
+ #:
164
+ #: These are counts and flags only, in keeping with the rest of NodeFacts: no source, no
165
+ #: identifiers, no literals.
166
+ #:
167
+ #: `has_guard_clause` is deliberately tri-state. The pruned tree can prove a guard is
168
+ #: present, but cannot prove one is absent — statement order survives pruning, operators
169
+ #: do not. So `None` means "not determinable here", NOT "no guard". Omitting beats
170
+ #: claiming; see uast.rs for the reasoning.
171
+ early_exit_count: int | None = None
172
+ negated_condition_count: int | None = None
173
+ has_guard_clause: bool | None = None
151
174
  #: The body mutates state — an augmented assignment or an assignment to an
152
175
  #: attribute/subscript (``self.x = …``, ``a[i] = …``). Flag only, no target name.
153
176
  mutates: bool | None = None
@@ -195,6 +218,20 @@ class NodeFacts(BaseModel, frozen=True):
195
218
  call_count: int | None = None
196
219
  recursive: bool | None = None
197
220
 
221
+ # Rust-owned keyed/collection/resource NodeFacts (#179, Core #203,
222
+ # Python #226). These are DTO fields only: the Python host must preserve
223
+ # Rust's privacy-safe counts, flags and shape categories without deriving
224
+ # them or silently dropping them through Pydantic's extra=ignore.
225
+ shape: str | None = None
226
+ child_count: int | None = None
227
+ value_kind: str | None = None
228
+ item_count: int | None = None
229
+ nesting: int | None = None
230
+ is_leaf: bool | None = None
231
+ attribute_count: int | None = None
232
+ nested_block_count: int | None = None
233
+ has_nested_block: bool | None = None
234
+
198
235
 
199
236
  class SemanticNode(BaseModel, frozen=True):
200
237
  """
@@ -401,14 +438,12 @@ class ChangeGroup(BaseModel, frozen=True):
401
438
  class ChangeStreamEvent(BaseModel, frozen=True):
402
439
  """A single event emitted by ``SemanticDiffer.diff_stream_progressive``.
403
440
 
404
- Events are ordered: STRUCTURAL (phase 1) events arrive first, followed by
405
- REFINED (phase 2) events that may revise or supersede them, and finally
406
- FINAL (phase 3) events from the classification and diff-analyzer passes.
441
+ The 0.0.2 Python adapter emits FINAL add events only, after Rust has
442
+ produced a complete diff. STRUCTURAL, REFINED, revise and remove remain in
443
+ the DTO schema for compatibility but are not synthesized by Python.
407
444
 
408
- ``replaced_ids`` contains the ``old_node.id`` / ``new_node.id`` values of
409
- Phase-1 changes that this event supersedes. When ``action="revise"`` the
410
- consumer should look up prior events by these IDs and replace them. When
411
- ``action="remove"`` there is no replacement — the change was absorbed.
445
+ ``replaced_ids`` carries identifiers supplied by a backend for revising or
446
+ removing earlier events; the 0.0.2 adapter leaves it empty.
412
447
  """
413
448
 
414
449
  phase: ChangeStreamPhase
@@ -570,6 +605,11 @@ class DiffConfig(BaseModel):
570
605
 
571
606
  Unlike result models, this is NOT frozen — callers may update it freely
572
607
  before passing to ``SemanticDiffer``.
608
+
609
+ Compatibility-only fields in 0.0.2: ``ignore_style``, ``detect_refactorings``,
610
+ ``approx_move_threshold``, ``leaf_text_diff`` and ``extra_trivia_types``.
611
+ Rust accepts their existing defaults but rejects unsupported overrides.
612
+ Configurable semantics are tracked in Core issues #220–#224.
573
613
  """
574
614
 
575
615
  # GumTree tuning
@@ -697,11 +737,10 @@ class DiffConfig(BaseModel):
697
737
  ``SemanticDiff`` will have ``is_fallback=True``."""
698
738
 
699
739
  stream_analysis: bool = False
700
- """Enable progressive per-phase streaming via ``diff_stream_progressive``.
740
+ """Legacy stream-analysis configuration flag.
701
741
 
702
- When ``True``, callers may iterate ``SemanticDiffer.diff_stream_progressive``
703
- to receive ``ChangeStreamEvent`` objects as each pipeline phase completes,
704
- rather than waiting for the entire pipeline to finish."""
742
+ In 0.0.2, ``diff_stream_progressive`` only adapts a completed Rust diff to
743
+ FINAL add events. This flag does not enable partial Rust phase snapshots."""
705
744
 
706
745
 
707
746
  # ---------------------------------------------------------------------------