intentumdiff-python 0.0.1__py3-none-macosx_11_0_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. intentumdiff/__init__.py +134 -0
  2. intentumdiff/_differ_gate.py +210 -0
  3. intentumdiff/_differ_presentation.py +786 -0
  4. intentumdiff/_differ_runtime.py +320 -0
  5. intentumdiff/analysis/__init__.py +1 -0
  6. intentumdiff/analysis/compile_commands.py +151 -0
  7. intentumdiff/analysis/cross_file.py +71 -0
  8. intentumdiff/analysis/diagnostics.py +309 -0
  9. intentumdiff/analysis/guardrail_reports.py +254 -0
  10. intentumdiff/analysis/guardrails.py +318 -0
  11. intentumdiff/analysis/invariances.py +1207 -0
  12. intentumdiff/analysis/keyed_profiles.py +25 -0
  13. intentumdiff/analysis/resource_profiles.py +14 -0
  14. intentumdiff/analysis/schema_resolver.py +816 -0
  15. intentumdiff/analysis/text_review.py +46 -0
  16. intentumdiff/analysis/user_schemas.py +430 -0
  17. intentumdiff/assets.py +45 -0
  18. intentumdiff/cache/__init__.py +37 -0
  19. intentumdiff/cache/duckdb_store.py +98 -0
  20. intentumdiff/cache/sqlite_store.py +149 -0
  21. intentumdiff/cache/store.py +190 -0
  22. intentumdiff/cli/__init__.py +199 -0
  23. intentumdiff/cli/_commands.py +2141 -0
  24. intentumdiff/cli/_parser.py +1061 -0
  25. intentumdiff/cli/_shared.py +644 -0
  26. intentumdiff/content_type.py +59 -0
  27. intentumdiff/core/__init__.py +1 -0
  28. intentumdiff/core/commit_differ.py +599 -0
  29. intentumdiff/core/config.py +51 -0
  30. intentumdiff/core/cst_serializer.py +101 -0
  31. intentumdiff/core/diffignore.py +87 -0
  32. intentumdiff/core/index.py +189 -0
  33. intentumdiff/core/indexer.py +792 -0
  34. intentumdiff/core/models.py +837 -0
  35. intentumdiff/differ.py +2195 -0
  36. intentumdiff/github_app.py +287 -0
  37. intentumdiff/intentumdiff_rust_core/__init__.py +7 -0
  38. intentumdiff/intentumdiff_rust_core/ffi.py +8 -0
  39. intentumdiff/intentumdiff_rust_core/libintentumdiff_rust_core.dylib +0 -0
  40. intentumdiff/invariances/__init__.py +2 -0
  41. intentumdiff/invariances/rules.schema.json +283 -0
  42. intentumdiff/invariances/rules.yaml +349 -0
  43. intentumdiff/live_server.py +1314 -0
  44. intentumdiff/lsp/__init__.py +67 -0
  45. intentumdiff/lsp/client.py +544 -0
  46. intentumdiff/lsp/config.py +96 -0
  47. intentumdiff/lsp/enricher.py +227 -0
  48. intentumdiff/lsp/exceptions.py +27 -0
  49. intentumdiff/lsp/launcher.py +273 -0
  50. intentumdiff/lsp/servers.py +356 -0
  51. intentumdiff/lsp_server/__init__.py +14 -0
  52. intentumdiff/lsp_server/_codelens.py +67 -0
  53. intentumdiff/lsp_server/_diagnostics.py +62 -0
  54. intentumdiff/lsp_server/_handlers.py +175 -0
  55. intentumdiff/lsp_server/server.py +205 -0
  56. intentumdiff/plugins/__init__.py +1 -0
  57. intentumdiff/plugins/adapter.py +397 -0
  58. intentumdiff/plugins/builtins.py +323 -0
  59. intentumdiff/plugins/exceptions.py +93 -0
  60. intentumdiff/plugins/hub.py +1155 -0
  61. intentumdiff/plugins/language_metadata.py +250 -0
  62. intentumdiff/plugins/loader.py +1252 -0
  63. intentumdiff/plugins/registry.py +1079 -0
  64. intentumdiff/plugins/registry.schema.json +99 -0
  65. intentumdiff/plugins/registry_schema.py +280 -0
  66. intentumdiff/plugins/wit/plugin.wit +351 -0
  67. intentumdiff/rust_core.py +2370 -0
  68. intentumdiff/serve/__init__.py +19 -0
  69. intentumdiff/serve/_app.py +485 -0
  70. intentumdiff/serve/_static/index.html +326 -0
  71. intentumdiff/serve/_static/playground.js +567 -0
  72. intentumdiff/sources/__init__.py +1 -0
  73. intentumdiff/sources/base.py +28 -0
  74. intentumdiff/sources/file_source.py +48 -0
  75. intentumdiff/sources/git_source.py +375 -0
  76. intentumdiff/sources/live_buffer_source.py +113 -0
  77. intentumdiff/sources/patch_source.py +126 -0
  78. intentumdiff/sources/string_source.py +44 -0
  79. intentumdiff/testing.py +165 -0
  80. intentumdiff/vcs/__init__.py +37 -0
  81. intentumdiff/vcs/_safe_path.py +50 -0
  82. intentumdiff/vcs/base.py +136 -0
  83. intentumdiff/vcs/git_backend.py +97 -0
  84. intentumdiff/vcs/git_cli.py +100 -0
  85. intentumdiff/vcs/hg_backend.py +94 -0
  86. intentumdiff/vcs/perforce_backend.py +109 -0
  87. intentumdiff/vcs/svn_backend.py +105 -0
  88. intentumdiff/wasm/.gitkeep +2 -0
  89. intentumdiff/wasm/abap_parser.wasm +0 -0
  90. intentumdiff/wasm/adf_parser.wasm +0 -0
  91. intentumdiff/wasm/asciidoc_parser.wasm +0 -0
  92. intentumdiff/wasm/asm_parser.wasm +0 -0
  93. intentumdiff/wasm/assemblyscript_parser.wasm +0 -0
  94. intentumdiff/wasm/astro_parser.wasm +0 -0
  95. intentumdiff/wasm/bash_parser.wasm +0 -0
  96. intentumdiff/wasm/clojure_parser.wasm +0 -0
  97. intentumdiff/wasm/cmake_parser.wasm +0 -0
  98. intentumdiff/wasm/cpp_parser.wasm +0 -0
  99. intentumdiff/wasm/csharp_parser.wasm +0 -0
  100. intentumdiff/wasm/css_parser.wasm +0 -0
  101. intentumdiff/wasm/dart_parser.wasm +0 -0
  102. intentumdiff/wasm/databricks_parser.wasm +0 -0
  103. intentumdiff/wasm/dax_parser.wasm +0 -0
  104. intentumdiff/wasm/delphi_parser.wasm +0 -0
  105. intentumdiff/wasm/dockerfile_parser.wasm +0 -0
  106. intentumdiff/wasm/elixir_parser.wasm +0 -0
  107. intentumdiff/wasm/freebasic_parser.wasm +0 -0
  108. intentumdiff/wasm/generic_parser.wasm +0 -0
  109. intentumdiff/wasm/gitignore_parser.wasm +0 -0
  110. intentumdiff/wasm/go_parser.wasm +0 -0
  111. intentumdiff/wasm/gomod_parser.wasm +0 -0
  112. intentumdiff/wasm/graphql_parser.wasm +0 -0
  113. intentumdiff/wasm/groovy_parser.wasm +0 -0
  114. intentumdiff/wasm/haskell_parser.wasm +0 -0
  115. intentumdiff/wasm/html_parser.wasm +0 -0
  116. intentumdiff/wasm/ini_parser.wasm +0 -0
  117. intentumdiff/wasm/java_parser.wasm +0 -0
  118. intentumdiff/wasm/js_ts_parser.wasm +0 -0
  119. intentumdiff/wasm/json_parser.wasm +0 -0
  120. intentumdiff/wasm/kotlin_parser.wasm +0 -0
  121. intentumdiff/wasm/latex_parser.wasm +0 -0
  122. intentumdiff/wasm/lua_parser.wasm +0 -0
  123. intentumdiff/wasm/make_parser.wasm +0 -0
  124. intentumdiff/wasm/markdown_parser.wasm +0 -0
  125. intentumdiff/wasm/mdx_parser.wasm +0 -0
  126. intentumdiff/wasm/ocaml_parser.wasm +0 -0
  127. intentumdiff/wasm/odin_parser.wasm +0 -0
  128. intentumdiff/wasm/parser_manifest.json +590 -0
  129. intentumdiff/wasm/perl_parser.wasm +0 -0
  130. intentumdiff/wasm/php_parser.wasm +0 -0
  131. intentumdiff/wasm/plsql_parser.wasm +0 -0
  132. intentumdiff/wasm/po_parser.wasm +0 -0
  133. intentumdiff/wasm/postscript_parser.wasm +0 -0
  134. intentumdiff/wasm/powershell_parser.wasm +0 -0
  135. intentumdiff/wasm/proto_parser.wasm +0 -0
  136. intentumdiff/wasm/puppet_parser.wasm +0 -0
  137. intentumdiff/wasm/python_parser.wasm +0 -0
  138. intentumdiff/wasm/qsharp_parser.wasm +0 -0
  139. intentumdiff/wasm/r_parser.wasm +0 -0
  140. intentumdiff/wasm/reasonml_parser.wasm +0 -0
  141. intentumdiff/wasm/ruby_parser.wasm +0 -0
  142. intentumdiff/wasm/rust_parser.wasm +0 -0
  143. intentumdiff/wasm/sas_parser.wasm +0 -0
  144. intentumdiff/wasm/scala_parser.wasm +0 -0
  145. intentumdiff/wasm/scss_parser.wasm +0 -0
  146. intentumdiff/wasm/sql_parser.wasm +0 -0
  147. intentumdiff/wasm/squirrel_parser.wasm +0 -0
  148. intentumdiff/wasm/svelte_parser.wasm +0 -0
  149. intentumdiff/wasm/swift_parser.wasm +0 -0
  150. intentumdiff/wasm/terraform_parser.wasm +0 -0
  151. intentumdiff/wasm/toml_parser.wasm +0 -0
  152. intentumdiff/wasm/tsql_parser.wasm +0 -0
  153. intentumdiff/wasm/vbnet_parser.wasm +0 -0
  154. intentumdiff/wasm/vue_parser.wasm +0 -0
  155. intentumdiff/wasm/wat_parser.wasm +0 -0
  156. intentumdiff/wasm/xml_parser.wasm +0 -0
  157. intentumdiff/wasm/yaml_parser.wasm +0 -0
  158. intentumdiff/wasm/zig_parser.wasm +0 -0
  159. intentumdiff/watcher.py +259 -0
  160. intentumdiff_python-0.0.1.dist-info/METADATA +112 -0
  161. intentumdiff_python-0.0.1.dist-info/RECORD +165 -0
  162. intentumdiff_python-0.0.1.dist-info/WHEEL +4 -0
  163. intentumdiff_python-0.0.1.dist-info/entry_points.txt +80 -0
  164. intentumdiff_python-0.0.1.dist-info/licenses/LICENSE +21 -0
  165. intentumdiff_python-0.0.1.dist-info/sboms/intentumdiff-rust-core.cyclonedx.json +10968 -0
@@ -0,0 +1,792 @@
1
+ """
2
+ intentumdiff.core.indexer
3
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ High-level indexing engine that walks a repository (or a set of files),
6
+ parses each file, and builds a :class:`~intentumdiff.core.index.SemanticIndex`.
7
+
8
+ Progress
9
+ --------
10
+ Every public method accepts an *on_progress* callback that fires before each
11
+ file is processed::
12
+
13
+ from intentumdiff.core.indexer import Indexer, IndexProgress
14
+
15
+ def my_callback(p: IndexProgress) -> None:
16
+ pct = int(p.fraction * 100)
17
+ print(f"[{pct:3d}%] {p.current_file}")
18
+
19
+ indexer = Indexer(differ, on_progress=my_callback)
20
+ result = indexer.index_repo(".")
21
+
22
+ The callback receives an :class:`IndexProgress` instance *before* each file is
23
+ processed (so the caller can display a spinner with the active filename) and a
24
+ final "done" event with ``current_file == ""`` when indexing completes.
25
+
26
+ A ``None`` per-call *on_progress* falls back to the constructor-level default.
27
+ Pass a no-op lambda to suppress events for a specific call::
28
+
29
+ result = indexer.index_repo(".", on_progress=lambda _: None)
30
+
31
+ Caching
32
+ -------
33
+ When the :class:`~intentumdiff.differ.SemanticDiffer` was created with a
34
+ ``DiffConfig.cache_path``, two levels of caching apply automatically:
35
+
36
+ * **Parse-tree cache** — each file's :class:`~intentumdiff.core.models.SemanticNode`
37
+ is cached keyed by content hash. Re-indexing the same file content skips
38
+ the Wasm call entirely.
39
+
40
+ * **Symbol-index cache** — after :meth:`Indexer.index_repo` finishes, the
41
+ resulting symbol and reference tables are stored in SQLite keyed by
42
+ ``(repo_root, commit_sha)``. A subsequent call for the *same* commit returns
43
+ the cached index immediately (unless *force=True*).
44
+ """
45
+
46
+ from __future__ import annotations
47
+
48
+ import hashlib
49
+ import json
50
+ import logging
51
+ from dataclasses import dataclass, field
52
+ from pathlib import Path
53
+ from typing import TYPE_CHECKING, Any, Callable
54
+
55
+ from intentumdiff.core.diffignore import load_diffignore
56
+
57
+ if TYPE_CHECKING:
58
+ from intentumdiff.core.index import SemanticIndex
59
+ from intentumdiff.core.models import SemanticNode
60
+ from intentumdiff.differ import SemanticDiffer
61
+
62
+ logger = logging.getLogger(__name__)
63
+
64
+ # Maximum number of files enriched concurrently by _index_files_lsp.
65
+ # Each slot holds one file open against the LSP server and fires its hover
66
+ # requests in parallel (bounded separately by TypeEnricher._MAX_CONCURRENT).
67
+ # Raise this if your server is fast and you have many files; lower it to
68
+ # reduce server load on shared/slow servers.
69
+ _FILE_CONCURRENCY = 8
70
+
71
+ # Number of threads used to read git blobs in parallel during file collection.
72
+ # Git blob reads are pure disk I/O, so threading gives near-linear speed-up
73
+ # up to the throughput limit of the storage device.
74
+ _BLOB_READ_WORKERS = 32
75
+
76
+ # Bytes read from each blob for language detection. Content-based detectors
77
+ # (tsql, plsql, adf, dax, postscript …) only need the file header; reading
78
+ # the full blob during a language-scan pass wastes bandwidth.
79
+ _DETECT_HEAD_BYTES = 2048
80
+
81
+ # ---------------------------------------------------------------------------
82
+ # Public types
83
+ # ---------------------------------------------------------------------------
84
+
85
+
86
+ @dataclass
87
+ class IndexProgress:
88
+ """
89
+ Snapshot of indexing progress, emitted via the *on_progress* callback.
90
+
91
+ The callback fires **before** each file is processed, allowing the UI to
92
+ show the name of the file currently being worked on. A final event with
93
+ ``current_file == ""`` signals that indexing has completed.
94
+ """
95
+
96
+ total: int
97
+ """Total number of candidate files discovered."""
98
+
99
+ done: int
100
+ """Files processed so far (includes skipped and errored files)."""
101
+
102
+ current_file: str
103
+ """Path of the file about to be processed; empty string in the final event."""
104
+
105
+ skipped: int = 0
106
+ """Files that had no registered parser and were silently skipped."""
107
+
108
+ errors: int = 0
109
+ """Files that raised an unexpected error during parsing."""
110
+
111
+ @property
112
+ def fraction(self) -> float:
113
+ """Progress as a float in ``[0.0, 1.0]``."""
114
+ return self.done / self.total if self.total > 0 else 0.0
115
+
116
+
117
+ @dataclass
118
+ class IndexResult:
119
+ """Returned by every :class:`Indexer` indexing method."""
120
+
121
+ files_indexed: int
122
+ """Number of files successfully parsed and added to the index."""
123
+
124
+ files_skipped: int
125
+ """Number of files skipped because no registered parser was found."""
126
+
127
+ errors: list[tuple[str, str]]
128
+ """``(filename, error_message)`` for files that failed to parse."""
129
+
130
+ semantic_index: "SemanticIndex"
131
+ """The fully built :class:`~intentumdiff.core.index.SemanticIndex`."""
132
+
133
+ from_cache: bool = False
134
+ """``True`` when the symbol index was loaded from the local cache."""
135
+
136
+ skipped_files: list[str] = field(default_factory=list)
137
+ """Filenames that were skipped (no registered parser)."""
138
+
139
+ type_enriched_count: int = 0
140
+ """Number of nodes that received ``type_info`` from an LSP hover call."""
141
+
142
+ languages_found: set[str] = field(default_factory=set)
143
+ """Set of language identifiers successfully parsed in this run."""
144
+
145
+ files_ignored: int = 0
146
+ """Number of files excluded by a ``.diffignore`` file."""
147
+
148
+ ignored_files: list[str] = field(default_factory=list)
149
+ """Relative paths excluded by ``.diffignore``."""
150
+
151
+
152
+ # Callback type alias — a plain callable so any UX layer can subscribe.
153
+ ProgressCallback = Callable[[IndexProgress], None]
154
+
155
+
156
+ # ---------------------------------------------------------------------------
157
+ # Internal helpers
158
+ # ---------------------------------------------------------------------------
159
+
160
+
161
+ def _make_index_key(repo_root: str, commit_sha: str) -> str:
162
+ """Return a stable cache key for ``(repo_root, commit_sha)``."""
163
+ encoded = repo_root.encode() + b"\x00" + commit_sha.encode()
164
+ return hashlib.sha256(encoded).hexdigest()
165
+
166
+
167
+ def _parse_catfile_output(
168
+ buf: bytes,
169
+ sha_to_path: "dict[str, str]",
170
+ ) -> "list[tuple[str, str]]":
171
+ """Parse ``git cat-file --batch`` stdout into ``(path, content)`` pairs."""
172
+ files: list[tuple[str, str]] = []
173
+ i = 0
174
+ n = len(buf)
175
+ while i < n:
176
+ nl = buf.find(b"\n", i)
177
+ if nl == -1:
178
+ break
179
+ header = buf[i:nl].decode("ascii", errors="replace")
180
+ i = nl + 1
181
+ parts = header.split()
182
+ # Expected: "<sha> blob <size>"; skip "missing" / non-blob entries.
183
+ if len(parts) < 3 or parts[1] != "blob":
184
+ continue
185
+ try:
186
+ size = int(parts[2])
187
+ except ValueError:
188
+ continue
189
+ content_bytes = buf[i : i + size]
190
+ i += size + 1 # skip trailing \n
191
+ if b"\x00" in content_bytes[:8192]: # binary heuristic
192
+ continue
193
+ path = sha_to_path.get(parts[0], "")
194
+ if not path:
195
+ continue
196
+ try:
197
+ content = content_bytes.decode("utf-8", errors="strict")
198
+ except UnicodeDecodeError:
199
+ content = content_bytes.decode("utf-8", errors="replace")
200
+ files.append((path, content))
201
+ return files
202
+
203
+
204
+ def _collect_blob_files(
205
+ repo_root: str,
206
+ ref: str,
207
+ diffignore: Any,
208
+ ) -> "tuple[list[tuple[str, str]], list[str]]":
209
+ """Return all text files at *ref* as ``(relative_path, decoded_content)``.
210
+
211
+ Shells to the git CLI (#98, A2.4.3): ``git ls-tree -r`` lists every blob and a
212
+ single ``git cat-file --batch`` reads them through git's optimised pack reader.
213
+ Binary blobs (NUL-byte heuristic in :func:`_parse_catfile_output`) are skipped.
214
+
215
+ Returns ``(files, ignored_files)`` where *ignored_files* are the paths excluded
216
+ by *diffignore*.
217
+ """
218
+ from intentumdiff.vcs.git_cli import run_git_bytes # noqa: PLC0415
219
+
220
+ ignored_files: list[str] = []
221
+ sha_to_path: dict[str, str] = {}
222
+ # ls-tree -r -z: "<mode> <type> <sha>\t<path>\0" per entry.
223
+ for entry in run_git_bytes(repo_root, ["ls-tree", "-r", "-z", ref]).split(b"\0"):
224
+ if not entry:
225
+ continue
226
+ meta, _tab, raw_path = entry.partition(b"\t")
227
+ parts = meta.split()
228
+ if len(parts) < 3 or parts[1] != b"blob":
229
+ continue
230
+ path = raw_path.decode("utf-8", errors="replace")
231
+ if diffignore and diffignore.is_ignored(path):
232
+ ignored_files.append(path)
233
+ continue
234
+ sha_to_path[parts[2].decode("ascii", errors="replace")] = path
235
+
236
+ if not sha_to_path:
237
+ return [], ignored_files
238
+
239
+ shas_input = "\n".join(sha_to_path).encode()
240
+ out = run_git_bytes(repo_root, ["cat-file", "--batch"], input_bytes=shas_input)
241
+ return _parse_catfile_output(out, sha_to_path), ignored_files
242
+
243
+
244
+ def _attach_type_info(
245
+ node: "SemanticNode",
246
+ type_map: dict[str, str],
247
+ ) -> "SemanticNode":
248
+ """Return a copy of *node* tree with ``type_info`` set from *type_map*.
249
+
250
+ ``SemanticNode`` is frozen so we must rebuild changed nodes from the leaves
251
+ up. Only nodes whose id appears in *type_map* are rebuilt.
252
+ """
253
+ from intentumdiff.core.models import SemanticNode # noqa: PLC0415
254
+
255
+ new_children = [_attach_type_info(c, type_map) for c in node.children]
256
+ if node.id in type_map or new_children != list(node.children):
257
+ return node.model_copy(
258
+ update={
259
+ "children": new_children,
260
+ **({"type_info": type_map[node.id]} if node.id in type_map else {}),
261
+ }
262
+ )
263
+ return node
264
+
265
+
266
+ # ---------------------------------------------------------------------------
267
+ # Indexer
268
+ # ---------------------------------------------------------------------------
269
+
270
+
271
+ class Indexer:
272
+ """
273
+ Walk a repository (or an explicit list of files) and build a
274
+ :class:`~intentumdiff.core.index.SemanticIndex`.
275
+
276
+ Parameters
277
+ ----------
278
+ differ:
279
+ A :class:`~intentumdiff.differ.SemanticDiffer` instance. Configure
280
+ it with ``DiffConfig.cache_path`` to activate parse-tree and
281
+ symbol-index caching for faster re-indexing.
282
+ on_progress:
283
+ Default :data:`ProgressCallback` applied to every method call that
284
+ accepts *on_progress*. Individual calls may override this or pass
285
+ ``None`` to suppress progress events for that call.
286
+ """
287
+
288
+ def __init__(
289
+ self,
290
+ differ: "SemanticDiffer",
291
+ *,
292
+ on_progress: ProgressCallback | None = None,
293
+ ) -> None:
294
+ self._differ = differ
295
+ self._on_progress = on_progress
296
+
297
+ # ── Public API ──────────────────────────────────────────────────────────
298
+
299
+ def detect_languages(
300
+ self,
301
+ repo_path: str | Path = ".",
302
+ ref: str = "HEAD",
303
+ *,
304
+ on_progress: "Callable[[int, int], None] | None" = None,
305
+ ) -> "tuple[set[str], int]":
306
+ """Return the set of languages present in *ref* without parsing any file.
307
+
308
+ Reads only the first :data:`_DETECT_HEAD_BYTES` of each blob (enough
309
+ for all content-based detectors) and calls the plugin
310
+ ``detect_language`` heuristics. Far faster than a full
311
+ :meth:`index_repo` scan — useful as a lightweight pre-pass to decide
312
+ which LSP servers to start before a type-enrichment run.
313
+
314
+ *on_progress(done, total)* is called after each blob is processed.
315
+ """
316
+ from intentumdiff.plugins.exceptions import ( # noqa: PLC0415
317
+ PluginNotFoundError,
318
+ )
319
+ from intentumdiff.vcs.git_cli import resolve_repo_root # noqa: PLC0415
320
+
321
+ repo_root = resolve_repo_root(repo_path)
322
+ diffignore = load_diffignore(repo_root)
323
+ files, _ignored = _collect_blob_files(repo_root, ref, diffignore)
324
+
325
+ total = len(files)
326
+ if on_progress:
327
+ on_progress(0, total)
328
+
329
+ # Detection inspects only the head of each blob (enough for all content-based
330
+ # detectors); content comes from the shared git cat-file batch collector.
331
+ languages: set[str] = set()
332
+ for i, (path, content) in enumerate(files):
333
+ if on_progress:
334
+ on_progress(i + 1, total)
335
+ content_head = content[:_DETECT_HEAD_BYTES]
336
+ try:
337
+ _, lang = self._differ._registry.detect_parser(path, content_head)
338
+ languages.add(lang)
339
+ except PluginNotFoundError:
340
+ pass
341
+
342
+ return languages, total
343
+
344
+ def index_repo(
345
+ self,
346
+ repo_path: str | Path = ".",
347
+ ref: str = "HEAD",
348
+ *,
349
+ on_progress: ProgressCallback | None = None,
350
+ force: bool = False,
351
+ ) -> IndexResult:
352
+ """
353
+ Parse every text file in a git repository at *ref* and build an index.
354
+
355
+ Parameters
356
+ ----------
357
+ repo_path:
358
+ Path to the repository root (or any subdirectory; GitPython will
359
+ locate the root automatically).
360
+ ref:
361
+ Git ref to index. Defaults to ``"HEAD"``.
362
+ on_progress:
363
+ Progress callback for this call. Falls back to the callback
364
+ passed to ``__init__`` when ``None``.
365
+ force:
366
+ Re-index even if a cached symbol index already exists for this
367
+ commit. Useful after a plugin or wasm binary update.
368
+
369
+ Returns
370
+ -------
371
+ IndexResult
372
+ Fully built index. Check :attr:`IndexResult.from_cache` to
373
+ determine whether the symbol table was loaded from cache.
374
+ """
375
+ from intentumdiff.vcs.git_cli import ( # noqa: PLC0415
376
+ resolve_repo_root,
377
+ run_git_bytes,
378
+ )
379
+
380
+ repo_root = resolve_repo_root(repo_path)
381
+ commit_sha = (
382
+ run_git_bytes(repo_root, ["rev-parse", "--verify", ref]).decode().strip()
383
+ )
384
+ callback = on_progress or self._on_progress
385
+
386
+ # ── Fast path: cached symbol index ────────────────────────────────
387
+ if not force and self._differ._cache is not None:
388
+ index_key = _make_index_key(repo_root, commit_sha)
389
+ cached = self._differ._cache.get_symbol_index(index_key)
390
+ if cached is not None:
391
+ logger.debug(
392
+ "Symbol index cache hit for %s @ %s",
393
+ repo_root,
394
+ commit_sha[:8],
395
+ )
396
+ from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
397
+
398
+ sem_index = SemanticIndex()
399
+ symbols_json, refs_json = cached
400
+ sem_index.load_symbol_table_json(symbols_json)
401
+ sem_index.load_reference_table_json(refs_json)
402
+ return IndexResult(
403
+ files_indexed=0,
404
+ files_skipped=0,
405
+ errors=[],
406
+ semantic_index=sem_index,
407
+ from_cache=True,
408
+ )
409
+
410
+ # ── Collect all text files in commit tree ─────────────────────────
411
+ diffignore = load_diffignore(repo_root)
412
+ files, ignored_files = _collect_blob_files(repo_root, ref, diffignore)
413
+
414
+ result = self._index_files(files, on_progress=callback)
415
+ result.files_ignored = len(ignored_files)
416
+ result.ignored_files = ignored_files
417
+
418
+ # ── Persist symbol index ───────────────────────────────────────────
419
+ if self._differ._cache is not None:
420
+ index_key = _make_index_key(repo_root, commit_sha)
421
+ self._store_symbol_index(
422
+ result.semantic_index, index_key, result.files_indexed
423
+ )
424
+
425
+ return result
426
+
427
+ def index_directory(
428
+ self,
429
+ directory: str | Path,
430
+ *,
431
+ on_progress: ProgressCallback | None = None,
432
+ ) -> IndexResult:
433
+ """
434
+ Parse every text file under *directory* (recursively) and build an index.
435
+
436
+ Parse-tree caching applies if the differ has a cache configured.
437
+ Symbol-index caching is **not** applied because there is no stable
438
+ commit key for a plain directory.
439
+
440
+ Parameters
441
+ ----------
442
+ directory:
443
+ Root directory to walk recursively.
444
+ on_progress:
445
+ Progress callback for this call. Falls back to the callback
446
+ passed to ``__init__`` when ``None``.
447
+ """
448
+ directory = Path(directory)
449
+ callback = on_progress or self._on_progress
450
+
451
+ diffignore = load_diffignore(directory)
452
+ ignored_files: list[str] = []
453
+ files: list[tuple[str, str]] = []
454
+ for path in sorted(directory.rglob("*")):
455
+ if not path.is_file():
456
+ continue
457
+ rel_posix = path.relative_to(directory).as_posix()
458
+ if diffignore and diffignore.is_ignored(rel_posix):
459
+ ignored_files.append(rel_posix)
460
+ continue
461
+ try:
462
+ content = path.read_text(encoding="utf-8", errors="replace")
463
+ except OSError:
464
+ continue
465
+ files.append((rel_posix, content))
466
+
467
+ result = self._index_files(files, on_progress=callback)
468
+ result.files_ignored = len(ignored_files)
469
+ result.ignored_files = ignored_files
470
+ return result
471
+
472
+ # ── Internal ────────────────────────────────────────────────────────────
473
+
474
+ def _index_files(
475
+ self,
476
+ files: list[tuple[str, str]],
477
+ *,
478
+ on_progress: ProgressCallback | None,
479
+ ) -> IndexResult:
480
+ """Core indexing loop — parse each (filename, content) pair."""
481
+ from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
482
+ from intentumdiff.plugins.exceptions import ( # noqa: PLC0415
483
+ PluginNotFoundError,
484
+ )
485
+
486
+ sem_index = SemanticIndex()
487
+ errors: list[tuple[str, str]] = []
488
+ skipped_files: list[str] = []
489
+ languages_found: set[str] = set()
490
+ done = 0
491
+ total = len(files)
492
+
493
+ for filename, content in files:
494
+ # Emit progress *before* processing so the UI shows the active file.
495
+ if on_progress is not None:
496
+ on_progress(
497
+ IndexProgress(
498
+ total=total,
499
+ done=done,
500
+ current_file=filename,
501
+ skipped=len(skipped_files),
502
+ errors=len(errors),
503
+ )
504
+ )
505
+
506
+ try:
507
+ node, language = self._differ.parse(content, filename)
508
+ sem_index.add_tree(filename, language, node)
509
+ languages_found.add(language)
510
+ except PluginNotFoundError:
511
+ skipped_files.append(filename)
512
+ except Exception as exc: # noqa: BLE001
513
+ errors.append((filename, str(exc)))
514
+ logger.debug("Failed to parse %r: %s", filename, exc)
515
+
516
+ done += 1
517
+
518
+ # Final event: done == total, current_file empty.
519
+ if on_progress is not None:
520
+ on_progress(
521
+ IndexProgress(
522
+ total=total,
523
+ done=done,
524
+ current_file="",
525
+ skipped=len(skipped_files),
526
+ errors=len(errors),
527
+ )
528
+ )
529
+
530
+ sem_index.build()
531
+ indexed = max(0, done - len(skipped_files) - len(errors))
532
+ return IndexResult(
533
+ files_indexed=indexed,
534
+ files_skipped=len(skipped_files),
535
+ skipped_files=skipped_files,
536
+ errors=errors,
537
+ semantic_index=sem_index,
538
+ languages_found=languages_found,
539
+ )
540
+
541
+ def _store_symbol_index(
542
+ self,
543
+ sem_index: "SemanticIndex",
544
+ index_key: str,
545
+ file_count: int,
546
+ ) -> None:
547
+ """Serialise and persist *sem_index* to the symbol-index cache."""
548
+ cache = self._differ._cache
549
+ if cache is None:
550
+ raise RuntimeError("symbol-index cache store is not configured")
551
+ try:
552
+ symbols_json = json.dumps(
553
+ {k: [d.model_dump() for d in v] for k, v in sem_index.symbols.items()}
554
+ )
555
+ refs_json = json.dumps(
556
+ {k: [r.model_dump() for r in v] for k, v in sem_index.references.items()}
557
+ )
558
+ cache.put_symbol_index(
559
+ index_key,
560
+ symbols_json,
561
+ refs_json,
562
+ file_count=file_count,
563
+ )
564
+ logger.debug("Stored symbol index in cache (key=%s…)", index_key[:12])
565
+ except Exception as exc: # noqa: BLE001
566
+ logger.warning("Failed to store symbol index in cache: %s", exc)
567
+
568
+ async def index_repo_lsp(
569
+ self,
570
+ repo_path: str | Path = ".",
571
+ ref: str = "HEAD",
572
+ *,
573
+ lsp_clients: "dict[str, Any]",
574
+ on_progress: ProgressCallback | None = None,
575
+ force: bool = False,
576
+ file_concurrency: int = _FILE_CONCURRENCY,
577
+ ) -> IndexResult:
578
+ """Like :meth:`index_repo` but attaches LSP type info to each parsed tree.
579
+
580
+ Parameters
581
+ ----------
582
+ lsp_clients:
583
+ Map of language name → connected ``AsyncLspClient`` instance.
584
+ Files whose language has no matching client are parsed normally.
585
+ file_concurrency:
586
+ Maximum number of files processed concurrently. Each slot holds
587
+ one file open against the LSP server while firing its hover
588
+ requests. Defaults to :data:`_FILE_CONCURRENCY` (8). Set higher
589
+ for fast local servers, lower for shared or rate-limited servers.
590
+ """
591
+ import asyncio
592
+ from intentumdiff.vcs.git_cli import ( # noqa: PLC0415
593
+ resolve_repo_root,
594
+ run_git_bytes,
595
+ )
596
+
597
+ repo_root = resolve_repo_root(repo_path)
598
+ commit_sha = (
599
+ run_git_bytes(repo_root, ["rev-parse", "--verify", ref]).decode().strip()
600
+ )
601
+ callback = on_progress or self._on_progress
602
+
603
+ # LSP-enriched indexes are stored under a separate key so they don't
604
+ # collide with plain (non-enriched) index_repo cache entries.
605
+ lsp_index_key = _make_index_key(repo_root, "lsp:" + commit_sha)
606
+
607
+ # ── Fast path: cached LSP symbol index ────────────────────────────
608
+ if not force and self._differ._cache is not None:
609
+ cached = self._differ._cache.get_symbol_index(lsp_index_key)
610
+ if cached is not None:
611
+ logger.debug(
612
+ "LSP symbol index cache hit for %s @ %s",
613
+ repo_root,
614
+ commit_sha[:8],
615
+ )
616
+ from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
617
+
618
+ sem_index = SemanticIndex()
619
+ symbols_json, refs_json = cached
620
+ sem_index.load_symbol_table_json(symbols_json)
621
+ sem_index.load_reference_table_json(refs_json)
622
+ return IndexResult(
623
+ files_indexed=0,
624
+ files_skipped=0,
625
+ errors=[],
626
+ semantic_index=sem_index,
627
+ from_cache=True,
628
+ )
629
+
630
+ # Collect files (parallel blob reads; same filtering as index_repo)
631
+ diffignore = load_diffignore(repo_root)
632
+ files, ignored_files = await asyncio.to_thread(
633
+ _collect_blob_files, repo_root, ref, diffignore
634
+ )
635
+
636
+ result = await self._index_files_lsp(
637
+ files,
638
+ lsp_clients=lsp_clients,
639
+ on_progress=callback,
640
+ file_concurrency=file_concurrency,
641
+ )
642
+ result.files_ignored = len(ignored_files)
643
+ result.ignored_files = ignored_files
644
+
645
+ if self._differ._cache is not None:
646
+ self._store_symbol_index(result.semantic_index, lsp_index_key, result.files_indexed)
647
+
648
+ return result
649
+
650
+ async def _index_files_lsp(
651
+ self,
652
+ files: list[tuple[str, str]],
653
+ *,
654
+ lsp_clients: "dict[str, Any]",
655
+ on_progress: ProgressCallback | None,
656
+ file_concurrency: int = _FILE_CONCURRENCY,
657
+ ) -> IndexResult:
658
+ """Async indexing loop — parse then type-enrich files concurrently.
659
+
660
+ Files are processed up to *file_concurrency* at a time. Within each
661
+ file all hover requests are already parallelised by
662
+ :class:`~intentumdiff.lsp.enricher.TypeEnricher`. After all tasks
663
+ complete, results are merged into the ``SemanticIndex`` sequentially so
664
+ the index itself never sees concurrent writes.
665
+ """
666
+ import asyncio # noqa: PLC0415
667
+ import concurrent.futures # noqa: PLC0415
668
+
669
+ from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
670
+ from intentumdiff.lsp.enricher import TypeEnricher # noqa: PLC0415
671
+ from intentumdiff.plugins.exceptions import PluginNotFoundError # noqa: PLC0415
672
+
673
+ total = len(files)
674
+
675
+ # Build enrichers once per language.
676
+ enrichers: dict[str, TypeEnricher] = {
677
+ lang: TypeEnricher(client, language=lang)
678
+ for lang, client in lsp_clients.items()
679
+ }
680
+
681
+ # Result type for each file task.
682
+ # Success: (filename, language, node, enriched_count)
683
+ # Skipped: PluginNotFoundError (re-raised so we can detect it)
684
+ # Error: any other exception
685
+
686
+ sem = asyncio.Semaphore(max(1, file_concurrency))
687
+ cache = self._differ._cache
688
+ differ = self._differ # captured for executor
689
+
690
+ # Use a single-worker executor so Wasm parse calls are serialised.
691
+ # Concurrent Wasm execution across multiple Stores/Engines triggers
692
+ # non-deterministic "bad parameter or other API misuse" errors from
693
+ # wasmtime. A single worker keeps parse calls sequential while still
694
+ # freeing the event loop so LSP enrich for completed files overlaps
695
+ # with the next file's parse (pipeline parallelism).
696
+ loop = asyncio.get_event_loop()
697
+ parse_executor = concurrent.futures.ThreadPoolExecutor(
698
+ max_workers=1, thread_name_prefix="wasm_parse"
699
+ )
700
+
701
+ async def _process_one(
702
+ filename: str,
703
+ content: str,
704
+ ) -> "tuple[str, str, Any, int]":
705
+ """Parse + optionally enrich one file; always returns a result tuple.
706
+
707
+ Returns ``(filename, language, node, enriched_count)`` on success,
708
+ ``(filename, 'SKIP', exc, 0)`` when no parser is registered, or
709
+ ``(filename, 'ERROR', exc, 0)`` on any other failure.
710
+ """
711
+ async with sem:
712
+ # Run the blocking Wasm parse in a dedicated single-worker
713
+ # thread so the event loop stays free for concurrent LSP I/O.
714
+ # A 1-worker pool prevents concurrent Wasm Store access which
715
+ # causes non-deterministic wasmtime errors.
716
+ try:
717
+ node, language = await loop.run_in_executor(
718
+ parse_executor, differ.parse, content, filename
719
+ )
720
+ except PluginNotFoundError as exc:
721
+ return filename, "SKIP", exc, 0
722
+ except Exception as exc:
723
+ return filename, "ERROR", exc, 0
724
+
725
+ enricher = enrichers.get(language)
726
+ enriched_count = 0
727
+ if enricher is not None:
728
+ # Check hover-map cache first.
729
+ type_map: dict[str, str] | None = None
730
+ hm_key: str | None = None
731
+ if cache is not None:
732
+ hm_key = cache.hover_map_key(content, language)
733
+ type_map = cache.get_hover_map(hm_key)
734
+ if type_map is None:
735
+ try:
736
+ type_map = await enricher.enrich(filename, content, node)
737
+ except Exception:
738
+ type_map = None
739
+ if type_map and hm_key is not None and cache is not None:
740
+ cache.put_hover_map(hm_key, type_map)
741
+ if type_map:
742
+ enriched_count = len(type_map)
743
+ node = _attach_type_info(node, type_map)
744
+ return filename, language, node, enriched_count
745
+
746
+ tasks = [asyncio.create_task(_process_one(f, c)) for f, c in files]
747
+
748
+ # Merge results as tasks complete so progress updates in real time.
749
+ sem_index = SemanticIndex()
750
+ errors: list[tuple[str, str]] = []
751
+ skipped_files: list[str] = []
752
+ languages_found: set[str] = set()
753
+ total_enriched = 0
754
+ done = 0
755
+
756
+ for future in asyncio.as_completed(tasks):
757
+ filename, language, payload, enriched_count = await future
758
+ done += 1
759
+
760
+ if language == "SKIP":
761
+ skipped_files.append(filename)
762
+ elif language == "ERROR":
763
+ errors.append((filename, str(payload)))
764
+ logger.debug("Failed to process %r: %s", filename, payload)
765
+ else:
766
+ sem_index.add_tree(filename, language, payload)
767
+ languages_found.add(language)
768
+ total_enriched += enriched_count
769
+
770
+ if on_progress is not None:
771
+ on_progress(
772
+ IndexProgress(
773
+ total=total,
774
+ done=done,
775
+ current_file=filename,
776
+ skipped=len(skipped_files),
777
+ errors=len(errors),
778
+ )
779
+ )
780
+
781
+ sem_index.build()
782
+ parse_executor.shutdown(wait=False)
783
+ indexed = max(0, done - len(skipped_files) - len(errors))
784
+ return IndexResult(
785
+ files_indexed=indexed,
786
+ files_skipped=len(skipped_files),
787
+ skipped_files=skipped_files,
788
+ errors=errors,
789
+ semantic_index=sem_index,
790
+ languages_found=languages_found,
791
+ type_enriched_count=total_enriched,
792
+ )