intentumdiff-python 0.0.1__py3-none-macosx_11_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- intentumdiff/__init__.py +134 -0
- intentumdiff/_differ_gate.py +210 -0
- intentumdiff/_differ_presentation.py +786 -0
- intentumdiff/_differ_runtime.py +320 -0
- intentumdiff/analysis/__init__.py +1 -0
- intentumdiff/analysis/compile_commands.py +151 -0
- intentumdiff/analysis/cross_file.py +71 -0
- intentumdiff/analysis/diagnostics.py +309 -0
- intentumdiff/analysis/guardrail_reports.py +254 -0
- intentumdiff/analysis/guardrails.py +318 -0
- intentumdiff/analysis/invariances.py +1207 -0
- intentumdiff/analysis/keyed_profiles.py +25 -0
- intentumdiff/analysis/resource_profiles.py +14 -0
- intentumdiff/analysis/schema_resolver.py +816 -0
- intentumdiff/analysis/text_review.py +46 -0
- intentumdiff/analysis/user_schemas.py +430 -0
- intentumdiff/assets.py +45 -0
- intentumdiff/cache/__init__.py +37 -0
- intentumdiff/cache/duckdb_store.py +98 -0
- intentumdiff/cache/sqlite_store.py +149 -0
- intentumdiff/cache/store.py +190 -0
- intentumdiff/cli/__init__.py +199 -0
- intentumdiff/cli/_commands.py +2141 -0
- intentumdiff/cli/_parser.py +1061 -0
- intentumdiff/cli/_shared.py +644 -0
- intentumdiff/content_type.py +59 -0
- intentumdiff/core/__init__.py +1 -0
- intentumdiff/core/commit_differ.py +599 -0
- intentumdiff/core/config.py +51 -0
- intentumdiff/core/cst_serializer.py +101 -0
- intentumdiff/core/diffignore.py +87 -0
- intentumdiff/core/index.py +189 -0
- intentumdiff/core/indexer.py +792 -0
- intentumdiff/core/models.py +837 -0
- intentumdiff/differ.py +2195 -0
- intentumdiff/github_app.py +287 -0
- intentumdiff/intentumdiff_rust_core/__init__.py +7 -0
- intentumdiff/intentumdiff_rust_core/ffi.py +8 -0
- intentumdiff/intentumdiff_rust_core/libintentumdiff_rust_core.dylib +0 -0
- intentumdiff/invariances/__init__.py +2 -0
- intentumdiff/invariances/rules.schema.json +283 -0
- intentumdiff/invariances/rules.yaml +349 -0
- intentumdiff/live_server.py +1314 -0
- intentumdiff/lsp/__init__.py +67 -0
- intentumdiff/lsp/client.py +544 -0
- intentumdiff/lsp/config.py +96 -0
- intentumdiff/lsp/enricher.py +227 -0
- intentumdiff/lsp/exceptions.py +27 -0
- intentumdiff/lsp/launcher.py +273 -0
- intentumdiff/lsp/servers.py +356 -0
- intentumdiff/lsp_server/__init__.py +14 -0
- intentumdiff/lsp_server/_codelens.py +67 -0
- intentumdiff/lsp_server/_diagnostics.py +62 -0
- intentumdiff/lsp_server/_handlers.py +175 -0
- intentumdiff/lsp_server/server.py +205 -0
- intentumdiff/plugins/__init__.py +1 -0
- intentumdiff/plugins/adapter.py +397 -0
- intentumdiff/plugins/builtins.py +323 -0
- intentumdiff/plugins/exceptions.py +93 -0
- intentumdiff/plugins/hub.py +1155 -0
- intentumdiff/plugins/language_metadata.py +250 -0
- intentumdiff/plugins/loader.py +1252 -0
- intentumdiff/plugins/registry.py +1079 -0
- intentumdiff/plugins/registry.schema.json +99 -0
- intentumdiff/plugins/registry_schema.py +280 -0
- intentumdiff/plugins/wit/plugin.wit +351 -0
- intentumdiff/rust_core.py +2370 -0
- intentumdiff/serve/__init__.py +19 -0
- intentumdiff/serve/_app.py +485 -0
- intentumdiff/serve/_static/index.html +326 -0
- intentumdiff/serve/_static/playground.js +567 -0
- intentumdiff/sources/__init__.py +1 -0
- intentumdiff/sources/base.py +28 -0
- intentumdiff/sources/file_source.py +48 -0
- intentumdiff/sources/git_source.py +375 -0
- intentumdiff/sources/live_buffer_source.py +113 -0
- intentumdiff/sources/patch_source.py +126 -0
- intentumdiff/sources/string_source.py +44 -0
- intentumdiff/testing.py +165 -0
- intentumdiff/vcs/__init__.py +37 -0
- intentumdiff/vcs/_safe_path.py +50 -0
- intentumdiff/vcs/base.py +136 -0
- intentumdiff/vcs/git_backend.py +97 -0
- intentumdiff/vcs/git_cli.py +100 -0
- intentumdiff/vcs/hg_backend.py +94 -0
- intentumdiff/vcs/perforce_backend.py +109 -0
- intentumdiff/vcs/svn_backend.py +105 -0
- intentumdiff/wasm/.gitkeep +2 -0
- intentumdiff/wasm/abap_parser.wasm +0 -0
- intentumdiff/wasm/adf_parser.wasm +0 -0
- intentumdiff/wasm/asciidoc_parser.wasm +0 -0
- intentumdiff/wasm/asm_parser.wasm +0 -0
- intentumdiff/wasm/assemblyscript_parser.wasm +0 -0
- intentumdiff/wasm/astro_parser.wasm +0 -0
- intentumdiff/wasm/bash_parser.wasm +0 -0
- intentumdiff/wasm/clojure_parser.wasm +0 -0
- intentumdiff/wasm/cmake_parser.wasm +0 -0
- intentumdiff/wasm/cpp_parser.wasm +0 -0
- intentumdiff/wasm/csharp_parser.wasm +0 -0
- intentumdiff/wasm/css_parser.wasm +0 -0
- intentumdiff/wasm/dart_parser.wasm +0 -0
- intentumdiff/wasm/databricks_parser.wasm +0 -0
- intentumdiff/wasm/dax_parser.wasm +0 -0
- intentumdiff/wasm/delphi_parser.wasm +0 -0
- intentumdiff/wasm/dockerfile_parser.wasm +0 -0
- intentumdiff/wasm/elixir_parser.wasm +0 -0
- intentumdiff/wasm/freebasic_parser.wasm +0 -0
- intentumdiff/wasm/generic_parser.wasm +0 -0
- intentumdiff/wasm/gitignore_parser.wasm +0 -0
- intentumdiff/wasm/go_parser.wasm +0 -0
- intentumdiff/wasm/gomod_parser.wasm +0 -0
- intentumdiff/wasm/graphql_parser.wasm +0 -0
- intentumdiff/wasm/groovy_parser.wasm +0 -0
- intentumdiff/wasm/haskell_parser.wasm +0 -0
- intentumdiff/wasm/html_parser.wasm +0 -0
- intentumdiff/wasm/ini_parser.wasm +0 -0
- intentumdiff/wasm/java_parser.wasm +0 -0
- intentumdiff/wasm/js_ts_parser.wasm +0 -0
- intentumdiff/wasm/json_parser.wasm +0 -0
- intentumdiff/wasm/kotlin_parser.wasm +0 -0
- intentumdiff/wasm/latex_parser.wasm +0 -0
- intentumdiff/wasm/lua_parser.wasm +0 -0
- intentumdiff/wasm/make_parser.wasm +0 -0
- intentumdiff/wasm/markdown_parser.wasm +0 -0
- intentumdiff/wasm/mdx_parser.wasm +0 -0
- intentumdiff/wasm/ocaml_parser.wasm +0 -0
- intentumdiff/wasm/odin_parser.wasm +0 -0
- intentumdiff/wasm/parser_manifest.json +590 -0
- intentumdiff/wasm/perl_parser.wasm +0 -0
- intentumdiff/wasm/php_parser.wasm +0 -0
- intentumdiff/wasm/plsql_parser.wasm +0 -0
- intentumdiff/wasm/po_parser.wasm +0 -0
- intentumdiff/wasm/postscript_parser.wasm +0 -0
- intentumdiff/wasm/powershell_parser.wasm +0 -0
- intentumdiff/wasm/proto_parser.wasm +0 -0
- intentumdiff/wasm/puppet_parser.wasm +0 -0
- intentumdiff/wasm/python_parser.wasm +0 -0
- intentumdiff/wasm/qsharp_parser.wasm +0 -0
- intentumdiff/wasm/r_parser.wasm +0 -0
- intentumdiff/wasm/reasonml_parser.wasm +0 -0
- intentumdiff/wasm/ruby_parser.wasm +0 -0
- intentumdiff/wasm/rust_parser.wasm +0 -0
- intentumdiff/wasm/sas_parser.wasm +0 -0
- intentumdiff/wasm/scala_parser.wasm +0 -0
- intentumdiff/wasm/scss_parser.wasm +0 -0
- intentumdiff/wasm/sql_parser.wasm +0 -0
- intentumdiff/wasm/squirrel_parser.wasm +0 -0
- intentumdiff/wasm/svelte_parser.wasm +0 -0
- intentumdiff/wasm/swift_parser.wasm +0 -0
- intentumdiff/wasm/terraform_parser.wasm +0 -0
- intentumdiff/wasm/toml_parser.wasm +0 -0
- intentumdiff/wasm/tsql_parser.wasm +0 -0
- intentumdiff/wasm/vbnet_parser.wasm +0 -0
- intentumdiff/wasm/vue_parser.wasm +0 -0
- intentumdiff/wasm/wat_parser.wasm +0 -0
- intentumdiff/wasm/xml_parser.wasm +0 -0
- intentumdiff/wasm/yaml_parser.wasm +0 -0
- intentumdiff/wasm/zig_parser.wasm +0 -0
- intentumdiff/watcher.py +259 -0
- intentumdiff_python-0.0.1.dist-info/METADATA +112 -0
- intentumdiff_python-0.0.1.dist-info/RECORD +165 -0
- intentumdiff_python-0.0.1.dist-info/WHEEL +4 -0
- intentumdiff_python-0.0.1.dist-info/entry_points.txt +80 -0
- intentumdiff_python-0.0.1.dist-info/licenses/LICENSE +21 -0
- intentumdiff_python-0.0.1.dist-info/sboms/intentumdiff-rust-core.cyclonedx.json +10968 -0
|
@@ -0,0 +1,792 @@
|
|
|
1
|
+
"""
|
|
2
|
+
intentumdiff.core.indexer
|
|
3
|
+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
4
|
+
|
|
5
|
+
High-level indexing engine that walks a repository (or a set of files),
|
|
6
|
+
parses each file, and builds a :class:`~intentumdiff.core.index.SemanticIndex`.
|
|
7
|
+
|
|
8
|
+
Progress
|
|
9
|
+
--------
|
|
10
|
+
Every public method accepts an *on_progress* callback that fires before each
|
|
11
|
+
file is processed::
|
|
12
|
+
|
|
13
|
+
from intentumdiff.core.indexer import Indexer, IndexProgress
|
|
14
|
+
|
|
15
|
+
def my_callback(p: IndexProgress) -> None:
|
|
16
|
+
pct = int(p.fraction * 100)
|
|
17
|
+
print(f"[{pct:3d}%] {p.current_file}")
|
|
18
|
+
|
|
19
|
+
indexer = Indexer(differ, on_progress=my_callback)
|
|
20
|
+
result = indexer.index_repo(".")
|
|
21
|
+
|
|
22
|
+
The callback receives an :class:`IndexProgress` instance *before* each file is
|
|
23
|
+
processed (so the caller can display a spinner with the active filename) and a
|
|
24
|
+
final "done" event with ``current_file == ""`` when indexing completes.
|
|
25
|
+
|
|
26
|
+
A ``None`` per-call *on_progress* falls back to the constructor-level default.
|
|
27
|
+
Pass a no-op lambda to suppress events for a specific call::
|
|
28
|
+
|
|
29
|
+
result = indexer.index_repo(".", on_progress=lambda _: None)
|
|
30
|
+
|
|
31
|
+
Caching
|
|
32
|
+
-------
|
|
33
|
+
When the :class:`~intentumdiff.differ.SemanticDiffer` was created with a
|
|
34
|
+
``DiffConfig.cache_path``, two levels of caching apply automatically:
|
|
35
|
+
|
|
36
|
+
* **Parse-tree cache** — each file's :class:`~intentumdiff.core.models.SemanticNode`
|
|
37
|
+
is cached keyed by content hash. Re-indexing the same file content skips
|
|
38
|
+
the Wasm call entirely.
|
|
39
|
+
|
|
40
|
+
* **Symbol-index cache** — after :meth:`Indexer.index_repo` finishes, the
|
|
41
|
+
resulting symbol and reference tables are stored in SQLite keyed by
|
|
42
|
+
``(repo_root, commit_sha)``. A subsequent call for the *same* commit returns
|
|
43
|
+
the cached index immediately (unless *force=True*).
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import hashlib
|
|
49
|
+
import json
|
|
50
|
+
import logging
|
|
51
|
+
from dataclasses import dataclass, field
|
|
52
|
+
from pathlib import Path
|
|
53
|
+
from typing import TYPE_CHECKING, Any, Callable
|
|
54
|
+
|
|
55
|
+
from intentumdiff.core.diffignore import load_diffignore
|
|
56
|
+
|
|
57
|
+
if TYPE_CHECKING:
|
|
58
|
+
from intentumdiff.core.index import SemanticIndex
|
|
59
|
+
from intentumdiff.core.models import SemanticNode
|
|
60
|
+
from intentumdiff.differ import SemanticDiffer
|
|
61
|
+
|
|
62
|
+
logger = logging.getLogger(__name__)
|
|
63
|
+
|
|
64
|
+
# Maximum number of files enriched concurrently by _index_files_lsp.
|
|
65
|
+
# Each slot holds one file open against the LSP server and fires its hover
|
|
66
|
+
# requests in parallel (bounded separately by TypeEnricher._MAX_CONCURRENT).
|
|
67
|
+
# Raise this if your server is fast and you have many files; lower it to
|
|
68
|
+
# reduce server load on shared/slow servers.
|
|
69
|
+
_FILE_CONCURRENCY = 8
|
|
70
|
+
|
|
71
|
+
# Number of threads used to read git blobs in parallel during file collection.
|
|
72
|
+
# Git blob reads are pure disk I/O, so threading gives near-linear speed-up
|
|
73
|
+
# up to the throughput limit of the storage device.
|
|
74
|
+
_BLOB_READ_WORKERS = 32
|
|
75
|
+
|
|
76
|
+
# Bytes read from each blob for language detection. Content-based detectors
|
|
77
|
+
# (tsql, plsql, adf, dax, postscript …) only need the file header; reading
|
|
78
|
+
# the full blob during a language-scan pass wastes bandwidth.
|
|
79
|
+
_DETECT_HEAD_BYTES = 2048
|
|
80
|
+
|
|
81
|
+
# ---------------------------------------------------------------------------
|
|
82
|
+
# Public types
|
|
83
|
+
# ---------------------------------------------------------------------------
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass
|
|
87
|
+
class IndexProgress:
|
|
88
|
+
"""
|
|
89
|
+
Snapshot of indexing progress, emitted via the *on_progress* callback.
|
|
90
|
+
|
|
91
|
+
The callback fires **before** each file is processed, allowing the UI to
|
|
92
|
+
show the name of the file currently being worked on. A final event with
|
|
93
|
+
``current_file == ""`` signals that indexing has completed.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
total: int
|
|
97
|
+
"""Total number of candidate files discovered."""
|
|
98
|
+
|
|
99
|
+
done: int
|
|
100
|
+
"""Files processed so far (includes skipped and errored files)."""
|
|
101
|
+
|
|
102
|
+
current_file: str
|
|
103
|
+
"""Path of the file about to be processed; empty string in the final event."""
|
|
104
|
+
|
|
105
|
+
skipped: int = 0
|
|
106
|
+
"""Files that had no registered parser and were silently skipped."""
|
|
107
|
+
|
|
108
|
+
errors: int = 0
|
|
109
|
+
"""Files that raised an unexpected error during parsing."""
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def fraction(self) -> float:
|
|
113
|
+
"""Progress as a float in ``[0.0, 1.0]``."""
|
|
114
|
+
return self.done / self.total if self.total > 0 else 0.0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass
|
|
118
|
+
class IndexResult:
|
|
119
|
+
"""Returned by every :class:`Indexer` indexing method."""
|
|
120
|
+
|
|
121
|
+
files_indexed: int
|
|
122
|
+
"""Number of files successfully parsed and added to the index."""
|
|
123
|
+
|
|
124
|
+
files_skipped: int
|
|
125
|
+
"""Number of files skipped because no registered parser was found."""
|
|
126
|
+
|
|
127
|
+
errors: list[tuple[str, str]]
|
|
128
|
+
"""``(filename, error_message)`` for files that failed to parse."""
|
|
129
|
+
|
|
130
|
+
semantic_index: "SemanticIndex"
|
|
131
|
+
"""The fully built :class:`~intentumdiff.core.index.SemanticIndex`."""
|
|
132
|
+
|
|
133
|
+
from_cache: bool = False
|
|
134
|
+
"""``True`` when the symbol index was loaded from the local cache."""
|
|
135
|
+
|
|
136
|
+
skipped_files: list[str] = field(default_factory=list)
|
|
137
|
+
"""Filenames that were skipped (no registered parser)."""
|
|
138
|
+
|
|
139
|
+
type_enriched_count: int = 0
|
|
140
|
+
"""Number of nodes that received ``type_info`` from an LSP hover call."""
|
|
141
|
+
|
|
142
|
+
languages_found: set[str] = field(default_factory=set)
|
|
143
|
+
"""Set of language identifiers successfully parsed in this run."""
|
|
144
|
+
|
|
145
|
+
files_ignored: int = 0
|
|
146
|
+
"""Number of files excluded by a ``.diffignore`` file."""
|
|
147
|
+
|
|
148
|
+
ignored_files: list[str] = field(default_factory=list)
|
|
149
|
+
"""Relative paths excluded by ``.diffignore``."""
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# Callback type alias — a plain callable so any UX layer can subscribe.
|
|
153
|
+
ProgressCallback = Callable[[IndexProgress], None]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# ---------------------------------------------------------------------------
|
|
157
|
+
# Internal helpers
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _make_index_key(repo_root: str, commit_sha: str) -> str:
|
|
162
|
+
"""Return a stable cache key for ``(repo_root, commit_sha)``."""
|
|
163
|
+
encoded = repo_root.encode() + b"\x00" + commit_sha.encode()
|
|
164
|
+
return hashlib.sha256(encoded).hexdigest()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _parse_catfile_output(
|
|
168
|
+
buf: bytes,
|
|
169
|
+
sha_to_path: "dict[str, str]",
|
|
170
|
+
) -> "list[tuple[str, str]]":
|
|
171
|
+
"""Parse ``git cat-file --batch`` stdout into ``(path, content)`` pairs."""
|
|
172
|
+
files: list[tuple[str, str]] = []
|
|
173
|
+
i = 0
|
|
174
|
+
n = len(buf)
|
|
175
|
+
while i < n:
|
|
176
|
+
nl = buf.find(b"\n", i)
|
|
177
|
+
if nl == -1:
|
|
178
|
+
break
|
|
179
|
+
header = buf[i:nl].decode("ascii", errors="replace")
|
|
180
|
+
i = nl + 1
|
|
181
|
+
parts = header.split()
|
|
182
|
+
# Expected: "<sha> blob <size>"; skip "missing" / non-blob entries.
|
|
183
|
+
if len(parts) < 3 or parts[1] != "blob":
|
|
184
|
+
continue
|
|
185
|
+
try:
|
|
186
|
+
size = int(parts[2])
|
|
187
|
+
except ValueError:
|
|
188
|
+
continue
|
|
189
|
+
content_bytes = buf[i : i + size]
|
|
190
|
+
i += size + 1 # skip trailing \n
|
|
191
|
+
if b"\x00" in content_bytes[:8192]: # binary heuristic
|
|
192
|
+
continue
|
|
193
|
+
path = sha_to_path.get(parts[0], "")
|
|
194
|
+
if not path:
|
|
195
|
+
continue
|
|
196
|
+
try:
|
|
197
|
+
content = content_bytes.decode("utf-8", errors="strict")
|
|
198
|
+
except UnicodeDecodeError:
|
|
199
|
+
content = content_bytes.decode("utf-8", errors="replace")
|
|
200
|
+
files.append((path, content))
|
|
201
|
+
return files
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _collect_blob_files(
|
|
205
|
+
repo_root: str,
|
|
206
|
+
ref: str,
|
|
207
|
+
diffignore: Any,
|
|
208
|
+
) -> "tuple[list[tuple[str, str]], list[str]]":
|
|
209
|
+
"""Return all text files at *ref* as ``(relative_path, decoded_content)``.
|
|
210
|
+
|
|
211
|
+
Shells to the git CLI (#98, A2.4.3): ``git ls-tree -r`` lists every blob and a
|
|
212
|
+
single ``git cat-file --batch`` reads them through git's optimised pack reader.
|
|
213
|
+
Binary blobs (NUL-byte heuristic in :func:`_parse_catfile_output`) are skipped.
|
|
214
|
+
|
|
215
|
+
Returns ``(files, ignored_files)`` where *ignored_files* are the paths excluded
|
|
216
|
+
by *diffignore*.
|
|
217
|
+
"""
|
|
218
|
+
from intentumdiff.vcs.git_cli import run_git_bytes # noqa: PLC0415
|
|
219
|
+
|
|
220
|
+
ignored_files: list[str] = []
|
|
221
|
+
sha_to_path: dict[str, str] = {}
|
|
222
|
+
# ls-tree -r -z: "<mode> <type> <sha>\t<path>\0" per entry.
|
|
223
|
+
for entry in run_git_bytes(repo_root, ["ls-tree", "-r", "-z", ref]).split(b"\0"):
|
|
224
|
+
if not entry:
|
|
225
|
+
continue
|
|
226
|
+
meta, _tab, raw_path = entry.partition(b"\t")
|
|
227
|
+
parts = meta.split()
|
|
228
|
+
if len(parts) < 3 or parts[1] != b"blob":
|
|
229
|
+
continue
|
|
230
|
+
path = raw_path.decode("utf-8", errors="replace")
|
|
231
|
+
if diffignore and diffignore.is_ignored(path):
|
|
232
|
+
ignored_files.append(path)
|
|
233
|
+
continue
|
|
234
|
+
sha_to_path[parts[2].decode("ascii", errors="replace")] = path
|
|
235
|
+
|
|
236
|
+
if not sha_to_path:
|
|
237
|
+
return [], ignored_files
|
|
238
|
+
|
|
239
|
+
shas_input = "\n".join(sha_to_path).encode()
|
|
240
|
+
out = run_git_bytes(repo_root, ["cat-file", "--batch"], input_bytes=shas_input)
|
|
241
|
+
return _parse_catfile_output(out, sha_to_path), ignored_files
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _attach_type_info(
|
|
245
|
+
node: "SemanticNode",
|
|
246
|
+
type_map: dict[str, str],
|
|
247
|
+
) -> "SemanticNode":
|
|
248
|
+
"""Return a copy of *node* tree with ``type_info`` set from *type_map*.
|
|
249
|
+
|
|
250
|
+
``SemanticNode`` is frozen so we must rebuild changed nodes from the leaves
|
|
251
|
+
up. Only nodes whose id appears in *type_map* are rebuilt.
|
|
252
|
+
"""
|
|
253
|
+
from intentumdiff.core.models import SemanticNode # noqa: PLC0415
|
|
254
|
+
|
|
255
|
+
new_children = [_attach_type_info(c, type_map) for c in node.children]
|
|
256
|
+
if node.id in type_map or new_children != list(node.children):
|
|
257
|
+
return node.model_copy(
|
|
258
|
+
update={
|
|
259
|
+
"children": new_children,
|
|
260
|
+
**({"type_info": type_map[node.id]} if node.id in type_map else {}),
|
|
261
|
+
}
|
|
262
|
+
)
|
|
263
|
+
return node
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# ---------------------------------------------------------------------------
|
|
267
|
+
# Indexer
|
|
268
|
+
# ---------------------------------------------------------------------------
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class Indexer:
|
|
272
|
+
"""
|
|
273
|
+
Walk a repository (or an explicit list of files) and build a
|
|
274
|
+
:class:`~intentumdiff.core.index.SemanticIndex`.
|
|
275
|
+
|
|
276
|
+
Parameters
|
|
277
|
+
----------
|
|
278
|
+
differ:
|
|
279
|
+
A :class:`~intentumdiff.differ.SemanticDiffer` instance. Configure
|
|
280
|
+
it with ``DiffConfig.cache_path`` to activate parse-tree and
|
|
281
|
+
symbol-index caching for faster re-indexing.
|
|
282
|
+
on_progress:
|
|
283
|
+
Default :data:`ProgressCallback` applied to every method call that
|
|
284
|
+
accepts *on_progress*. Individual calls may override this or pass
|
|
285
|
+
``None`` to suppress progress events for that call.
|
|
286
|
+
"""
|
|
287
|
+
|
|
288
|
+
def __init__(
|
|
289
|
+
self,
|
|
290
|
+
differ: "SemanticDiffer",
|
|
291
|
+
*,
|
|
292
|
+
on_progress: ProgressCallback | None = None,
|
|
293
|
+
) -> None:
|
|
294
|
+
self._differ = differ
|
|
295
|
+
self._on_progress = on_progress
|
|
296
|
+
|
|
297
|
+
# ── Public API ──────────────────────────────────────────────────────────
|
|
298
|
+
|
|
299
|
+
def detect_languages(
|
|
300
|
+
self,
|
|
301
|
+
repo_path: str | Path = ".",
|
|
302
|
+
ref: str = "HEAD",
|
|
303
|
+
*,
|
|
304
|
+
on_progress: "Callable[[int, int], None] | None" = None,
|
|
305
|
+
) -> "tuple[set[str], int]":
|
|
306
|
+
"""Return the set of languages present in *ref* without parsing any file.
|
|
307
|
+
|
|
308
|
+
Reads only the first :data:`_DETECT_HEAD_BYTES` of each blob (enough
|
|
309
|
+
for all content-based detectors) and calls the plugin
|
|
310
|
+
``detect_language`` heuristics. Far faster than a full
|
|
311
|
+
:meth:`index_repo` scan — useful as a lightweight pre-pass to decide
|
|
312
|
+
which LSP servers to start before a type-enrichment run.
|
|
313
|
+
|
|
314
|
+
*on_progress(done, total)* is called after each blob is processed.
|
|
315
|
+
"""
|
|
316
|
+
from intentumdiff.plugins.exceptions import ( # noqa: PLC0415
|
|
317
|
+
PluginNotFoundError,
|
|
318
|
+
)
|
|
319
|
+
from intentumdiff.vcs.git_cli import resolve_repo_root # noqa: PLC0415
|
|
320
|
+
|
|
321
|
+
repo_root = resolve_repo_root(repo_path)
|
|
322
|
+
diffignore = load_diffignore(repo_root)
|
|
323
|
+
files, _ignored = _collect_blob_files(repo_root, ref, diffignore)
|
|
324
|
+
|
|
325
|
+
total = len(files)
|
|
326
|
+
if on_progress:
|
|
327
|
+
on_progress(0, total)
|
|
328
|
+
|
|
329
|
+
# Detection inspects only the head of each blob (enough for all content-based
|
|
330
|
+
# detectors); content comes from the shared git cat-file batch collector.
|
|
331
|
+
languages: set[str] = set()
|
|
332
|
+
for i, (path, content) in enumerate(files):
|
|
333
|
+
if on_progress:
|
|
334
|
+
on_progress(i + 1, total)
|
|
335
|
+
content_head = content[:_DETECT_HEAD_BYTES]
|
|
336
|
+
try:
|
|
337
|
+
_, lang = self._differ._registry.detect_parser(path, content_head)
|
|
338
|
+
languages.add(lang)
|
|
339
|
+
except PluginNotFoundError:
|
|
340
|
+
pass
|
|
341
|
+
|
|
342
|
+
return languages, total
|
|
343
|
+
|
|
344
|
+
def index_repo(
|
|
345
|
+
self,
|
|
346
|
+
repo_path: str | Path = ".",
|
|
347
|
+
ref: str = "HEAD",
|
|
348
|
+
*,
|
|
349
|
+
on_progress: ProgressCallback | None = None,
|
|
350
|
+
force: bool = False,
|
|
351
|
+
) -> IndexResult:
|
|
352
|
+
"""
|
|
353
|
+
Parse every text file in a git repository at *ref* and build an index.
|
|
354
|
+
|
|
355
|
+
Parameters
|
|
356
|
+
----------
|
|
357
|
+
repo_path:
|
|
358
|
+
Path to the repository root (or any subdirectory; GitPython will
|
|
359
|
+
locate the root automatically).
|
|
360
|
+
ref:
|
|
361
|
+
Git ref to index. Defaults to ``"HEAD"``.
|
|
362
|
+
on_progress:
|
|
363
|
+
Progress callback for this call. Falls back to the callback
|
|
364
|
+
passed to ``__init__`` when ``None``.
|
|
365
|
+
force:
|
|
366
|
+
Re-index even if a cached symbol index already exists for this
|
|
367
|
+
commit. Useful after a plugin or wasm binary update.
|
|
368
|
+
|
|
369
|
+
Returns
|
|
370
|
+
-------
|
|
371
|
+
IndexResult
|
|
372
|
+
Fully built index. Check :attr:`IndexResult.from_cache` to
|
|
373
|
+
determine whether the symbol table was loaded from cache.
|
|
374
|
+
"""
|
|
375
|
+
from intentumdiff.vcs.git_cli import ( # noqa: PLC0415
|
|
376
|
+
resolve_repo_root,
|
|
377
|
+
run_git_bytes,
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
repo_root = resolve_repo_root(repo_path)
|
|
381
|
+
commit_sha = (
|
|
382
|
+
run_git_bytes(repo_root, ["rev-parse", "--verify", ref]).decode().strip()
|
|
383
|
+
)
|
|
384
|
+
callback = on_progress or self._on_progress
|
|
385
|
+
|
|
386
|
+
# ── Fast path: cached symbol index ────────────────────────────────
|
|
387
|
+
if not force and self._differ._cache is not None:
|
|
388
|
+
index_key = _make_index_key(repo_root, commit_sha)
|
|
389
|
+
cached = self._differ._cache.get_symbol_index(index_key)
|
|
390
|
+
if cached is not None:
|
|
391
|
+
logger.debug(
|
|
392
|
+
"Symbol index cache hit for %s @ %s",
|
|
393
|
+
repo_root,
|
|
394
|
+
commit_sha[:8],
|
|
395
|
+
)
|
|
396
|
+
from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
|
|
397
|
+
|
|
398
|
+
sem_index = SemanticIndex()
|
|
399
|
+
symbols_json, refs_json = cached
|
|
400
|
+
sem_index.load_symbol_table_json(symbols_json)
|
|
401
|
+
sem_index.load_reference_table_json(refs_json)
|
|
402
|
+
return IndexResult(
|
|
403
|
+
files_indexed=0,
|
|
404
|
+
files_skipped=0,
|
|
405
|
+
errors=[],
|
|
406
|
+
semantic_index=sem_index,
|
|
407
|
+
from_cache=True,
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
# ── Collect all text files in commit tree ─────────────────────────
|
|
411
|
+
diffignore = load_diffignore(repo_root)
|
|
412
|
+
files, ignored_files = _collect_blob_files(repo_root, ref, diffignore)
|
|
413
|
+
|
|
414
|
+
result = self._index_files(files, on_progress=callback)
|
|
415
|
+
result.files_ignored = len(ignored_files)
|
|
416
|
+
result.ignored_files = ignored_files
|
|
417
|
+
|
|
418
|
+
# ── Persist symbol index ───────────────────────────────────────────
|
|
419
|
+
if self._differ._cache is not None:
|
|
420
|
+
index_key = _make_index_key(repo_root, commit_sha)
|
|
421
|
+
self._store_symbol_index(
|
|
422
|
+
result.semantic_index, index_key, result.files_indexed
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
return result
|
|
426
|
+
|
|
427
|
+
def index_directory(
|
|
428
|
+
self,
|
|
429
|
+
directory: str | Path,
|
|
430
|
+
*,
|
|
431
|
+
on_progress: ProgressCallback | None = None,
|
|
432
|
+
) -> IndexResult:
|
|
433
|
+
"""
|
|
434
|
+
Parse every text file under *directory* (recursively) and build an index.
|
|
435
|
+
|
|
436
|
+
Parse-tree caching applies if the differ has a cache configured.
|
|
437
|
+
Symbol-index caching is **not** applied because there is no stable
|
|
438
|
+
commit key for a plain directory.
|
|
439
|
+
|
|
440
|
+
Parameters
|
|
441
|
+
----------
|
|
442
|
+
directory:
|
|
443
|
+
Root directory to walk recursively.
|
|
444
|
+
on_progress:
|
|
445
|
+
Progress callback for this call. Falls back to the callback
|
|
446
|
+
passed to ``__init__`` when ``None``.
|
|
447
|
+
"""
|
|
448
|
+
directory = Path(directory)
|
|
449
|
+
callback = on_progress or self._on_progress
|
|
450
|
+
|
|
451
|
+
diffignore = load_diffignore(directory)
|
|
452
|
+
ignored_files: list[str] = []
|
|
453
|
+
files: list[tuple[str, str]] = []
|
|
454
|
+
for path in sorted(directory.rglob("*")):
|
|
455
|
+
if not path.is_file():
|
|
456
|
+
continue
|
|
457
|
+
rel_posix = path.relative_to(directory).as_posix()
|
|
458
|
+
if diffignore and diffignore.is_ignored(rel_posix):
|
|
459
|
+
ignored_files.append(rel_posix)
|
|
460
|
+
continue
|
|
461
|
+
try:
|
|
462
|
+
content = path.read_text(encoding="utf-8", errors="replace")
|
|
463
|
+
except OSError:
|
|
464
|
+
continue
|
|
465
|
+
files.append((rel_posix, content))
|
|
466
|
+
|
|
467
|
+
result = self._index_files(files, on_progress=callback)
|
|
468
|
+
result.files_ignored = len(ignored_files)
|
|
469
|
+
result.ignored_files = ignored_files
|
|
470
|
+
return result
|
|
471
|
+
|
|
472
|
+
# ── Internal ────────────────────────────────────────────────────────────
|
|
473
|
+
|
|
474
|
+
def _index_files(
|
|
475
|
+
self,
|
|
476
|
+
files: list[tuple[str, str]],
|
|
477
|
+
*,
|
|
478
|
+
on_progress: ProgressCallback | None,
|
|
479
|
+
) -> IndexResult:
|
|
480
|
+
"""Core indexing loop — parse each (filename, content) pair."""
|
|
481
|
+
from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
|
|
482
|
+
from intentumdiff.plugins.exceptions import ( # noqa: PLC0415
|
|
483
|
+
PluginNotFoundError,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
sem_index = SemanticIndex()
|
|
487
|
+
errors: list[tuple[str, str]] = []
|
|
488
|
+
skipped_files: list[str] = []
|
|
489
|
+
languages_found: set[str] = set()
|
|
490
|
+
done = 0
|
|
491
|
+
total = len(files)
|
|
492
|
+
|
|
493
|
+
for filename, content in files:
|
|
494
|
+
# Emit progress *before* processing so the UI shows the active file.
|
|
495
|
+
if on_progress is not None:
|
|
496
|
+
on_progress(
|
|
497
|
+
IndexProgress(
|
|
498
|
+
total=total,
|
|
499
|
+
done=done,
|
|
500
|
+
current_file=filename,
|
|
501
|
+
skipped=len(skipped_files),
|
|
502
|
+
errors=len(errors),
|
|
503
|
+
)
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
try:
|
|
507
|
+
node, language = self._differ.parse(content, filename)
|
|
508
|
+
sem_index.add_tree(filename, language, node)
|
|
509
|
+
languages_found.add(language)
|
|
510
|
+
except PluginNotFoundError:
|
|
511
|
+
skipped_files.append(filename)
|
|
512
|
+
except Exception as exc: # noqa: BLE001
|
|
513
|
+
errors.append((filename, str(exc)))
|
|
514
|
+
logger.debug("Failed to parse %r: %s", filename, exc)
|
|
515
|
+
|
|
516
|
+
done += 1
|
|
517
|
+
|
|
518
|
+
# Final event: done == total, current_file empty.
|
|
519
|
+
if on_progress is not None:
|
|
520
|
+
on_progress(
|
|
521
|
+
IndexProgress(
|
|
522
|
+
total=total,
|
|
523
|
+
done=done,
|
|
524
|
+
current_file="",
|
|
525
|
+
skipped=len(skipped_files),
|
|
526
|
+
errors=len(errors),
|
|
527
|
+
)
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
sem_index.build()
|
|
531
|
+
indexed = max(0, done - len(skipped_files) - len(errors))
|
|
532
|
+
return IndexResult(
|
|
533
|
+
files_indexed=indexed,
|
|
534
|
+
files_skipped=len(skipped_files),
|
|
535
|
+
skipped_files=skipped_files,
|
|
536
|
+
errors=errors,
|
|
537
|
+
semantic_index=sem_index,
|
|
538
|
+
languages_found=languages_found,
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
def _store_symbol_index(
|
|
542
|
+
self,
|
|
543
|
+
sem_index: "SemanticIndex",
|
|
544
|
+
index_key: str,
|
|
545
|
+
file_count: int,
|
|
546
|
+
) -> None:
|
|
547
|
+
"""Serialise and persist *sem_index* to the symbol-index cache."""
|
|
548
|
+
cache = self._differ._cache
|
|
549
|
+
if cache is None:
|
|
550
|
+
raise RuntimeError("symbol-index cache store is not configured")
|
|
551
|
+
try:
|
|
552
|
+
symbols_json = json.dumps(
|
|
553
|
+
{k: [d.model_dump() for d in v] for k, v in sem_index.symbols.items()}
|
|
554
|
+
)
|
|
555
|
+
refs_json = json.dumps(
|
|
556
|
+
{k: [r.model_dump() for r in v] for k, v in sem_index.references.items()}
|
|
557
|
+
)
|
|
558
|
+
cache.put_symbol_index(
|
|
559
|
+
index_key,
|
|
560
|
+
symbols_json,
|
|
561
|
+
refs_json,
|
|
562
|
+
file_count=file_count,
|
|
563
|
+
)
|
|
564
|
+
logger.debug("Stored symbol index in cache (key=%s…)", index_key[:12])
|
|
565
|
+
except Exception as exc: # noqa: BLE001
|
|
566
|
+
logger.warning("Failed to store symbol index in cache: %s", exc)
|
|
567
|
+
|
|
568
|
+
async def index_repo_lsp(
|
|
569
|
+
self,
|
|
570
|
+
repo_path: str | Path = ".",
|
|
571
|
+
ref: str = "HEAD",
|
|
572
|
+
*,
|
|
573
|
+
lsp_clients: "dict[str, Any]",
|
|
574
|
+
on_progress: ProgressCallback | None = None,
|
|
575
|
+
force: bool = False,
|
|
576
|
+
file_concurrency: int = _FILE_CONCURRENCY,
|
|
577
|
+
) -> IndexResult:
|
|
578
|
+
"""Like :meth:`index_repo` but attaches LSP type info to each parsed tree.
|
|
579
|
+
|
|
580
|
+
Parameters
|
|
581
|
+
----------
|
|
582
|
+
lsp_clients:
|
|
583
|
+
Map of language name → connected ``AsyncLspClient`` instance.
|
|
584
|
+
Files whose language has no matching client are parsed normally.
|
|
585
|
+
file_concurrency:
|
|
586
|
+
Maximum number of files processed concurrently. Each slot holds
|
|
587
|
+
one file open against the LSP server while firing its hover
|
|
588
|
+
requests. Defaults to :data:`_FILE_CONCURRENCY` (8). Set higher
|
|
589
|
+
for fast local servers, lower for shared or rate-limited servers.
|
|
590
|
+
"""
|
|
591
|
+
import asyncio
|
|
592
|
+
from intentumdiff.vcs.git_cli import ( # noqa: PLC0415
|
|
593
|
+
resolve_repo_root,
|
|
594
|
+
run_git_bytes,
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
repo_root = resolve_repo_root(repo_path)
|
|
598
|
+
commit_sha = (
|
|
599
|
+
run_git_bytes(repo_root, ["rev-parse", "--verify", ref]).decode().strip()
|
|
600
|
+
)
|
|
601
|
+
callback = on_progress or self._on_progress
|
|
602
|
+
|
|
603
|
+
# LSP-enriched indexes are stored under a separate key so they don't
|
|
604
|
+
# collide with plain (non-enriched) index_repo cache entries.
|
|
605
|
+
lsp_index_key = _make_index_key(repo_root, "lsp:" + commit_sha)
|
|
606
|
+
|
|
607
|
+
# ── Fast path: cached LSP symbol index ────────────────────────────
|
|
608
|
+
if not force and self._differ._cache is not None:
|
|
609
|
+
cached = self._differ._cache.get_symbol_index(lsp_index_key)
|
|
610
|
+
if cached is not None:
|
|
611
|
+
logger.debug(
|
|
612
|
+
"LSP symbol index cache hit for %s @ %s",
|
|
613
|
+
repo_root,
|
|
614
|
+
commit_sha[:8],
|
|
615
|
+
)
|
|
616
|
+
from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
|
|
617
|
+
|
|
618
|
+
sem_index = SemanticIndex()
|
|
619
|
+
symbols_json, refs_json = cached
|
|
620
|
+
sem_index.load_symbol_table_json(symbols_json)
|
|
621
|
+
sem_index.load_reference_table_json(refs_json)
|
|
622
|
+
return IndexResult(
|
|
623
|
+
files_indexed=0,
|
|
624
|
+
files_skipped=0,
|
|
625
|
+
errors=[],
|
|
626
|
+
semantic_index=sem_index,
|
|
627
|
+
from_cache=True,
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
# Collect files (parallel blob reads; same filtering as index_repo)
|
|
631
|
+
diffignore = load_diffignore(repo_root)
|
|
632
|
+
files, ignored_files = await asyncio.to_thread(
|
|
633
|
+
_collect_blob_files, repo_root, ref, diffignore
|
|
634
|
+
)
|
|
635
|
+
|
|
636
|
+
result = await self._index_files_lsp(
|
|
637
|
+
files,
|
|
638
|
+
lsp_clients=lsp_clients,
|
|
639
|
+
on_progress=callback,
|
|
640
|
+
file_concurrency=file_concurrency,
|
|
641
|
+
)
|
|
642
|
+
result.files_ignored = len(ignored_files)
|
|
643
|
+
result.ignored_files = ignored_files
|
|
644
|
+
|
|
645
|
+
if self._differ._cache is not None:
|
|
646
|
+
self._store_symbol_index(result.semantic_index, lsp_index_key, result.files_indexed)
|
|
647
|
+
|
|
648
|
+
return result
|
|
649
|
+
|
|
650
|
+
async def _index_files_lsp(
|
|
651
|
+
self,
|
|
652
|
+
files: list[tuple[str, str]],
|
|
653
|
+
*,
|
|
654
|
+
lsp_clients: "dict[str, Any]",
|
|
655
|
+
on_progress: ProgressCallback | None,
|
|
656
|
+
file_concurrency: int = _FILE_CONCURRENCY,
|
|
657
|
+
) -> IndexResult:
|
|
658
|
+
"""Async indexing loop — parse then type-enrich files concurrently.
|
|
659
|
+
|
|
660
|
+
Files are processed up to *file_concurrency* at a time. Within each
|
|
661
|
+
file all hover requests are already parallelised by
|
|
662
|
+
:class:`~intentumdiff.lsp.enricher.TypeEnricher`. After all tasks
|
|
663
|
+
complete, results are merged into the ``SemanticIndex`` sequentially so
|
|
664
|
+
the index itself never sees concurrent writes.
|
|
665
|
+
"""
|
|
666
|
+
import asyncio # noqa: PLC0415
|
|
667
|
+
import concurrent.futures # noqa: PLC0415
|
|
668
|
+
|
|
669
|
+
from intentumdiff.core.index import SemanticIndex # noqa: PLC0415
|
|
670
|
+
from intentumdiff.lsp.enricher import TypeEnricher # noqa: PLC0415
|
|
671
|
+
from intentumdiff.plugins.exceptions import PluginNotFoundError # noqa: PLC0415
|
|
672
|
+
|
|
673
|
+
total = len(files)
|
|
674
|
+
|
|
675
|
+
# Build enrichers once per language.
|
|
676
|
+
enrichers: dict[str, TypeEnricher] = {
|
|
677
|
+
lang: TypeEnricher(client, language=lang)
|
|
678
|
+
for lang, client in lsp_clients.items()
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
# Result type for each file task.
|
|
682
|
+
# Success: (filename, language, node, enriched_count)
|
|
683
|
+
# Skipped: PluginNotFoundError (re-raised so we can detect it)
|
|
684
|
+
# Error: any other exception
|
|
685
|
+
|
|
686
|
+
sem = asyncio.Semaphore(max(1, file_concurrency))
|
|
687
|
+
cache = self._differ._cache
|
|
688
|
+
differ = self._differ # captured for executor
|
|
689
|
+
|
|
690
|
+
# Use a single-worker executor so Wasm parse calls are serialised.
|
|
691
|
+
# Concurrent Wasm execution across multiple Stores/Engines triggers
|
|
692
|
+
# non-deterministic "bad parameter or other API misuse" errors from
|
|
693
|
+
# wasmtime. A single worker keeps parse calls sequential while still
|
|
694
|
+
# freeing the event loop so LSP enrich for completed files overlaps
|
|
695
|
+
# with the next file's parse (pipeline parallelism).
|
|
696
|
+
loop = asyncio.get_event_loop()
|
|
697
|
+
parse_executor = concurrent.futures.ThreadPoolExecutor(
|
|
698
|
+
max_workers=1, thread_name_prefix="wasm_parse"
|
|
699
|
+
)
|
|
700
|
+
|
|
701
|
+
async def _process_one(
|
|
702
|
+
filename: str,
|
|
703
|
+
content: str,
|
|
704
|
+
) -> "tuple[str, str, Any, int]":
|
|
705
|
+
"""Parse + optionally enrich one file; always returns a result tuple.
|
|
706
|
+
|
|
707
|
+
Returns ``(filename, language, node, enriched_count)`` on success,
|
|
708
|
+
``(filename, 'SKIP', exc, 0)`` when no parser is registered, or
|
|
709
|
+
``(filename, 'ERROR', exc, 0)`` on any other failure.
|
|
710
|
+
"""
|
|
711
|
+
async with sem:
|
|
712
|
+
# Run the blocking Wasm parse in a dedicated single-worker
|
|
713
|
+
# thread so the event loop stays free for concurrent LSP I/O.
|
|
714
|
+
# A 1-worker pool prevents concurrent Wasm Store access which
|
|
715
|
+
# causes non-deterministic wasmtime errors.
|
|
716
|
+
try:
|
|
717
|
+
node, language = await loop.run_in_executor(
|
|
718
|
+
parse_executor, differ.parse, content, filename
|
|
719
|
+
)
|
|
720
|
+
except PluginNotFoundError as exc:
|
|
721
|
+
return filename, "SKIP", exc, 0
|
|
722
|
+
except Exception as exc:
|
|
723
|
+
return filename, "ERROR", exc, 0
|
|
724
|
+
|
|
725
|
+
enricher = enrichers.get(language)
|
|
726
|
+
enriched_count = 0
|
|
727
|
+
if enricher is not None:
|
|
728
|
+
# Check hover-map cache first.
|
|
729
|
+
type_map: dict[str, str] | None = None
|
|
730
|
+
hm_key: str | None = None
|
|
731
|
+
if cache is not None:
|
|
732
|
+
hm_key = cache.hover_map_key(content, language)
|
|
733
|
+
type_map = cache.get_hover_map(hm_key)
|
|
734
|
+
if type_map is None:
|
|
735
|
+
try:
|
|
736
|
+
type_map = await enricher.enrich(filename, content, node)
|
|
737
|
+
except Exception:
|
|
738
|
+
type_map = None
|
|
739
|
+
if type_map and hm_key is not None and cache is not None:
|
|
740
|
+
cache.put_hover_map(hm_key, type_map)
|
|
741
|
+
if type_map:
|
|
742
|
+
enriched_count = len(type_map)
|
|
743
|
+
node = _attach_type_info(node, type_map)
|
|
744
|
+
return filename, language, node, enriched_count
|
|
745
|
+
|
|
746
|
+
tasks = [asyncio.create_task(_process_one(f, c)) for f, c in files]
|
|
747
|
+
|
|
748
|
+
# Merge results as tasks complete so progress updates in real time.
|
|
749
|
+
sem_index = SemanticIndex()
|
|
750
|
+
errors: list[tuple[str, str]] = []
|
|
751
|
+
skipped_files: list[str] = []
|
|
752
|
+
languages_found: set[str] = set()
|
|
753
|
+
total_enriched = 0
|
|
754
|
+
done = 0
|
|
755
|
+
|
|
756
|
+
for future in asyncio.as_completed(tasks):
|
|
757
|
+
filename, language, payload, enriched_count = await future
|
|
758
|
+
done += 1
|
|
759
|
+
|
|
760
|
+
if language == "SKIP":
|
|
761
|
+
skipped_files.append(filename)
|
|
762
|
+
elif language == "ERROR":
|
|
763
|
+
errors.append((filename, str(payload)))
|
|
764
|
+
logger.debug("Failed to process %r: %s", filename, payload)
|
|
765
|
+
else:
|
|
766
|
+
sem_index.add_tree(filename, language, payload)
|
|
767
|
+
languages_found.add(language)
|
|
768
|
+
total_enriched += enriched_count
|
|
769
|
+
|
|
770
|
+
if on_progress is not None:
|
|
771
|
+
on_progress(
|
|
772
|
+
IndexProgress(
|
|
773
|
+
total=total,
|
|
774
|
+
done=done,
|
|
775
|
+
current_file=filename,
|
|
776
|
+
skipped=len(skipped_files),
|
|
777
|
+
errors=len(errors),
|
|
778
|
+
)
|
|
779
|
+
)
|
|
780
|
+
|
|
781
|
+
sem_index.build()
|
|
782
|
+
parse_executor.shutdown(wait=False)
|
|
783
|
+
indexed = max(0, done - len(skipped_files) - len(errors))
|
|
784
|
+
return IndexResult(
|
|
785
|
+
files_indexed=indexed,
|
|
786
|
+
files_skipped=len(skipped_files),
|
|
787
|
+
skipped_files=skipped_files,
|
|
788
|
+
errors=errors,
|
|
789
|
+
semantic_index=sem_index,
|
|
790
|
+
languages_found=languages_found,
|
|
791
|
+
type_enriched_count=total_enriched,
|
|
792
|
+
)
|