msdev 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- msdev/__init__.py +3 -0
- msdev/cli.py +1460 -0
- msdev/core/__init__.py +1 -0
- msdev/core/config.py +193 -0
- msdev/core/guides.py +100 -0
- msdev/core/inventory.py +1390 -0
- msdev/core/limits.py +44 -0
- msdev/core/resources.py +300 -0
- msdev/core/services/__init__.py +111 -0
- msdev/core/services/context.py +61 -0
- msdev/core/services/environment.py +84 -0
- msdev/core/services/execution.py +136 -0
- msdev/core/services/model.py +1085 -0
- msdev/core/services/node.py +210 -0
- msdev/core/services/npu.py +116 -0
- msdev/core/services/workspace.py +567 -0
- msdev/core/transport.py +1225 -0
- msdev/core/workspace/__init__.py +36 -0
- msdev/core/workspace/access.py +1216 -0
- msdev/core/workspace/client.py +274 -0
- msdev/core/workspace/paths.py +45 -0
- msdev/core/workspace/registry.py +151 -0
- msdev/daemon.py +1322 -0
- msdev/session/__init__.py +13 -0
- msdev/session/export.py +190 -0
- msdev/session/log.py +269 -0
- msdev-0.9.0.dist-info/METADATA +295 -0
- msdev-0.9.0.dist-info/RECORD +31 -0
- msdev-0.9.0.dist-info/WHEEL +5 -0
- msdev-0.9.0.dist-info/entry_points.txt +3 -0
- msdev-0.9.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1216 @@
|
|
|
1
|
+
"""Daemon-side, descriptor-relative workspace filesystem operations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import binascii
|
|
7
|
+
import errno
|
|
8
|
+
import fnmatch
|
|
9
|
+
import functools
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import secrets
|
|
15
|
+
import selectors
|
|
16
|
+
import shutil
|
|
17
|
+
import stat
|
|
18
|
+
import subprocess
|
|
19
|
+
import threading
|
|
20
|
+
import time
|
|
21
|
+
from contextlib import contextmanager
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any, Generator, Iterable, Iterator
|
|
25
|
+
|
|
26
|
+
from .paths import normalize_relative
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
MAX_READ_BYTES = 16 * 1024 * 1024
|
|
30
|
+
MAX_STAT_HASH_BYTES = 64 * 1024 * 1024
|
|
31
|
+
MAX_RESULTS = 10_000
|
|
32
|
+
MAX_LIST_RESULTS = 10_000
|
|
33
|
+
MAX_TRAVERSAL_ENTRIES = 100_000
|
|
34
|
+
MAX_SEARCH_SECONDS = 10.0
|
|
35
|
+
MAX_SEARCH_FILES = 10_000
|
|
36
|
+
MAX_SEARCH_BYTES = 256 * 1024 * 1024
|
|
37
|
+
MAX_SEARCH_LINE_BYTES = 1024 * 1024
|
|
38
|
+
_READ_CHUNK_SIZE = 64 * 1024
|
|
39
|
+
_RG_JSON_LINE_BYTES = (2 * MAX_SEARCH_LINE_BYTES) + 64 * 1024
|
|
40
|
+
_REGEX_METACHARACTERS = frozenset(r".^$*+?{}[]\|()")
|
|
41
|
+
|
|
42
|
+
_O_CLOEXEC = getattr(os, "O_CLOEXEC", 0)
|
|
43
|
+
_O_DIRECTORY = getattr(os, "O_DIRECTORY", 0)
|
|
44
|
+
_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
|
|
45
|
+
_O_NONBLOCK = getattr(os, "O_NONBLOCK", 0)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def workspace_root(root: str | os.PathLike[str] | None) -> Path:
|
|
49
|
+
"""Resolve and validate the root supplied at the RPC boundary."""
|
|
50
|
+
if not isinstance(root, (str, os.PathLike)):
|
|
51
|
+
raise ValueError("workspace root must be a path")
|
|
52
|
+
raw = os.fspath(root)
|
|
53
|
+
if "\0" in raw:
|
|
54
|
+
raise ValueError("workspace root must not contain NUL")
|
|
55
|
+
candidate = Path(raw).expanduser()
|
|
56
|
+
if not candidate.is_absolute():
|
|
57
|
+
raise ValueError("workspace root must be absolute")
|
|
58
|
+
resolved = candidate.resolve(strict=True)
|
|
59
|
+
if not resolved.is_dir():
|
|
60
|
+
raise NotADirectoryError(f"workspace root is not a directory: {resolved}")
|
|
61
|
+
return resolved
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _assert_contained(root: Path, candidate: Path) -> Path:
|
|
65
|
+
try:
|
|
66
|
+
canonical = candidate.resolve(strict=False)
|
|
67
|
+
canonical.relative_to(root)
|
|
68
|
+
except (OSError, RuntimeError, ValueError) as exc:
|
|
69
|
+
raise PermissionError(
|
|
70
|
+
f"workspace path escapes workspace root: {candidate}"
|
|
71
|
+
) from exc
|
|
72
|
+
return canonical
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def resolve_path(
|
|
76
|
+
root: str | os.PathLike[str] | None,
|
|
77
|
+
relative: str | None,
|
|
78
|
+
) -> tuple[str, Path]:
|
|
79
|
+
"""Resolve a path for compatibility callers.
|
|
80
|
+
|
|
81
|
+
Filesystem operations below do not rely on this check-then-use result; they
|
|
82
|
+
traverse from an opened root directory descriptor with ``O_NOFOLLOW``.
|
|
83
|
+
"""
|
|
84
|
+
root_path = workspace_root(root)
|
|
85
|
+
normalized = normalize_relative("." if relative is None else relative)
|
|
86
|
+
candidate = (
|
|
87
|
+
root_path
|
|
88
|
+
if normalized == "."
|
|
89
|
+
else root_path.joinpath(*normalized.split("/"))
|
|
90
|
+
)
|
|
91
|
+
return normalized, _assert_contained(root_path, candidate)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _require_secure_open_flags() -> None:
|
|
95
|
+
if not _O_NOFOLLOW or not _O_DIRECTORY:
|
|
96
|
+
raise RuntimeError(
|
|
97
|
+
"secure workspace access requires O_NOFOLLOW and O_DIRECTORY"
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _secure_open_error(exc: OSError, relative: str) -> None:
|
|
102
|
+
if exc.errno in {errno.ELOOP, errno.ENOTDIR}:
|
|
103
|
+
raise PermissionError(
|
|
104
|
+
f"workspace path contains a symlink or non-directory ancestor: "
|
|
105
|
+
f"{relative}"
|
|
106
|
+
) from exc
|
|
107
|
+
raise exc
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _open_workspace(
|
|
111
|
+
root: str | os.PathLike[str] | None,
|
|
112
|
+
) -> tuple[Path, int]:
|
|
113
|
+
_require_secure_open_flags()
|
|
114
|
+
root_path = workspace_root(root)
|
|
115
|
+
flags = os.O_RDONLY | _O_DIRECTORY | _O_NOFOLLOW | _O_CLOEXEC
|
|
116
|
+
try:
|
|
117
|
+
descriptor = os.open(root_path.anchor, flags)
|
|
118
|
+
except OSError as exc:
|
|
119
|
+
_secure_open_error(exc, ".")
|
|
120
|
+
raise AssertionError("unreachable")
|
|
121
|
+
try:
|
|
122
|
+
for component in root_path.parts[1:]:
|
|
123
|
+
try:
|
|
124
|
+
child = os.open(component, flags, dir_fd=descriptor)
|
|
125
|
+
except OSError as exc:
|
|
126
|
+
_secure_open_error(exc, str(root_path))
|
|
127
|
+
raise AssertionError("unreachable")
|
|
128
|
+
os.close(descriptor)
|
|
129
|
+
descriptor = child
|
|
130
|
+
except Exception:
|
|
131
|
+
os.close(descriptor)
|
|
132
|
+
raise
|
|
133
|
+
if not stat.S_ISDIR(os.fstat(descriptor).st_mode):
|
|
134
|
+
os.close(descriptor)
|
|
135
|
+
raise NotADirectoryError(f"workspace root is not a directory: {root_path}")
|
|
136
|
+
return root_path, descriptor
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _components(relative: str) -> list[str]:
|
|
140
|
+
return [] if relative == "." else relative.split("/")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _open_directory_components(
|
|
144
|
+
root_fd: int,
|
|
145
|
+
components: Iterable[str],
|
|
146
|
+
relative: str,
|
|
147
|
+
) -> int:
|
|
148
|
+
descriptor = os.dup(root_fd)
|
|
149
|
+
try:
|
|
150
|
+
for component in components:
|
|
151
|
+
try:
|
|
152
|
+
child = os.open(
|
|
153
|
+
component,
|
|
154
|
+
os.O_RDONLY | _O_DIRECTORY | _O_NOFOLLOW | _O_CLOEXEC,
|
|
155
|
+
dir_fd=descriptor,
|
|
156
|
+
)
|
|
157
|
+
except OSError as exc:
|
|
158
|
+
_secure_open_error(exc, relative)
|
|
159
|
+
raise AssertionError("unreachable")
|
|
160
|
+
os.close(descriptor)
|
|
161
|
+
descriptor = child
|
|
162
|
+
return descriptor
|
|
163
|
+
except Exception:
|
|
164
|
+
os.close(descriptor)
|
|
165
|
+
raise
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _open_parent(root_fd: int, relative: str) -> tuple[int, str]:
|
|
169
|
+
parts = _components(relative)
|
|
170
|
+
if not parts:
|
|
171
|
+
raise IsADirectoryError("workspace root cannot be used as a file")
|
|
172
|
+
return (
|
|
173
|
+
_open_directory_components(root_fd, parts[:-1], relative),
|
|
174
|
+
parts[-1],
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _open_relative(
|
|
179
|
+
root_fd: int,
|
|
180
|
+
relative: str,
|
|
181
|
+
flags: int,
|
|
182
|
+
*,
|
|
183
|
+
mode: int = 0o600,
|
|
184
|
+
) -> int:
|
|
185
|
+
parts = _components(relative)
|
|
186
|
+
if not parts:
|
|
187
|
+
return os.dup(root_fd)
|
|
188
|
+
parent_fd = _open_directory_components(root_fd, parts[:-1], relative)
|
|
189
|
+
try:
|
|
190
|
+
try:
|
|
191
|
+
return os.open(
|
|
192
|
+
parts[-1],
|
|
193
|
+
flags | _O_NOFOLLOW | _O_CLOEXEC,
|
|
194
|
+
mode,
|
|
195
|
+
dir_fd=parent_fd,
|
|
196
|
+
)
|
|
197
|
+
except OSError as exc:
|
|
198
|
+
_secure_open_error(exc, relative)
|
|
199
|
+
raise AssertionError("unreachable")
|
|
200
|
+
finally:
|
|
201
|
+
os.close(parent_fd)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _join_relative(parent: str, name: str) -> str:
|
|
205
|
+
return name if parent == "." else f"{parent}/{name}"
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _path_type(metadata: os.stat_result) -> str:
|
|
209
|
+
mode = metadata.st_mode
|
|
210
|
+
if stat.S_ISLNK(mode):
|
|
211
|
+
return "symlink"
|
|
212
|
+
if stat.S_ISREG(mode):
|
|
213
|
+
return "file"
|
|
214
|
+
if stat.S_ISDIR(mode):
|
|
215
|
+
return "directory"
|
|
216
|
+
return "other"
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _metadata(
|
|
220
|
+
relative: str,
|
|
221
|
+
metadata: os.stat_result,
|
|
222
|
+
*,
|
|
223
|
+
sha256: str | None = None,
|
|
224
|
+
) -> dict[str, Any]:
|
|
225
|
+
kind = _path_type(metadata)
|
|
226
|
+
value: dict[str, Any] = {
|
|
227
|
+
"path": relative,
|
|
228
|
+
"type": kind,
|
|
229
|
+
"size": metadata.st_size if kind != "directory" else None,
|
|
230
|
+
"mtime_ns": metadata.st_mtime_ns,
|
|
231
|
+
}
|
|
232
|
+
if sha256 is not None:
|
|
233
|
+
value["sha256"] = sha256
|
|
234
|
+
return value
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _sha256_fd(descriptor: int, max_bytes: int | None = None) -> str | None:
|
|
238
|
+
digest = hashlib.sha256()
|
|
239
|
+
total = 0
|
|
240
|
+
while True:
|
|
241
|
+
read_size = 1024 * 1024
|
|
242
|
+
if max_bytes is not None:
|
|
243
|
+
read_size = min(read_size, max_bytes - total + 1)
|
|
244
|
+
chunk = os.read(descriptor, read_size)
|
|
245
|
+
if not chunk:
|
|
246
|
+
return digest.hexdigest()
|
|
247
|
+
total += len(chunk)
|
|
248
|
+
if max_bytes is not None and total > max_bytes:
|
|
249
|
+
return None
|
|
250
|
+
digest.update(chunk)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def stat_path(
|
|
254
|
+
root: str | os.PathLike[str] | None,
|
|
255
|
+
relative: str | None = ".",
|
|
256
|
+
) -> dict[str, Any]:
|
|
257
|
+
normalized = normalize_relative("." if relative is None else relative)
|
|
258
|
+
_, root_fd = _open_workspace(root)
|
|
259
|
+
try:
|
|
260
|
+
if normalized == ".":
|
|
261
|
+
return _metadata(".", os.fstat(root_fd))
|
|
262
|
+
parent_fd, leaf = _open_parent(root_fd, normalized)
|
|
263
|
+
try:
|
|
264
|
+
metadata = os.stat(leaf, dir_fd=parent_fd, follow_symlinks=False)
|
|
265
|
+
kind = _path_type(metadata)
|
|
266
|
+
if kind == "file":
|
|
267
|
+
descriptor = os.open(
|
|
268
|
+
leaf,
|
|
269
|
+
os.O_RDONLY | _O_NONBLOCK | _O_NOFOLLOW | _O_CLOEXEC,
|
|
270
|
+
dir_fd=parent_fd,
|
|
271
|
+
)
|
|
272
|
+
try:
|
|
273
|
+
metadata = os.fstat(descriptor)
|
|
274
|
+
if metadata.st_size > MAX_STAT_HASH_BYTES:
|
|
275
|
+
value = _metadata(normalized, metadata)
|
|
276
|
+
value.update(
|
|
277
|
+
{
|
|
278
|
+
"sha256": None,
|
|
279
|
+
"sha256_truncated": True,
|
|
280
|
+
"sha256_reason": (
|
|
281
|
+
"file exceeds stat hash limit of "
|
|
282
|
+
f"{MAX_STAT_HASH_BYTES} bytes"
|
|
283
|
+
),
|
|
284
|
+
}
|
|
285
|
+
)
|
|
286
|
+
return value
|
|
287
|
+
digest = _sha256_fd(descriptor, MAX_STAT_HASH_BYTES)
|
|
288
|
+
if digest is None:
|
|
289
|
+
value = _metadata(normalized, metadata)
|
|
290
|
+
value.update(
|
|
291
|
+
{
|
|
292
|
+
"sha256": None,
|
|
293
|
+
"sha256_truncated": True,
|
|
294
|
+
"sha256_reason": (
|
|
295
|
+
"file exceeded stat hash limit of "
|
|
296
|
+
f"{MAX_STAT_HASH_BYTES} bytes while reading"
|
|
297
|
+
),
|
|
298
|
+
}
|
|
299
|
+
)
|
|
300
|
+
return value
|
|
301
|
+
return _metadata(normalized, metadata, sha256=digest)
|
|
302
|
+
finally:
|
|
303
|
+
os.close(descriptor)
|
|
304
|
+
if kind == "directory":
|
|
305
|
+
descriptor = os.open(
|
|
306
|
+
leaf,
|
|
307
|
+
os.O_RDONLY | _O_DIRECTORY | _O_NOFOLLOW | _O_CLOEXEC,
|
|
308
|
+
dir_fd=parent_fd,
|
|
309
|
+
)
|
|
310
|
+
try:
|
|
311
|
+
metadata = os.fstat(descriptor)
|
|
312
|
+
finally:
|
|
313
|
+
os.close(descriptor)
|
|
314
|
+
return _metadata(normalized, metadata)
|
|
315
|
+
finally:
|
|
316
|
+
os.close(parent_fd)
|
|
317
|
+
except OSError as exc:
|
|
318
|
+
_secure_open_error(exc, normalized)
|
|
319
|
+
raise AssertionError("unreachable")
|
|
320
|
+
finally:
|
|
321
|
+
os.close(root_fd)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _positive_limit(value: Any, name: str, maximum: int) -> int:
|
|
325
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
326
|
+
raise ValueError(f"{name} must be an integer")
|
|
327
|
+
if not 1 <= value <= maximum:
|
|
328
|
+
raise ValueError(f"{name} must be between 1 and {maximum}")
|
|
329
|
+
return value
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def read_file(
|
|
333
|
+
root: str | os.PathLike[str] | None,
|
|
334
|
+
relative: str | None,
|
|
335
|
+
max_bytes: int = 1024 * 1024,
|
|
336
|
+
) -> dict[str, Any]:
|
|
337
|
+
limit = _positive_limit(max_bytes, "max_bytes", MAX_READ_BYTES)
|
|
338
|
+
normalized = normalize_relative("." if relative is None else relative)
|
|
339
|
+
_, root_fd = _open_workspace(root)
|
|
340
|
+
descriptor = None
|
|
341
|
+
try:
|
|
342
|
+
descriptor = _open_relative(
|
|
343
|
+
root_fd,
|
|
344
|
+
normalized,
|
|
345
|
+
os.O_RDONLY | _O_NONBLOCK,
|
|
346
|
+
)
|
|
347
|
+
metadata = os.fstat(descriptor)
|
|
348
|
+
if not stat.S_ISREG(metadata.st_mode):
|
|
349
|
+
if stat.S_ISDIR(metadata.st_mode):
|
|
350
|
+
raise IsADirectoryError(
|
|
351
|
+
f"workspace path is a directory: {normalized}"
|
|
352
|
+
)
|
|
353
|
+
raise ValueError(
|
|
354
|
+
f"workspace path is not a regular file: {normalized}"
|
|
355
|
+
)
|
|
356
|
+
content = bytearray()
|
|
357
|
+
while len(content) <= limit:
|
|
358
|
+
chunk = os.read(
|
|
359
|
+
descriptor,
|
|
360
|
+
min(_READ_CHUNK_SIZE, limit + 1 - len(content)),
|
|
361
|
+
)
|
|
362
|
+
if not chunk:
|
|
363
|
+
break
|
|
364
|
+
content.extend(chunk)
|
|
365
|
+
if len(content) > limit:
|
|
366
|
+
raise ValueError(
|
|
367
|
+
f"refusing to read {normalized!r}: file size is exceeding "
|
|
368
|
+
f"read limit of {limit} bytes"
|
|
369
|
+
)
|
|
370
|
+
payload = bytes(content)
|
|
371
|
+
return {
|
|
372
|
+
"path": normalized,
|
|
373
|
+
"size": len(payload),
|
|
374
|
+
"sha256": hashlib.sha256(payload).hexdigest(),
|
|
375
|
+
"content_base64": base64.b64encode(payload).decode("ascii"),
|
|
376
|
+
}
|
|
377
|
+
finally:
|
|
378
|
+
if descriptor is not None:
|
|
379
|
+
os.close(descriptor)
|
|
380
|
+
os.close(root_fd)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _iter_sorted_names_fd(
|
|
384
|
+
directory_fd: int,
|
|
385
|
+
*,
|
|
386
|
+
budget: "_TraversalBudget | None" = None,
|
|
387
|
+
) -> Iterator[str]:
|
|
388
|
+
"""Yield a sorted, bounded snapshot of directory names."""
|
|
389
|
+
scan = budget or _TraversalBudget(MAX_TRAVERSAL_ENTRIES)
|
|
390
|
+
names: list[str] = []
|
|
391
|
+
os.lseek(directory_fd, 0, os.SEEK_SET)
|
|
392
|
+
with os.scandir(directory_fd) as entries:
|
|
393
|
+
for entry in entries:
|
|
394
|
+
if scan.deadline is not None and time.monotonic() >= scan.deadline:
|
|
395
|
+
scan.truncated = True
|
|
396
|
+
scan.deadline_exceeded = True
|
|
397
|
+
break
|
|
398
|
+
if scan.visited >= scan.maximum:
|
|
399
|
+
scan.truncated = True
|
|
400
|
+
break
|
|
401
|
+
scan.visited += 1
|
|
402
|
+
names.append(entry.name)
|
|
403
|
+
names.sort()
|
|
404
|
+
yield from names
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def list_directory(
|
|
408
|
+
root: str | os.PathLike[str] | None,
|
|
409
|
+
relative: str | None = ".",
|
|
410
|
+
) -> dict[str, Any]:
|
|
411
|
+
normalized = normalize_relative("." if relative is None else relative)
|
|
412
|
+
_, root_fd = _open_workspace(root)
|
|
413
|
+
directory_fd = None
|
|
414
|
+
try:
|
|
415
|
+
directory_fd = _open_relative(
|
|
416
|
+
root_fd,
|
|
417
|
+
normalized,
|
|
418
|
+
os.O_RDONLY | _O_DIRECTORY,
|
|
419
|
+
)
|
|
420
|
+
entries: list[dict[str, Any]] = []
|
|
421
|
+
truncated = False
|
|
422
|
+
budget = _TraversalBudget(MAX_TRAVERSAL_ENTRIES)
|
|
423
|
+
for name in _iter_sorted_names_fd(directory_fd, budget=budget):
|
|
424
|
+
try:
|
|
425
|
+
metadata = os.stat(
|
|
426
|
+
name,
|
|
427
|
+
dir_fd=directory_fd,
|
|
428
|
+
follow_symlinks=False,
|
|
429
|
+
)
|
|
430
|
+
except FileNotFoundError:
|
|
431
|
+
continue
|
|
432
|
+
if len(entries) == MAX_LIST_RESULTS:
|
|
433
|
+
truncated = True
|
|
434
|
+
break
|
|
435
|
+
entries.append(
|
|
436
|
+
_metadata(_join_relative(normalized, name), metadata)
|
|
437
|
+
)
|
|
438
|
+
return {
|
|
439
|
+
"path": normalized,
|
|
440
|
+
"entries": entries,
|
|
441
|
+
"truncated": truncated or budget.truncated,
|
|
442
|
+
"visited_entries": budget.visited,
|
|
443
|
+
}
|
|
444
|
+
finally:
|
|
445
|
+
if directory_fd is not None:
|
|
446
|
+
os.close(directory_fd)
|
|
447
|
+
os.close(root_fd)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _max_results(value: Any) -> int:
|
|
451
|
+
return _positive_limit(value, "max_results", MAX_RESULTS)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
@dataclass
|
|
455
|
+
class _TraversalBudget:
|
|
456
|
+
maximum: int = MAX_TRAVERSAL_ENTRIES
|
|
457
|
+
visited: int = 0
|
|
458
|
+
truncated: bool = False
|
|
459
|
+
deadline: float | None = None
|
|
460
|
+
deadline_exceeded: bool = False
|
|
461
|
+
|
|
462
|
+
def _walk_tree_fd(
|
|
463
|
+
directory_fd: int,
|
|
464
|
+
relative: str = ".",
|
|
465
|
+
budget: _TraversalBudget | None = None,
|
|
466
|
+
) -> Generator[tuple[str, os.stat_result, int, str], None, None]:
|
|
467
|
+
budget = budget or _TraversalBudget()
|
|
468
|
+
for name in _iter_sorted_names_fd(directory_fd, budget=budget):
|
|
469
|
+
try:
|
|
470
|
+
metadata = os.stat(
|
|
471
|
+
name,
|
|
472
|
+
dir_fd=directory_fd,
|
|
473
|
+
follow_symlinks=False,
|
|
474
|
+
)
|
|
475
|
+
except FileNotFoundError:
|
|
476
|
+
continue
|
|
477
|
+
child_relative = _join_relative(relative, name)
|
|
478
|
+
yield child_relative, metadata, directory_fd, name
|
|
479
|
+
if not stat.S_ISDIR(metadata.st_mode):
|
|
480
|
+
continue
|
|
481
|
+
try:
|
|
482
|
+
child_fd = os.open(
|
|
483
|
+
name,
|
|
484
|
+
os.O_RDONLY | _O_DIRECTORY | _O_NOFOLLOW | _O_CLOEXEC,
|
|
485
|
+
dir_fd=directory_fd,
|
|
486
|
+
)
|
|
487
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
488
|
+
continue
|
|
489
|
+
except OSError as exc:
|
|
490
|
+
if exc.errno == errno.ELOOP:
|
|
491
|
+
continue
|
|
492
|
+
raise
|
|
493
|
+
try:
|
|
494
|
+
yield from _walk_tree_fd(child_fd, child_relative, budget)
|
|
495
|
+
finally:
|
|
496
|
+
os.close(child_fd)
|
|
497
|
+
if budget.truncated:
|
|
498
|
+
return
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _glob_match(relative: str, pattern: str) -> bool:
|
|
502
|
+
path_parts = () if relative == "." else tuple(relative.split("/"))
|
|
503
|
+
pattern_parts = () if pattern == "." else tuple(pattern.split("/"))
|
|
504
|
+
|
|
505
|
+
@functools.lru_cache(maxsize=None)
|
|
506
|
+
def match(path_index: int, pattern_index: int) -> bool:
|
|
507
|
+
if pattern_index == len(pattern_parts):
|
|
508
|
+
return path_index == len(path_parts)
|
|
509
|
+
current = pattern_parts[pattern_index]
|
|
510
|
+
if current == "**":
|
|
511
|
+
return match(path_index, pattern_index + 1) or (
|
|
512
|
+
path_index < len(path_parts)
|
|
513
|
+
and match(path_index + 1, pattern_index)
|
|
514
|
+
)
|
|
515
|
+
return (
|
|
516
|
+
path_index < len(path_parts)
|
|
517
|
+
and fnmatch.fnmatchcase(path_parts[path_index], current)
|
|
518
|
+
and match(path_index + 1, pattern_index + 1)
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
return match(0, 0)
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def glob_paths(
|
|
525
|
+
root: str | os.PathLike[str] | None,
|
|
526
|
+
pattern: str,
|
|
527
|
+
max_results: int = 1000,
|
|
528
|
+
) -> dict[str, Any]:
|
|
529
|
+
limit = _max_results(max_results)
|
|
530
|
+
normalized_pattern = normalize_relative(pattern)
|
|
531
|
+
_, root_fd = _open_workspace(root)
|
|
532
|
+
matches: list[dict[str, Any]] = []
|
|
533
|
+
truncated = False
|
|
534
|
+
budget = _TraversalBudget(MAX_TRAVERSAL_ENTRIES)
|
|
535
|
+
walker = _walk_tree_fd(root_fd, budget=budget)
|
|
536
|
+
try:
|
|
537
|
+
if normalized_pattern == ".":
|
|
538
|
+
matches.append(_metadata(".", os.fstat(root_fd)))
|
|
539
|
+
else:
|
|
540
|
+
for relative, metadata, _parent_fd, _name in walker:
|
|
541
|
+
if not _glob_match(relative, normalized_pattern):
|
|
542
|
+
continue
|
|
543
|
+
if len(matches) == limit:
|
|
544
|
+
truncated = True
|
|
545
|
+
break
|
|
546
|
+
matches.append(_metadata(relative, metadata))
|
|
547
|
+
finally:
|
|
548
|
+
walker.close()
|
|
549
|
+
os.close(root_fd)
|
|
550
|
+
return {
|
|
551
|
+
"pattern": normalized_pattern,
|
|
552
|
+
"matches": matches,
|
|
553
|
+
"truncated": truncated or budget.truncated,
|
|
554
|
+
"visited_entries": budget.visited,
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def _strict_base64(value: Any) -> bytes:
|
|
559
|
+
if not isinstance(value, str):
|
|
560
|
+
raise ValueError("content_base64 must be a base64 string")
|
|
561
|
+
try:
|
|
562
|
+
decoded = base64.b64decode(value, validate=True)
|
|
563
|
+
except (binascii.Error, ValueError) as exc:
|
|
564
|
+
raise ValueError("content_base64 is not valid base64") from exc
|
|
565
|
+
if base64.b64encode(decoded).decode("ascii") != value:
|
|
566
|
+
raise ValueError("content_base64 is not canonical base64")
|
|
567
|
+
return decoded
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _expected_digest(value: Any) -> str | None:
|
|
571
|
+
if value is None:
|
|
572
|
+
return None
|
|
573
|
+
if not isinstance(value, str) or not re.fullmatch(r"[0-9a-fA-F]{64}", value):
|
|
574
|
+
raise ValueError(
|
|
575
|
+
"expected_sha256 must be a 64-character hexadecimal digest"
|
|
576
|
+
)
|
|
577
|
+
return value.lower()
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
@dataclass
|
|
581
|
+
class _PathLockEntry:
|
|
582
|
+
lock: threading.Lock
|
|
583
|
+
users: int = 0
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
_PATH_LOCK_GUARD = threading.Lock()
|
|
587
|
+
_PATH_LOCKS: dict[tuple[str, str], _PathLockEntry] = {}
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
@contextmanager
|
|
591
|
+
def _path_lock(key: tuple[str, str]) -> Iterator[None]:
|
|
592
|
+
with _PATH_LOCK_GUARD:
|
|
593
|
+
entry = _PATH_LOCKS.get(key)
|
|
594
|
+
if entry is None:
|
|
595
|
+
entry = _PathLockEntry(threading.Lock())
|
|
596
|
+
_PATH_LOCKS[key] = entry
|
|
597
|
+
entry.users += 1
|
|
598
|
+
entry.lock.acquire()
|
|
599
|
+
try:
|
|
600
|
+
yield
|
|
601
|
+
finally:
|
|
602
|
+
entry.lock.release()
|
|
603
|
+
with _PATH_LOCK_GUARD:
|
|
604
|
+
entry.users -= 1
|
|
605
|
+
if entry.users == 0 and _PATH_LOCKS.get(key) is entry:
|
|
606
|
+
del _PATH_LOCKS[key]
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def _existing_digest_and_mode(
|
|
610
|
+
parent_fd: int,
|
|
611
|
+
leaf: str,
|
|
612
|
+
) -> tuple[str | None, int | None]:
|
|
613
|
+
try:
|
|
614
|
+
descriptor = os.open(
|
|
615
|
+
leaf,
|
|
616
|
+
os.O_RDONLY | _O_NONBLOCK | _O_NOFOLLOW | _O_CLOEXEC,
|
|
617
|
+
dir_fd=parent_fd,
|
|
618
|
+
)
|
|
619
|
+
except FileNotFoundError:
|
|
620
|
+
return None, None
|
|
621
|
+
except OSError as exc:
|
|
622
|
+
if exc.errno == errno.ELOOP:
|
|
623
|
+
return None, None
|
|
624
|
+
raise
|
|
625
|
+
try:
|
|
626
|
+
metadata = os.fstat(descriptor)
|
|
627
|
+
if not stat.S_ISREG(metadata.st_mode):
|
|
628
|
+
return None, None
|
|
629
|
+
return _sha256_fd(descriptor), stat.S_IMODE(metadata.st_mode)
|
|
630
|
+
finally:
|
|
631
|
+
os.close(descriptor)
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def _create_temporary_file(parent_fd: int) -> tuple[int, str]:
|
|
635
|
+
for _ in range(100):
|
|
636
|
+
name = f".msdev-write-{secrets.token_hex(12)}"
|
|
637
|
+
try:
|
|
638
|
+
descriptor = os.open(
|
|
639
|
+
name,
|
|
640
|
+
os.O_WRONLY
|
|
641
|
+
| os.O_CREAT
|
|
642
|
+
| os.O_EXCL
|
|
643
|
+
| _O_NOFOLLOW
|
|
644
|
+
| _O_CLOEXEC,
|
|
645
|
+
0o600,
|
|
646
|
+
dir_fd=parent_fd,
|
|
647
|
+
)
|
|
648
|
+
return descriptor, name
|
|
649
|
+
except FileExistsError:
|
|
650
|
+
continue
|
|
651
|
+
raise FileExistsError("could not allocate a unique workspace temp file")
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def _write_all(descriptor: int, content: bytes) -> None:
|
|
655
|
+
view = memoryview(content)
|
|
656
|
+
while view:
|
|
657
|
+
written = os.write(descriptor, view)
|
|
658
|
+
if written <= 0:
|
|
659
|
+
raise OSError("short write to workspace temporary file")
|
|
660
|
+
view = view[written:]
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def atomic_write(
|
|
664
|
+
root: str | os.PathLike[str] | None,
|
|
665
|
+
relative: str | None,
|
|
666
|
+
content_base64: str,
|
|
667
|
+
expected_sha256: str | None = None,
|
|
668
|
+
) -> dict[str, Any]:
|
|
669
|
+
root_path = workspace_root(root)
|
|
670
|
+
normalized = normalize_relative("." if relative is None else relative)
|
|
671
|
+
key = (str(root_path), normalized)
|
|
672
|
+
with _path_lock(key):
|
|
673
|
+
content = _strict_base64(content_base64)
|
|
674
|
+
expected = _expected_digest(expected_sha256)
|
|
675
|
+
_, root_fd = _open_workspace(root_path)
|
|
676
|
+
parent_fd = None
|
|
677
|
+
temporary_name = None
|
|
678
|
+
temporary_fd = None
|
|
679
|
+
try:
|
|
680
|
+
parent_fd, leaf = _open_parent(root_fd, normalized)
|
|
681
|
+
try:
|
|
682
|
+
existing = os.stat(
|
|
683
|
+
leaf,
|
|
684
|
+
dir_fd=parent_fd,
|
|
685
|
+
follow_symlinks=False,
|
|
686
|
+
)
|
|
687
|
+
except FileNotFoundError:
|
|
688
|
+
existing = None
|
|
689
|
+
if existing is not None and stat.S_ISDIR(existing.st_mode):
|
|
690
|
+
raise IsADirectoryError(
|
|
691
|
+
f"workspace path is a directory: {normalized}"
|
|
692
|
+
)
|
|
693
|
+
if (
|
|
694
|
+
existing is not None
|
|
695
|
+
and not stat.S_ISREG(existing.st_mode)
|
|
696
|
+
and not stat.S_ISLNK(existing.st_mode)
|
|
697
|
+
):
|
|
698
|
+
raise ValueError(
|
|
699
|
+
f"workspace path is not a regular file: {normalized}"
|
|
700
|
+
)
|
|
701
|
+
|
|
702
|
+
current, existing_mode = _existing_digest_and_mode(parent_fd, leaf)
|
|
703
|
+
if expected is not None and current != expected:
|
|
704
|
+
raise RuntimeError(
|
|
705
|
+
f"workspace file changed before write: {normalized} "
|
|
706
|
+
f"(expected {expected}, found {current or 'missing'})"
|
|
707
|
+
)
|
|
708
|
+
|
|
709
|
+
temporary_fd, temporary_name = _create_temporary_file(parent_fd)
|
|
710
|
+
if existing_mode is not None:
|
|
711
|
+
os.fchmod(temporary_fd, existing_mode)
|
|
712
|
+
_write_all(temporary_fd, content)
|
|
713
|
+
os.fsync(temporary_fd)
|
|
714
|
+
os.close(temporary_fd)
|
|
715
|
+
temporary_fd = None
|
|
716
|
+
|
|
717
|
+
if expected is not None:
|
|
718
|
+
latest, _ = _existing_digest_and_mode(parent_fd, leaf)
|
|
719
|
+
if latest != expected:
|
|
720
|
+
raise RuntimeError(
|
|
721
|
+
f"workspace file changed before replace: {normalized} "
|
|
722
|
+
f"(expected {expected}, found {latest or 'missing'})"
|
|
723
|
+
)
|
|
724
|
+
os.replace(
|
|
725
|
+
temporary_name,
|
|
726
|
+
leaf,
|
|
727
|
+
src_dir_fd=parent_fd,
|
|
728
|
+
dst_dir_fd=parent_fd,
|
|
729
|
+
)
|
|
730
|
+
temporary_name = None
|
|
731
|
+
os.fsync(parent_fd)
|
|
732
|
+
finally:
|
|
733
|
+
if temporary_fd is not None:
|
|
734
|
+
os.close(temporary_fd)
|
|
735
|
+
if temporary_name is not None and parent_fd is not None:
|
|
736
|
+
try:
|
|
737
|
+
os.unlink(temporary_name, dir_fd=parent_fd)
|
|
738
|
+
except FileNotFoundError:
|
|
739
|
+
pass
|
|
740
|
+
if parent_fd is not None:
|
|
741
|
+
os.close(parent_fd)
|
|
742
|
+
os.close(root_fd)
|
|
743
|
+
return {
|
|
744
|
+
"path": normalized,
|
|
745
|
+
"size": len(content),
|
|
746
|
+
"sha256": hashlib.sha256(content).hexdigest(),
|
|
747
|
+
"atomic": True,
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def delete_file(
|
|
752
|
+
root: str | os.PathLike[str] | None,
|
|
753
|
+
relative: str | None,
|
|
754
|
+
) -> dict[str, Any]:
|
|
755
|
+
root_path = workspace_root(root)
|
|
756
|
+
normalized = normalize_relative("." if relative is None else relative)
|
|
757
|
+
key = (str(root_path), normalized)
|
|
758
|
+
with _path_lock(key):
|
|
759
|
+
_, root_fd = _open_workspace(root_path)
|
|
760
|
+
parent_fd = None
|
|
761
|
+
try:
|
|
762
|
+
parent_fd, leaf = _open_parent(root_fd, normalized)
|
|
763
|
+
try:
|
|
764
|
+
metadata = os.stat(
|
|
765
|
+
leaf,
|
|
766
|
+
dir_fd=parent_fd,
|
|
767
|
+
follow_symlinks=False,
|
|
768
|
+
)
|
|
769
|
+
except FileNotFoundError as exc:
|
|
770
|
+
raise FileNotFoundError(
|
|
771
|
+
f"workspace file does not exist: {normalized}"
|
|
772
|
+
) from exc
|
|
773
|
+
if stat.S_ISDIR(metadata.st_mode):
|
|
774
|
+
raise IsADirectoryError(
|
|
775
|
+
"workspace delete only supports files, not directories: "
|
|
776
|
+
f"{normalized}"
|
|
777
|
+
)
|
|
778
|
+
if not (
|
|
779
|
+
stat.S_ISREG(metadata.st_mode)
|
|
780
|
+
or stat.S_ISLNK(metadata.st_mode)
|
|
781
|
+
):
|
|
782
|
+
raise ValueError(
|
|
783
|
+
f"workspace path is not a regular file: {normalized}"
|
|
784
|
+
)
|
|
785
|
+
os.unlink(leaf, dir_fd=parent_fd)
|
|
786
|
+
finally:
|
|
787
|
+
if parent_fd is not None:
|
|
788
|
+
os.close(parent_fd)
|
|
789
|
+
os.close(root_fd)
|
|
790
|
+
return {"path": normalized, "deleted": True}
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
def _search_scopes(paths: Any) -> list[str]:
|
|
794
|
+
raw_paths = ["."] if paths is None else paths
|
|
795
|
+
if (
|
|
796
|
+
not isinstance(raw_paths, list)
|
|
797
|
+
or not raw_paths
|
|
798
|
+
or not all(isinstance(item, str) for item in raw_paths)
|
|
799
|
+
):
|
|
800
|
+
raise ValueError("search paths must be a non-empty string array")
|
|
801
|
+
return sorted({normalize_relative(item) for item in raw_paths})
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
@dataclass
|
|
805
|
+
class _SearchBudget:
|
|
806
|
+
started: float
|
|
807
|
+
deadline: float
|
|
808
|
+
files_visited: int = 0
|
|
809
|
+
bytes_read: int = 0
|
|
810
|
+
reasons: set[str] | None = None
|
|
811
|
+
|
|
812
|
+
@classmethod
|
|
813
|
+
def create(cls) -> "_SearchBudget":
|
|
814
|
+
started = time.monotonic()
|
|
815
|
+
return cls(started, started + MAX_SEARCH_SECONDS, reasons=set())
|
|
816
|
+
|
|
817
|
+
def mark(self, reason: str) -> None:
|
|
818
|
+
assert self.reasons is not None
|
|
819
|
+
self.reasons.add(reason)
|
|
820
|
+
|
|
821
|
+
def has_reason(self, reason: str) -> bool:
|
|
822
|
+
assert self.reasons is not None
|
|
823
|
+
return reason in self.reasons
|
|
824
|
+
|
|
825
|
+
def remaining_seconds(self) -> float:
|
|
826
|
+
return self.deadline - time.monotonic()
|
|
827
|
+
|
|
828
|
+
def check_deadline(self) -> bool:
|
|
829
|
+
if self.remaining_seconds() <= 0:
|
|
830
|
+
self.mark("deadline")
|
|
831
|
+
return False
|
|
832
|
+
return True
|
|
833
|
+
|
|
834
|
+
def claim_file(self, size: int) -> bool:
|
|
835
|
+
if not self.check_deadline():
|
|
836
|
+
return False
|
|
837
|
+
if self.files_visited >= MAX_SEARCH_FILES:
|
|
838
|
+
self.mark("files")
|
|
839
|
+
return False
|
|
840
|
+
if size > MAX_SEARCH_BYTES - self.bytes_read:
|
|
841
|
+
self.mark("bytes")
|
|
842
|
+
return False
|
|
843
|
+
self.files_visited += 1
|
|
844
|
+
self.bytes_read += size
|
|
845
|
+
return True
|
|
846
|
+
|
|
847
|
+
|
|
848
|
+
def _iter_search_files(
|
|
849
|
+
root_fd: int,
|
|
850
|
+
scopes: Iterable[str],
|
|
851
|
+
traversal: _TraversalBudget,
|
|
852
|
+
) -> Generator[tuple[str, int], None, None]:
|
|
853
|
+
for scope in scopes:
|
|
854
|
+
try:
|
|
855
|
+
scope_fd = _open_relative(
|
|
856
|
+
root_fd,
|
|
857
|
+
scope,
|
|
858
|
+
os.O_RDONLY | _O_NONBLOCK,
|
|
859
|
+
)
|
|
860
|
+
except FileNotFoundError as exc:
|
|
861
|
+
raise FileNotFoundError(f"search path does not exist: {scope}") from exc
|
|
862
|
+
metadata = os.fstat(scope_fd)
|
|
863
|
+
if stat.S_ISREG(metadata.st_mode):
|
|
864
|
+
try:
|
|
865
|
+
yield scope, scope_fd
|
|
866
|
+
finally:
|
|
867
|
+
os.close(scope_fd)
|
|
868
|
+
continue
|
|
869
|
+
if not stat.S_ISDIR(metadata.st_mode):
|
|
870
|
+
os.close(scope_fd)
|
|
871
|
+
continue
|
|
872
|
+
try:
|
|
873
|
+
walker = _walk_tree_fd(scope_fd, scope, traversal)
|
|
874
|
+
try:
|
|
875
|
+
for relative, child_metadata, parent_fd, name in walker:
|
|
876
|
+
if not stat.S_ISREG(child_metadata.st_mode):
|
|
877
|
+
continue
|
|
878
|
+
try:
|
|
879
|
+
file_fd = os.open(
|
|
880
|
+
name,
|
|
881
|
+
os.O_RDONLY
|
|
882
|
+
| _O_NONBLOCK
|
|
883
|
+
| _O_NOFOLLOW
|
|
884
|
+
| _O_CLOEXEC,
|
|
885
|
+
dir_fd=parent_fd,
|
|
886
|
+
)
|
|
887
|
+
except (FileNotFoundError, PermissionError):
|
|
888
|
+
continue
|
|
889
|
+
except OSError as exc:
|
|
890
|
+
if exc.errno == errno.ELOOP:
|
|
891
|
+
continue
|
|
892
|
+
raise
|
|
893
|
+
try:
|
|
894
|
+
if stat.S_ISREG(os.fstat(file_fd).st_mode):
|
|
895
|
+
yield relative, file_fd
|
|
896
|
+
finally:
|
|
897
|
+
os.close(file_fd)
|
|
898
|
+
finally:
|
|
899
|
+
walker.close()
|
|
900
|
+
finally:
|
|
901
|
+
os.close(scope_fd)
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
def _search_glob_match(relative: str, pattern: str | None) -> bool:
|
|
905
|
+
if pattern is None:
|
|
906
|
+
return True
|
|
907
|
+
if "/" not in pattern:
|
|
908
|
+
return fnmatch.fnmatchcase(relative.rsplit("/", 1)[-1], pattern)
|
|
909
|
+
return _glob_match(relative, pattern)
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
def _python_file_matches(
|
|
913
|
+
descriptor: int,
|
|
914
|
+
relative: str,
|
|
915
|
+
literal: str,
|
|
916
|
+
budget: _SearchBudget,
|
|
917
|
+
) -> Iterator[dict[str, Any]]:
|
|
918
|
+
with os.fdopen(
|
|
919
|
+
os.dup(descriptor),
|
|
920
|
+
"rb",
|
|
921
|
+
) as handle:
|
|
922
|
+
line_number = 0
|
|
923
|
+
while budget.check_deadline():
|
|
924
|
+
line = handle.readline(MAX_SEARCH_LINE_BYTES + 1)
|
|
925
|
+
if not line:
|
|
926
|
+
return
|
|
927
|
+
line_number += 1
|
|
928
|
+
oversized = len(line) > MAX_SEARCH_LINE_BYTES
|
|
929
|
+
while oversized and not line.endswith(b"\n"):
|
|
930
|
+
if not budget.check_deadline():
|
|
931
|
+
return
|
|
932
|
+
fragment = handle.readline(MAX_SEARCH_LINE_BYTES + 1)
|
|
933
|
+
if not fragment or fragment.endswith(b"\n"):
|
|
934
|
+
break
|
|
935
|
+
if oversized:
|
|
936
|
+
budget.mark("line_size")
|
|
937
|
+
continue
|
|
938
|
+
text = line.decode("utf-8", errors="replace")
|
|
939
|
+
if literal in text:
|
|
940
|
+
yield {
|
|
941
|
+
"path": relative,
|
|
942
|
+
"line": line_number,
|
|
943
|
+
"text": text.rstrip("\r\n"),
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
|
|
947
|
+
def _rg_match_message(line: bytes, relative: str) -> dict[str, Any] | None:
|
|
948
|
+
try:
|
|
949
|
+
message = json.loads(line)
|
|
950
|
+
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
951
|
+
return None
|
|
952
|
+
if message.get("type") != "match":
|
|
953
|
+
return None
|
|
954
|
+
data = message.get("data") or {}
|
|
955
|
+
line_number = data.get("line_number")
|
|
956
|
+
text = (data.get("lines") or {}).get("text")
|
|
957
|
+
if not isinstance(line_number, int) or not isinstance(text, str):
|
|
958
|
+
return None
|
|
959
|
+
return {
|
|
960
|
+
"path": relative,
|
|
961
|
+
"line": line_number,
|
|
962
|
+
"text": text.rstrip("\r\n"),
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _rg_file_matches(
|
|
967
|
+
executable: str,
|
|
968
|
+
descriptor: int,
|
|
969
|
+
relative: str,
|
|
970
|
+
pattern: str,
|
|
971
|
+
max_matches: int,
|
|
972
|
+
budget: _SearchBudget,
|
|
973
|
+
) -> Generator[dict[str, Any], None, None]:
|
|
974
|
+
os.lseek(descriptor, 0, os.SEEK_SET)
|
|
975
|
+
process = subprocess.Popen(
|
|
976
|
+
[
|
|
977
|
+
executable,
|
|
978
|
+
"--json",
|
|
979
|
+
"--line-number",
|
|
980
|
+
"--color",
|
|
981
|
+
"never",
|
|
982
|
+
"--text",
|
|
983
|
+
"--no-config",
|
|
984
|
+
"--max-columns",
|
|
985
|
+
str(MAX_SEARCH_LINE_BYTES),
|
|
986
|
+
"--max-filesize",
|
|
987
|
+
str(MAX_SEARCH_BYTES),
|
|
988
|
+
"--max-count",
|
|
989
|
+
str(max_matches),
|
|
990
|
+
"--",
|
|
991
|
+
pattern,
|
|
992
|
+
"-",
|
|
993
|
+
],
|
|
994
|
+
stdin=descriptor,
|
|
995
|
+
stdout=subprocess.PIPE,
|
|
996
|
+
stderr=subprocess.DEVNULL,
|
|
997
|
+
)
|
|
998
|
+
assert process.stdout is not None
|
|
999
|
+
buffer = bytearray()
|
|
1000
|
+
discarding_line = False
|
|
1001
|
+
stream_finished = False
|
|
1002
|
+
try:
|
|
1003
|
+
with selectors.DefaultSelector() as selector:
|
|
1004
|
+
selector.register(process.stdout, selectors.EVENT_READ)
|
|
1005
|
+
while not stream_finished:
|
|
1006
|
+
remaining = budget.remaining_seconds()
|
|
1007
|
+
if remaining <= 0:
|
|
1008
|
+
budget.mark("deadline")
|
|
1009
|
+
break
|
|
1010
|
+
events = selector.select(min(remaining, 0.1))
|
|
1011
|
+
if not events:
|
|
1012
|
+
if process.poll() is not None:
|
|
1013
|
+
break
|
|
1014
|
+
continue
|
|
1015
|
+
for _key, _mask in events:
|
|
1016
|
+
chunk = os.read(process.stdout.fileno(), _READ_CHUNK_SIZE)
|
|
1017
|
+
if not chunk:
|
|
1018
|
+
stream_finished = True
|
|
1019
|
+
break
|
|
1020
|
+
while chunk:
|
|
1021
|
+
newline = chunk.find(b"\n")
|
|
1022
|
+
if discarding_line:
|
|
1023
|
+
if newline < 0:
|
|
1024
|
+
chunk = b""
|
|
1025
|
+
else:
|
|
1026
|
+
discarding_line = False
|
|
1027
|
+
chunk = chunk[newline + 1 :]
|
|
1028
|
+
continue
|
|
1029
|
+
if newline < 0:
|
|
1030
|
+
if len(buffer) + len(chunk) > _RG_JSON_LINE_BYTES:
|
|
1031
|
+
buffer.clear()
|
|
1032
|
+
discarding_line = True
|
|
1033
|
+
budget.mark("line_size")
|
|
1034
|
+
else:
|
|
1035
|
+
buffer.extend(chunk)
|
|
1036
|
+
chunk = b""
|
|
1037
|
+
continue
|
|
1038
|
+
segment = chunk[:newline]
|
|
1039
|
+
chunk = chunk[newline + 1 :]
|
|
1040
|
+
if len(buffer) + len(segment) > _RG_JSON_LINE_BYTES:
|
|
1041
|
+
buffer.clear()
|
|
1042
|
+
budget.mark("line_size")
|
|
1043
|
+
continue
|
|
1044
|
+
buffer.extend(segment)
|
|
1045
|
+
match = _rg_match_message(bytes(buffer), relative)
|
|
1046
|
+
buffer.clear()
|
|
1047
|
+
if match is not None:
|
|
1048
|
+
yield match
|
|
1049
|
+
if budget.has_reason("deadline"):
|
|
1050
|
+
if process.poll() is None:
|
|
1051
|
+
process.kill()
|
|
1052
|
+
returncode = process.wait()
|
|
1053
|
+
else:
|
|
1054
|
+
try:
|
|
1055
|
+
returncode = process.wait(
|
|
1056
|
+
timeout=max(0.001, budget.remaining_seconds())
|
|
1057
|
+
)
|
|
1058
|
+
except subprocess.TimeoutExpired:
|
|
1059
|
+
budget.mark("deadline")
|
|
1060
|
+
if process.poll() is None:
|
|
1061
|
+
process.kill()
|
|
1062
|
+
returncode = process.wait()
|
|
1063
|
+
if returncode not in {0, 1} and not budget.has_reason("deadline"):
|
|
1064
|
+
raise ValueError(f"rg search failed with exit code {returncode}")
|
|
1065
|
+
finally:
|
|
1066
|
+
if process.poll() is None:
|
|
1067
|
+
process.kill()
|
|
1068
|
+
process.wait()
|
|
1069
|
+
process.stdout.close()
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
def _search_python(
|
|
1073
|
+
root_fd: int,
|
|
1074
|
+
pattern: str,
|
|
1075
|
+
scopes: list[str],
|
|
1076
|
+
glob_filter: str | None,
|
|
1077
|
+
limit: int,
|
|
1078
|
+
) -> dict[str, Any]:
|
|
1079
|
+
if any(character in _REGEX_METACHARACTERS for character in pattern):
|
|
1080
|
+
raise ValueError(
|
|
1081
|
+
"Python fallback search only supports literal substring patterns; "
|
|
1082
|
+
"ripgrep is required for regular-expression metacharacters"
|
|
1083
|
+
)
|
|
1084
|
+
matches: list[dict[str, Any]] = []
|
|
1085
|
+
budget = _SearchBudget.create()
|
|
1086
|
+
traversal = _TraversalBudget(
|
|
1087
|
+
MAX_TRAVERSAL_ENTRIES,
|
|
1088
|
+
deadline=budget.deadline,
|
|
1089
|
+
)
|
|
1090
|
+
files = _iter_search_files(root_fd, scopes, traversal)
|
|
1091
|
+
try:
|
|
1092
|
+
for relative, descriptor in files:
|
|
1093
|
+
if not _search_glob_match(relative, glob_filter):
|
|
1094
|
+
continue
|
|
1095
|
+
if not budget.claim_file(os.fstat(descriptor).st_size):
|
|
1096
|
+
break
|
|
1097
|
+
for match in _python_file_matches(
|
|
1098
|
+
descriptor,
|
|
1099
|
+
relative,
|
|
1100
|
+
pattern,
|
|
1101
|
+
budget,
|
|
1102
|
+
):
|
|
1103
|
+
if len(matches) == limit:
|
|
1104
|
+
budget.mark("results")
|
|
1105
|
+
break
|
|
1106
|
+
matches.append(match)
|
|
1107
|
+
if budget.reasons:
|
|
1108
|
+
break
|
|
1109
|
+
finally:
|
|
1110
|
+
files.close()
|
|
1111
|
+
if traversal.deadline_exceeded:
|
|
1112
|
+
budget.mark("deadline")
|
|
1113
|
+
elif traversal.truncated:
|
|
1114
|
+
budget.mark("entries")
|
|
1115
|
+
return {
|
|
1116
|
+
"matches": matches,
|
|
1117
|
+
"truncated": bool(budget.reasons),
|
|
1118
|
+
"engine": "python",
|
|
1119
|
+
"semantics": "literal",
|
|
1120
|
+
"files_visited": budget.files_visited,
|
|
1121
|
+
"bytes_read": budget.bytes_read,
|
|
1122
|
+
"truncation_reasons": sorted(budget.reasons or ()),
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
def _search_rg(
|
|
1127
|
+
executable: str,
|
|
1128
|
+
root_fd: int,
|
|
1129
|
+
pattern: str,
|
|
1130
|
+
scopes: list[str],
|
|
1131
|
+
glob_filter: str | None,
|
|
1132
|
+
limit: int,
|
|
1133
|
+
) -> dict[str, Any]:
|
|
1134
|
+
matches: list[dict[str, Any]] = []
|
|
1135
|
+
budget = _SearchBudget.create()
|
|
1136
|
+
traversal = _TraversalBudget(
|
|
1137
|
+
MAX_TRAVERSAL_ENTRIES,
|
|
1138
|
+
deadline=budget.deadline,
|
|
1139
|
+
)
|
|
1140
|
+
files = _iter_search_files(root_fd, scopes, traversal)
|
|
1141
|
+
try:
|
|
1142
|
+
for relative, descriptor in files:
|
|
1143
|
+
if not _search_glob_match(relative, glob_filter):
|
|
1144
|
+
continue
|
|
1145
|
+
if not budget.claim_file(os.fstat(descriptor).st_size):
|
|
1146
|
+
break
|
|
1147
|
+
file_matches = _rg_file_matches(
|
|
1148
|
+
executable,
|
|
1149
|
+
descriptor,
|
|
1150
|
+
relative,
|
|
1151
|
+
pattern,
|
|
1152
|
+
limit - len(matches) + 1,
|
|
1153
|
+
budget,
|
|
1154
|
+
)
|
|
1155
|
+
try:
|
|
1156
|
+
for match in file_matches:
|
|
1157
|
+
if len(matches) == limit:
|
|
1158
|
+
budget.mark("results")
|
|
1159
|
+
break
|
|
1160
|
+
matches.append(match)
|
|
1161
|
+
finally:
|
|
1162
|
+
file_matches.close()
|
|
1163
|
+
if budget.reasons:
|
|
1164
|
+
break
|
|
1165
|
+
finally:
|
|
1166
|
+
files.close()
|
|
1167
|
+
if traversal.deadline_exceeded:
|
|
1168
|
+
budget.mark("deadline")
|
|
1169
|
+
elif traversal.truncated:
|
|
1170
|
+
budget.mark("entries")
|
|
1171
|
+
return {
|
|
1172
|
+
"matches": matches,
|
|
1173
|
+
"truncated": bool(budget.reasons),
|
|
1174
|
+
"engine": "rg",
|
|
1175
|
+
"files_visited": budget.files_visited,
|
|
1176
|
+
"bytes_read": budget.bytes_read,
|
|
1177
|
+
"truncation_reasons": sorted(budget.reasons or ()),
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
|
|
1181
|
+
def search(
|
|
1182
|
+
root: str | os.PathLike[str] | None,
|
|
1183
|
+
pattern: str,
|
|
1184
|
+
paths: list[str] | None = None,
|
|
1185
|
+
glob: str | None = None,
|
|
1186
|
+
max_results: int = 1000,
|
|
1187
|
+
) -> dict[str, Any]:
|
|
1188
|
+
if not isinstance(pattern, str) or "\0" in pattern:
|
|
1189
|
+
raise ValueError("search pattern must be a NUL-free string")
|
|
1190
|
+
limit = _max_results(max_results)
|
|
1191
|
+
scopes = _search_scopes(paths)
|
|
1192
|
+
glob_filter = normalize_relative(glob) if glob is not None else None
|
|
1193
|
+
_, root_fd = _open_workspace(root)
|
|
1194
|
+
try:
|
|
1195
|
+
executable = shutil.which("rg")
|
|
1196
|
+
if executable is not None:
|
|
1197
|
+
try:
|
|
1198
|
+
return _search_rg(
|
|
1199
|
+
executable,
|
|
1200
|
+
root_fd,
|
|
1201
|
+
pattern,
|
|
1202
|
+
scopes,
|
|
1203
|
+
glob_filter,
|
|
1204
|
+
limit,
|
|
1205
|
+
)
|
|
1206
|
+
except OSError:
|
|
1207
|
+
pass
|
|
1208
|
+
return _search_python(
|
|
1209
|
+
root_fd,
|
|
1210
|
+
pattern,
|
|
1211
|
+
scopes,
|
|
1212
|
+
glob_filter,
|
|
1213
|
+
limit,
|
|
1214
|
+
)
|
|
1215
|
+
finally:
|
|
1216
|
+
os.close(root_fd)
|