python-delphi-lsp 3.6.0__cp310-abi3-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- delphi_lsp/__init__.py +184 -0
- delphi_lsp/_version.py +1 -0
- delphi_lsp/agent_cache.py +1574 -0
- delphi_lsp/agent_cli.py +902 -0
- delphi_lsp/agent_context.py +3907 -0
- delphi_lsp/agent_cpg.py +349 -0
- delphi_lsp/agent_cpg_builder.py +666 -0
- delphi_lsp/agent_layers.py +1410 -0
- delphi_lsp/agent_metrics.py +526 -0
- delphi_lsp/agent_protocol.py +445 -0
- delphi_lsp/agent_relations.py +1881 -0
- delphi_lsp/agent_templates.py +614 -0
- delphi_lsp/agent_wiki.py +2853 -0
- delphi_lsp/agent_workspace.py +860 -0
- delphi_lsp/ast_serialize.py +126 -0
- delphi_lsp/binary.py +250 -0
- delphi_lsp/comment_builder.py +28 -0
- delphi_lsp/consts.py +345 -0
- delphi_lsp/delphi_lsp_native.pyd +0 -0
- delphi_lsp/delphiast_lexer.py +399 -0
- delphi_lsp/delphiast_parser.py +2189 -0
- delphi_lsp/delphiast_tokens.py +41 -0
- delphi_lsp/grammar.py +459 -0
- delphi_lsp/incremental.py +1139 -0
- delphi_lsp/lark_builder.py +2684 -0
- delphi_lsp/lark_tokens.py +236 -0
- delphi_lsp/lexical_scanner.py +30 -0
- delphi_lsp/lsp_server.py +6698 -0
- delphi_lsp/metrics.py +1452 -0
- delphi_lsp/native_bridge.py +300 -0
- delphi_lsp/navigation_cache.py +189 -0
- delphi_lsp/nodes.py +462 -0
- delphi_lsp/parallel_outline.py +662 -0
- delphi_lsp/parse_artifacts.py +969 -0
- delphi_lsp/parser.py +342 -0
- delphi_lsp/parser_backend.py +139 -0
- delphi_lsp/preprocessor.py +1122 -0
- delphi_lsp/progress.py +26 -0
- delphi_lsp/project_config.py +285 -0
- delphi_lsp/project_discovery.py +932 -0
- delphi_lsp/project_indexer.py +470 -0
- delphi_lsp/py.typed +1 -0
- delphi_lsp/response_cache.py +465 -0
- delphi_lsp/semantic.py +394 -0
- delphi_lsp/semantic_builder.py +2302 -0
- delphi_lsp/source_reader.py +17 -0
- delphi_lsp/wiki_external_sort.py +392 -0
- delphi_lsp/wiki_memory.py +548 -0
- delphi_lsp/workspace.py +83 -0
- delphi_lsp/writer.py +73 -0
- python_delphi_lsp-3.6.0.dist-info/METADATA +766 -0
- python_delphi_lsp-3.6.0.dist-info/RECORD +56 -0
- python_delphi_lsp-3.6.0.dist-info/WHEEL +4 -0
- python_delphi_lsp-3.6.0.dist-info/entry_points.txt +3 -0
- python_delphi_lsp-3.6.0.dist-info/licenses/LICENSE +373 -0
- python_delphi_lsp-3.6.0.dist-info/sboms/python-delphi-lsp.cyclonedx.json +770 -0
|
@@ -0,0 +1,1574 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import contextlib
|
|
5
|
+
import hmac
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import secrets
|
|
11
|
+
import select
|
|
12
|
+
import signal
|
|
13
|
+
import socket
|
|
14
|
+
import stat
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
import tempfile
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
from collections.abc import Callable, Collection, Mapping
|
|
21
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
22
|
+
from dataclasses import dataclass, fields, is_dataclass, replace
|
|
23
|
+
from functools import lru_cache
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from types import FrameType, ModuleType
|
|
26
|
+
|
|
27
|
+
from watchfiles import watch
|
|
28
|
+
|
|
29
|
+
from ._version import __version__
|
|
30
|
+
from .agent_context import AgentContext
|
|
31
|
+
from .agent_protocol import AgentProtocolError
|
|
32
|
+
from .project_config import ProjectPathConfig, load_project_path_config
|
|
33
|
+
from .project_discovery import SKIP_DIRS
|
|
34
|
+
|
|
35
|
+
DEFAULT_MAX_MEMORY_BYTES = 512 * 1024**2
|
|
36
|
+
DEFAULT_MAX_DISK_CACHE_BYTES = 512 * 1024**2
|
|
37
|
+
WARNING_THRESHOLD_PERCENT = 80
|
|
38
|
+
DAEMON_SCHEMA = 2
|
|
39
|
+
DEFAULT_IDLE_TIMEOUT = 1800
|
|
40
|
+
DEFAULT_STARTUP_TIMEOUT = 120.0
|
|
41
|
+
_MAX_MESSAGE_BYTES = 1024 * 1024
|
|
42
|
+
_CONNECTION_TIMEOUT = 2.0
|
|
43
|
+
_PREAUTH_CONNECTION_TIMEOUT = 0.25
|
|
44
|
+
_CONNECTION_WORKERS = 8
|
|
45
|
+
_CLIENT_RESPONSE_TIMEOUT = 120.0
|
|
46
|
+
_MAX_CLIENT_RESPONSE_TIMEOUT = 600.0
|
|
47
|
+
_CLIENT_SECONDS_PER_SOURCE = 0.25
|
|
48
|
+
_MEMORY_SIZE = re.compile(r"^(?P<count>[1-9][0-9]*)(?P<suffix>[KMG]?)$", re.IGNORECASE)
|
|
49
|
+
_STARTUP_DIAGNOSTIC_BYTES = 16 * 1024
|
|
50
|
+
_STARTUP_TOKEN_RE = re.compile(r"(?i)(token\b[^\n\r]*?:?\s*['\"]?[A-Za-z0-9_-]+['\"]?|\b[a-zA-Z0-9_-]{32,})")
|
|
51
|
+
_START_LOCK_INCOMPLETE_GRACE_SECONDS = 1.0
|
|
52
|
+
_CACHE_REVISION_CHECK_INTERVAL_SECONDS = 3600.0
|
|
53
|
+
_WATCHER_READY_TIMEOUT_SECONDS = 1.0
|
|
54
|
+
_WATCHER_READY_POLL_MILLISECONDS = 50
|
|
55
|
+
_STOP_TIMEOUT_SECONDS = 15.0
|
|
56
|
+
_STOP_SIGNAL_DELAY_SECONDS = 3.0
|
|
57
|
+
_STOP_FORCE_TIMEOUT_SECONDS = 2.0
|
|
58
|
+
_WATCHED_SUFFIXES = frozenset(
|
|
59
|
+
{".pas", ".pp", ".inc", ".dpr", ".dpk", ".dproj", ".cfg"}
|
|
60
|
+
)
|
|
61
|
+
_PROCESS_START_LOCKS_GUARD = threading.Lock()
|
|
62
|
+
_PROCESS_START_LOCKS: dict[str, tuple[threading.Lock, int]] = {}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def current_process_rss_bytes() -> int:
|
|
66
|
+
"""Return the current resident set size, or zero when unavailable."""
|
|
67
|
+
try:
|
|
68
|
+
if sys.platform == "darwin":
|
|
69
|
+
return _darwin_process_rss_bytes()
|
|
70
|
+
if sys.platform.startswith("linux"):
|
|
71
|
+
statm = Path("/proc/self/statm").read_text(encoding="ascii").split()
|
|
72
|
+
return int(statm[1]) * int(os.sysconf("SC_PAGE_SIZE"))
|
|
73
|
+
if os.name == "nt":
|
|
74
|
+
return _windows_process_rss_bytes()
|
|
75
|
+
except Exception:
|
|
76
|
+
pass
|
|
77
|
+
return 0
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _darwin_process_rss_bytes() -> int:
|
|
81
|
+
import ctypes
|
|
82
|
+
|
|
83
|
+
class TimeValue(ctypes.Structure):
|
|
84
|
+
_fields_ = [("seconds", ctypes.c_int), ("microseconds", ctypes.c_int)]
|
|
85
|
+
|
|
86
|
+
class MachTaskBasicInfo(ctypes.Structure):
|
|
87
|
+
_fields_ = [
|
|
88
|
+
("virtual_size", ctypes.c_uint64),
|
|
89
|
+
("resident_size", ctypes.c_uint64),
|
|
90
|
+
("resident_size_max", ctypes.c_uint64),
|
|
91
|
+
("user_time", TimeValue),
|
|
92
|
+
("system_time", TimeValue),
|
|
93
|
+
("policy", ctypes.c_int),
|
|
94
|
+
("suspend_count", ctypes.c_int),
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
library = ctypes.CDLL(None)
|
|
98
|
+
library.mach_task_self.restype = ctypes.c_uint
|
|
99
|
+
library.task_info.argtypes = (
|
|
100
|
+
ctypes.c_uint,
|
|
101
|
+
ctypes.c_int,
|
|
102
|
+
ctypes.c_void_p,
|
|
103
|
+
ctypes.POINTER(ctypes.c_uint),
|
|
104
|
+
)
|
|
105
|
+
library.task_info.restype = ctypes.c_int
|
|
106
|
+
info = MachTaskBasicInfo()
|
|
107
|
+
count = ctypes.c_uint(ctypes.sizeof(info) // ctypes.sizeof(ctypes.c_int))
|
|
108
|
+
result = library.task_info(
|
|
109
|
+
library.mach_task_self(),
|
|
110
|
+
20,
|
|
111
|
+
ctypes.byref(info),
|
|
112
|
+
ctypes.byref(count),
|
|
113
|
+
)
|
|
114
|
+
return int(info.resident_size) if result == 0 else 0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _windows_process_rss_bytes() -> int:
|
|
118
|
+
import ctypes
|
|
119
|
+
from ctypes import wintypes
|
|
120
|
+
|
|
121
|
+
class ProcessMemoryCounters(ctypes.Structure):
|
|
122
|
+
_fields_ = [
|
|
123
|
+
("cb", wintypes.DWORD),
|
|
124
|
+
("PageFaultCount", wintypes.DWORD),
|
|
125
|
+
("PeakWorkingSetSize", ctypes.c_size_t),
|
|
126
|
+
("WorkingSetSize", ctypes.c_size_t),
|
|
127
|
+
("QuotaPeakPagedPoolUsage", ctypes.c_size_t),
|
|
128
|
+
("QuotaPagedPoolUsage", ctypes.c_size_t),
|
|
129
|
+
("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t),
|
|
130
|
+
("QuotaNonPagedPoolUsage", ctypes.c_size_t),
|
|
131
|
+
("PagefileUsage", ctypes.c_size_t),
|
|
132
|
+
("PeakPagefileUsage", ctypes.c_size_t),
|
|
133
|
+
]
|
|
134
|
+
|
|
135
|
+
counters = ProcessMemoryCounters()
|
|
136
|
+
counters.cb = ctypes.sizeof(counters)
|
|
137
|
+
process = ctypes.windll.kernel32.GetCurrentProcess()
|
|
138
|
+
if not ctypes.windll.psapi.GetProcessMemoryInfo(
|
|
139
|
+
process,
|
|
140
|
+
ctypes.byref(counters),
|
|
141
|
+
counters.cb,
|
|
142
|
+
):
|
|
143
|
+
return 0
|
|
144
|
+
return int(counters.WorkingSetSize)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def estimate_deep_size(value: object) -> int:
|
|
148
|
+
"""Estimate the memory retained by an owned object graph.
|
|
149
|
+
|
|
150
|
+
The traversal is intentionally best-effort: unsupported or opaque values
|
|
151
|
+
still count themselves, but do not cause cache accounting to fail.
|
|
152
|
+
"""
|
|
153
|
+
total = 0
|
|
154
|
+
seen: set[int] = set()
|
|
155
|
+
pending: list[object] = [value]
|
|
156
|
+
|
|
157
|
+
while pending:
|
|
158
|
+
current = pending.pop()
|
|
159
|
+
if isinstance(current, (ModuleType, type)) or callable(current):
|
|
160
|
+
continue
|
|
161
|
+
identifier = id(current)
|
|
162
|
+
if identifier in seen:
|
|
163
|
+
continue
|
|
164
|
+
seen.add(identifier)
|
|
165
|
+
try:
|
|
166
|
+
total += sys.getsizeof(current)
|
|
167
|
+
except Exception:
|
|
168
|
+
pass
|
|
169
|
+
|
|
170
|
+
try:
|
|
171
|
+
pending.extend(_children(current))
|
|
172
|
+
except Exception:
|
|
173
|
+
continue
|
|
174
|
+
return total
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@dataclass
|
|
178
|
+
class CacheStats:
|
|
179
|
+
requests: int = 0
|
|
180
|
+
warm_hits: int = 0
|
|
181
|
+
rebuilds: int = 0
|
|
182
|
+
invalidations: int = 0
|
|
183
|
+
evictions: int = 0
|
|
184
|
+
parallel_fallbacks: int = 0
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@dataclass(frozen=True)
|
|
188
|
+
class BudgetResult:
|
|
189
|
+
retained_bytes: int
|
|
190
|
+
utilization_percent: float
|
|
191
|
+
peak_utilization_percent: float
|
|
192
|
+
warning_active: bool
|
|
193
|
+
warning_triggered: bool
|
|
194
|
+
compacted: bool
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@dataclass(frozen=True)
|
|
198
|
+
class CacheBudget:
|
|
199
|
+
max_bytes: int = DEFAULT_MAX_MEMORY_BYTES
|
|
200
|
+
warning_percent: int = WARNING_THRESHOLD_PERCENT
|
|
201
|
+
|
|
202
|
+
def __post_init__(self) -> None:
|
|
203
|
+
if self.max_bytes <= 0:
|
|
204
|
+
raise ValueError("max_bytes must be greater than zero.")
|
|
205
|
+
if not 0 < self.warning_percent <= 100:
|
|
206
|
+
raise ValueError("warning_percent must be between 1 and 100.")
|
|
207
|
+
|
|
208
|
+
def enforce(
|
|
209
|
+
self,
|
|
210
|
+
*,
|
|
211
|
+
measure: Callable[[], int],
|
|
212
|
+
evict_auxiliary: Callable[[], None],
|
|
213
|
+
evict_navigation: Callable[[], None],
|
|
214
|
+
) -> BudgetResult:
|
|
215
|
+
initial_retained = measure()
|
|
216
|
+
initial_utilization_percent = initial_retained * 100.0 / self.max_bytes
|
|
217
|
+
compacted = False
|
|
218
|
+
retained = initial_retained
|
|
219
|
+
|
|
220
|
+
if initial_retained > self.max_bytes:
|
|
221
|
+
evict_auxiliary()
|
|
222
|
+
compacted = True
|
|
223
|
+
retained = measure()
|
|
224
|
+
if retained > self.max_bytes:
|
|
225
|
+
evict_navigation()
|
|
226
|
+
retained = measure()
|
|
227
|
+
|
|
228
|
+
current_utilization_percent = retained * 100.0 / self.max_bytes
|
|
229
|
+
return BudgetResult(
|
|
230
|
+
retained_bytes=retained,
|
|
231
|
+
utilization_percent=current_utilization_percent,
|
|
232
|
+
peak_utilization_percent=initial_utilization_percent,
|
|
233
|
+
warning_active=current_utilization_percent >= self.warning_percent,
|
|
234
|
+
warning_triggered=initial_utilization_percent >= self.warning_percent,
|
|
235
|
+
compacted=compacted,
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def parse_memory_size(value: str) -> int:
|
|
240
|
+
match = _MEMORY_SIZE.fullmatch(value.strip())
|
|
241
|
+
if match is None:
|
|
242
|
+
raise ValueError("Memory size must be a positive integer with optional K, M, or G suffix.")
|
|
243
|
+
|
|
244
|
+
multiplier = {
|
|
245
|
+
"": 1,
|
|
246
|
+
"K": 1024,
|
|
247
|
+
"M": 1024**2,
|
|
248
|
+
"G": 1024**3,
|
|
249
|
+
}
|
|
250
|
+
return int(match.group("count")) * multiplier[match.group("suffix").upper()]
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def cache_warning(result: BudgetResult, max_bytes: int) -> str:
|
|
254
|
+
if not result.warning_triggered:
|
|
255
|
+
return ""
|
|
256
|
+
|
|
257
|
+
if result.compacted:
|
|
258
|
+
state = (
|
|
259
|
+
f"Warning: Delphi cache peaked at {result.peak_utilization_percent:.1f}% of the {max_bytes} byte budget; "
|
|
260
|
+
f"{result.retained_bytes} bytes remain retained after compaction."
|
|
261
|
+
f"{' Cache compacted.' if result.compacted else ''}"
|
|
262
|
+
)
|
|
263
|
+
else:
|
|
264
|
+
state = (
|
|
265
|
+
f"Warning: Delphi cache currently at {result.utilization_percent:.1f}% of the {max_bytes} byte budget; "
|
|
266
|
+
f"{result.retained_bytes} bytes retained."
|
|
267
|
+
)
|
|
268
|
+
return (
|
|
269
|
+
f"{state} "
|
|
270
|
+
"Increase --max-memory, stop unused daemons, or allow compact mode."
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _children(value: object) -> tuple[object, ...]:
|
|
275
|
+
children: list[object] = []
|
|
276
|
+
if isinstance(value, Mapping):
|
|
277
|
+
try:
|
|
278
|
+
for key, item in value.items():
|
|
279
|
+
children.extend((key, item))
|
|
280
|
+
except Exception:
|
|
281
|
+
pass
|
|
282
|
+
elif isinstance(value, Collection) and not isinstance(value, (str, bytes, bytearray)):
|
|
283
|
+
try:
|
|
284
|
+
children.extend(value)
|
|
285
|
+
except Exception:
|
|
286
|
+
pass
|
|
287
|
+
|
|
288
|
+
is_dataclass_instance = is_dataclass(value) and not isinstance(value, type)
|
|
289
|
+
dataclass_field_names: frozenset[str] = frozenset()
|
|
290
|
+
if is_dataclass_instance:
|
|
291
|
+
try:
|
|
292
|
+
dataclass_fields = fields(value)
|
|
293
|
+
except Exception:
|
|
294
|
+
dataclass_fields = ()
|
|
295
|
+
dataclass_field_names = frozenset(field.name for field in dataclass_fields)
|
|
296
|
+
for field in dataclass_fields:
|
|
297
|
+
try:
|
|
298
|
+
children.append(getattr(value, field.name))
|
|
299
|
+
except Exception:
|
|
300
|
+
continue
|
|
301
|
+
else:
|
|
302
|
+
try:
|
|
303
|
+
instance_values = vars(value)
|
|
304
|
+
except Exception:
|
|
305
|
+
instance_values = None
|
|
306
|
+
if isinstance(instance_values, Mapping):
|
|
307
|
+
children.extend(instance_values.values())
|
|
308
|
+
|
|
309
|
+
for slot in _slot_names(type(value)):
|
|
310
|
+
if slot in dataclass_field_names:
|
|
311
|
+
continue
|
|
312
|
+
try:
|
|
313
|
+
children.append(getattr(value, slot))
|
|
314
|
+
except Exception:
|
|
315
|
+
continue
|
|
316
|
+
return tuple(children)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _slot_names(value_type: type[object]) -> tuple[str, ...]:
|
|
320
|
+
names: list[str] = []
|
|
321
|
+
for base in value_type.__mro__:
|
|
322
|
+
try:
|
|
323
|
+
slots = getattr(base, "__slots__", ())
|
|
324
|
+
except Exception:
|
|
325
|
+
continue
|
|
326
|
+
if isinstance(slots, str):
|
|
327
|
+
slots = (slots,)
|
|
328
|
+
try:
|
|
329
|
+
names.extend(name for name in slots if isinstance(name, str))
|
|
330
|
+
except Exception:
|
|
331
|
+
continue
|
|
332
|
+
return tuple(names)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
@dataclass(frozen=True)
|
|
336
|
+
class CacheMetadata:
|
|
337
|
+
schema: int
|
|
338
|
+
root: str
|
|
339
|
+
pid: int
|
|
340
|
+
port: int
|
|
341
|
+
token: str
|
|
342
|
+
version: str
|
|
343
|
+
project_file: str
|
|
344
|
+
max_memory_bytes: int
|
|
345
|
+
workers: int
|
|
346
|
+
idle_timeout: int
|
|
347
|
+
started_at: float
|
|
348
|
+
max_disk_cache_bytes: int = DEFAULT_MAX_DISK_CACHE_BYTES
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
@dataclass(frozen=True)
|
|
352
|
+
class CacheClientResponse:
|
|
353
|
+
payload: dict[str, object]
|
|
354
|
+
warning: str = ""
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
class CacheClientError(RuntimeError):
|
|
358
|
+
def __init__(self, code: str, message: str) -> None:
|
|
359
|
+
self.code = code
|
|
360
|
+
self.message = message
|
|
361
|
+
super().__init__(message)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
class _RequestGuard:
|
|
365
|
+
def __init__(
|
|
366
|
+
self,
|
|
367
|
+
service: _CacheService,
|
|
368
|
+
connection: socket.socket,
|
|
369
|
+
*,
|
|
370
|
+
enforce_rss_budget: bool = True,
|
|
371
|
+
) -> None:
|
|
372
|
+
self._service = service
|
|
373
|
+
self._connection = connection
|
|
374
|
+
self._enforce_rss_budget = enforce_rss_budget
|
|
375
|
+
|
|
376
|
+
def check(self) -> None:
|
|
377
|
+
if self._service.shutdown.is_set():
|
|
378
|
+
raise CacheClientError(
|
|
379
|
+
"request_cancelled",
|
|
380
|
+
"Cache request was cancelled.",
|
|
381
|
+
)
|
|
382
|
+
try:
|
|
383
|
+
readable, _, _ = select.select(
|
|
384
|
+
(self._connection,),
|
|
385
|
+
(),
|
|
386
|
+
(),
|
|
387
|
+
0,
|
|
388
|
+
)
|
|
389
|
+
if readable and not self._connection.recv(1, socket.MSG_PEEK):
|
|
390
|
+
raise CacheClientError(
|
|
391
|
+
"client_disconnected",
|
|
392
|
+
"Cache client disconnected.",
|
|
393
|
+
)
|
|
394
|
+
except CacheClientError:
|
|
395
|
+
raise
|
|
396
|
+
except (OSError, ValueError) as error:
|
|
397
|
+
raise CacheClientError(
|
|
398
|
+
"client_disconnected",
|
|
399
|
+
"Cache client disconnected.",
|
|
400
|
+
) from error
|
|
401
|
+
if (
|
|
402
|
+
self._enforce_rss_budget
|
|
403
|
+
and self._service._measure_retained_bytes()
|
|
404
|
+
> self._service.metadata.max_memory_bytes
|
|
405
|
+
):
|
|
406
|
+
raise CacheClientError(
|
|
407
|
+
"memory_budget_exceeded",
|
|
408
|
+
"Cache request exceeded the configured retained-memory budget.",
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def cache_metadata_path(root: str | Path) -> Path:
|
|
413
|
+
return Path(root).resolve() / ".delphi-lsp" / "agent-cache" / "daemon.json"
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _safe_metadata_path(root: str | Path, *, create: bool = False) -> Path:
|
|
417
|
+
root_path = Path(root).resolve()
|
|
418
|
+
for path in (root_path / ".delphi-lsp", root_path / ".delphi-lsp" / "agent-cache"):
|
|
419
|
+
if path.exists() and path.is_symlink():
|
|
420
|
+
raise CacheClientError("unsafe_metadata", "Cache metadata path is unsafe.")
|
|
421
|
+
if create:
|
|
422
|
+
path.mkdir(mode=0o700, exist_ok=True)
|
|
423
|
+
if os.name != "nt":
|
|
424
|
+
os.chmod(path, 0o700)
|
|
425
|
+
result = root_path / ".delphi-lsp" / "agent-cache" / "daemon.json"
|
|
426
|
+
if result.exists() and result.is_symlink():
|
|
427
|
+
raise CacheClientError("unsafe_metadata", "Cache metadata path is unsafe.")
|
|
428
|
+
return result
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _safe_navigation_cache_path(root: str | Path, *, create: bool = False) -> Path:
|
|
432
|
+
parent = _safe_metadata_path(root, create=create).parent
|
|
433
|
+
result = parent / "navigation-v1"
|
|
434
|
+
if result.exists() and result.is_symlink():
|
|
435
|
+
raise CacheClientError("unsafe_metadata", "Navigation cache path is unsafe.")
|
|
436
|
+
if create:
|
|
437
|
+
result.mkdir(mode=0o700, exist_ok=True)
|
|
438
|
+
if os.name != "nt":
|
|
439
|
+
os.chmod(result, 0o700)
|
|
440
|
+
return result
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def navigation_cache_path(root: str | Path, *, create: bool = False) -> Path:
|
|
444
|
+
"""Return the validated persistent navigation-cache directory."""
|
|
445
|
+
return _safe_navigation_cache_path(root, create=create)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _metadata_mapping(metadata: CacheMetadata) -> dict[str, object]:
|
|
449
|
+
return {field.name: getattr(metadata, field.name) for field in fields(metadata)}
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _write_metadata(metadata: CacheMetadata) -> None:
|
|
453
|
+
path = _safe_metadata_path(metadata.root, create=True)
|
|
454
|
+
data = json.dumps(_metadata_mapping(metadata), sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
455
|
+
descriptor, temporary = tempfile.mkstemp(prefix=".daemon-", dir=path.parent)
|
|
456
|
+
try:
|
|
457
|
+
if os.name != "nt":
|
|
458
|
+
os.fchmod(descriptor, 0o600)
|
|
459
|
+
with os.fdopen(descriptor, "wb") as stream:
|
|
460
|
+
stream.write(data)
|
|
461
|
+
stream.flush()
|
|
462
|
+
os.fsync(stream.fileno())
|
|
463
|
+
os.replace(temporary, path)
|
|
464
|
+
if os.name != "nt":
|
|
465
|
+
os.chmod(path, 0o600)
|
|
466
|
+
finally:
|
|
467
|
+
with contextlib.suppress(FileNotFoundError):
|
|
468
|
+
os.unlink(temporary)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _read_metadata_record(root: str | Path) -> object | None:
|
|
472
|
+
path = _safe_metadata_path(root)
|
|
473
|
+
if not path.exists():
|
|
474
|
+
return None
|
|
475
|
+
try:
|
|
476
|
+
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
|
477
|
+
descriptor = os.open(path, flags)
|
|
478
|
+
try:
|
|
479
|
+
info = os.fstat(descriptor)
|
|
480
|
+
if os.name != "nt" and (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077):
|
|
481
|
+
raise CacheClientError("unsafe_metadata", "Cache metadata path is unsafe.")
|
|
482
|
+
raw = json.loads(os.read(descriptor, _MAX_MESSAGE_BYTES).decode("utf-8"))
|
|
483
|
+
finally:
|
|
484
|
+
os.close(descriptor)
|
|
485
|
+
except CacheClientError:
|
|
486
|
+
raise
|
|
487
|
+
except (OSError, ValueError, UnicodeError):
|
|
488
|
+
return None
|
|
489
|
+
return raw
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _read_metadata(root: str | Path) -> CacheMetadata | None:
|
|
493
|
+
raw = _read_metadata_record(root)
|
|
494
|
+
required = {field.name for field in fields(CacheMetadata)}
|
|
495
|
+
legacy_required = required - {"max_disk_cache_bytes"}
|
|
496
|
+
if not isinstance(raw, dict) or frozenset(raw) not in {
|
|
497
|
+
frozenset(required),
|
|
498
|
+
frozenset(legacy_required),
|
|
499
|
+
}:
|
|
500
|
+
return None
|
|
501
|
+
if "max_disk_cache_bytes" not in raw:
|
|
502
|
+
raw = {**raw, "max_disk_cache_bytes": DEFAULT_MAX_DISK_CACHE_BYTES}
|
|
503
|
+
values = tuple(
|
|
504
|
+
raw[name]
|
|
505
|
+
for name in (
|
|
506
|
+
"schema",
|
|
507
|
+
"root",
|
|
508
|
+
"pid",
|
|
509
|
+
"port",
|
|
510
|
+
"token",
|
|
511
|
+
"version",
|
|
512
|
+
"project_file",
|
|
513
|
+
"max_memory_bytes",
|
|
514
|
+
"workers",
|
|
515
|
+
"idle_timeout",
|
|
516
|
+
"started_at",
|
|
517
|
+
"max_disk_cache_bytes",
|
|
518
|
+
)
|
|
519
|
+
)
|
|
520
|
+
if (type(values[0]) is not int or values[0] != DAEMON_SCHEMA or not isinstance(values[1], str)
|
|
521
|
+
or type(values[2]) is not int or values[2] <= 0 or type(values[3]) is not int or not 0 < values[3] < 65536
|
|
522
|
+
or not isinstance(values[4], str) or len(values[4]) < 32 or not isinstance(values[5], str)
|
|
523
|
+
or not isinstance(values[6], str) or type(values[7]) is not int or values[7] <= 0
|
|
524
|
+
or type(values[8]) is not int or not 0 <= values[8] <= 32
|
|
525
|
+
or type(values[9]) is not int or values[9] <= 0 or type(values[10]) not in (int, float)
|
|
526
|
+
or type(values[11]) is not int or values[11] < 0):
|
|
527
|
+
return None
|
|
528
|
+
canonical = str(Path(root).resolve())
|
|
529
|
+
if values[1] != canonical:
|
|
530
|
+
return None
|
|
531
|
+
return CacheMetadata(*values)
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def _remove_incompatible_metadata_if_stale(root: str | Path) -> None:
|
|
535
|
+
path = _safe_metadata_path(root)
|
|
536
|
+
if not path.exists():
|
|
537
|
+
return
|
|
538
|
+
raw = _read_metadata_record(root)
|
|
539
|
+
canonical = str(Path(root).resolve())
|
|
540
|
+
pid = raw.get("pid") if isinstance(raw, dict) and raw.get("root") == canonical else None
|
|
541
|
+
if type(pid) is int and pid > 0 and _pid_alive(pid):
|
|
542
|
+
raise CacheClientError("unavailable", "Live cache daemon is unavailable.")
|
|
543
|
+
with contextlib.suppress(FileNotFoundError):
|
|
544
|
+
path.unlink()
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def _metadata_is_owned(metadata: CacheMetadata) -> bool:
|
|
548
|
+
current = _read_metadata(metadata.root)
|
|
549
|
+
return current is not None and current.pid == metadata.pid and hmac.compare_digest(current.token, metadata.token)
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _remove_metadata_if_owned(metadata: CacheMetadata) -> None:
|
|
553
|
+
path = _safe_metadata_path(metadata.root)
|
|
554
|
+
if _metadata_is_owned(metadata):
|
|
555
|
+
with contextlib.suppress(FileNotFoundError):
|
|
556
|
+
path.unlink()
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def _windows_pid_alive(pid: int) -> bool:
|
|
560
|
+
import ctypes
|
|
561
|
+
from ctypes import wintypes
|
|
562
|
+
|
|
563
|
+
synchronize = 0x00100000
|
|
564
|
+
wait_timeout = 0x00000102
|
|
565
|
+
kernel32 = ctypes.windll.kernel32
|
|
566
|
+
kernel32.OpenProcess.argtypes = (wintypes.DWORD, wintypes.BOOL, wintypes.DWORD)
|
|
567
|
+
kernel32.OpenProcess.restype = wintypes.HANDLE
|
|
568
|
+
kernel32.WaitForSingleObject.argtypes = (wintypes.HANDLE, wintypes.DWORD)
|
|
569
|
+
kernel32.WaitForSingleObject.restype = wintypes.DWORD
|
|
570
|
+
kernel32.CloseHandle.argtypes = (wintypes.HANDLE,)
|
|
571
|
+
kernel32.CloseHandle.restype = wintypes.BOOL
|
|
572
|
+
handle = kernel32.OpenProcess(synchronize, False, pid)
|
|
573
|
+
if not handle:
|
|
574
|
+
return False
|
|
575
|
+
try:
|
|
576
|
+
return kernel32.WaitForSingleObject(handle, 0) == wait_timeout
|
|
577
|
+
finally:
|
|
578
|
+
kernel32.CloseHandle(handle)
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _pid_alive(pid: int) -> bool:
|
|
582
|
+
if os.name == "nt":
|
|
583
|
+
return _windows_pid_alive(pid)
|
|
584
|
+
try:
|
|
585
|
+
os.kill(pid, 0)
|
|
586
|
+
except OSError:
|
|
587
|
+
return False
|
|
588
|
+
return True
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def _reap_child_if_exited(pid: int) -> bool:
|
|
592
|
+
waitpid = getattr(os, "waitpid", None)
|
|
593
|
+
nohang = getattr(os, "WNOHANG", None)
|
|
594
|
+
if not callable(waitpid) or nohang is None:
|
|
595
|
+
return False
|
|
596
|
+
try:
|
|
597
|
+
return waitpid(pid, nohang)[0] == pid
|
|
598
|
+
except (ChildProcessError, OSError):
|
|
599
|
+
return False
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def _start_lock_path(root: str | Path) -> Path:
|
|
603
|
+
return _safe_metadata_path(root, create=True).with_name("start.lock")
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def _start_lock_is_stale(path: Path) -> bool:
|
|
607
|
+
try:
|
|
608
|
+
age = max(0.0, time.time() - path.stat().st_mtime)
|
|
609
|
+
except OSError:
|
|
610
|
+
return True
|
|
611
|
+
try:
|
|
612
|
+
record = json.loads(path.read_text(encoding="utf-8"))
|
|
613
|
+
except (OSError, ValueError, UnicodeError):
|
|
614
|
+
return age >= _START_LOCK_INCOMPLETE_GRACE_SECONDS
|
|
615
|
+
owner = record.get("pid") if isinstance(record, dict) else None
|
|
616
|
+
if type(owner) is int and owner > 0:
|
|
617
|
+
return not _pid_alive(owner)
|
|
618
|
+
return age >= _START_LOCK_INCOMPLETE_GRACE_SECONDS
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
@contextlib.contextmanager
|
|
622
|
+
def _start_lock(root: str | Path, timeout: float):
|
|
623
|
+
path = _start_lock_path(root)
|
|
624
|
+
token = secrets.token_urlsafe(24)
|
|
625
|
+
deadline = time.monotonic() + timeout
|
|
626
|
+
while True:
|
|
627
|
+
try:
|
|
628
|
+
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
629
|
+
try:
|
|
630
|
+
payload = json.dumps({"pid": os.getpid(), "token": token, "started_at": time.time()}).encode("utf-8")
|
|
631
|
+
os.write(descriptor, payload)
|
|
632
|
+
os.fsync(descriptor)
|
|
633
|
+
finally:
|
|
634
|
+
os.close(descriptor)
|
|
635
|
+
break
|
|
636
|
+
except FileExistsError:
|
|
637
|
+
if _start_lock_is_stale(path):
|
|
638
|
+
with contextlib.suppress(FileNotFoundError):
|
|
639
|
+
path.unlink()
|
|
640
|
+
continue
|
|
641
|
+
if time.monotonic() >= deadline:
|
|
642
|
+
raise CacheClientError("startup_locked", "Cache startup is busy.")
|
|
643
|
+
time.sleep(0.05)
|
|
644
|
+
try:
|
|
645
|
+
yield
|
|
646
|
+
finally:
|
|
647
|
+
try:
|
|
648
|
+
current = json.loads(path.read_text(encoding="utf-8"))
|
|
649
|
+
if isinstance(current, dict) and hmac.compare_digest(str(current.get("token", "")), token):
|
|
650
|
+
path.unlink()
|
|
651
|
+
except (OSError, ValueError, UnicodeError):
|
|
652
|
+
pass
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
@contextlib.contextmanager
|
|
656
|
+
def _process_start_lock(root: str | Path):
|
|
657
|
+
canonical = str(Path(root).resolve())
|
|
658
|
+
with _PROCESS_START_LOCKS_GUARD:
|
|
659
|
+
lock, users = _PROCESS_START_LOCKS.get(
|
|
660
|
+
canonical,
|
|
661
|
+
(threading.Lock(), 0),
|
|
662
|
+
)
|
|
663
|
+
_PROCESS_START_LOCKS[canonical] = (lock, users + 1)
|
|
664
|
+
try:
|
|
665
|
+
with lock:
|
|
666
|
+
yield
|
|
667
|
+
finally:
|
|
668
|
+
with _PROCESS_START_LOCKS_GUARD:
|
|
669
|
+
current_lock, current_users = _PROCESS_START_LOCKS[canonical]
|
|
670
|
+
if current_users == 1:
|
|
671
|
+
del _PROCESS_START_LOCKS[canonical]
|
|
672
|
+
else:
|
|
673
|
+
_PROCESS_START_LOCKS[canonical] = (
|
|
674
|
+
current_lock,
|
|
675
|
+
current_users - 1,
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def _client_exchange(metadata: CacheMetadata, request: dict[str, object]) -> CacheClientResponse:
|
|
680
|
+
try:
|
|
681
|
+
with socket.create_connection(("127.0.0.1", metadata.port), timeout=2) as connection:
|
|
682
|
+
connection.settimeout(_client_response_timeout(metadata, request))
|
|
683
|
+
request_without_token = {key: value for key, value in request.items() if key != "token"}
|
|
684
|
+
connection.sendall(json.dumps({"token": metadata.token, **request_without_token}, separators=(",", ":")).encode("utf-8") + b"\n")
|
|
685
|
+
response = _read_line(connection)
|
|
686
|
+
except OSError as error:
|
|
687
|
+
raise CacheClientError("unavailable", "Cache daemon is unavailable.") from error
|
|
688
|
+
try:
|
|
689
|
+
decoded = json.loads(response.decode("utf-8"))
|
|
690
|
+
except (UnicodeError, ValueError) as error:
|
|
691
|
+
raise CacheClientError("invalid_response", "Cache daemon returned an invalid response.") from error
|
|
692
|
+
if not isinstance(decoded, dict):
|
|
693
|
+
raise CacheClientError("invalid_response", "Cache daemon returned an invalid response.")
|
|
694
|
+
if "error" in decoded:
|
|
695
|
+
error = decoded["error"]
|
|
696
|
+
if isinstance(error, dict):
|
|
697
|
+
raise CacheClientError(str(error.get("code", "error")), str(error.get("message", "Cache request failed.")))
|
|
698
|
+
raise CacheClientError("error", "Cache request failed.")
|
|
699
|
+
payload = decoded.get("payload")
|
|
700
|
+
if not isinstance(payload, dict):
|
|
701
|
+
raise CacheClientError("invalid_response", "Cache daemon returned an invalid response.")
|
|
702
|
+
return CacheClientResponse(payload, str(decoded.get("warning", "")))
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def _client_response_timeout(
|
|
706
|
+
metadata: CacheMetadata,
|
|
707
|
+
request: Mapping[str, object],
|
|
708
|
+
) -> float:
|
|
709
|
+
if request.get("action") in {"status", "stop"}:
|
|
710
|
+
return _CLIENT_RESPONSE_TIMEOUT
|
|
711
|
+
return min(
|
|
712
|
+
_MAX_CLIENT_RESPONSE_TIMEOUT,
|
|
713
|
+
_CLIENT_RESPONSE_TIMEOUT
|
|
714
|
+
+ _workspace_source_count(metadata.root) * _CLIENT_SECONDS_PER_SOURCE,
|
|
715
|
+
)
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
@lru_cache(maxsize=64)
|
|
719
|
+
def _workspace_source_count(root: str) -> int:
|
|
720
|
+
count = 0
|
|
721
|
+
source_suffixes = {".pas", ".pp", ".inc", ".dpr", ".dpk"}
|
|
722
|
+
maximum_relevant_count = int(
|
|
723
|
+
(_MAX_CLIENT_RESPONSE_TIMEOUT - _CLIENT_RESPONSE_TIMEOUT)
|
|
724
|
+
/ _CLIENT_SECONDS_PER_SOURCE
|
|
725
|
+
)
|
|
726
|
+
for _current, directories, files in os.walk(Path(root).expanduser().resolve()):
|
|
727
|
+
directories[:] = [
|
|
728
|
+
directory
|
|
729
|
+
for directory in directories
|
|
730
|
+
if directory not in SKIP_DIRS
|
|
731
|
+
]
|
|
732
|
+
count += sum(
|
|
733
|
+
1
|
|
734
|
+
for file_name in files
|
|
735
|
+
if Path(file_name).suffix.casefold() in source_suffixes
|
|
736
|
+
)
|
|
737
|
+
if count >= maximum_relevant_count:
|
|
738
|
+
return maximum_relevant_count
|
|
739
|
+
return count
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def _read_line(connection: socket.socket) -> bytes:
|
|
743
|
+
chunks = bytearray()
|
|
744
|
+
while len(chunks) <= _MAX_MESSAGE_BYTES:
|
|
745
|
+
piece = connection.recv(min(65536, _MAX_MESSAGE_BYTES + 1 - len(chunks)))
|
|
746
|
+
if not piece:
|
|
747
|
+
break
|
|
748
|
+
chunks.extend(piece)
|
|
749
|
+
if b"\n" in piece:
|
|
750
|
+
line, _, _ = chunks.partition(b"\n")
|
|
751
|
+
return line
|
|
752
|
+
raise CacheClientError("invalid_response", "Cache daemon returned an invalid response.")
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
def _truncate_and_sanitize_startup_diagnostics(raw: bytes, *, max_bytes: int = _STARTUP_DIAGNOSTIC_BYTES) -> str:
|
|
756
|
+
if not raw:
|
|
757
|
+
return ""
|
|
758
|
+
text = raw.decode("utf-8", "replace")
|
|
759
|
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
760
|
+
if max_bytes > 0 and len(text) > max_bytes:
|
|
761
|
+
text = text[-max_bytes:]
|
|
762
|
+
text = re.sub(r"[\x00-\x09\x0b-\x1f\x7f-\x9f]", " ", text)
|
|
763
|
+
text = _STARTUP_TOKEN_RE.sub("<redacted>", text)
|
|
764
|
+
text = re.sub(r"\s+", " ", text).strip()
|
|
765
|
+
return text
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
def _read_startup_tail(diagnostics: object, *, max_bytes: int = _STARTUP_DIAGNOSTIC_BYTES) -> bytes:
|
|
769
|
+
with contextlib.suppress(Exception):
|
|
770
|
+
diagnostics.seek(0, os.SEEK_END)
|
|
771
|
+
end = diagnostics.tell()
|
|
772
|
+
diagnostics.seek(max(0, end - max_bytes), os.SEEK_SET)
|
|
773
|
+
return diagnostics.read()
|
|
774
|
+
return b""
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def _daemon_process_options() -> dict[str, object]:
|
|
778
|
+
options: dict[str, object] = {
|
|
779
|
+
"stdin": subprocess.DEVNULL,
|
|
780
|
+
"stdout": subprocess.DEVNULL,
|
|
781
|
+
}
|
|
782
|
+
if os.name == "nt":
|
|
783
|
+
options["creationflags"] = (
|
|
784
|
+
subprocess.CREATE_NO_WINDOW
|
|
785
|
+
| subprocess.CREATE_NEW_PROCESS_GROUP
|
|
786
|
+
)
|
|
787
|
+
else:
|
|
788
|
+
options["start_new_session"] = True
|
|
789
|
+
return options
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
class _CacheService:
|
|
793
|
+
def __init__(
|
|
794
|
+
self,
|
|
795
|
+
metadata: CacheMetadata,
|
|
796
|
+
*,
|
|
797
|
+
defer_context: bool = False,
|
|
798
|
+
) -> None:
|
|
799
|
+
self.metadata = metadata
|
|
800
|
+
self._rss_baseline_bytes = current_process_rss_bytes()
|
|
801
|
+
self._context: AgentContext | None = None
|
|
802
|
+
self.context_ready = threading.Event()
|
|
803
|
+
self.watcher_ready = threading.Event()
|
|
804
|
+
self.watcher_failed = threading.Event()
|
|
805
|
+
self.budget = CacheBudget(metadata.max_memory_bytes)
|
|
806
|
+
self.stats = CacheStats()
|
|
807
|
+
self.lock = threading.Lock()
|
|
808
|
+
self._request_state_lock = threading.Lock()
|
|
809
|
+
self._active_requests = 0
|
|
810
|
+
self.started = time.monotonic()
|
|
811
|
+
self.last_activity = self.started
|
|
812
|
+
self.cache_state = "warming"
|
|
813
|
+
self.last_revision = ""
|
|
814
|
+
self.last_budget = BudgetResult(0, 0.0, 0.0, False, False, False)
|
|
815
|
+
self.prewarm_seconds = 0.0
|
|
816
|
+
self.startup_error = ""
|
|
817
|
+
self.shutdown = threading.Event()
|
|
818
|
+
self.prewarm_thread: threading.Thread | None = None
|
|
819
|
+
if not defer_context:
|
|
820
|
+
self._initialize_context()
|
|
821
|
+
|
|
822
|
+
@property
|
|
823
|
+
def context(self) -> AgentContext:
|
|
824
|
+
if self._context is None:
|
|
825
|
+
raise RuntimeError("Cache workspace is still being discovered.")
|
|
826
|
+
return self._context
|
|
827
|
+
|
|
828
|
+
def _initialize_context(self) -> AgentContext:
|
|
829
|
+
if self._context is None:
|
|
830
|
+
context = AgentContext.open(
|
|
831
|
+
self.metadata.root,
|
|
832
|
+
self.metadata.project_file or None,
|
|
833
|
+
workers=self.metadata.workers,
|
|
834
|
+
worker_memory_budget_bytes=self.metadata.max_memory_bytes,
|
|
835
|
+
revision_check_interval_seconds=_CACHE_REVISION_CHECK_INTERVAL_SECONDS,
|
|
836
|
+
navigation_cache_dir=navigation_cache_path(
|
|
837
|
+
self.metadata.root,
|
|
838
|
+
create=True,
|
|
839
|
+
),
|
|
840
|
+
navigation_cache_max_bytes=self.metadata.max_disk_cache_bytes,
|
|
841
|
+
)
|
|
842
|
+
self._context = context
|
|
843
|
+
self.last_revision = context.workspace.workspace_revision
|
|
844
|
+
self.context_ready.set()
|
|
845
|
+
return self._context
|
|
846
|
+
|
|
847
|
+
def _measure_retained_bytes(self) -> int:
|
|
848
|
+
process_rss = current_process_rss_bytes()
|
|
849
|
+
process_growth = (
|
|
850
|
+
max(0, process_rss - self._rss_baseline_bytes)
|
|
851
|
+
if process_rss > 0 and self._rss_baseline_bytes > 0
|
|
852
|
+
else 0
|
|
853
|
+
)
|
|
854
|
+
return max(self.context.estimated_cache_bytes, process_growth)
|
|
855
|
+
|
|
856
|
+
def prewarm(self) -> None:
|
|
857
|
+
started = time.monotonic()
|
|
858
|
+
context = self._initialize_context()
|
|
859
|
+
try:
|
|
860
|
+
self.last_revision = context.prewarm_navigation()
|
|
861
|
+
except AgentProtocolError as error:
|
|
862
|
+
if error.code != "project_required":
|
|
863
|
+
raise
|
|
864
|
+
prewarm_rss = current_process_rss_bytes()
|
|
865
|
+
if prewarm_rss > 0:
|
|
866
|
+
# Parsing can leave a high allocator RSS watermark even though the
|
|
867
|
+
# retained navigation data is already covered by its own estimate.
|
|
868
|
+
# Track only subsequent, otherwise unaccounted-for RSS growth.
|
|
869
|
+
self._rss_baseline_bytes = prewarm_rss
|
|
870
|
+
self._enforce_budget()
|
|
871
|
+
self.prewarm_seconds = time.monotonic() - started
|
|
872
|
+
self.stats.parallel_fallbacks = context.parallel_stats.fallbacks
|
|
873
|
+
|
|
874
|
+
def start_prewarm(self) -> None:
|
|
875
|
+
def run() -> None:
|
|
876
|
+
try:
|
|
877
|
+
with self.lock:
|
|
878
|
+
self.prewarm()
|
|
879
|
+
except Exception as error:
|
|
880
|
+
detail = _truncate_and_sanitize_startup_diagnostics(
|
|
881
|
+
f"{type(error).__name__}: {error}".encode(
|
|
882
|
+
"utf-8",
|
|
883
|
+
"replace",
|
|
884
|
+
)
|
|
885
|
+
)
|
|
886
|
+
self.startup_error = detail or type(error).__name__
|
|
887
|
+
self.cache_state = "failed"
|
|
888
|
+
finally:
|
|
889
|
+
self.context_ready.set()
|
|
890
|
+
|
|
891
|
+
self.prewarm_thread = threading.Thread(
|
|
892
|
+
target=run,
|
|
893
|
+
name="delphi-cache-prewarm",
|
|
894
|
+
daemon=True,
|
|
895
|
+
)
|
|
896
|
+
self.prewarm_thread.start()
|
|
897
|
+
|
|
898
|
+
def request(
|
|
899
|
+
self,
|
|
900
|
+
request: dict[str, object],
|
|
901
|
+
*,
|
|
902
|
+
cancel_check: Callable[[], None] | None = None,
|
|
903
|
+
) -> CacheClientResponse:
|
|
904
|
+
action = request.get("action")
|
|
905
|
+
if action == "status":
|
|
906
|
+
warning = "" if request.get("_startup_probe") is True else self._consume_warning()
|
|
907
|
+
return CacheClientResponse(self.status(), warning)
|
|
908
|
+
if action == "stop":
|
|
909
|
+
self.shutdown.set()
|
|
910
|
+
return CacheClientResponse({"stopping": True})
|
|
911
|
+
if self.cache_state == "warming":
|
|
912
|
+
self.last_activity = time.monotonic()
|
|
913
|
+
raise CacheClientError(
|
|
914
|
+
"cache_warming",
|
|
915
|
+
"Cache is still warming. Retry after cache status reports ready.",
|
|
916
|
+
)
|
|
917
|
+
if self.cache_state == "failed":
|
|
918
|
+
detail = (
|
|
919
|
+
f" {self.startup_error}"
|
|
920
|
+
if self.startup_error
|
|
921
|
+
else ""
|
|
922
|
+
)
|
|
923
|
+
raise CacheClientError(
|
|
924
|
+
"cache_failed",
|
|
925
|
+
f"Cache workspace discovery or warm-up failed.{detail}",
|
|
926
|
+
)
|
|
927
|
+
self.watcher_ready.wait(timeout=_WATCHER_READY_TIMEOUT_SECONDS)
|
|
928
|
+
if self.watcher_failed.is_set():
|
|
929
|
+
self.context.invalidate_revision_cache()
|
|
930
|
+
with self._request_state_lock:
|
|
931
|
+
self._active_requests += 1
|
|
932
|
+
lock_acquired = False
|
|
933
|
+
try:
|
|
934
|
+
while not self.lock.acquire(timeout=0.05):
|
|
935
|
+
if self.shutdown.is_set():
|
|
936
|
+
raise CacheClientError(
|
|
937
|
+
"request_cancelled",
|
|
938
|
+
"Cache request was cancelled.",
|
|
939
|
+
)
|
|
940
|
+
if cancel_check is not None:
|
|
941
|
+
cancel_check()
|
|
942
|
+
lock_acquired = True
|
|
943
|
+
if self.shutdown.is_set():
|
|
944
|
+
raise CacheClientError(
|
|
945
|
+
"request_cancelled",
|
|
946
|
+
"Cache request was cancelled.",
|
|
947
|
+
)
|
|
948
|
+
if cancel_check is not None:
|
|
949
|
+
cancel_check()
|
|
950
|
+
self.last_activity = time.monotonic()
|
|
951
|
+
before = self.last_revision
|
|
952
|
+
was_warm = self.context.navigation_cache_is_warm
|
|
953
|
+
request_aborted = False
|
|
954
|
+
try:
|
|
955
|
+
try:
|
|
956
|
+
handled = (
|
|
957
|
+
self.context.handle(request)
|
|
958
|
+
if cancel_check is None
|
|
959
|
+
else self.context.handle(
|
|
960
|
+
request,
|
|
961
|
+
cancel_check=cancel_check,
|
|
962
|
+
)
|
|
963
|
+
)
|
|
964
|
+
if self.shutdown.is_set():
|
|
965
|
+
raise CacheClientError(
|
|
966
|
+
"request_cancelled",
|
|
967
|
+
"Cache request was cancelled.",
|
|
968
|
+
)
|
|
969
|
+
if cancel_check is not None:
|
|
970
|
+
cancel_check()
|
|
971
|
+
response = handled.to_mapping()
|
|
972
|
+
except CacheClientError:
|
|
973
|
+
request_aborted = True
|
|
974
|
+
raise
|
|
975
|
+
except AgentProtocolError as error:
|
|
976
|
+
raise CacheClientError(
|
|
977
|
+
error.code,
|
|
978
|
+
error.message,
|
|
979
|
+
) from None
|
|
980
|
+
except (OSError, UnicodeError):
|
|
981
|
+
raise CacheClientError(
|
|
982
|
+
"source_unavailable",
|
|
983
|
+
"Source is unavailable.",
|
|
984
|
+
) from None
|
|
985
|
+
except Exception:
|
|
986
|
+
raise CacheClientError(
|
|
987
|
+
"internal_error",
|
|
988
|
+
"Internal cache error.",
|
|
989
|
+
) from None
|
|
990
|
+
finally:
|
|
991
|
+
if not request_aborted and not self.shutdown.is_set():
|
|
992
|
+
self._enforce_budget()
|
|
993
|
+
after = str(response["workspace_revision"])
|
|
994
|
+
self.last_revision = after
|
|
995
|
+
self.stats.requests += 1
|
|
996
|
+
if was_warm:
|
|
997
|
+
self.stats.warm_hits += 1
|
|
998
|
+
else:
|
|
999
|
+
self.stats.rebuilds += 1
|
|
1000
|
+
self.stats.parallel_fallbacks += (
|
|
1001
|
+
self.context.parallel_stats.fallbacks
|
|
1002
|
+
)
|
|
1003
|
+
if before != after:
|
|
1004
|
+
self.stats.invalidations += 1
|
|
1005
|
+
return CacheClientResponse(
|
|
1006
|
+
response,
|
|
1007
|
+
self._consume_warning(),
|
|
1008
|
+
)
|
|
1009
|
+
finally:
|
|
1010
|
+
if lock_acquired:
|
|
1011
|
+
self.lock.release()
|
|
1012
|
+
with self._request_state_lock:
|
|
1013
|
+
self._active_requests -= 1
|
|
1014
|
+
|
|
1015
|
+
def _enforce_budget(self) -> None:
|
|
1016
|
+
self.last_budget = self.budget.enforce(
|
|
1017
|
+
measure=self._measure_retained_bytes,
|
|
1018
|
+
evict_auxiliary=self.context.evict_auxiliary_caches,
|
|
1019
|
+
evict_navigation=self.context.evict_navigation_caches,
|
|
1020
|
+
)
|
|
1021
|
+
if self.last_budget.compacted:
|
|
1022
|
+
self.stats.evictions += 1
|
|
1023
|
+
self.cache_state = "compact"
|
|
1024
|
+
else:
|
|
1025
|
+
self.cache_state = (
|
|
1026
|
+
"warm"
|
|
1027
|
+
if self.context.navigation_cache_is_warm
|
|
1028
|
+
else "ready"
|
|
1029
|
+
)
|
|
1030
|
+
|
|
1031
|
+
def _consume_warning(self) -> str:
|
|
1032
|
+
warning = cache_warning(self.last_budget, self.budget.max_bytes)
|
|
1033
|
+
if self.last_budget.warning_triggered and not self.last_budget.warning_active:
|
|
1034
|
+
self.last_budget = replace(self.last_budget, warning_triggered=False)
|
|
1035
|
+
return warning
|
|
1036
|
+
|
|
1037
|
+
def status(self) -> dict[str, object]:
|
|
1038
|
+
now = time.monotonic()
|
|
1039
|
+
idle = max(0.0, now - self.last_activity)
|
|
1040
|
+
context = self._context
|
|
1041
|
+
live_rss = current_process_rss_bytes()
|
|
1042
|
+
current_bytes = (
|
|
1043
|
+
live_rss
|
|
1044
|
+
if live_rss > 0
|
|
1045
|
+
else self.last_budget.retained_bytes
|
|
1046
|
+
)
|
|
1047
|
+
current_utilization = current_bytes * 100.0 / self.budget.max_bytes
|
|
1048
|
+
peak_utilization = max(
|
|
1049
|
+
self.last_budget.peak_utilization_percent,
|
|
1050
|
+
current_utilization,
|
|
1051
|
+
)
|
|
1052
|
+
parallel_stats = (
|
|
1053
|
+
context.parallel_stats
|
|
1054
|
+
if context is not None
|
|
1055
|
+
else None
|
|
1056
|
+
)
|
|
1057
|
+
with self._request_state_lock:
|
|
1058
|
+
busy = self._active_requests > 0
|
|
1059
|
+
return {
|
|
1060
|
+
"pid": self.metadata.pid, "root": self.metadata.root, "project_file": self.metadata.project_file,
|
|
1061
|
+
"version": self.metadata.version, "uptime": now - self.started, "idle_seconds": idle,
|
|
1062
|
+
"last_activity_at": time.time() - idle,
|
|
1063
|
+
"max_memory_bytes": self.budget.max_bytes, "current_bytes": current_bytes,
|
|
1064
|
+
"current_utilization_percent": current_utilization,
|
|
1065
|
+
"peak_utilization_percent": peak_utilization,
|
|
1066
|
+
"warning_active": current_utilization >= self.budget.warning_percent,
|
|
1067
|
+
"warning_threshold_percent": WARNING_THRESHOLD_PERCENT,
|
|
1068
|
+
"cache_state": self.cache_state, "requests": self.stats.requests, "warm_hits": self.stats.warm_hits,
|
|
1069
|
+
"rebuilds": self.stats.rebuilds, "invalidations": self.stats.invalidations, "evictions": self.stats.evictions,
|
|
1070
|
+
"workers_configured": "auto" if self.metadata.workers == 0 else self.metadata.workers,
|
|
1071
|
+
"workers_effective": parallel_stats.effective_workers if parallel_stats else 0,
|
|
1072
|
+
"parallel_files_completed": parallel_stats.files_completed if parallel_stats else 0,
|
|
1073
|
+
"prewarm_seconds": self.prewarm_seconds,
|
|
1074
|
+
"parallel_seconds": parallel_stats.elapsed_seconds if parallel_stats else 0.0,
|
|
1075
|
+
"parallel_fallbacks": self.stats.parallel_fallbacks,
|
|
1076
|
+
"navigation_disk_hits": context.navigation_disk_hits if context else 0,
|
|
1077
|
+
"navigation_disk_misses": context.navigation_disk_misses if context else 0,
|
|
1078
|
+
"max_disk_cache_bytes": self.metadata.max_disk_cache_bytes,
|
|
1079
|
+
"cpg_cache_entries": context.cpg_cache_entries if context else 0,
|
|
1080
|
+
"cpg_cache_bytes": context.cpg_cache_bytes if context else 0,
|
|
1081
|
+
"idle_timeout": self.metadata.idle_timeout, "idle_remaining": max(0.0, self.metadata.idle_timeout - idle),
|
|
1082
|
+
"workspace_revision": self.last_revision,
|
|
1083
|
+
"startup_error": self.startup_error,
|
|
1084
|
+
"watcher_active": not self.watcher_failed.is_set(),
|
|
1085
|
+
"busy": busy,
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
def _watch_filter(_change: object, path: str) -> bool:
|
|
1090
|
+
return Path(path).suffix.casefold() in _WATCHED_SUFFIXES
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def _workspace_watch_filter(
|
|
1094
|
+
root: str | Path,
|
|
1095
|
+
project_config: ProjectPathConfig | None,
|
|
1096
|
+
) -> Callable[[object, str], bool]:
|
|
1097
|
+
root_path = Path(root).expanduser().resolve()
|
|
1098
|
+
|
|
1099
|
+
def include(change: object, path: str) -> bool:
|
|
1100
|
+
candidate = Path(path)
|
|
1101
|
+
try:
|
|
1102
|
+
relative = candidate.relative_to(root_path)
|
|
1103
|
+
except ValueError:
|
|
1104
|
+
return False
|
|
1105
|
+
if any(part in SKIP_DIRS for part in relative.parts):
|
|
1106
|
+
return False
|
|
1107
|
+
if (
|
|
1108
|
+
project_config is not None
|
|
1109
|
+
and project_config.excludes_workspace_path(candidate)
|
|
1110
|
+
):
|
|
1111
|
+
return False
|
|
1112
|
+
return _watch_filter(change, path)
|
|
1113
|
+
|
|
1114
|
+
return include
|
|
1115
|
+
|
|
1116
|
+
|
|
1117
|
+
def watch_workspace_changes(
|
|
1118
|
+
root: str | Path,
|
|
1119
|
+
*,
|
|
1120
|
+
stop_event: threading.Event,
|
|
1121
|
+
on_change: Callable[[], None],
|
|
1122
|
+
on_ready: Callable[[], None] | None = None,
|
|
1123
|
+
watch_filter: Callable[[object, str], bool] | None = None,
|
|
1124
|
+
) -> None:
|
|
1125
|
+
ready = False
|
|
1126
|
+
try:
|
|
1127
|
+
for changes in watch(
|
|
1128
|
+
root,
|
|
1129
|
+
watch_filter=watch_filter or _watch_filter,
|
|
1130
|
+
stop_event=stop_event,
|
|
1131
|
+
debounce=50,
|
|
1132
|
+
step=20,
|
|
1133
|
+
rust_timeout=_WATCHER_READY_POLL_MILLISECONDS,
|
|
1134
|
+
yield_on_timeout=True,
|
|
1135
|
+
recursive=True,
|
|
1136
|
+
raise_interrupt=False,
|
|
1137
|
+
):
|
|
1138
|
+
if not ready:
|
|
1139
|
+
ready = True
|
|
1140
|
+
if on_ready is not None:
|
|
1141
|
+
on_ready()
|
|
1142
|
+
if changes:
|
|
1143
|
+
on_change()
|
|
1144
|
+
except (OSError, RuntimeError):
|
|
1145
|
+
on_change()
|
|
1146
|
+
finally:
|
|
1147
|
+
if not ready and on_ready is not None:
|
|
1148
|
+
on_ready()
|
|
1149
|
+
|
|
1150
|
+
|
|
1151
|
+
def _watch_workspace(service: _CacheService) -> None:
|
|
1152
|
+
service.context_ready.wait()
|
|
1153
|
+
if service.shutdown.is_set() or service._context is None:
|
|
1154
|
+
return
|
|
1155
|
+
try:
|
|
1156
|
+
watch_workspace_changes(
|
|
1157
|
+
service.metadata.root,
|
|
1158
|
+
stop_event=service.shutdown,
|
|
1159
|
+
on_change=service._context.invalidate_revision_cache,
|
|
1160
|
+
on_ready=service.watcher_ready.set,
|
|
1161
|
+
watch_filter=_workspace_watch_filter(
|
|
1162
|
+
service.metadata.root,
|
|
1163
|
+
service._context.workspace.project_config,
|
|
1164
|
+
),
|
|
1165
|
+
)
|
|
1166
|
+
finally:
|
|
1167
|
+
if not service.shutdown.is_set():
|
|
1168
|
+
service.watcher_failed.set()
|
|
1169
|
+
|
|
1170
|
+
|
|
1171
|
+
def _serve_connection(connection: socket.socket, service: _CacheService) -> None:
|
|
1172
|
+
try:
|
|
1173
|
+
connection.settimeout(_PREAUTH_CONNECTION_TIMEOUT)
|
|
1174
|
+
line = _read_line(connection)
|
|
1175
|
+
connection.settimeout(_CONNECTION_TIMEOUT)
|
|
1176
|
+
request = json.loads(line.decode("utf-8"))
|
|
1177
|
+
if not isinstance(request, dict) or not hmac.compare_digest(str(request.pop("token", "")), service.metadata.token):
|
|
1178
|
+
response: dict[str, object] = {"error": {"code": "authentication_failed", "message": "authentication failed"}}
|
|
1179
|
+
else:
|
|
1180
|
+
try:
|
|
1181
|
+
action = request.get("action")
|
|
1182
|
+
guard = None if action in {"status", "stop"} else _RequestGuard(
|
|
1183
|
+
service,
|
|
1184
|
+
connection,
|
|
1185
|
+
enforce_rss_budget=action in {"trace", "cpg"},
|
|
1186
|
+
)
|
|
1187
|
+
result = service.request(
|
|
1188
|
+
request,
|
|
1189
|
+
cancel_check=guard.check if guard is not None else None,
|
|
1190
|
+
)
|
|
1191
|
+
response = {"payload": result.payload, "warning": result.warning}
|
|
1192
|
+
except CacheClientError as error:
|
|
1193
|
+
response = {"error": {"code": error.code, "message": error.message}}
|
|
1194
|
+
except Exception:
|
|
1195
|
+
response = {"error": {"code": "invalid_request", "message": "Invalid cache request."}}
|
|
1196
|
+
with contextlib.suppress(OSError):
|
|
1197
|
+
connection.sendall(json.dumps(response, separators=(",", ":")).encode("utf-8") + b"\n")
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
def _serve_owned_connection(
|
|
1201
|
+
connection: socket.socket,
|
|
1202
|
+
service: _CacheService,
|
|
1203
|
+
slots: threading.BoundedSemaphore,
|
|
1204
|
+
) -> None:
|
|
1205
|
+
try:
|
|
1206
|
+
with connection:
|
|
1207
|
+
_serve_connection(connection, service)
|
|
1208
|
+
finally:
|
|
1209
|
+
slots.release()
|
|
1210
|
+
|
|
1211
|
+
|
|
1212
|
+
def run_cache_daemon(
|
|
1213
|
+
root: str | Path,
|
|
1214
|
+
*,
|
|
1215
|
+
project_file: str = "",
|
|
1216
|
+
max_memory_bytes: int = DEFAULT_MAX_MEMORY_BYTES,
|
|
1217
|
+
workers: int = 0,
|
|
1218
|
+
idle_timeout: int = DEFAULT_IDLE_TIMEOUT,
|
|
1219
|
+
max_disk_cache_bytes: int = DEFAULT_MAX_DISK_CACHE_BYTES,
|
|
1220
|
+
) -> None:
|
|
1221
|
+
canonical = str(Path(root).resolve())
|
|
1222
|
+
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
1223
|
+
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
1224
|
+
listener.bind(("127.0.0.1", 0))
|
|
1225
|
+
listener.listen(16)
|
|
1226
|
+
listener.settimeout(0.25)
|
|
1227
|
+
metadata = CacheMetadata(
|
|
1228
|
+
DAEMON_SCHEMA,
|
|
1229
|
+
canonical,
|
|
1230
|
+
os.getpid(),
|
|
1231
|
+
listener.getsockname()[1],
|
|
1232
|
+
secrets.token_urlsafe(32),
|
|
1233
|
+
__version__,
|
|
1234
|
+
project_file,
|
|
1235
|
+
max_memory_bytes,
|
|
1236
|
+
workers,
|
|
1237
|
+
idle_timeout,
|
|
1238
|
+
time.time(),
|
|
1239
|
+
max_disk_cache_bytes,
|
|
1240
|
+
)
|
|
1241
|
+
watcher: threading.Thread | None = None
|
|
1242
|
+
previous_sigterm: (
|
|
1243
|
+
Callable[[int, FrameType | None], object]
|
|
1244
|
+
| int
|
|
1245
|
+
| None
|
|
1246
|
+
) = None
|
|
1247
|
+
connection_slots = threading.BoundedSemaphore(_CONNECTION_WORKERS)
|
|
1248
|
+
connection_pool = ThreadPoolExecutor(
|
|
1249
|
+
max_workers=_CONNECTION_WORKERS,
|
|
1250
|
+
thread_name_prefix="delphi-cache-client",
|
|
1251
|
+
)
|
|
1252
|
+
try:
|
|
1253
|
+
service = _CacheService(metadata, defer_context=True)
|
|
1254
|
+
if threading.current_thread() is threading.main_thread():
|
|
1255
|
+
previous_sigterm = signal.getsignal(signal.SIGTERM)
|
|
1256
|
+
|
|
1257
|
+
def request_shutdown(
|
|
1258
|
+
_signum: int,
|
|
1259
|
+
_frame: FrameType | None,
|
|
1260
|
+
) -> None:
|
|
1261
|
+
service.shutdown.set()
|
|
1262
|
+
|
|
1263
|
+
signal.signal(signal.SIGTERM, request_shutdown)
|
|
1264
|
+
_write_metadata(metadata)
|
|
1265
|
+
watcher = threading.Thread(
|
|
1266
|
+
target=_watch_workspace,
|
|
1267
|
+
args=(service,),
|
|
1268
|
+
name="delphi-cache-watcher",
|
|
1269
|
+
daemon=True,
|
|
1270
|
+
)
|
|
1271
|
+
watcher.start()
|
|
1272
|
+
service.start_prewarm()
|
|
1273
|
+
while (
|
|
1274
|
+
not service.shutdown.is_set()
|
|
1275
|
+
and time.monotonic() - service.last_activity < idle_timeout
|
|
1276
|
+
):
|
|
1277
|
+
try:
|
|
1278
|
+
connection, _ = listener.accept()
|
|
1279
|
+
except TimeoutError:
|
|
1280
|
+
continue
|
|
1281
|
+
if not connection_slots.acquire(blocking=False):
|
|
1282
|
+
connection.close()
|
|
1283
|
+
continue
|
|
1284
|
+
try:
|
|
1285
|
+
connection_pool.submit(
|
|
1286
|
+
_serve_owned_connection,
|
|
1287
|
+
connection,
|
|
1288
|
+
service,
|
|
1289
|
+
connection_slots,
|
|
1290
|
+
)
|
|
1291
|
+
except RuntimeError:
|
|
1292
|
+
connection_slots.release()
|
|
1293
|
+
connection.close()
|
|
1294
|
+
finally:
|
|
1295
|
+
if "service" in locals():
|
|
1296
|
+
service.shutdown.set()
|
|
1297
|
+
listener.close()
|
|
1298
|
+
connection_pool.shutdown(wait=True, cancel_futures=True)
|
|
1299
|
+
_remove_metadata_if_owned(metadata)
|
|
1300
|
+
if watcher is not None:
|
|
1301
|
+
watcher.join(timeout=2.0)
|
|
1302
|
+
if previous_sigterm is not None:
|
|
1303
|
+
signal.signal(signal.SIGTERM, previous_sigterm)
|
|
1304
|
+
|
|
1305
|
+
|
|
1306
|
+
def _start_cache_unlocked(
|
|
1307
|
+
root: str | Path,
|
|
1308
|
+
*,
|
|
1309
|
+
project_file: str | Path | None = None,
|
|
1310
|
+
max_memory_bytes: int = DEFAULT_MAX_MEMORY_BYTES,
|
|
1311
|
+
workers: int = 0,
|
|
1312
|
+
idle_timeout: int = DEFAULT_IDLE_TIMEOUT,
|
|
1313
|
+
startup_timeout: float = DEFAULT_STARTUP_TIMEOUT,
|
|
1314
|
+
max_disk_cache_bytes: int = DEFAULT_MAX_DISK_CACHE_BYTES,
|
|
1315
|
+
) -> CacheMetadata:
|
|
1316
|
+
canonical = str(Path(root).resolve())
|
|
1317
|
+
project = str((Path(canonical) / project_file).resolve()) if project_file and not Path(project_file).is_absolute() else (str(Path(project_file).resolve()) if project_file else "")
|
|
1318
|
+
existing = _read_metadata(canonical)
|
|
1319
|
+
if existing and _pid_alive(existing.pid):
|
|
1320
|
+
try:
|
|
1321
|
+
_client_exchange(existing, {"action": "status", "_startup_probe": True})
|
|
1322
|
+
except CacheClientError as error:
|
|
1323
|
+
raise CacheClientError("unavailable", "Live cache daemon is unavailable.") from error
|
|
1324
|
+
if (
|
|
1325
|
+
existing.project_file,
|
|
1326
|
+
existing.max_memory_bytes,
|
|
1327
|
+
existing.workers,
|
|
1328
|
+
existing.idle_timeout,
|
|
1329
|
+
existing.max_disk_cache_bytes,
|
|
1330
|
+
) != (
|
|
1331
|
+
project,
|
|
1332
|
+
max_memory_bytes,
|
|
1333
|
+
workers,
|
|
1334
|
+
idle_timeout,
|
|
1335
|
+
max_disk_cache_bytes,
|
|
1336
|
+
):
|
|
1337
|
+
raise CacheClientError("configuration_conflict", "A live cache daemon has conflicting configuration.")
|
|
1338
|
+
return existing
|
|
1339
|
+
if existing:
|
|
1340
|
+
_remove_metadata_if_owned(existing)
|
|
1341
|
+
else:
|
|
1342
|
+
_remove_incompatible_metadata_if_stale(canonical)
|
|
1343
|
+
command = [
|
|
1344
|
+
sys.executable,
|
|
1345
|
+
"-m",
|
|
1346
|
+
"delphi_lsp.agent_cache",
|
|
1347
|
+
"serve",
|
|
1348
|
+
"--root",
|
|
1349
|
+
canonical,
|
|
1350
|
+
"--max-memory",
|
|
1351
|
+
str(max_memory_bytes),
|
|
1352
|
+
"--workers",
|
|
1353
|
+
str(workers),
|
|
1354
|
+
"--idle-timeout",
|
|
1355
|
+
str(idle_timeout),
|
|
1356
|
+
"--max-disk-cache",
|
|
1357
|
+
str(max_disk_cache_bytes),
|
|
1358
|
+
]
|
|
1359
|
+
if project:
|
|
1360
|
+
command.extend(("--project-file", project))
|
|
1361
|
+
options = _daemon_process_options()
|
|
1362
|
+
with tempfile.TemporaryFile() as diagnostics:
|
|
1363
|
+
options["stderr"] = diagnostics
|
|
1364
|
+
process = subprocess.Popen(command, **options)
|
|
1365
|
+
deadline = time.monotonic() + startup_timeout
|
|
1366
|
+
startup_ready = False
|
|
1367
|
+
timed_out = False
|
|
1368
|
+
return_code: int | None = None
|
|
1369
|
+
try:
|
|
1370
|
+
while time.monotonic() < deadline:
|
|
1371
|
+
metadata = _read_metadata(canonical)
|
|
1372
|
+
if metadata:
|
|
1373
|
+
try:
|
|
1374
|
+
_client_exchange(metadata, {"action": "status", "_startup_probe": True})
|
|
1375
|
+
startup_ready = True
|
|
1376
|
+
return metadata
|
|
1377
|
+
except CacheClientError:
|
|
1378
|
+
pass
|
|
1379
|
+
if process.poll() is not None:
|
|
1380
|
+
break
|
|
1381
|
+
time.sleep(0.05)
|
|
1382
|
+
return_code = process.poll()
|
|
1383
|
+
timed_out = return_code is None
|
|
1384
|
+
if timed_out:
|
|
1385
|
+
process.kill()
|
|
1386
|
+
with contextlib.suppress(Exception):
|
|
1387
|
+
return_code = process.wait()
|
|
1388
|
+
metadata = _read_metadata(canonical)
|
|
1389
|
+
if metadata and metadata.pid == process.pid:
|
|
1390
|
+
_remove_metadata_if_owned(metadata)
|
|
1391
|
+
diagnostics.seek(0)
|
|
1392
|
+
raw = _read_startup_tail(diagnostics)
|
|
1393
|
+
message = _truncate_and_sanitize_startup_diagnostics(raw)
|
|
1394
|
+
base = "Cache daemon did not become ready."
|
|
1395
|
+
if timed_out:
|
|
1396
|
+
reason = (
|
|
1397
|
+
f" Startup timed out after {startup_timeout:.1f}s before "
|
|
1398
|
+
"readiness metadata was published."
|
|
1399
|
+
)
|
|
1400
|
+
else:
|
|
1401
|
+
reason = (
|
|
1402
|
+
f" Child process exited with code {return_code} before "
|
|
1403
|
+
"readiness metadata was published."
|
|
1404
|
+
)
|
|
1405
|
+
if message:
|
|
1406
|
+
raise CacheClientError(
|
|
1407
|
+
"startup_failed",
|
|
1408
|
+
f"{base}{reason} Child stderr: {message}",
|
|
1409
|
+
)
|
|
1410
|
+
raise CacheClientError(
|
|
1411
|
+
"startup_failed",
|
|
1412
|
+
(
|
|
1413
|
+
f"{base}{reason} No child diagnostics were emitted. "
|
|
1414
|
+
f"Python executable: {sys.executable}. Workspace: {canonical}."
|
|
1415
|
+
),
|
|
1416
|
+
)
|
|
1417
|
+
finally:
|
|
1418
|
+
if not startup_ready:
|
|
1419
|
+
if process.poll() is None:
|
|
1420
|
+
process.kill()
|
|
1421
|
+
with contextlib.suppress(Exception):
|
|
1422
|
+
process.wait()
|
|
1423
|
+
|
|
1424
|
+
|
|
1425
|
+
def start_cache(
|
|
1426
|
+
root: str | Path,
|
|
1427
|
+
*,
|
|
1428
|
+
project_file: str | Path | None = None,
|
|
1429
|
+
max_memory_bytes: int = DEFAULT_MAX_MEMORY_BYTES,
|
|
1430
|
+
workers: int = 0,
|
|
1431
|
+
idle_timeout: int = DEFAULT_IDLE_TIMEOUT,
|
|
1432
|
+
startup_timeout: float = DEFAULT_STARTUP_TIMEOUT,
|
|
1433
|
+
max_disk_cache_bytes: int = DEFAULT_MAX_DISK_CACHE_BYTES,
|
|
1434
|
+
) -> CacheMetadata:
|
|
1435
|
+
if type(workers) is not int or not 0 <= workers <= 32:
|
|
1436
|
+
raise ValueError("workers must be auto or an integer from 1 through 32.")
|
|
1437
|
+
if type(idle_timeout) is not int or idle_timeout <= 0:
|
|
1438
|
+
raise ValueError("idle_timeout must be greater than zero.")
|
|
1439
|
+
if not math.isfinite(startup_timeout) or startup_timeout <= 0:
|
|
1440
|
+
raise ValueError("startup_timeout must be greater than zero.")
|
|
1441
|
+
if (
|
|
1442
|
+
isinstance(max_disk_cache_bytes, bool)
|
|
1443
|
+
or not isinstance(max_disk_cache_bytes, int)
|
|
1444
|
+
or max_disk_cache_bytes < 0
|
|
1445
|
+
):
|
|
1446
|
+
raise ValueError("max_disk_cache_bytes must be non-negative.")
|
|
1447
|
+
load_project_path_config(root)
|
|
1448
|
+
with _process_start_lock(root), _start_lock(root, startup_timeout):
|
|
1449
|
+
return _start_cache_unlocked(
|
|
1450
|
+
root,
|
|
1451
|
+
project_file=project_file,
|
|
1452
|
+
max_memory_bytes=max_memory_bytes,
|
|
1453
|
+
workers=workers,
|
|
1454
|
+
idle_timeout=idle_timeout,
|
|
1455
|
+
startup_timeout=startup_timeout,
|
|
1456
|
+
max_disk_cache_bytes=max_disk_cache_bytes,
|
|
1457
|
+
)
|
|
1458
|
+
|
|
1459
|
+
|
|
1460
|
+
def query_cache(root: str | Path, request: dict[str, object]) -> CacheClientResponse:
|
|
1461
|
+
metadata = _read_metadata(root)
|
|
1462
|
+
if metadata is None:
|
|
1463
|
+
_remove_incompatible_metadata_if_stale(root)
|
|
1464
|
+
raise CacheClientError("cache_not_running", "Cache daemon is not running.")
|
|
1465
|
+
if not _pid_alive(metadata.pid):
|
|
1466
|
+
_remove_metadata_if_owned(metadata)
|
|
1467
|
+
raise CacheClientError("cache_not_running", "Cache daemon is not running.")
|
|
1468
|
+
while True:
|
|
1469
|
+
try:
|
|
1470
|
+
return _client_exchange(metadata, request)
|
|
1471
|
+
except CacheClientError as error:
|
|
1472
|
+
if error.code != "cache_warming":
|
|
1473
|
+
raise
|
|
1474
|
+
time.sleep(0.05)
|
|
1475
|
+
|
|
1476
|
+
|
|
1477
|
+
def cache_status(root: str | Path) -> dict[str, object]:
|
|
1478
|
+
return query_cache(root, {"action": "status"}).payload
|
|
1479
|
+
|
|
1480
|
+
|
|
1481
|
+
def stop_cache(root: str | Path) -> None:
|
|
1482
|
+
metadata = _read_metadata(root)
|
|
1483
|
+
if metadata is None:
|
|
1484
|
+
_remove_incompatible_metadata_if_stale(root)
|
|
1485
|
+
return
|
|
1486
|
+
if not _pid_alive(metadata.pid):
|
|
1487
|
+
_remove_metadata_if_owned(metadata)
|
|
1488
|
+
return
|
|
1489
|
+
try:
|
|
1490
|
+
_client_exchange(metadata, {"action": "stop"})
|
|
1491
|
+
except CacheClientError:
|
|
1492
|
+
pass
|
|
1493
|
+
started = time.monotonic()
|
|
1494
|
+
deadline = started + _STOP_TIMEOUT_SECONDS
|
|
1495
|
+
signal_sent = False
|
|
1496
|
+
stopped = False
|
|
1497
|
+
while True:
|
|
1498
|
+
now = time.monotonic()
|
|
1499
|
+
if now >= deadline:
|
|
1500
|
+
break
|
|
1501
|
+
if _reap_child_if_exited(metadata.pid):
|
|
1502
|
+
stopped = True
|
|
1503
|
+
break
|
|
1504
|
+
if not _pid_alive(metadata.pid):
|
|
1505
|
+
stopped = True
|
|
1506
|
+
break
|
|
1507
|
+
if (
|
|
1508
|
+
not signal_sent
|
|
1509
|
+
and os.name != "nt"
|
|
1510
|
+
and now - started >= _STOP_SIGNAL_DELAY_SECONDS
|
|
1511
|
+
):
|
|
1512
|
+
with contextlib.suppress(OSError):
|
|
1513
|
+
os.kill(metadata.pid, signal.SIGTERM)
|
|
1514
|
+
signal_sent = True
|
|
1515
|
+
time.sleep(0.05)
|
|
1516
|
+
if not stopped and _pid_alive(metadata.pid):
|
|
1517
|
+
current = _read_metadata(root)
|
|
1518
|
+
if current == metadata:
|
|
1519
|
+
force_signal = (
|
|
1520
|
+
signal.SIGKILL
|
|
1521
|
+
if os.name != "nt" and hasattr(signal, "SIGKILL")
|
|
1522
|
+
else signal.SIGTERM
|
|
1523
|
+
)
|
|
1524
|
+
with contextlib.suppress(OSError):
|
|
1525
|
+
os.kill(metadata.pid, force_signal)
|
|
1526
|
+
force_deadline = time.monotonic() + _STOP_FORCE_TIMEOUT_SECONDS
|
|
1527
|
+
while time.monotonic() < force_deadline:
|
|
1528
|
+
if _reap_child_if_exited(metadata.pid) or not _pid_alive(
|
|
1529
|
+
metadata.pid
|
|
1530
|
+
):
|
|
1531
|
+
stopped = True
|
|
1532
|
+
break
|
|
1533
|
+
time.sleep(0.05)
|
|
1534
|
+
if not stopped and _pid_alive(metadata.pid):
|
|
1535
|
+
raise CacheClientError("stop_failed", "Cache daemon did not stop.")
|
|
1536
|
+
_remove_metadata_if_owned(metadata)
|
|
1537
|
+
|
|
1538
|
+
|
|
1539
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1540
|
+
parser = argparse.ArgumentParser()
|
|
1541
|
+
command = parser.add_subparsers(dest="command", required=True)
|
|
1542
|
+
serve = command.add_parser("serve")
|
|
1543
|
+
serve.add_argument("--root", required=True)
|
|
1544
|
+
serve.add_argument("--project-file", default="")
|
|
1545
|
+
serve.add_argument("--max-memory", type=int, default=DEFAULT_MAX_MEMORY_BYTES)
|
|
1546
|
+
serve.add_argument("--workers", type=int, default=0)
|
|
1547
|
+
serve.add_argument("--idle-timeout", type=_positive_integer, default=DEFAULT_IDLE_TIMEOUT)
|
|
1548
|
+
serve.add_argument(
|
|
1549
|
+
"--max-disk-cache",
|
|
1550
|
+
type=int,
|
|
1551
|
+
default=DEFAULT_MAX_DISK_CACHE_BYTES,
|
|
1552
|
+
)
|
|
1553
|
+
args = parser.parse_args(argv)
|
|
1554
|
+
if args.command == "serve":
|
|
1555
|
+
run_cache_daemon(
|
|
1556
|
+
args.root,
|
|
1557
|
+
project_file=args.project_file,
|
|
1558
|
+
max_memory_bytes=args.max_memory,
|
|
1559
|
+
workers=args.workers,
|
|
1560
|
+
idle_timeout=args.idle_timeout,
|
|
1561
|
+
max_disk_cache_bytes=args.max_disk_cache,
|
|
1562
|
+
)
|
|
1563
|
+
return 0
|
|
1564
|
+
|
|
1565
|
+
|
|
1566
|
+
def _positive_integer(value: str) -> int:
|
|
1567
|
+
parsed = int(value)
|
|
1568
|
+
if parsed <= 0:
|
|
1569
|
+
raise argparse.ArgumentTypeError("value must be greater than zero")
|
|
1570
|
+
return parsed
|
|
1571
|
+
|
|
1572
|
+
|
|
1573
|
+
if __name__ == "__main__":
|
|
1574
|
+
raise SystemExit(main())
|