fbuild 1.2.8__py3-none-any.whl → 1.2.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fbuild/__init__.py +5 -1
- fbuild/build/configurable_compiler.py +49 -6
- fbuild/build/configurable_linker.py +14 -9
- fbuild/build/orchestrator_esp32.py +6 -3
- fbuild/build/orchestrator_rp2040.py +6 -2
- fbuild/cli.py +300 -5
- fbuild/config/ini_parser.py +13 -1
- fbuild/daemon/__init__.py +11 -0
- fbuild/daemon/async_client.py +5 -4
- fbuild/daemon/async_client_lib.py +1543 -0
- fbuild/daemon/async_protocol.py +825 -0
- fbuild/daemon/async_server.py +2100 -0
- fbuild/daemon/client.py +425 -13
- fbuild/daemon/configuration_lock.py +13 -13
- fbuild/daemon/connection.py +508 -0
- fbuild/daemon/connection_registry.py +579 -0
- fbuild/daemon/daemon.py +517 -164
- fbuild/daemon/daemon_context.py +72 -1
- fbuild/daemon/device_discovery.py +477 -0
- fbuild/daemon/device_manager.py +821 -0
- fbuild/daemon/error_collector.py +263 -263
- fbuild/daemon/file_cache.py +332 -332
- fbuild/daemon/firmware_ledger.py +46 -123
- fbuild/daemon/lock_manager.py +508 -508
- fbuild/daemon/messages.py +431 -0
- fbuild/daemon/operation_registry.py +288 -288
- fbuild/daemon/processors/build_processor.py +34 -1
- fbuild/daemon/processors/deploy_processor.py +1 -3
- fbuild/daemon/processors/locking_processor.py +7 -7
- fbuild/daemon/request_processor.py +457 -457
- fbuild/daemon/shared_serial.py +7 -7
- fbuild/daemon/status_manager.py +238 -238
- fbuild/daemon/subprocess_manager.py +316 -316
- fbuild/deploy/docker_utils.py +182 -2
- fbuild/deploy/monitor.py +1 -1
- fbuild/deploy/qemu_runner.py +71 -13
- fbuild/ledger/board_ledger.py +46 -122
- fbuild/output.py +238 -2
- fbuild/packages/library_compiler.py +15 -5
- fbuild/packages/library_manager.py +12 -6
- fbuild-1.2.15.dist-info/METADATA +569 -0
- {fbuild-1.2.8.dist-info → fbuild-1.2.15.dist-info}/RECORD +46 -39
- fbuild-1.2.8.dist-info/METADATA +0 -468
- {fbuild-1.2.8.dist-info → fbuild-1.2.15.dist-info}/WHEEL +0 -0
- {fbuild-1.2.8.dist-info → fbuild-1.2.15.dist-info}/entry_points.txt +0 -0
- {fbuild-1.2.8.dist-info → fbuild-1.2.15.dist-info}/licenses/LICENSE +0 -0
- {fbuild-1.2.8.dist-info → fbuild-1.2.15.dist-info}/top_level.txt +0 -0
fbuild/daemon/lock_manager.py
CHANGED
|
@@ -1,508 +1,508 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Resource Lock Manager - Unified lock management for daemon operations.
|
|
3
|
-
|
|
4
|
-
This module provides the ResourceLockManager class which centralizes all
|
|
5
|
-
lock management logic. Key features:
|
|
6
|
-
- Per-port and per-project locks with context managers
|
|
7
|
-
- Lock timeout/expiry for automatic stale lock detection
|
|
8
|
-
- Lock holder tracking for better error messages
|
|
9
|
-
- Force-release capability for stuck locks
|
|
10
|
-
- Automatic cleanup of stale locks
|
|
11
|
-
"""
|
|
12
|
-
|
|
13
|
-
import logging
|
|
14
|
-
import threading
|
|
15
|
-
import time
|
|
16
|
-
from contextlib import contextmanager
|
|
17
|
-
from dataclasses import dataclass, field
|
|
18
|
-
from typing import Any, Iterator
|
|
19
|
-
|
|
20
|
-
# Default lock timeout: 30 minutes (for long builds)
|
|
21
|
-
DEFAULT_LOCK_TIMEOUT = 1800.0
|
|
22
|
-
|
|
23
|
-
# Stale lock threshold: locks older than this with no activity are candidates for cleanup
|
|
24
|
-
STALE_LOCK_THRESHOLD = 3600.0 # 1 hour
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
@dataclass
|
|
28
|
-
class LockInfo:
|
|
29
|
-
"""Information about a lock for debugging, timeout detection, and cleanup.
|
|
30
|
-
|
|
31
|
-
Attributes:
|
|
32
|
-
lock: The actual threading.Lock object
|
|
33
|
-
created_at: Unix timestamp when lock was created
|
|
34
|
-
acquired_at: Unix timestamp when lock was last acquired (None if not held)
|
|
35
|
-
last_released_at: Unix timestamp when lock was last released
|
|
36
|
-
acquisition_count: Number of times lock has been acquired
|
|
37
|
-
holder_thread_id: Thread ID currently holding the lock (None if not held)
|
|
38
|
-
holder_operation_id: Operation ID currently holding the lock
|
|
39
|
-
holder_description: Human-readable description of what's holding the lock
|
|
40
|
-
timeout: Maximum time in seconds the lock can be held before considered stale
|
|
41
|
-
"""
|
|
42
|
-
|
|
43
|
-
lock: threading.Lock
|
|
44
|
-
created_at: float = field(default_factory=time.time)
|
|
45
|
-
acquired_at: float | None = None
|
|
46
|
-
last_released_at: float | None = None
|
|
47
|
-
acquisition_count: int = 0
|
|
48
|
-
holder_thread_id: int | None = None
|
|
49
|
-
holder_operation_id: str | None = None
|
|
50
|
-
holder_description: str | None = None
|
|
51
|
-
timeout: float = DEFAULT_LOCK_TIMEOUT
|
|
52
|
-
|
|
53
|
-
def is_held(self) -> bool:
|
|
54
|
-
"""Check if lock is currently held."""
|
|
55
|
-
return self.acquired_at is not None and self.last_released_at is None or (self.acquired_at is not None and self.last_released_at is not None and self.acquired_at > self.last_released_at)
|
|
56
|
-
|
|
57
|
-
def is_stale(self) -> bool:
|
|
58
|
-
"""Check if lock is stale (held beyond timeout)."""
|
|
59
|
-
if not self.is_held():
|
|
60
|
-
return False
|
|
61
|
-
if self.acquired_at is None:
|
|
62
|
-
return False
|
|
63
|
-
hold_time = time.time() - self.acquired_at
|
|
64
|
-
return hold_time > self.timeout
|
|
65
|
-
|
|
66
|
-
def hold_duration(self) -> float | None:
|
|
67
|
-
"""Get how long the lock has been held."""
|
|
68
|
-
if not self.is_held() or self.acquired_at is None:
|
|
69
|
-
return None
|
|
70
|
-
return time.time() - self.acquired_at
|
|
71
|
-
|
|
72
|
-
def to_dict(self) -> dict[str, Any]:
|
|
73
|
-
"""Convert to dictionary for JSON serialization."""
|
|
74
|
-
return {
|
|
75
|
-
"created_at": self.created_at,
|
|
76
|
-
"acquired_at": self.acquired_at,
|
|
77
|
-
"last_released_at": self.last_released_at,
|
|
78
|
-
"acquisition_count": self.acquisition_count,
|
|
79
|
-
"holder_thread_id": self.holder_thread_id,
|
|
80
|
-
"holder_operation_id": self.holder_operation_id,
|
|
81
|
-
"holder_description": self.holder_description,
|
|
82
|
-
"timeout": self.timeout,
|
|
83
|
-
"is_held": self.is_held(),
|
|
84
|
-
"is_stale": self.is_stale(),
|
|
85
|
-
"hold_duration": self.hold_duration(),
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
class LockAcquisitionError(RuntimeError):
|
|
90
|
-
"""Error raised when a lock cannot be acquired.
|
|
91
|
-
|
|
92
|
-
Provides detailed information about what's holding the lock.
|
|
93
|
-
"""
|
|
94
|
-
|
|
95
|
-
def __init__(
|
|
96
|
-
self,
|
|
97
|
-
resource_type: str,
|
|
98
|
-
resource_id: str,
|
|
99
|
-
lock_info: LockInfo | None = None,
|
|
100
|
-
):
|
|
101
|
-
self.resource_type = resource_type
|
|
102
|
-
self.resource_id = resource_id
|
|
103
|
-
self.lock_info = lock_info
|
|
104
|
-
|
|
105
|
-
# Build detailed error message
|
|
106
|
-
if lock_info is not None and lock_info.is_held():
|
|
107
|
-
holder_desc = lock_info.holder_description or "unknown operation"
|
|
108
|
-
hold_duration = lock_info.hold_duration()
|
|
109
|
-
duration_str = f" (held for {hold_duration:.1f}s)" if hold_duration else ""
|
|
110
|
-
if lock_info.is_stale():
|
|
111
|
-
message = (
|
|
112
|
-
f"{resource_type.capitalize()} lock unavailable for: {resource_id}. " + f"STALE lock held by: {holder_desc}{duration_str}. " + "Consider force-releasing with clear_stale_locks()."
|
|
113
|
-
)
|
|
114
|
-
else:
|
|
115
|
-
message = f"{resource_type.capitalize()} lock unavailable for: {resource_id}. " + f"Currently held by: {holder_desc}{duration_str}."
|
|
116
|
-
else:
|
|
117
|
-
message = f"{resource_type.capitalize()} lock unavailable for: {resource_id}"
|
|
118
|
-
|
|
119
|
-
super().__init__(message)
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
class ResourceLockManager:
|
|
123
|
-
"""Manages per-port and per-project locks with timeout detection and cleanup.
|
|
124
|
-
|
|
125
|
-
This class provides a unified interface for managing locks that protect
|
|
126
|
-
shared resources (serial ports and project directories). Features:
|
|
127
|
-
- Context managers for automatic lock acquisition/release
|
|
128
|
-
- Lock timeout detection for stale lock cleanup
|
|
129
|
-
- Lock holder tracking for informative error messages
|
|
130
|
-
- Force-release capability for stuck locks
|
|
131
|
-
- Thread-safe operations
|
|
132
|
-
|
|
133
|
-
Example:
|
|
134
|
-
>>> manager = ResourceLockManager()
|
|
135
|
-
>>>
|
|
136
|
-
>>> # Acquire port lock for serial operations
|
|
137
|
-
>>> with manager.acquire_port_lock("COM3", operation_id="deploy_123",
|
|
138
|
-
... description="Deploy to ESP32"):
|
|
139
|
-
... upload_firmware_to_port("COM3")
|
|
140
|
-
>>>
|
|
141
|
-
>>> # Check for stale locks
|
|
142
|
-
>>> stale = manager.get_stale_locks()
|
|
143
|
-
>>> if stale:
|
|
144
|
-
... print(f"Found {len(stale)} stale locks")
|
|
145
|
-
... manager.force_release_stale_locks()
|
|
146
|
-
"""
|
|
147
|
-
|
|
148
|
-
def __init__(self) -> None:
|
|
149
|
-
"""Initialize the ResourceLockManager."""
|
|
150
|
-
self._master_lock = threading.Lock() # Protects the lock dictionaries
|
|
151
|
-
self._port_locks: dict[str, LockInfo] = {} # Per-port locks
|
|
152
|
-
self._project_locks: dict[str, LockInfo] = {} # Per-project locks
|
|
153
|
-
|
|
154
|
-
@contextmanager
|
|
155
|
-
def acquire_port_lock(
|
|
156
|
-
self,
|
|
157
|
-
port: str,
|
|
158
|
-
blocking: bool = True,
|
|
159
|
-
timeout: float = DEFAULT_LOCK_TIMEOUT,
|
|
160
|
-
operation_id: str | None = None,
|
|
161
|
-
description: str | None = None,
|
|
162
|
-
) -> Iterator[None]:
|
|
163
|
-
"""Acquire a lock for a specific serial port.
|
|
164
|
-
|
|
165
|
-
This ensures that only one operation can use a serial port at a time,
|
|
166
|
-
preventing conflicts between deploy and monitor operations.
|
|
167
|
-
|
|
168
|
-
Args:
|
|
169
|
-
port: Serial port identifier (e.g., "COM3", "/dev/ttyUSB0")
|
|
170
|
-
blocking: If True, wait for lock. If False, raise LockAcquisitionError if unavailable.
|
|
171
|
-
timeout: Maximum time the lock can be held before considered stale.
|
|
172
|
-
operation_id: Identifier for the operation holding the lock.
|
|
173
|
-
description: Human-readable description of what's holding the lock.
|
|
174
|
-
|
|
175
|
-
Yields:
|
|
176
|
-
None (the lock is held for the duration of the context)
|
|
177
|
-
|
|
178
|
-
Raises:
|
|
179
|
-
LockAcquisitionError: If blocking=False and lock is not available
|
|
180
|
-
"""
|
|
181
|
-
lock_info = self._get_or_create_port_lock(port, timeout)
|
|
182
|
-
logging.debug(f"Acquiring port lock for: {port} (blocking={blocking})")
|
|
183
|
-
|
|
184
|
-
acquired = lock_info.lock.acquire(blocking=blocking)
|
|
185
|
-
if not acquired:
|
|
186
|
-
raise LockAcquisitionError("port", port, lock_info)
|
|
187
|
-
|
|
188
|
-
try:
|
|
189
|
-
# Record lock acquisition details
|
|
190
|
-
with self._master_lock:
|
|
191
|
-
lock_info.acquired_at = time.time()
|
|
192
|
-
lock_info.acquisition_count += 1
|
|
193
|
-
lock_info.holder_thread_id = threading.get_ident()
|
|
194
|
-
lock_info.holder_operation_id = operation_id
|
|
195
|
-
lock_info.holder_description = description or f"Operation on port {port}"
|
|
196
|
-
lock_info.timeout = timeout
|
|
197
|
-
|
|
198
|
-
logging.debug(f"Port lock acquired for: {port} " + f"(count={lock_info.acquisition_count}, operation={operation_id})")
|
|
199
|
-
yield
|
|
200
|
-
finally:
|
|
201
|
-
# Clear holder info before releasing
|
|
202
|
-
with self._master_lock:
|
|
203
|
-
lock_info.last_released_at = time.time()
|
|
204
|
-
lock_info.holder_thread_id = None
|
|
205
|
-
lock_info.holder_operation_id = None
|
|
206
|
-
lock_info.holder_description = None
|
|
207
|
-
lock_info.lock.release()
|
|
208
|
-
logging.debug(f"Port lock released for: {port}")
|
|
209
|
-
|
|
210
|
-
@contextmanager
|
|
211
|
-
def acquire_project_lock(
|
|
212
|
-
self,
|
|
213
|
-
project_dir: str,
|
|
214
|
-
blocking: bool = True,
|
|
215
|
-
timeout: float = DEFAULT_LOCK_TIMEOUT,
|
|
216
|
-
operation_id: str | None = None,
|
|
217
|
-
description: str | None = None,
|
|
218
|
-
) -> Iterator[None]:
|
|
219
|
-
"""Acquire a lock for a specific project directory.
|
|
220
|
-
|
|
221
|
-
This ensures that only one build operation can run for a project at a time,
|
|
222
|
-
preventing file conflicts and race conditions during compilation.
|
|
223
|
-
|
|
224
|
-
Args:
|
|
225
|
-
project_dir: Absolute path to project directory
|
|
226
|
-
blocking: If True, wait for lock. If False, raise LockAcquisitionError if unavailable.
|
|
227
|
-
timeout: Maximum time the lock can be held before considered stale.
|
|
228
|
-
operation_id: Identifier for the operation holding the lock.
|
|
229
|
-
description: Human-readable description of what's holding the lock.
|
|
230
|
-
|
|
231
|
-
Yields:
|
|
232
|
-
None (the lock is held for the duration of the context)
|
|
233
|
-
|
|
234
|
-
Raises:
|
|
235
|
-
LockAcquisitionError: If blocking=False and lock is not available
|
|
236
|
-
"""
|
|
237
|
-
lock_info = self._get_or_create_project_lock(project_dir, timeout)
|
|
238
|
-
logging.debug(f"Acquiring project lock for: {project_dir} (blocking={blocking})")
|
|
239
|
-
|
|
240
|
-
acquired = lock_info.lock.acquire(blocking=blocking)
|
|
241
|
-
if not acquired:
|
|
242
|
-
raise LockAcquisitionError("project", project_dir, lock_info)
|
|
243
|
-
|
|
244
|
-
try:
|
|
245
|
-
# Record lock acquisition details
|
|
246
|
-
with self._master_lock:
|
|
247
|
-
lock_info.acquired_at = time.time()
|
|
248
|
-
lock_info.acquisition_count += 1
|
|
249
|
-
lock_info.holder_thread_id = threading.get_ident()
|
|
250
|
-
lock_info.holder_operation_id = operation_id
|
|
251
|
-
lock_info.holder_description = description or f"Build for {project_dir}"
|
|
252
|
-
lock_info.timeout = timeout
|
|
253
|
-
|
|
254
|
-
logging.debug(f"Project lock acquired for: {project_dir} " + f"(count={lock_info.acquisition_count}, operation={operation_id})")
|
|
255
|
-
yield
|
|
256
|
-
finally:
|
|
257
|
-
# Clear holder info before releasing
|
|
258
|
-
with self._master_lock:
|
|
259
|
-
lock_info.last_released_at = time.time()
|
|
260
|
-
lock_info.holder_thread_id = None
|
|
261
|
-
lock_info.holder_operation_id = None
|
|
262
|
-
lock_info.holder_description = None
|
|
263
|
-
lock_info.lock.release()
|
|
264
|
-
logging.debug(f"Project lock released for: {project_dir}")
|
|
265
|
-
|
|
266
|
-
def _get_or_create_port_lock(self, port: str, timeout: float = DEFAULT_LOCK_TIMEOUT) -> LockInfo:
|
|
267
|
-
"""Get or create a lock for the given port."""
|
|
268
|
-
with self._master_lock:
|
|
269
|
-
if port not in self._port_locks:
|
|
270
|
-
self._port_locks[port] = LockInfo(lock=threading.Lock(), timeout=timeout)
|
|
271
|
-
return self._port_locks[port]
|
|
272
|
-
|
|
273
|
-
def _get_or_create_project_lock(self, project_dir: str, timeout: float = DEFAULT_LOCK_TIMEOUT) -> LockInfo:
|
|
274
|
-
"""Get or create a lock for the given project directory."""
|
|
275
|
-
with self._master_lock:
|
|
276
|
-
if project_dir not in self._project_locks:
|
|
277
|
-
self._project_locks[project_dir] = LockInfo(lock=threading.Lock(), timeout=timeout)
|
|
278
|
-
return self._project_locks[project_dir]
|
|
279
|
-
|
|
280
|
-
def get_stale_locks(self) -> dict[str, list[tuple[str, LockInfo]]]:
|
|
281
|
-
"""Get all locks that are stale (held beyond their timeout).
|
|
282
|
-
|
|
283
|
-
Returns:
|
|
284
|
-
Dictionary with 'port_locks' and 'project_locks' keys, each containing
|
|
285
|
-
a list of (resource_id, lock_info) tuples for stale locks.
|
|
286
|
-
"""
|
|
287
|
-
with self._master_lock:
|
|
288
|
-
stale_ports = [(port, info) for port, info in self._port_locks.items() if info.is_stale()]
|
|
289
|
-
stale_projects = [(project, info) for project, info in self._project_locks.items() if info.is_stale()]
|
|
290
|
-
return {
|
|
291
|
-
"port_locks": stale_ports,
|
|
292
|
-
"project_locks": stale_projects,
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
def get_held_locks(self) -> dict[str, list[tuple[str, LockInfo]]]:
|
|
296
|
-
"""Get all locks that are currently held.
|
|
297
|
-
|
|
298
|
-
Returns:
|
|
299
|
-
Dictionary with 'port_locks' and 'project_locks' keys, each containing
|
|
300
|
-
a list of (resource_id, lock_info) tuples for held locks.
|
|
301
|
-
"""
|
|
302
|
-
with self._master_lock:
|
|
303
|
-
held_ports = [(port, info) for port, info in self._port_locks.items() if info.is_held()]
|
|
304
|
-
held_projects = [(project, info) for project, info in self._project_locks.items() if info.is_held()]
|
|
305
|
-
return {
|
|
306
|
-
"port_locks": held_ports,
|
|
307
|
-
"project_locks": held_projects,
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
def force_release_lock(self, resource_type: str, resource_id: str) -> bool:
|
|
311
|
-
"""Force-release a lock (use with caution - may cause race conditions).
|
|
312
|
-
|
|
313
|
-
This should only be used to clear stale locks from stuck operations.
|
|
314
|
-
Force-releasing an active lock may cause data corruption.
|
|
315
|
-
|
|
316
|
-
Args:
|
|
317
|
-
resource_type: "port" or "project"
|
|
318
|
-
resource_id: The port or project directory identifier
|
|
319
|
-
|
|
320
|
-
Returns:
|
|
321
|
-
True if lock was force-released, False if lock not found
|
|
322
|
-
"""
|
|
323
|
-
with self._master_lock:
|
|
324
|
-
if resource_type == "port":
|
|
325
|
-
locks_dict = self._port_locks
|
|
326
|
-
elif resource_type == "project":
|
|
327
|
-
locks_dict = self._project_locks
|
|
328
|
-
else:
|
|
329
|
-
logging.error(f"Unknown resource type: {resource_type}")
|
|
330
|
-
return False
|
|
331
|
-
|
|
332
|
-
if resource_id not in locks_dict:
|
|
333
|
-
logging.warning(f"Lock not found for {resource_type}: {resource_id}")
|
|
334
|
-
return False
|
|
335
|
-
|
|
336
|
-
lock_info = locks_dict[resource_id]
|
|
337
|
-
if not lock_info.is_held():
|
|
338
|
-
logging.info(f"Lock for {resource_type} {resource_id} is not held")
|
|
339
|
-
return False
|
|
340
|
-
|
|
341
|
-
# Clear holder info and mark as released
|
|
342
|
-
logging.warning(f"Force-releasing {resource_type} lock for: {resource_id} " + f"(was held by: {lock_info.holder_description})")
|
|
343
|
-
lock_info.last_released_at = time.time()
|
|
344
|
-
lock_info.holder_thread_id = None
|
|
345
|
-
lock_info.holder_operation_id = None
|
|
346
|
-
lock_info.holder_description = None
|
|
347
|
-
|
|
348
|
-
# Try to release the lock if it's actually held
|
|
349
|
-
# Note: This may fail if the lock isn't held by this thread
|
|
350
|
-
try:
|
|
351
|
-
lock_info.lock.release()
|
|
352
|
-
except RuntimeError:
|
|
353
|
-
# Lock wasn't held - this is OK for force-release
|
|
354
|
-
pass
|
|
355
|
-
|
|
356
|
-
return True
|
|
357
|
-
|
|
358
|
-
def force_release_stale_locks(self) -> int:
|
|
359
|
-
"""Force-release all stale locks.
|
|
360
|
-
|
|
361
|
-
Returns:
|
|
362
|
-
Number of locks force-released
|
|
363
|
-
"""
|
|
364
|
-
stale = self.get_stale_locks()
|
|
365
|
-
released = 0
|
|
366
|
-
|
|
367
|
-
for port, _ in stale["port_locks"]:
|
|
368
|
-
if self.force_release_lock("port", port):
|
|
369
|
-
released += 1
|
|
370
|
-
|
|
371
|
-
for project, _ in stale["project_locks"]:
|
|
372
|
-
if self.force_release_lock("project", project):
|
|
373
|
-
released += 1
|
|
374
|
-
|
|
375
|
-
if released > 0:
|
|
376
|
-
logging.info(f"Force-released {released} stale locks")
|
|
377
|
-
|
|
378
|
-
return released
|
|
379
|
-
|
|
380
|
-
def cleanup_unused_locks(self, older_than: float = STALE_LOCK_THRESHOLD) -> int:
|
|
381
|
-
"""Clean up locks that haven't been acquired recently.
|
|
382
|
-
|
|
383
|
-
This prevents memory leaks from locks that were created for operations
|
|
384
|
-
that are no longer running. A lock is considered unused if it:
|
|
385
|
-
- Is not currently held AND
|
|
386
|
-
- Hasn't been acquired in the specified time period
|
|
387
|
-
|
|
388
|
-
Args:
|
|
389
|
-
older_than: Time in seconds. Locks not acquired in this period are removed.
|
|
390
|
-
|
|
391
|
-
Returns:
|
|
392
|
-
Number of locks removed
|
|
393
|
-
"""
|
|
394
|
-
current_time = time.time()
|
|
395
|
-
removed_count = 0
|
|
396
|
-
|
|
397
|
-
with self._master_lock:
|
|
398
|
-
# Clean up port locks
|
|
399
|
-
ports_to_remove = []
|
|
400
|
-
for port, lock_info in self._port_locks.items():
|
|
401
|
-
if lock_info.is_held():
|
|
402
|
-
continue # Don't remove held locks
|
|
403
|
-
|
|
404
|
-
# Check last activity time
|
|
405
|
-
last_activity = lock_info.last_released_at or lock_info.created_at
|
|
406
|
-
if current_time - last_activity > older_than:
|
|
407
|
-
ports_to_remove.append(port)
|
|
408
|
-
|
|
409
|
-
for port in ports_to_remove:
|
|
410
|
-
del self._port_locks[port]
|
|
411
|
-
removed_count += 1
|
|
412
|
-
logging.debug(f"Cleaned up unused port lock: {port}")
|
|
413
|
-
|
|
414
|
-
# Clean up project locks
|
|
415
|
-
projects_to_remove = []
|
|
416
|
-
for project_dir, lock_info in self._project_locks.items():
|
|
417
|
-
if lock_info.is_held():
|
|
418
|
-
continue # Don't remove held locks
|
|
419
|
-
|
|
420
|
-
# Check last activity time
|
|
421
|
-
last_activity = lock_info.last_released_at or lock_info.created_at
|
|
422
|
-
if current_time - last_activity > older_than:
|
|
423
|
-
projects_to_remove.append(project_dir)
|
|
424
|
-
|
|
425
|
-
for project_dir in projects_to_remove:
|
|
426
|
-
del self._project_locks[project_dir]
|
|
427
|
-
removed_count += 1
|
|
428
|
-
logging.debug(f"Cleaned up unused project lock: {project_dir}")
|
|
429
|
-
|
|
430
|
-
if removed_count > 0:
|
|
431
|
-
logging.info(f"Cleaned up {removed_count} unused locks")
|
|
432
|
-
|
|
433
|
-
return removed_count
|
|
434
|
-
|
|
435
|
-
def get_lock_status(self) -> dict[str, dict[str, int]]:
|
|
436
|
-
"""Get current lock status for debugging.
|
|
437
|
-
|
|
438
|
-
Returns:
|
|
439
|
-
Dictionary with 'port_locks' and 'project_locks' keys, each containing
|
|
440
|
-
a mapping of resource identifier to acquisition count.
|
|
441
|
-
"""
|
|
442
|
-
with self._master_lock:
|
|
443
|
-
return {
|
|
444
|
-
"port_locks": {port: info.acquisition_count for port, info in self._port_locks.items()},
|
|
445
|
-
"project_locks": {project: info.acquisition_count for project, info in self._project_locks.items()},
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
def get_lock_details(self) -> dict[str, dict[str, dict[str, Any]]]:
|
|
449
|
-
"""Get detailed lock information for debugging and status reporting.
|
|
450
|
-
|
|
451
|
-
Returns:
|
|
452
|
-
Dictionary with 'port_locks' and 'project_locks' keys, each containing
|
|
453
|
-
a mapping of resource identifier to detailed lock info dict.
|
|
454
|
-
"""
|
|
455
|
-
with self._master_lock:
|
|
456
|
-
return {
|
|
457
|
-
"port_locks": {port: info.to_dict() for port, info in self._port_locks.items()},
|
|
458
|
-
"project_locks": {project: info.to_dict() for project, info in self._project_locks.items()},
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
def get_lock_count(self) -> dict[str, int]:
|
|
462
|
-
"""Get the total number of locks currently tracked.
|
|
463
|
-
|
|
464
|
-
Returns:
|
|
465
|
-
Dictionary with 'port_locks' and 'project_locks' counts.
|
|
466
|
-
"""
|
|
467
|
-
with self._master_lock:
|
|
468
|
-
return {
|
|
469
|
-
"port_locks": len(self._port_locks),
|
|
470
|
-
"project_locks": len(self._project_locks),
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
def clear_all_locks(self) -> int:
|
|
474
|
-
"""Clear all locks (use with extreme caution - only for daemon restart).
|
|
475
|
-
|
|
476
|
-
This force-releases all locks and clears the lock dictionaries.
|
|
477
|
-
Should only be used during daemon shutdown/restart.
|
|
478
|
-
|
|
479
|
-
Returns:
|
|
480
|
-
Number of locks cleared
|
|
481
|
-
"""
|
|
482
|
-
with self._master_lock:
|
|
483
|
-
count = len(self._port_locks) + len(self._project_locks)
|
|
484
|
-
|
|
485
|
-
# Force release any held locks
|
|
486
|
-
for port, lock_info in self._port_locks.items():
|
|
487
|
-
if lock_info.is_held():
|
|
488
|
-
logging.warning(f"Clearing held port lock: {port}")
|
|
489
|
-
try:
|
|
490
|
-
lock_info.lock.release()
|
|
491
|
-
except RuntimeError:
|
|
492
|
-
pass
|
|
493
|
-
|
|
494
|
-
for project, lock_info in self._project_locks.items():
|
|
495
|
-
if lock_info.is_held():
|
|
496
|
-
logging.warning(f"Clearing held project lock: {project}")
|
|
497
|
-
try:
|
|
498
|
-
lock_info.lock.release()
|
|
499
|
-
except RuntimeError:
|
|
500
|
-
pass
|
|
501
|
-
|
|
502
|
-
self._port_locks.clear()
|
|
503
|
-
self._project_locks.clear()
|
|
504
|
-
|
|
505
|
-
if count > 0:
|
|
506
|
-
logging.info(f"Cleared all {count} locks")
|
|
507
|
-
|
|
508
|
-
return count
|
|
1
|
+
"""
|
|
2
|
+
Resource Lock Manager - Unified lock management for daemon operations.
|
|
3
|
+
|
|
4
|
+
This module provides the ResourceLockManager class which centralizes all
|
|
5
|
+
lock management logic. Key features:
|
|
6
|
+
- Per-port and per-project locks with context managers
|
|
7
|
+
- Lock timeout/expiry for automatic stale lock detection
|
|
8
|
+
- Lock holder tracking for better error messages
|
|
9
|
+
- Force-release capability for stuck locks
|
|
10
|
+
- Automatic cleanup of stale locks
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import logging
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from typing import Any, Iterator
|
|
19
|
+
|
|
20
|
+
# Default lock timeout: 30 minutes (for long builds)
|
|
21
|
+
DEFAULT_LOCK_TIMEOUT = 1800.0
|
|
22
|
+
|
|
23
|
+
# Stale lock threshold: locks older than this with no activity are candidates for cleanup
|
|
24
|
+
STALE_LOCK_THRESHOLD = 3600.0 # 1 hour
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class LockInfo:
|
|
29
|
+
"""Information about a lock for debugging, timeout detection, and cleanup.
|
|
30
|
+
|
|
31
|
+
Attributes:
|
|
32
|
+
lock: The actual threading.Lock object
|
|
33
|
+
created_at: Unix timestamp when lock was created
|
|
34
|
+
acquired_at: Unix timestamp when lock was last acquired (None if not held)
|
|
35
|
+
last_released_at: Unix timestamp when lock was last released
|
|
36
|
+
acquisition_count: Number of times lock has been acquired
|
|
37
|
+
holder_thread_id: Thread ID currently holding the lock (None if not held)
|
|
38
|
+
holder_operation_id: Operation ID currently holding the lock
|
|
39
|
+
holder_description: Human-readable description of what's holding the lock
|
|
40
|
+
timeout: Maximum time in seconds the lock can be held before considered stale
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
lock: threading.Lock
|
|
44
|
+
created_at: float = field(default_factory=time.time)
|
|
45
|
+
acquired_at: float | None = None
|
|
46
|
+
last_released_at: float | None = None
|
|
47
|
+
acquisition_count: int = 0
|
|
48
|
+
holder_thread_id: int | None = None
|
|
49
|
+
holder_operation_id: str | None = None
|
|
50
|
+
holder_description: str | None = None
|
|
51
|
+
timeout: float = DEFAULT_LOCK_TIMEOUT
|
|
52
|
+
|
|
53
|
+
def is_held(self) -> bool:
|
|
54
|
+
"""Check if lock is currently held."""
|
|
55
|
+
return self.acquired_at is not None and self.last_released_at is None or (self.acquired_at is not None and self.last_released_at is not None and self.acquired_at > self.last_released_at)
|
|
56
|
+
|
|
57
|
+
def is_stale(self) -> bool:
|
|
58
|
+
"""Check if lock is stale (held beyond timeout)."""
|
|
59
|
+
if not self.is_held():
|
|
60
|
+
return False
|
|
61
|
+
if self.acquired_at is None:
|
|
62
|
+
return False
|
|
63
|
+
hold_time = time.time() - self.acquired_at
|
|
64
|
+
return hold_time > self.timeout
|
|
65
|
+
|
|
66
|
+
def hold_duration(self) -> float | None:
|
|
67
|
+
"""Get how long the lock has been held."""
|
|
68
|
+
if not self.is_held() or self.acquired_at is None:
|
|
69
|
+
return None
|
|
70
|
+
return time.time() - self.acquired_at
|
|
71
|
+
|
|
72
|
+
def to_dict(self) -> dict[str, Any]:
|
|
73
|
+
"""Convert to dictionary for JSON serialization."""
|
|
74
|
+
return {
|
|
75
|
+
"created_at": self.created_at,
|
|
76
|
+
"acquired_at": self.acquired_at,
|
|
77
|
+
"last_released_at": self.last_released_at,
|
|
78
|
+
"acquisition_count": self.acquisition_count,
|
|
79
|
+
"holder_thread_id": self.holder_thread_id,
|
|
80
|
+
"holder_operation_id": self.holder_operation_id,
|
|
81
|
+
"holder_description": self.holder_description,
|
|
82
|
+
"timeout": self.timeout,
|
|
83
|
+
"is_held": self.is_held(),
|
|
84
|
+
"is_stale": self.is_stale(),
|
|
85
|
+
"hold_duration": self.hold_duration(),
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class LockAcquisitionError(RuntimeError):
|
|
90
|
+
"""Error raised when a lock cannot be acquired.
|
|
91
|
+
|
|
92
|
+
Provides detailed information about what's holding the lock.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
def __init__(
|
|
96
|
+
self,
|
|
97
|
+
resource_type: str,
|
|
98
|
+
resource_id: str,
|
|
99
|
+
lock_info: LockInfo | None = None,
|
|
100
|
+
):
|
|
101
|
+
self.resource_type = resource_type
|
|
102
|
+
self.resource_id = resource_id
|
|
103
|
+
self.lock_info = lock_info
|
|
104
|
+
|
|
105
|
+
# Build detailed error message
|
|
106
|
+
if lock_info is not None and lock_info.is_held():
|
|
107
|
+
holder_desc = lock_info.holder_description or "unknown operation"
|
|
108
|
+
hold_duration = lock_info.hold_duration()
|
|
109
|
+
duration_str = f" (held for {hold_duration:.1f}s)" if hold_duration else ""
|
|
110
|
+
if lock_info.is_stale():
|
|
111
|
+
message = (
|
|
112
|
+
f"{resource_type.capitalize()} lock unavailable for: {resource_id}. " + f"STALE lock held by: {holder_desc}{duration_str}. " + "Consider force-releasing with clear_stale_locks()."
|
|
113
|
+
)
|
|
114
|
+
else:
|
|
115
|
+
message = f"{resource_type.capitalize()} lock unavailable for: {resource_id}. " + f"Currently held by: {holder_desc}{duration_str}."
|
|
116
|
+
else:
|
|
117
|
+
message = f"{resource_type.capitalize()} lock unavailable for: {resource_id}"
|
|
118
|
+
|
|
119
|
+
super().__init__(message)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class ResourceLockManager:
|
|
123
|
+
"""Manages per-port and per-project locks with timeout detection and cleanup.
|
|
124
|
+
|
|
125
|
+
This class provides a unified interface for managing locks that protect
|
|
126
|
+
shared resources (serial ports and project directories). Features:
|
|
127
|
+
- Context managers for automatic lock acquisition/release
|
|
128
|
+
- Lock timeout detection for stale lock cleanup
|
|
129
|
+
- Lock holder tracking for informative error messages
|
|
130
|
+
- Force-release capability for stuck locks
|
|
131
|
+
- Thread-safe operations
|
|
132
|
+
|
|
133
|
+
Example:
|
|
134
|
+
>>> manager = ResourceLockManager()
|
|
135
|
+
>>>
|
|
136
|
+
>>> # Acquire port lock for serial operations
|
|
137
|
+
>>> with manager.acquire_port_lock("COM3", operation_id="deploy_123",
|
|
138
|
+
... description="Deploy to ESP32"):
|
|
139
|
+
... upload_firmware_to_port("COM3")
|
|
140
|
+
>>>
|
|
141
|
+
>>> # Check for stale locks
|
|
142
|
+
>>> stale = manager.get_stale_locks()
|
|
143
|
+
>>> if stale:
|
|
144
|
+
... print(f"Found {len(stale)} stale locks")
|
|
145
|
+
... manager.force_release_stale_locks()
|
|
146
|
+
"""
|
|
147
|
+
|
|
148
|
+
def __init__(self) -> None:
|
|
149
|
+
"""Initialize the ResourceLockManager."""
|
|
150
|
+
self._master_lock = threading.Lock() # Protects the lock dictionaries
|
|
151
|
+
self._port_locks: dict[str, LockInfo] = {} # Per-port locks
|
|
152
|
+
self._project_locks: dict[str, LockInfo] = {} # Per-project locks
|
|
153
|
+
|
|
154
|
+
@contextmanager
|
|
155
|
+
def acquire_port_lock(
|
|
156
|
+
self,
|
|
157
|
+
port: str,
|
|
158
|
+
blocking: bool = True,
|
|
159
|
+
timeout: float = DEFAULT_LOCK_TIMEOUT,
|
|
160
|
+
operation_id: str | None = None,
|
|
161
|
+
description: str | None = None,
|
|
162
|
+
) -> Iterator[None]:
|
|
163
|
+
"""Acquire a lock for a specific serial port.
|
|
164
|
+
|
|
165
|
+
This ensures that only one operation can use a serial port at a time,
|
|
166
|
+
preventing conflicts between deploy and monitor operations.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
port: Serial port identifier (e.g., "COM3", "/dev/ttyUSB0")
|
|
170
|
+
blocking: If True, wait for lock. If False, raise LockAcquisitionError if unavailable.
|
|
171
|
+
timeout: Maximum time the lock can be held before considered stale.
|
|
172
|
+
operation_id: Identifier for the operation holding the lock.
|
|
173
|
+
description: Human-readable description of what's holding the lock.
|
|
174
|
+
|
|
175
|
+
Yields:
|
|
176
|
+
None (the lock is held for the duration of the context)
|
|
177
|
+
|
|
178
|
+
Raises:
|
|
179
|
+
LockAcquisitionError: If blocking=False and lock is not available
|
|
180
|
+
"""
|
|
181
|
+
lock_info = self._get_or_create_port_lock(port, timeout)
|
|
182
|
+
logging.debug(f"Acquiring port lock for: {port} (blocking={blocking})")
|
|
183
|
+
|
|
184
|
+
acquired = lock_info.lock.acquire(blocking=blocking)
|
|
185
|
+
if not acquired:
|
|
186
|
+
raise LockAcquisitionError("port", port, lock_info)
|
|
187
|
+
|
|
188
|
+
try:
|
|
189
|
+
# Record lock acquisition details
|
|
190
|
+
with self._master_lock:
|
|
191
|
+
lock_info.acquired_at = time.time()
|
|
192
|
+
lock_info.acquisition_count += 1
|
|
193
|
+
lock_info.holder_thread_id = threading.get_ident()
|
|
194
|
+
lock_info.holder_operation_id = operation_id
|
|
195
|
+
lock_info.holder_description = description or f"Operation on port {port}"
|
|
196
|
+
lock_info.timeout = timeout
|
|
197
|
+
|
|
198
|
+
logging.debug(f"Port lock acquired for: {port} " + f"(count={lock_info.acquisition_count}, operation={operation_id})")
|
|
199
|
+
yield
|
|
200
|
+
finally:
|
|
201
|
+
# Clear holder info before releasing
|
|
202
|
+
with self._master_lock:
|
|
203
|
+
lock_info.last_released_at = time.time()
|
|
204
|
+
lock_info.holder_thread_id = None
|
|
205
|
+
lock_info.holder_operation_id = None
|
|
206
|
+
lock_info.holder_description = None
|
|
207
|
+
lock_info.lock.release()
|
|
208
|
+
logging.debug(f"Port lock released for: {port}")
|
|
209
|
+
|
|
210
|
+
@contextmanager
|
|
211
|
+
def acquire_project_lock(
|
|
212
|
+
self,
|
|
213
|
+
project_dir: str,
|
|
214
|
+
blocking: bool = True,
|
|
215
|
+
timeout: float = DEFAULT_LOCK_TIMEOUT,
|
|
216
|
+
operation_id: str | None = None,
|
|
217
|
+
description: str | None = None,
|
|
218
|
+
) -> Iterator[None]:
|
|
219
|
+
"""Acquire a lock for a specific project directory.
|
|
220
|
+
|
|
221
|
+
This ensures that only one build operation can run for a project at a time,
|
|
222
|
+
preventing file conflicts and race conditions during compilation.
|
|
223
|
+
|
|
224
|
+
Args:
|
|
225
|
+
project_dir: Absolute path to project directory
|
|
226
|
+
blocking: If True, wait for lock. If False, raise LockAcquisitionError if unavailable.
|
|
227
|
+
timeout: Maximum time the lock can be held before considered stale.
|
|
228
|
+
operation_id: Identifier for the operation holding the lock.
|
|
229
|
+
description: Human-readable description of what's holding the lock.
|
|
230
|
+
|
|
231
|
+
Yields:
|
|
232
|
+
None (the lock is held for the duration of the context)
|
|
233
|
+
|
|
234
|
+
Raises:
|
|
235
|
+
LockAcquisitionError: If blocking=False and lock is not available
|
|
236
|
+
"""
|
|
237
|
+
lock_info = self._get_or_create_project_lock(project_dir, timeout)
|
|
238
|
+
logging.debug(f"Acquiring project lock for: {project_dir} (blocking={blocking})")
|
|
239
|
+
|
|
240
|
+
acquired = lock_info.lock.acquire(blocking=blocking)
|
|
241
|
+
if not acquired:
|
|
242
|
+
raise LockAcquisitionError("project", project_dir, lock_info)
|
|
243
|
+
|
|
244
|
+
try:
|
|
245
|
+
# Record lock acquisition details
|
|
246
|
+
with self._master_lock:
|
|
247
|
+
lock_info.acquired_at = time.time()
|
|
248
|
+
lock_info.acquisition_count += 1
|
|
249
|
+
lock_info.holder_thread_id = threading.get_ident()
|
|
250
|
+
lock_info.holder_operation_id = operation_id
|
|
251
|
+
lock_info.holder_description = description or f"Build for {project_dir}"
|
|
252
|
+
lock_info.timeout = timeout
|
|
253
|
+
|
|
254
|
+
logging.debug(f"Project lock acquired for: {project_dir} " + f"(count={lock_info.acquisition_count}, operation={operation_id})")
|
|
255
|
+
yield
|
|
256
|
+
finally:
|
|
257
|
+
# Clear holder info before releasing
|
|
258
|
+
with self._master_lock:
|
|
259
|
+
lock_info.last_released_at = time.time()
|
|
260
|
+
lock_info.holder_thread_id = None
|
|
261
|
+
lock_info.holder_operation_id = None
|
|
262
|
+
lock_info.holder_description = None
|
|
263
|
+
lock_info.lock.release()
|
|
264
|
+
logging.debug(f"Project lock released for: {project_dir}")
|
|
265
|
+
|
|
266
|
+
def _get_or_create_port_lock(self, port: str, timeout: float = DEFAULT_LOCK_TIMEOUT) -> LockInfo:
|
|
267
|
+
"""Get or create a lock for the given port."""
|
|
268
|
+
with self._master_lock:
|
|
269
|
+
if port not in self._port_locks:
|
|
270
|
+
self._port_locks[port] = LockInfo(lock=threading.Lock(), timeout=timeout)
|
|
271
|
+
return self._port_locks[port]
|
|
272
|
+
|
|
273
|
+
def _get_or_create_project_lock(self, project_dir: str, timeout: float = DEFAULT_LOCK_TIMEOUT) -> LockInfo:
|
|
274
|
+
"""Get or create a lock for the given project directory."""
|
|
275
|
+
with self._master_lock:
|
|
276
|
+
if project_dir not in self._project_locks:
|
|
277
|
+
self._project_locks[project_dir] = LockInfo(lock=threading.Lock(), timeout=timeout)
|
|
278
|
+
return self._project_locks[project_dir]
|
|
279
|
+
|
|
280
|
+
def get_stale_locks(self) -> dict[str, list[tuple[str, LockInfo]]]:
|
|
281
|
+
"""Get all locks that are stale (held beyond their timeout).
|
|
282
|
+
|
|
283
|
+
Returns:
|
|
284
|
+
Dictionary with 'port_locks' and 'project_locks' keys, each containing
|
|
285
|
+
a list of (resource_id, lock_info) tuples for stale locks.
|
|
286
|
+
"""
|
|
287
|
+
with self._master_lock:
|
|
288
|
+
stale_ports = [(port, info) for port, info in self._port_locks.items() if info.is_stale()]
|
|
289
|
+
stale_projects = [(project, info) for project, info in self._project_locks.items() if info.is_stale()]
|
|
290
|
+
return {
|
|
291
|
+
"port_locks": stale_ports,
|
|
292
|
+
"project_locks": stale_projects,
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
def get_held_locks(self) -> dict[str, list[tuple[str, LockInfo]]]:
|
|
296
|
+
"""Get all locks that are currently held.
|
|
297
|
+
|
|
298
|
+
Returns:
|
|
299
|
+
Dictionary with 'port_locks' and 'project_locks' keys, each containing
|
|
300
|
+
a list of (resource_id, lock_info) tuples for held locks.
|
|
301
|
+
"""
|
|
302
|
+
with self._master_lock:
|
|
303
|
+
held_ports = [(port, info) for port, info in self._port_locks.items() if info.is_held()]
|
|
304
|
+
held_projects = [(project, info) for project, info in self._project_locks.items() if info.is_held()]
|
|
305
|
+
return {
|
|
306
|
+
"port_locks": held_ports,
|
|
307
|
+
"project_locks": held_projects,
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
def force_release_lock(self, resource_type: str, resource_id: str) -> bool:
|
|
311
|
+
"""Force-release a lock (use with caution - may cause race conditions).
|
|
312
|
+
|
|
313
|
+
This should only be used to clear stale locks from stuck operations.
|
|
314
|
+
Force-releasing an active lock may cause data corruption.
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
resource_type: "port" or "project"
|
|
318
|
+
resource_id: The port or project directory identifier
|
|
319
|
+
|
|
320
|
+
Returns:
|
|
321
|
+
True if lock was force-released, False if lock not found
|
|
322
|
+
"""
|
|
323
|
+
with self._master_lock:
|
|
324
|
+
if resource_type == "port":
|
|
325
|
+
locks_dict = self._port_locks
|
|
326
|
+
elif resource_type == "project":
|
|
327
|
+
locks_dict = self._project_locks
|
|
328
|
+
else:
|
|
329
|
+
logging.error(f"Unknown resource type: {resource_type}")
|
|
330
|
+
return False
|
|
331
|
+
|
|
332
|
+
if resource_id not in locks_dict:
|
|
333
|
+
logging.warning(f"Lock not found for {resource_type}: {resource_id}")
|
|
334
|
+
return False
|
|
335
|
+
|
|
336
|
+
lock_info = locks_dict[resource_id]
|
|
337
|
+
if not lock_info.is_held():
|
|
338
|
+
logging.info(f"Lock for {resource_type} {resource_id} is not held")
|
|
339
|
+
return False
|
|
340
|
+
|
|
341
|
+
# Clear holder info and mark as released
|
|
342
|
+
logging.warning(f"Force-releasing {resource_type} lock for: {resource_id} " + f"(was held by: {lock_info.holder_description})")
|
|
343
|
+
lock_info.last_released_at = time.time()
|
|
344
|
+
lock_info.holder_thread_id = None
|
|
345
|
+
lock_info.holder_operation_id = None
|
|
346
|
+
lock_info.holder_description = None
|
|
347
|
+
|
|
348
|
+
# Try to release the lock if it's actually held
|
|
349
|
+
# Note: This may fail if the lock isn't held by this thread
|
|
350
|
+
try:
|
|
351
|
+
lock_info.lock.release()
|
|
352
|
+
except RuntimeError:
|
|
353
|
+
# Lock wasn't held - this is OK for force-release
|
|
354
|
+
pass
|
|
355
|
+
|
|
356
|
+
return True
|
|
357
|
+
|
|
358
|
+
def force_release_stale_locks(self) -> int:
|
|
359
|
+
"""Force-release all stale locks.
|
|
360
|
+
|
|
361
|
+
Returns:
|
|
362
|
+
Number of locks force-released
|
|
363
|
+
"""
|
|
364
|
+
stale = self.get_stale_locks()
|
|
365
|
+
released = 0
|
|
366
|
+
|
|
367
|
+
for port, _ in stale["port_locks"]:
|
|
368
|
+
if self.force_release_lock("port", port):
|
|
369
|
+
released += 1
|
|
370
|
+
|
|
371
|
+
for project, _ in stale["project_locks"]:
|
|
372
|
+
if self.force_release_lock("project", project):
|
|
373
|
+
released += 1
|
|
374
|
+
|
|
375
|
+
if released > 0:
|
|
376
|
+
logging.info(f"Force-released {released} stale locks")
|
|
377
|
+
|
|
378
|
+
return released
|
|
379
|
+
|
|
380
|
+
def cleanup_unused_locks(self, older_than: float = STALE_LOCK_THRESHOLD) -> int:
|
|
381
|
+
"""Clean up locks that haven't been acquired recently.
|
|
382
|
+
|
|
383
|
+
This prevents memory leaks from locks that were created for operations
|
|
384
|
+
that are no longer running. A lock is considered unused if it:
|
|
385
|
+
- Is not currently held AND
|
|
386
|
+
- Hasn't been acquired in the specified time period
|
|
387
|
+
|
|
388
|
+
Args:
|
|
389
|
+
older_than: Time in seconds. Locks not acquired in this period are removed.
|
|
390
|
+
|
|
391
|
+
Returns:
|
|
392
|
+
Number of locks removed
|
|
393
|
+
"""
|
|
394
|
+
current_time = time.time()
|
|
395
|
+
removed_count = 0
|
|
396
|
+
|
|
397
|
+
with self._master_lock:
|
|
398
|
+
# Clean up port locks
|
|
399
|
+
ports_to_remove = []
|
|
400
|
+
for port, lock_info in self._port_locks.items():
|
|
401
|
+
if lock_info.is_held():
|
|
402
|
+
continue # Don't remove held locks
|
|
403
|
+
|
|
404
|
+
# Check last activity time
|
|
405
|
+
last_activity = lock_info.last_released_at or lock_info.created_at
|
|
406
|
+
if current_time - last_activity > older_than:
|
|
407
|
+
ports_to_remove.append(port)
|
|
408
|
+
|
|
409
|
+
for port in ports_to_remove:
|
|
410
|
+
del self._port_locks[port]
|
|
411
|
+
removed_count += 1
|
|
412
|
+
logging.debug(f"Cleaned up unused port lock: {port}")
|
|
413
|
+
|
|
414
|
+
# Clean up project locks
|
|
415
|
+
projects_to_remove = []
|
|
416
|
+
for project_dir, lock_info in self._project_locks.items():
|
|
417
|
+
if lock_info.is_held():
|
|
418
|
+
continue # Don't remove held locks
|
|
419
|
+
|
|
420
|
+
# Check last activity time
|
|
421
|
+
last_activity = lock_info.last_released_at or lock_info.created_at
|
|
422
|
+
if current_time - last_activity > older_than:
|
|
423
|
+
projects_to_remove.append(project_dir)
|
|
424
|
+
|
|
425
|
+
for project_dir in projects_to_remove:
|
|
426
|
+
del self._project_locks[project_dir]
|
|
427
|
+
removed_count += 1
|
|
428
|
+
logging.debug(f"Cleaned up unused project lock: {project_dir}")
|
|
429
|
+
|
|
430
|
+
if removed_count > 0:
|
|
431
|
+
logging.info(f"Cleaned up {removed_count} unused locks")
|
|
432
|
+
|
|
433
|
+
return removed_count
|
|
434
|
+
|
|
435
|
+
def get_lock_status(self) -> dict[str, dict[str, int]]:
|
|
436
|
+
"""Get current lock status for debugging.
|
|
437
|
+
|
|
438
|
+
Returns:
|
|
439
|
+
Dictionary with 'port_locks' and 'project_locks' keys, each containing
|
|
440
|
+
a mapping of resource identifier to acquisition count.
|
|
441
|
+
"""
|
|
442
|
+
with self._master_lock:
|
|
443
|
+
return {
|
|
444
|
+
"port_locks": {port: info.acquisition_count for port, info in self._port_locks.items()},
|
|
445
|
+
"project_locks": {project: info.acquisition_count for project, info in self._project_locks.items()},
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
def get_lock_details(self) -> dict[str, dict[str, dict[str, Any]]]:
|
|
449
|
+
"""Get detailed lock information for debugging and status reporting.
|
|
450
|
+
|
|
451
|
+
Returns:
|
|
452
|
+
Dictionary with 'port_locks' and 'project_locks' keys, each containing
|
|
453
|
+
a mapping of resource identifier to detailed lock info dict.
|
|
454
|
+
"""
|
|
455
|
+
with self._master_lock:
|
|
456
|
+
return {
|
|
457
|
+
"port_locks": {port: info.to_dict() for port, info in self._port_locks.items()},
|
|
458
|
+
"project_locks": {project: info.to_dict() for project, info in self._project_locks.items()},
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
def get_lock_count(self) -> dict[str, int]:
|
|
462
|
+
"""Get the total number of locks currently tracked.
|
|
463
|
+
|
|
464
|
+
Returns:
|
|
465
|
+
Dictionary with 'port_locks' and 'project_locks' counts.
|
|
466
|
+
"""
|
|
467
|
+
with self._master_lock:
|
|
468
|
+
return {
|
|
469
|
+
"port_locks": len(self._port_locks),
|
|
470
|
+
"project_locks": len(self._project_locks),
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
def clear_all_locks(self) -> int:
|
|
474
|
+
"""Clear all locks (use with extreme caution - only for daemon restart).
|
|
475
|
+
|
|
476
|
+
This force-releases all locks and clears the lock dictionaries.
|
|
477
|
+
Should only be used during daemon shutdown/restart.
|
|
478
|
+
|
|
479
|
+
Returns:
|
|
480
|
+
Number of locks cleared
|
|
481
|
+
"""
|
|
482
|
+
with self._master_lock:
|
|
483
|
+
count = len(self._port_locks) + len(self._project_locks)
|
|
484
|
+
|
|
485
|
+
# Force release any held locks
|
|
486
|
+
for port, lock_info in self._port_locks.items():
|
|
487
|
+
if lock_info.is_held():
|
|
488
|
+
logging.warning(f"Clearing held port lock: {port}")
|
|
489
|
+
try:
|
|
490
|
+
lock_info.lock.release()
|
|
491
|
+
except RuntimeError:
|
|
492
|
+
pass
|
|
493
|
+
|
|
494
|
+
for project, lock_info in self._project_locks.items():
|
|
495
|
+
if lock_info.is_held():
|
|
496
|
+
logging.warning(f"Clearing held project lock: {project}")
|
|
497
|
+
try:
|
|
498
|
+
lock_info.lock.release()
|
|
499
|
+
except RuntimeError:
|
|
500
|
+
pass
|
|
501
|
+
|
|
502
|
+
self._port_locks.clear()
|
|
503
|
+
self._project_locks.clear()
|
|
504
|
+
|
|
505
|
+
if count > 0:
|
|
506
|
+
logging.info(f"Cleared all {count} locks")
|
|
507
|
+
|
|
508
|
+
return count
|