sedlib 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sedlib/__init__.py +47 -0
- sedlib/bol2rad.py +552 -0
- sedlib/catalog.py +1141 -0
- sedlib/core.py +6059 -0
- sedlib/data/__init__.py +2 -0
- sedlib/data/temp_to_bc_coefficients.yaml +62 -0
- sedlib/filter/__init__.py +5 -0
- sedlib/filter/core.py +1064 -0
- sedlib/filter/data/__init__.py +2 -0
- sedlib/filter/data/svo_all_filter_database.pickle +0 -0
- sedlib/filter/data/svo_filter_catalog.pickle +0 -0
- sedlib/filter/data/svo_meta_data.xml +1282 -0
- sedlib/filter/utils.py +71 -0
- sedlib/helper.py +361 -0
- sedlib/utils.py +789 -0
- sedlib/version.py +12 -0
- sedlib-1.0.0.dist-info/METADATA +611 -0
- sedlib-1.0.0.dist-info/RECORD +21 -0
- sedlib-1.0.0.dist-info/WHEEL +5 -0
- sedlib-1.0.0.dist-info/licenses/LICENSE +201 -0
- sedlib-1.0.0.dist-info/top_level.txt +1 -0
sedlib/utils.py
ADDED
@@ -0,0 +1,789 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
|
3
|
+
"""
|
4
|
+
Utility functions for sedlib
|
5
|
+
"""
|
6
|
+
|
7
|
+
__all__ = [
|
8
|
+
'tqdm', 'tqdm_joblib', 'ParallelPbar',
|
9
|
+
'InMemoryHandler', 'get_local_ip', 'get_system_ram',
|
10
|
+
'generate_unique_worker_name', 'MacDaemon', 'Daemon'
|
11
|
+
]
|
12
|
+
|
13
|
+
# System and OS
|
14
|
+
import os
|
15
|
+
import platform
|
16
|
+
import socket
|
17
|
+
import subprocess
|
18
|
+
import sys
|
19
|
+
import uuid
|
20
|
+
|
21
|
+
# Python utilities
|
22
|
+
import contextlib
|
23
|
+
import logging
|
24
|
+
from typing import List, Optional
|
25
|
+
|
26
|
+
# Progress and parallel processing
|
27
|
+
import joblib
|
28
|
+
import psutil
|
29
|
+
from tqdm import tqdm as terminal_tqdm
|
30
|
+
from tqdm.auto import tqdm as notebook_tqdm
|
31
|
+
|
32
|
+
# Time and process management
|
33
|
+
import time
|
34
|
+
import signal
|
35
|
+
import atexit
|
36
|
+
|
37
|
+
# Automatically choose the correct tqdm implementation
|
38
|
+
if "ipykernel" in sys.modules:
|
39
|
+
tqdm = notebook_tqdm
|
40
|
+
else:
|
41
|
+
tqdm = terminal_tqdm
|
42
|
+
|
43
|
+
|
44
|
+
@contextlib.contextmanager
|
45
|
+
def tqdm_joblib(*args, **kwargs):
|
46
|
+
"""Context manager to patch joblib to report into tqdm progress bar
|
47
|
+
given as argument"""
|
48
|
+
|
49
|
+
tqdm_object = tqdm(*args, **kwargs)
|
50
|
+
|
51
|
+
class TqdmBatchCompletionCallback(joblib.parallel.BatchCompletionCallBack):
|
52
|
+
def __init__(self, *args, **kwargs):
|
53
|
+
super().__init__(*args, **kwargs)
|
54
|
+
|
55
|
+
def __call__(self, *args, **kwargs):
|
56
|
+
tqdm_object.update(n=self.batch_size)
|
57
|
+
return super().__call__(*args, **kwargs)
|
58
|
+
|
59
|
+
old_batch_callback = joblib.parallel.BatchCompletionCallBack
|
60
|
+
joblib.parallel.BatchCompletionCallBack = TqdmBatchCompletionCallback
|
61
|
+
try:
|
62
|
+
yield tqdm_object
|
63
|
+
finally:
|
64
|
+
joblib.parallel.BatchCompletionCallBack = old_batch_callback
|
65
|
+
tqdm_object.close()
|
66
|
+
|
67
|
+
|
68
|
+
def ParallelPbar(desc=None, **tqdm_kwargs):
|
69
|
+
|
70
|
+
class Parallel(joblib.Parallel):
|
71
|
+
def __call__(self, it):
|
72
|
+
it = list(it)
|
73
|
+
with tqdm_joblib(total=len(it), desc=desc, **tqdm_kwargs):
|
74
|
+
return super().__call__(it)
|
75
|
+
|
76
|
+
return Parallel
|
77
|
+
|
78
|
+
|
79
|
+
class InMemoryHandler(logging.Handler):
|
80
|
+
"""A logging handler that stores log records in memory."""
|
81
|
+
|
82
|
+
def __init__(self, capacity: Optional[int] = None):
|
83
|
+
"""Initialize the handler with optional capacity limit.
|
84
|
+
|
85
|
+
Parameters
|
86
|
+
----------
|
87
|
+
capacity : Optional[int]
|
88
|
+
Maximum number of log records to store. If None, no limit is applied.
|
89
|
+
"""
|
90
|
+
super().__init__()
|
91
|
+
self.capacity = capacity
|
92
|
+
self.logs: List[str] = []
|
93
|
+
|
94
|
+
def emit(self, record: logging.LogRecord) -> None:
|
95
|
+
"""Store the log record in memory.
|
96
|
+
|
97
|
+
Parameters
|
98
|
+
----------
|
99
|
+
record : logging.LogRecord
|
100
|
+
The log record to store
|
101
|
+
"""
|
102
|
+
log_entry = self.format(record)
|
103
|
+
self.logs.append(log_entry)
|
104
|
+
if self.capacity and len(self.logs) > self.capacity:
|
105
|
+
self.logs.pop(0)
|
106
|
+
|
107
|
+
def get_logs(self, log_type: str = 'all') -> List[str]:
|
108
|
+
"""Get all stored log records.
|
109
|
+
|
110
|
+
Parameters
|
111
|
+
----------
|
112
|
+
log_type : str
|
113
|
+
log type to get
|
114
|
+
possible values are 'all', 'info', 'debug', 'warning', 'error'
|
115
|
+
(default: 'all')
|
116
|
+
|
117
|
+
Returns
|
118
|
+
-------
|
119
|
+
List[str]
|
120
|
+
List of formatted log records
|
121
|
+
"""
|
122
|
+
if log_type == 'all':
|
123
|
+
return self.logs.copy()
|
124
|
+
elif log_type == 'info':
|
125
|
+
return [log for log in self.logs if log[22:].startswith('INFO')]
|
126
|
+
elif log_type == 'debug':
|
127
|
+
return [log for log in self.logs if log[22:].startswith('DEBUG')]
|
128
|
+
elif log_type == 'warning':
|
129
|
+
return [log for log in self.logs if log[22:].startswith('WARNING')]
|
130
|
+
elif log_type == 'error':
|
131
|
+
return [log for log in self.logs if log[22:].startswith('ERROR')]
|
132
|
+
else:
|
133
|
+
raise ValueError(f"Invalid log type: {log_type}")
|
134
|
+
|
135
|
+
def clear(self) -> None:
|
136
|
+
"""Clear all stored log records."""
|
137
|
+
self.logs.clear()
|
138
|
+
|
139
|
+
|
140
|
+
def old_get_local_ip():
|
141
|
+
"""Get the local network IP address.
|
142
|
+
|
143
|
+
Attempts to find a private network IP address (192.168.x.x, 172.16-31.x.x,
|
144
|
+
or 10.x.x.x) by checking network interfaces and socket connections.
|
145
|
+
|
146
|
+
Returns:
|
147
|
+
str: Local network IP address, or "127.0.0.1" if none found.
|
148
|
+
"""
|
149
|
+
def is_local_network_ip(ip):
|
150
|
+
"""Check if IP is in private network ranges."""
|
151
|
+
parts = [int(part) for part in ip.split('.')]
|
152
|
+
return ((parts[0] == 192 and parts[1] == 168) or # 192.168.x.x
|
153
|
+
(parts[0] == 172 and 16 <= parts[1] <= 31) or # 172.16-31.x.x
|
154
|
+
(parts[0] == 10)) # 10.x.x.x
|
155
|
+
|
156
|
+
try:
|
157
|
+
# Try network interfaces first
|
158
|
+
for interface in socket.getaddrinfo(socket.gethostname(), None):
|
159
|
+
ip = interface[4][0]
|
160
|
+
if '.' in ip and is_local_network_ip(ip):
|
161
|
+
return ip
|
162
|
+
|
163
|
+
# Try socket connection method
|
164
|
+
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
165
|
+
try:
|
166
|
+
s.bind(('', 0))
|
167
|
+
s.connect(('10.255.255.255', 1))
|
168
|
+
ip = s.getsockname()[0]
|
169
|
+
if is_local_network_ip(ip):
|
170
|
+
return ip
|
171
|
+
finally:
|
172
|
+
s.close()
|
173
|
+
except Exception:
|
174
|
+
pass
|
175
|
+
|
176
|
+
return "127.0.0.1"
|
177
|
+
|
178
|
+
|
179
|
+
def old_get_system_ram():
|
180
|
+
"""Get total system RAM in GB.
|
181
|
+
|
182
|
+
Raises:
|
183
|
+
ValueError: If the OS type is not supported
|
184
|
+
|
185
|
+
Returns:
|
186
|
+
float: Total RAM in GB
|
187
|
+
"""
|
188
|
+
|
189
|
+
os_type = sys.platform
|
190
|
+
if os_type in ['linux', 'darwin']:
|
191
|
+
ram_gb = round(
|
192
|
+
os.sysconf('SC_PAGE_SIZE') *
|
193
|
+
os.sysconf('SC_PHYS_PAGES') / (1024.**3)
|
194
|
+
)
|
195
|
+
elif os_type == 'windows':
|
196
|
+
ram_gb = psutil.virtual_memory().total / (1024.**3)
|
197
|
+
else:
|
198
|
+
raise ValueError(f"Unsupported OS type: {os_type}")
|
199
|
+
|
200
|
+
return ram_gb
|
201
|
+
|
202
|
+
|
203
|
+
def get_local_ip() -> str:
|
204
|
+
"""Return the local IP address by scanning local network interfaces (no external queries)."""
|
205
|
+
try:
|
206
|
+
# Examine all network interfaces
|
207
|
+
addrs = psutil.net_if_addrs()
|
208
|
+
for iface, snics in addrs.items():
|
209
|
+
for snic in snics:
|
210
|
+
if snic.family == socket.AF_INET:
|
211
|
+
ip = snic.address
|
212
|
+
# Skip loopback and link-local addresses
|
213
|
+
if ip.startswith("127.") or ip.startswith("169.254."):
|
214
|
+
continue
|
215
|
+
return ip
|
216
|
+
except Exception:
|
217
|
+
pass
|
218
|
+
|
219
|
+
# Fallback: hostname lookup
|
220
|
+
try:
|
221
|
+
ip = socket.gethostbyname(socket.gethostname())
|
222
|
+
if ip and not ip.startswith("127."):
|
223
|
+
return ip
|
224
|
+
except Exception:
|
225
|
+
pass
|
226
|
+
|
227
|
+
# Final fallback to loopback
|
228
|
+
return "127.0.0.1"
|
229
|
+
|
230
|
+
|
231
|
+
def get_system_ram() -> float:
|
232
|
+
"""Return total physical RAM in gigabytes across all platforms."""
|
233
|
+
total_bytes = psutil.virtual_memory().total
|
234
|
+
return round(total_bytes / (1024 ** 3), 0)
|
235
|
+
|
236
|
+
|
237
|
+
def generate_unique_worker_name():
|
238
|
+
"""
|
239
|
+
Generate a unique, memorable name for a worker process on this machine.
|
240
|
+
|
241
|
+
This function always returns the same name when run on the same host,
|
242
|
+
regardless of network configuration, yet yields different names on
|
243
|
+
different machines. The format is:
|
244
|
+
|
245
|
+
{OS}_{hostname}_{Constellation}{NN}
|
246
|
+
|
247
|
+
- OS : Single-letter code for the operating system
|
248
|
+
('L' = Linux, 'W' = Windows, 'M' = macOS, 'U' = Unknown)
|
249
|
+
- hostname : The full machine hostname, lowercased (dots removed)
|
250
|
+
- Constellation : A full constellation name chosen deterministically
|
251
|
+
from a fixed list, never truncated
|
252
|
+
- NN : A two-digit number (00–99) to avoid collisions
|
253
|
+
|
254
|
+
By using intact constellation names, nothing is ever cut off, making
|
255
|
+
the result easier to remember. Example output:
|
256
|
+
|
257
|
+
"L_my-laptop_Orion07"
|
258
|
+
"""
|
259
|
+
# Determine OS code
|
260
|
+
os_name = platform.system()
|
261
|
+
os_code = {'Linux': 'L', 'Windows': 'W', 'Darwin': 'M'}.get(os_name, 'U')
|
262
|
+
|
263
|
+
# Get the hostname (remove domain parts, lowercase)
|
264
|
+
raw_host = platform.node().split('.')[0]
|
265
|
+
hostname = raw_host.lower() if raw_host else 'unknown'
|
266
|
+
|
267
|
+
# Try to read a stable machine identifier
|
268
|
+
machine_id = None
|
269
|
+
if os_name == 'Linux':
|
270
|
+
try:
|
271
|
+
with open('/etc/machine-id', 'r') as f:
|
272
|
+
machine_id = f.read().strip()
|
273
|
+
except Exception:
|
274
|
+
pass
|
275
|
+
elif os_name == 'Windows':
|
276
|
+
try:
|
277
|
+
import winreg
|
278
|
+
key = winreg.OpenKey(
|
279
|
+
winreg.HKEY_LOCAL_MACHINE,
|
280
|
+
r"SOFTWARE\Microsoft\Cryptography",
|
281
|
+
0,
|
282
|
+
winreg.KEY_READ | winreg.KEY_WOW64_64KEY
|
283
|
+
)
|
284
|
+
machine_id, _ = winreg.QueryValueEx(key, "MachineGuid")
|
285
|
+
except Exception:
|
286
|
+
pass
|
287
|
+
elif os_name == 'Darwin':
|
288
|
+
try:
|
289
|
+
out = subprocess.check_output(
|
290
|
+
["ioreg", "-rd1", "-c", "IOPlatformExpertDevice"],
|
291
|
+
text=True
|
292
|
+
)
|
293
|
+
for line in out.splitlines():
|
294
|
+
if "IOPlatformUUID" in line:
|
295
|
+
machine_id = line.split('=')[1].strip().strip('"')
|
296
|
+
break
|
297
|
+
except Exception:
|
298
|
+
pass
|
299
|
+
|
300
|
+
# Fallback to MAC address if no platform-specific ID found
|
301
|
+
if not machine_id:
|
302
|
+
machine_id = f"{uuid.getnode():x}"
|
303
|
+
|
304
|
+
# Build a deterministic UUID5 hash from OS, hostname, and machine_id
|
305
|
+
name_input = f"{os_name}-{platform.node()}-{machine_id}"
|
306
|
+
hash_hex = uuid.uuid5(uuid.NAMESPACE_OID, name_input).hex
|
307
|
+
|
308
|
+
# Full list of constellations (names never truncated)
|
309
|
+
CONSTELLATIONS = [
|
310
|
+
"Andromeda", "Antlia", "Apus", "Aquarius", "Aquila", "Ara", "Aries",
|
311
|
+
"Auriga", "Bootes", "Caelum", "Camelopardalis", "Cancer", "CanesVenatici",
|
312
|
+
"CanisMajor", "CanisMinor", "Capricornus", "Carina", "Cassiopeia",
|
313
|
+
"Centaurus", "Cepheus", "Cetus", "Chamaeleon", "Circinus", "Columba",
|
314
|
+
"ComaBerenices", "CoronaAustralis", "CoronaBorealis", "Corvus",
|
315
|
+
"Crater", "Crux", "Cygnus", "Delphinus", "Dorado", "Draco", "Equuleus",
|
316
|
+
"Eridanus", "Fornax", "Gemini", "Grus", "Hercules", "Horologium",
|
317
|
+
"Hydra", "Hydrus", "Indus", "Lacerta", "Leo", "LeoMinor", "Lepus",
|
318
|
+
"Libra", "Lupus", "Lynx", "Lyra", "Mensa", "Microscopium", "Monoceros",
|
319
|
+
"Musca", "Norma", "Octans", "Ophiuchus", "Orion", "Pavo", "Pegasus",
|
320
|
+
"Perseus", "Phoenix", "Pictor", "Pisces", "Puppis", "Pyxis", "Reticulum",
|
321
|
+
"Sagitta", "Sagittarius", "Scorpius", "Sculptor", "Scutum", "Serpens",
|
322
|
+
"Sextans", "Taurus", "Telescopium", "Triangulum", "Tucana", "UrsaMajor",
|
323
|
+
"UrsaMinor", "Vela", "Virgo", "Volans", "Vulpecula"
|
324
|
+
]
|
325
|
+
|
326
|
+
# Pick one constellation and a two-digit suffix deterministically
|
327
|
+
idx = int(hash_hex[:4], 16) % len(CONSTELLATIONS)
|
328
|
+
num = int(hash_hex[4:6], 16) % 100
|
329
|
+
constellation = CONSTELLATIONS[idx]
|
330
|
+
|
331
|
+
# Assemble the final name
|
332
|
+
return f"{os_code}_{hostname}_{constellation}{num:02d}"
|
333
|
+
|
334
|
+
|
335
|
+
class MacDaemon:
|
336
|
+
"""macOS daemon implementation that avoids fork() to prevent deadlocks.
|
337
|
+
|
338
|
+
Uses subprocess.Popen similar to Windows implementation instead of fork()
|
339
|
+
to create a detached process on macOS.
|
340
|
+
"""
|
341
|
+
|
342
|
+
def __init__(self, pidfile, worker, working_dir=None):
|
343
|
+
"""Initialize the macOS daemon.
|
344
|
+
|
345
|
+
Args:
|
346
|
+
pidfile (str): Path to file for storing process ID
|
347
|
+
worker (Worker): Worker instance to run as daemon
|
348
|
+
working_dir (str, optional): Working directory for daemon process
|
349
|
+
"""
|
350
|
+
self.pidfile = os.path.abspath(pidfile)
|
351
|
+
self.worker = worker
|
352
|
+
self.working_dir = working_dir or os.getcwd()
|
353
|
+
self.logger = logging.getLogger("worker.macdaemon")
|
354
|
+
|
355
|
+
def is_running(self):
|
356
|
+
"""Check if the daemon process is currently running.
|
357
|
+
|
358
|
+
Returns:
|
359
|
+
bool: True if daemon is running, False otherwise
|
360
|
+
"""
|
361
|
+
if not os.path.exists(self.pidfile):
|
362
|
+
return False
|
363
|
+
|
364
|
+
try:
|
365
|
+
with open(self.pidfile, "r") as pf:
|
366
|
+
pid = int(pf.read().strip())
|
367
|
+
|
368
|
+
if not psutil.pid_exists(pid):
|
369
|
+
return False
|
370
|
+
|
371
|
+
# Check if process is a Python process
|
372
|
+
try:
|
373
|
+
process = psutil.Process(pid)
|
374
|
+
cmdline = " ".join(process.cmdline()).lower()
|
375
|
+
|
376
|
+
# Check if it's our worker script
|
377
|
+
if "python" in process.name().lower() and "worker.py" in cmdline:
|
378
|
+
self.logger.debug(f"Found worker process with PID {pid}")
|
379
|
+
return True
|
380
|
+
|
381
|
+
self.logger.warning(
|
382
|
+
f"Found process with PID {pid} but not our worker."
|
383
|
+
)
|
384
|
+
return False
|
385
|
+
except (psutil.NoSuchProcess, psutil.AccessDenied) as e:
|
386
|
+
self.logger.error(f"Error checking process: {e}")
|
387
|
+
return False
|
388
|
+
except (IOError, ValueError) as e:
|
389
|
+
self.logger.error(f"Error reading PID file: {e}")
|
390
|
+
return False
|
391
|
+
|
392
|
+
def _write_pid(self, pid):
|
393
|
+
"""Write PID to the PID file.
|
394
|
+
|
395
|
+
Args:
|
396
|
+
pid (int): Process ID to write
|
397
|
+
"""
|
398
|
+
# Create parent directories for pidfile if needed
|
399
|
+
pidfile_dir = os.path.dirname(self.pidfile)
|
400
|
+
if pidfile_dir and not os.path.exists(pidfile_dir):
|
401
|
+
os.makedirs(pidfile_dir)
|
402
|
+
|
403
|
+
with open(self.pidfile, 'w') as pf:
|
404
|
+
pf.write(f"{pid}\n")
|
405
|
+
|
406
|
+
self.logger.info(f"macOS daemon started with PID: {pid}")
|
407
|
+
|
408
|
+
def delpid(self):
|
409
|
+
"""Remove the PID file when daemon exits."""
|
410
|
+
if os.path.exists(self.pidfile):
|
411
|
+
try:
|
412
|
+
os.remove(self.pidfile)
|
413
|
+
self.logger.info("PID file removed")
|
414
|
+
except Exception as e:
|
415
|
+
self.logger.error(f"Error removing PID file: {e}")
|
416
|
+
|
417
|
+
def start(self):
|
418
|
+
"""Start the daemon process."""
|
419
|
+
self.logger.info("Starting macOS daemon...")
|
420
|
+
|
421
|
+
if self.is_running():
|
422
|
+
self.logger.error("Daemon already running according to PID file")
|
423
|
+
sys.exit(1)
|
424
|
+
|
425
|
+
try:
|
426
|
+
# Get script path
|
427
|
+
script_path = os.path.abspath(sys.argv[0])
|
428
|
+
|
429
|
+
# Build command with daemon flag
|
430
|
+
cmd = [
|
431
|
+
sys.executable,
|
432
|
+
script_path,
|
433
|
+
"run",
|
434
|
+
"--daemon",
|
435
|
+
"--config", getattr(self.worker, "_config_file", "config.json"),
|
436
|
+
"--logfile", getattr(self.worker, "_logfile", "worker.log"),
|
437
|
+
"--loglevel", getattr(self.worker, "_loglevel", "INFO"),
|
438
|
+
"--pidfile", self.pidfile
|
439
|
+
]
|
440
|
+
|
441
|
+
# Create a detached process
|
442
|
+
self.logger.info(f"Starting worker as detached process: {' '.join(cmd)}")
|
443
|
+
|
444
|
+
# On macOS, we need to use subprocess but handle differently than Windows
|
445
|
+
with open(os.devnull, 'w') as devnull_fh:
|
446
|
+
process = subprocess.Popen(
|
447
|
+
cmd,
|
448
|
+
cwd=self.working_dir,
|
449
|
+
stdout=devnull_fh,
|
450
|
+
stderr=devnull_fh,
|
451
|
+
stdin=subprocess.PIPE,
|
452
|
+
# No creationflags parameter on macOS
|
453
|
+
)
|
454
|
+
|
455
|
+
# Wait briefly to see if process exits immediately (indicating error)
|
456
|
+
time.sleep(2)
|
457
|
+
|
458
|
+
# process.poll() returns None if running, or exit code if terminated
|
459
|
+
exit_code = process.poll()
|
460
|
+
if exit_code is not None:
|
461
|
+
# If process exited, its stdout/stderr went to devnull.
|
462
|
+
# Guide user to the daemon's own log file.
|
463
|
+
self.logger.error(
|
464
|
+
f"Process exited immediately with code {exit_code}. "
|
465
|
+
f"Check daemon's log file (e.g., worker.log) for details."
|
466
|
+
)
|
467
|
+
raise RuntimeError(
|
468
|
+
f"Failed to start daemon (exited with code {exit_code}). "
|
469
|
+
f"Check daemon's log file."
|
470
|
+
)
|
471
|
+
|
472
|
+
# Process is still running, so it should be our daemon process
|
473
|
+
pid = process.pid
|
474
|
+
self._write_pid(pid)
|
475
|
+
|
476
|
+
self.logger.info(f"macOS daemon started with PID {pid}")
|
477
|
+
|
478
|
+
except Exception as e:
|
479
|
+
self.logger.error(f"Failed to start daemon: {e}", exc_info=True)
|
480
|
+
self.delpid()
|
481
|
+
sys.exit(1)
|
482
|
+
|
483
|
+
def stop(self):
|
484
|
+
"""Stop the daemon process."""
|
485
|
+
self.logger.info("Stopping macOS daemon...")
|
486
|
+
|
487
|
+
if not os.path.exists(self.pidfile):
|
488
|
+
self.logger.warning("PID file not found. Daemon not running?")
|
489
|
+
return
|
490
|
+
|
491
|
+
try:
|
492
|
+
with open(self.pidfile, "r") as pf:
|
493
|
+
pid = int(pf.read().strip())
|
494
|
+
|
495
|
+
if not psutil.pid_exists(pid):
|
496
|
+
self.logger.warning(
|
497
|
+
f"Process {pid} not found. Removing stale PID file."
|
498
|
+
)
|
499
|
+
self.delpid()
|
500
|
+
return
|
501
|
+
|
502
|
+
# Try graceful termination first
|
503
|
+
try:
|
504
|
+
process = psutil.Process(pid)
|
505
|
+
self.logger.info(f"Sending termination signal to process {pid}")
|
506
|
+
process.terminate()
|
507
|
+
|
508
|
+
# Wait for process to terminate
|
509
|
+
gone, alive = psutil.wait_procs([process], timeout=10)
|
510
|
+
|
511
|
+
# Force kill if still running
|
512
|
+
if alive:
|
513
|
+
self.logger.warning(f"Process {pid} did not terminate gracefully. Forcing termination.")
|
514
|
+
for p in alive:
|
515
|
+
p.kill()
|
516
|
+
except psutil.NoSuchProcess:
|
517
|
+
self.logger.warning(f"Process {pid} not found")
|
518
|
+
|
519
|
+
self.delpid()
|
520
|
+
self.logger.info("macOS daemon stopped")
|
521
|
+
|
522
|
+
except (IOError, ValueError) as err:
|
523
|
+
self.logger.error(f"Error reading PID file: {err}")
|
524
|
+
self.delpid()
|
525
|
+
except Exception as err:
|
526
|
+
self.logger.error(f"Error stopping daemon: {err}")
|
527
|
+
sys.exit(1)
|
528
|
+
|
529
|
+
def restart(self):
|
530
|
+
"""Restart the daemon process."""
|
531
|
+
self.logger.info("Restarting macOS daemon...")
|
532
|
+
self.stop()
|
533
|
+
time.sleep(2) # Give it time to fully stop
|
534
|
+
self.start()
|
535
|
+
|
536
|
+
def run(self):
|
537
|
+
"""Run the worker daemon loop.
|
538
|
+
|
539
|
+
This method is called in the child process when started with the --daemon flag.
|
540
|
+
"""
|
541
|
+
# Create and register cleanup handler
|
542
|
+
def cleanup_handler(signum=None, frame=None):
|
543
|
+
self.logger.info("Received shutdown signal. Cleaning up...")
|
544
|
+
if hasattr(self.worker, 'send_heartbeat'):
|
545
|
+
try:
|
546
|
+
self.worker.send_heartbeat(status="offline", message="shutting down")
|
547
|
+
except Exception as e:
|
548
|
+
self.logger.error(f"Error sending final heartbeat: {e}")
|
549
|
+
|
550
|
+
self.delpid()
|
551
|
+
sys.exit(0)
|
552
|
+
|
553
|
+
# Register cleanup for normal exit
|
554
|
+
atexit.register(cleanup_handler)
|
555
|
+
|
556
|
+
# Register signal handlers
|
557
|
+
signal.signal(signal.SIGTERM, cleanup_handler)
|
558
|
+
signal.signal(signal.SIGINT, cleanup_handler)
|
559
|
+
|
560
|
+
# If args supplied by main() calling this directly, write PID
|
561
|
+
if os.path.exists(self.pidfile):
|
562
|
+
self.logger.debug("PID file already exists")
|
563
|
+
else:
|
564
|
+
self._write_pid(os.getpid())
|
565
|
+
|
566
|
+
try:
|
567
|
+
self.logger.info("Starting macOS worker's main processing loop...")
|
568
|
+
self.worker.loop() # Worker's infinite loop
|
569
|
+
except Exception as e:
|
570
|
+
self.logger.critical(f"Critical error in worker loop: {e}", exc_info=True)
|
571
|
+
self.delpid()
|
572
|
+
sys.exit(1)
|
573
|
+
|
574
|
+
|
575
|
+
class Daemon:
|
576
|
+
"""Unix daemon supporting process management.
|
577
|
+
|
578
|
+
Implements the Unix double-fork idiom for creating daemon processes
|
579
|
+
that run detached from the controlling terminal.
|
580
|
+
"""
|
581
|
+
|
582
|
+
def __init__(self, pidfile, worker, working_dir=None):
|
583
|
+
"""Initialize the daemon.
|
584
|
+
|
585
|
+
Args:
|
586
|
+
pidfile (str): Path to file for storing process ID
|
587
|
+
worker (Worker): Worker instance to run as daemon
|
588
|
+
working_dir (str, optional): Working directory for daemon process
|
589
|
+
"""
|
590
|
+
self.pidfile = os.path.abspath(pidfile)
|
591
|
+
self.worker = worker
|
592
|
+
self.working_dir = working_dir or os.getcwd()
|
593
|
+
self.logger = logging.getLogger("worker.daemon")
|
594
|
+
|
595
|
+
def daemonize(self):
|
596
|
+
"""Create background process using double-fork method.
|
597
|
+
|
598
|
+
Creates a detached process that runs independently from the
|
599
|
+
terminal. Sets up proper file descriptors and process hierarchy.
|
600
|
+
|
601
|
+
Raises:
|
602
|
+
OSError: If fork operations fail
|
603
|
+
"""
|
604
|
+
# First fork
|
605
|
+
try:
|
606
|
+
pid = os.fork()
|
607
|
+
if pid > 0:
|
608
|
+
sys.exit(0) # Exit first parent
|
609
|
+
except OSError as err:
|
610
|
+
self.logger.error(f"First fork failed: {err}")
|
611
|
+
sys.exit(1)
|
612
|
+
|
613
|
+
# Decouple from parent environment
|
614
|
+
os.chdir(self.working_dir)
|
615
|
+
os.setsid()
|
616
|
+
os.umask(0)
|
617
|
+
|
618
|
+
# Second fork
|
619
|
+
try:
|
620
|
+
pid = os.fork()
|
621
|
+
if pid > 0:
|
622
|
+
sys.exit(0) # Exit second parent
|
623
|
+
except OSError as err:
|
624
|
+
self.logger.error(f"Second fork failed: {err}")
|
625
|
+
sys.exit(1)
|
626
|
+
|
627
|
+
# Redirect standard file descriptors
|
628
|
+
sys.stdout.flush()
|
629
|
+
sys.stderr.flush()
|
630
|
+
si = open(os.devnull, 'r')
|
631
|
+
so = open(os.devnull, 'a+')
|
632
|
+
se = open(os.devnull, 'a+')
|
633
|
+
os.dup2(si.fileno(), sys.stdin.fileno())
|
634
|
+
os.dup2(so.fileno(), sys.stdout.fileno())
|
635
|
+
os.dup2(se.fileno(), sys.stderr.fileno())
|
636
|
+
|
637
|
+
# Register cleanup function and record PID
|
638
|
+
atexit.register(self.delpid)
|
639
|
+
pid = str(os.getpid())
|
640
|
+
|
641
|
+
# Create parent directories for pidfile if needed
|
642
|
+
pidfile_dir = os.path.dirname(self.pidfile)
|
643
|
+
if pidfile_dir and not os.path.exists(pidfile_dir):
|
644
|
+
os.makedirs(pidfile_dir)
|
645
|
+
|
646
|
+
with open(self.pidfile, 'w+') as pf:
|
647
|
+
pf.write(f"{pid}\n")
|
648
|
+
|
649
|
+
self.logger.info(f"Daemon started with PID: {pid}")
|
650
|
+
|
651
|
+
def delpid(self):
|
652
|
+
"""Remove the PID file when daemon exits."""
|
653
|
+
if os.path.exists(self.pidfile):
|
654
|
+
try:
|
655
|
+
os.remove(self.pidfile)
|
656
|
+
self.logger.info("PID file removed")
|
657
|
+
except Exception as e:
|
658
|
+
self.logger.error(f"Error removing PID file: {e}")
|
659
|
+
|
660
|
+
def is_running(self):
|
661
|
+
"""Check if the daemon process is currently running.
|
662
|
+
|
663
|
+
Returns:
|
664
|
+
bool: True if daemon is running, False otherwise
|
665
|
+
"""
|
666
|
+
if not os.path.exists(self.pidfile):
|
667
|
+
return False
|
668
|
+
|
669
|
+
try:
|
670
|
+
with open(self.pidfile, "r") as pf:
|
671
|
+
pid = int(pf.read().strip())
|
672
|
+
|
673
|
+
if not psutil.pid_exists(pid):
|
674
|
+
return False
|
675
|
+
|
676
|
+
# Check process command line to ensure it's our daemon
|
677
|
+
process = psutil.Process(pid)
|
678
|
+
cmdline = " ".join(process.cmdline()).lower()
|
679
|
+
if "python" in process.name().lower() or "python" in cmdline:
|
680
|
+
return True
|
681
|
+
|
682
|
+
self.logger.warning(
|
683
|
+
f"Found stale PID file. Process {pid} not our daemon."
|
684
|
+
)
|
685
|
+
return False
|
686
|
+
except (IOError, ValueError, psutil.NoSuchProcess,
|
687
|
+
psutil.AccessDenied) as e:
|
688
|
+
self.logger.error(f"Error checking daemon status: {e}")
|
689
|
+
return False
|
690
|
+
|
691
|
+
def start(self):
|
692
|
+
"""Start the daemon process."""
|
693
|
+
self.logger.info("Starting daemon...")
|
694
|
+
|
695
|
+
if self.is_running():
|
696
|
+
self.logger.error("Daemon already running according to PID file")
|
697
|
+
sys.exit(1)
|
698
|
+
|
699
|
+
try:
|
700
|
+
self.daemonize()
|
701
|
+
self.run()
|
702
|
+
except Exception as e:
|
703
|
+
self.logger.error(f"Failed to start daemon: {e}")
|
704
|
+
self.delpid()
|
705
|
+
sys.exit(1)
|
706
|
+
|
707
|
+
def stop(self):
|
708
|
+
"""Stop the daemon process."""
|
709
|
+
self.logger.info("Stopping daemon...")
|
710
|
+
|
711
|
+
if not os.path.exists(self.pidfile):
|
712
|
+
self.logger.warning("PID file not found. Daemon not running?")
|
713
|
+
return
|
714
|
+
|
715
|
+
try:
|
716
|
+
with open(self.pidfile, "r") as pf:
|
717
|
+
pid = int(pf.read().strip())
|
718
|
+
|
719
|
+
if not psutil.pid_exists(pid):
|
720
|
+
self.logger.warning(
|
721
|
+
f"Process {pid} not found. Removing stale PID file."
|
722
|
+
)
|
723
|
+
self.delpid()
|
724
|
+
return
|
725
|
+
|
726
|
+
# Send termination signal
|
727
|
+
self.logger.info(f"Sending termination signal to process {pid}")
|
728
|
+
os.kill(pid, signal.SIGTERM)
|
729
|
+
|
730
|
+
# Wait for process to terminate
|
731
|
+
timeout = 10 # seconds
|
732
|
+
start_time = time.time()
|
733
|
+
while (psutil.pid_exists(pid) and
|
734
|
+
time.time() - start_time < timeout):
|
735
|
+
time.sleep(0.5)
|
736
|
+
|
737
|
+
# Force kill if still running
|
738
|
+
if psutil.pid_exists(pid):
|
739
|
+
self.logger.warning(
|
740
|
+
f"Process {pid} did not terminate after {timeout}s. "
|
741
|
+
f"Forcing termination."
|
742
|
+
)
|
743
|
+
os.kill(pid, signal.SIGKILL)
|
744
|
+
|
745
|
+
self.delpid()
|
746
|
+
self.logger.info("Daemon stopped")
|
747
|
+
|
748
|
+
except (IOError, ValueError) as err:
|
749
|
+
self.logger.error(f"Error reading PID file: {err}")
|
750
|
+
self.delpid()
|
751
|
+
except Exception as err:
|
752
|
+
self.logger.error(f"Error stopping daemon: {err}")
|
753
|
+
sys.exit(1)
|
754
|
+
|
755
|
+
def restart(self):
|
756
|
+
"""Restart the daemon process."""
|
757
|
+
self.logger.info("Restarting daemon...")
|
758
|
+
self.stop()
|
759
|
+
time.sleep(2) # Give it time to fully stop
|
760
|
+
self.start()
|
761
|
+
|
762
|
+
def run(self):
|
763
|
+
"""Run the worker daemon loop with signal handling."""
|
764
|
+
# Set up signal handlers
|
765
|
+
signal.signal(signal.SIGTERM, self._handle_signal)
|
766
|
+
signal.signal(signal.SIGINT, self._handle_signal)
|
767
|
+
signal.signal(signal.SIGHUP, self._handle_signal)
|
768
|
+
|
769
|
+
try:
|
770
|
+
self.logger.info("Starting worker's main processing loop...")
|
771
|
+
self.worker.loop() # This now contains the infinite loop
|
772
|
+
except SystemExit:
|
773
|
+
self.logger.info("Daemon run loop exited.") # Normal exit via sys.exit
|
774
|
+
except Exception as e:
|
775
|
+
self.logger.critical(f"Critical error in daemon run loop: {e}", exc_info=True)
|
776
|
+
self.delpid()
|
777
|
+
sys.exit(1) # Ensure daemon exits on critical failure
|
778
|
+
|
779
|
+
def _handle_signal(self, signum, frame):
|
780
|
+
"""Handle termination signals for clean shutdown.
|
781
|
+
|
782
|
+
Args:
|
783
|
+
signum (int): Signal number received
|
784
|
+
frame (frame): Current stack frame
|
785
|
+
"""
|
786
|
+
signame = signal.Signals(signum).name
|
787
|
+
self.logger.info(f"Received signal {signame}. Shutting down...")
|
788
|
+
self.delpid()
|
789
|
+
sys.exit(0)
|