meerschaum 2.2.6__py3-none-any.whl → 2.2.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__main__.py +10 -5
- meerschaum/_internal/entry.py +13 -13
- meerschaum/_internal/shell/Shell.py +26 -22
- meerschaum/_internal/shell/updates.py +175 -0
- meerschaum/actions/register.py +19 -5
- meerschaum/actions/sync.py +3 -3
- meerschaum/actions/upgrade.py +28 -36
- meerschaum/api/routes/_pipes.py +20 -20
- meerschaum/config/_formatting.py +1 -0
- meerschaum/config/_paths.py +4 -0
- meerschaum/config/_shell.py +78 -66
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +1 -0
- meerschaum/connectors/api/_misc.py +1 -1
- meerschaum/connectors/api/_request.py +13 -9
- meerschaum/core/Pipe/_sync.py +3 -0
- meerschaum/utils/daemon/Daemon.py +88 -129
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +14 -5
- meerschaum/utils/daemon/RotatingFile.py +8 -1
- meerschaum/utils/daemon/__init__.py +28 -21
- meerschaum/utils/formatting/__init__.py +81 -36
- meerschaum/utils/formatting/_jobs.py +47 -9
- meerschaum/utils/packages/__init__.py +21 -15
- meerschaum/utils/prompt.py +5 -0
- meerschaum/utils/schedule.py +21 -15
- {meerschaum-2.2.6.dist-info → meerschaum-2.2.7.dist-info}/METADATA +1 -1
- {meerschaum-2.2.6.dist-info → meerschaum-2.2.7.dist-info}/RECORD +33 -32
- {meerschaum-2.2.6.dist-info → meerschaum-2.2.7.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.2.7.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.2.7.dist-info}/WHEEL +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.2.7.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.2.7.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.2.7.dist-info}/zip-safe +0 -0
@@ -17,10 +17,17 @@ import time
|
|
17
17
|
import traceback
|
18
18
|
from functools import partial
|
19
19
|
from datetime import datetime, timezone
|
20
|
-
|
20
|
+
|
21
|
+
import meerschaum as mrsm
|
22
|
+
from meerschaum.utils.typing import (
|
23
|
+
Optional, Dict, Any, SuccessTuple, Callable, List, Union,
|
24
|
+
is_success_tuple,
|
25
|
+
)
|
21
26
|
from meerschaum.config import get_config
|
22
27
|
from meerschaum.config.static import STATIC_CONFIG
|
23
|
-
from meerschaum.config._paths import
|
28
|
+
from meerschaum.config._paths import (
|
29
|
+
DAEMON_RESOURCES_PATH, LOGS_RESOURCES_PATH, DAEMON_ERROR_LOG_PATH,
|
30
|
+
)
|
24
31
|
from meerschaum.config._patch import apply_patch_to_config
|
25
32
|
from meerschaum.utils.warnings import warn, error
|
26
33
|
from meerschaum.utils.packages import attempt_import
|
@@ -30,6 +37,9 @@ from meerschaum.utils.daemon.RotatingFile import RotatingFile
|
|
30
37
|
from meerschaum.utils.threading import RepeatTimer
|
31
38
|
from meerschaum.__main__ import _close_pools
|
32
39
|
|
40
|
+
_daemons = []
|
41
|
+
_results = {}
|
42
|
+
|
33
43
|
class Daemon:
|
34
44
|
"""
|
35
45
|
Daemonize Python functions into background processes.
|
@@ -94,7 +104,7 @@ class Daemon:
|
|
94
104
|
)
|
95
105
|
if 'target' not in self.__dict__:
|
96
106
|
if target is None:
|
97
|
-
error(
|
107
|
+
error("Cannot create a Daemon without a target.")
|
98
108
|
self.target = target
|
99
109
|
if 'target_args' not in self.__dict__:
|
100
110
|
self.target_args = target_args if target_args is not None else []
|
@@ -113,16 +123,16 @@ class Daemon:
|
|
113
123
|
self._properties = properties
|
114
124
|
if self._properties is None:
|
115
125
|
self._properties = {}
|
116
|
-
self._properties.update({'label'
|
126
|
+
self._properties.update({'label': self.label})
|
117
127
|
### Instantiate the process and if it doesn't exist, make sure the PID is removed.
|
118
128
|
_ = self.process
|
119
129
|
|
120
130
|
|
121
131
|
def _run_exit(
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
132
|
+
self,
|
133
|
+
keep_daemon_output: bool = True,
|
134
|
+
allow_dirty_run: bool = False,
|
135
|
+
) -> Any:
|
126
136
|
"""Run the daemon's target function.
|
127
137
|
NOTE: This WILL EXIT the parent process!
|
128
138
|
|
@@ -130,7 +140,7 @@ class Daemon:
|
|
130
140
|
----------
|
131
141
|
keep_daemon_output: bool, default True
|
132
142
|
If `False`, delete the daemon's output directory upon exiting.
|
133
|
-
|
143
|
+
|
134
144
|
allow_dirty_run, bool, default False:
|
135
145
|
If `True`, run the daemon, even if the `daemon_id` directory exists.
|
136
146
|
This option is dangerous because if the same `daemon_id` runs twice,
|
@@ -141,31 +151,34 @@ class Daemon:
|
|
141
151
|
Nothing — this will exit the parent process.
|
142
152
|
"""
|
143
153
|
import platform, sys, os, traceback
|
144
|
-
from meerschaum.config._paths import DAEMON_ERROR_LOG_PATH
|
145
154
|
from meerschaum.utils.warnings import warn
|
146
155
|
from meerschaum.config import get_config
|
147
156
|
daemon = attempt_import('daemon')
|
148
157
|
lines = get_config('jobs', 'terminal', 'lines')
|
149
|
-
columns = get_config('jobs','terminal', 'columns')
|
158
|
+
columns = get_config('jobs', 'terminal', 'columns')
|
150
159
|
|
151
160
|
if platform.system() == 'Windows':
|
152
161
|
return False, "Windows is no longer supported."
|
153
162
|
|
154
163
|
self._setup(allow_dirty_run)
|
155
164
|
|
165
|
+
### NOTE: The SIGINT handler has been removed so that child processes may handle
|
166
|
+
### KeyboardInterrupts themselves.
|
167
|
+
### The previous aggressive approach was redundant because of the SIGTERM handler.
|
156
168
|
self._daemon_context = daemon.DaemonContext(
|
157
|
-
pidfile
|
158
|
-
stdout
|
159
|
-
stderr
|
160
|
-
working_directory
|
161
|
-
detach_process
|
162
|
-
files_preserve
|
163
|
-
signal_map
|
164
|
-
signal.SIGINT: self._handle_interrupt,
|
169
|
+
pidfile=self.pid_lock,
|
170
|
+
stdout=self.rotating_log,
|
171
|
+
stderr=self.rotating_log,
|
172
|
+
working_directory=os.getcwd(),
|
173
|
+
detach_process=True,
|
174
|
+
files_preserve=list(self.rotating_log.subfile_objects.values()),
|
175
|
+
signal_map={
|
165
176
|
signal.SIGTERM: self._handle_sigterm,
|
166
177
|
},
|
167
178
|
)
|
168
179
|
|
180
|
+
_daemons.append(self)
|
181
|
+
|
169
182
|
log_refresh_seconds = get_config('jobs', 'logs', 'refresh_files_seconds')
|
170
183
|
self._log_refresh_timer = RepeatTimer(
|
171
184
|
log_refresh_seconds,
|
@@ -177,53 +190,52 @@ class Daemon:
|
|
177
190
|
with self._daemon_context:
|
178
191
|
os.environ[STATIC_CONFIG['environment']['daemon_id']] = self.daemon_id
|
179
192
|
self.rotating_log.refresh_files(start_interception=True)
|
193
|
+
result = None
|
180
194
|
try:
|
181
195
|
with open(self.pid_path, 'w+', encoding='utf-8') as f:
|
182
196
|
f.write(str(os.getpid()))
|
183
197
|
|
184
198
|
self._log_refresh_timer.start()
|
199
|
+
self.properties['result'] = None
|
200
|
+
self.write_properties()
|
185
201
|
result = self.target(*self.target_args, **self.target_kw)
|
186
202
|
self.properties['result'] = result
|
203
|
+
except (BrokenPipeError, KeyboardInterrupt, SystemExit):
|
204
|
+
pass
|
187
205
|
except Exception as e:
|
188
|
-
warn(
|
206
|
+
warn(
|
207
|
+
f"Exception in daemon target function: {traceback.format_exc()}",
|
208
|
+
)
|
189
209
|
result = e
|
190
210
|
finally:
|
211
|
+
_results[self.daemon_id] = result
|
212
|
+
|
213
|
+
if keep_daemon_output:
|
214
|
+
self._capture_process_timestamp('ended')
|
215
|
+
else:
|
216
|
+
self.cleanup()
|
217
|
+
|
191
218
|
self._log_refresh_timer.cancel()
|
192
|
-
self.rotating_log.close()
|
193
219
|
if self.pid is None and self.pid_path.exists():
|
194
220
|
self.pid_path.unlink()
|
195
221
|
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
222
|
+
if is_success_tuple(result):
|
223
|
+
try:
|
224
|
+
mrsm.pprint(result)
|
225
|
+
except BrokenPipeError:
|
226
|
+
pass
|
200
227
|
|
201
|
-
return result
|
202
228
|
except Exception as e:
|
203
229
|
daemon_error = traceback.format_exc()
|
204
230
|
with open(DAEMON_ERROR_LOG_PATH, 'a+', encoding='utf-8') as f:
|
205
231
|
f.write(daemon_error)
|
206
232
|
warn(f"Encountered an error while running the daemon '{self}':\n{daemon_error}")
|
207
|
-
finally:
|
208
|
-
self._cleanup_on_exit()
|
209
|
-
|
210
|
-
def _cleanup_on_exit(self):
|
211
|
-
"""Perform cleanup operations when the daemon exits."""
|
212
|
-
try:
|
213
|
-
self._log_refresh_timer.cancel()
|
214
|
-
self.rotating_log.close()
|
215
|
-
if self.pid is None and self.pid_path.exists():
|
216
|
-
self.pid_path.unlink()
|
217
|
-
self._capture_process_timestamp('ended')
|
218
|
-
except Exception as e:
|
219
|
-
warn(f"Error during daemon cleanup: {e}")
|
220
|
-
|
221
233
|
|
222
234
|
def _capture_process_timestamp(
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
235
|
+
self,
|
236
|
+
process_key: str,
|
237
|
+
write_properties: bool = True,
|
238
|
+
) -> None:
|
227
239
|
"""
|
228
240
|
Record the current timestamp to the parameters `process:<process_key>`.
|
229
241
|
|
@@ -247,13 +259,12 @@ class Daemon:
|
|
247
259
|
if write_properties:
|
248
260
|
self.write_properties()
|
249
261
|
|
250
|
-
|
251
262
|
def run(
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
263
|
+
self,
|
264
|
+
keep_daemon_output: bool = True,
|
265
|
+
allow_dirty_run: bool = False,
|
266
|
+
debug: bool = False,
|
267
|
+
) -> SuccessTuple:
|
257
268
|
"""Run the daemon as a child process and continue executing the parent.
|
258
269
|
|
259
270
|
Parameters
|
@@ -288,7 +299,7 @@ class Daemon:
|
|
288
299
|
"from meerschaum.utils.daemon import Daemon; "
|
289
300
|
+ f"daemon = Daemon(daemon_id='{self.daemon_id}'); "
|
290
301
|
+ f"daemon._run_exit(keep_daemon_output={keep_daemon_output}, "
|
291
|
-
+
|
302
|
+
+ "allow_dirty_run=True)"
|
292
303
|
)
|
293
304
|
env = dict(os.environ)
|
294
305
|
env['MRSM_NOASK'] = 'true'
|
@@ -298,11 +309,9 @@ class Daemon:
|
|
298
309
|
if _launch_success_bool
|
299
310
|
else f"Failed to start daemon '{self.daemon_id}'."
|
300
311
|
)
|
301
|
-
self._capture_process_timestamp('began')
|
302
312
|
return _launch_success_bool, msg
|
303
313
|
|
304
|
-
|
305
|
-
def kill(self, timeout: Optional[int] = 8) -> SuccessTuple:
|
314
|
+
def kill(self, timeout: Union[int, float, None] = 8) -> SuccessTuple:
|
306
315
|
"""Forcibly terminate a running daemon.
|
307
316
|
Sends a SIGTERM signal to the process.
|
308
317
|
|
@@ -318,7 +327,6 @@ class Daemon:
|
|
318
327
|
if self.status != 'paused':
|
319
328
|
success, msg = self._send_signal(signal.SIGTERM, timeout=timeout)
|
320
329
|
if success:
|
321
|
-
self._capture_process_timestamp('ended')
|
322
330
|
return success, msg
|
323
331
|
|
324
332
|
if self.status == 'stopped':
|
@@ -332,7 +340,6 @@ class Daemon:
|
|
332
340
|
except Exception as e:
|
333
341
|
return False, f"Failed to kill job {self} with exception: {e}"
|
334
342
|
|
335
|
-
self._capture_process_timestamp('ended')
|
336
343
|
if self.pid_path.exists():
|
337
344
|
try:
|
338
345
|
self.pid_path.unlink()
|
@@ -340,22 +347,18 @@ class Daemon:
|
|
340
347
|
pass
|
341
348
|
return True, "Success"
|
342
349
|
|
343
|
-
|
344
350
|
def quit(self, timeout: Union[int, float, None] = None) -> SuccessTuple:
|
345
351
|
"""Gracefully quit a running daemon."""
|
346
352
|
if self.status == 'paused':
|
347
353
|
return self.kill(timeout)
|
348
354
|
signal_success, signal_msg = self._send_signal(signal.SIGINT, timeout=timeout)
|
349
|
-
if signal_success:
|
350
|
-
self._capture_process_timestamp('ended')
|
351
355
|
return signal_success, signal_msg
|
352
356
|
|
353
|
-
|
354
357
|
def pause(
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
358
|
+
self,
|
359
|
+
timeout: Union[int, float, None] = None,
|
360
|
+
check_timeout_interval: Union[float, int, None] = None,
|
361
|
+
) -> SuccessTuple:
|
359
362
|
"""
|
360
363
|
Pause the daemon if it is running.
|
361
364
|
|
@@ -414,7 +417,6 @@ class Daemon:
|
|
414
417
|
+ ('s' if timeout != 1 else '') + '.'
|
415
418
|
)
|
416
419
|
|
417
|
-
|
418
420
|
def resume(
|
419
421
|
self,
|
420
422
|
timeout: Union[int, float, None] = None,
|
@@ -471,38 +473,6 @@ class Daemon:
|
|
471
473
|
)
|
472
474
|
|
473
475
|
|
474
|
-
def _handle_interrupt(self, signal_number: int, stack_frame: 'frame') -> None:
|
475
|
-
"""
|
476
|
-
Handle `SIGINT` within the Daemon context.
|
477
|
-
This method is injected into the `DaemonContext`.
|
478
|
-
"""
|
479
|
-
from meerschaum.utils.process import signal_handler
|
480
|
-
signal_handler(signal_number, stack_frame)
|
481
|
-
|
482
|
-
self.rotating_log.stop_log_fd_interception(unused_only=False)
|
483
|
-
timer = self.__dict__.get('_log_refresh_timer', None)
|
484
|
-
if timer is not None:
|
485
|
-
timer.cancel()
|
486
|
-
|
487
|
-
daemon_context = self.__dict__.get('_daemon_context', None)
|
488
|
-
if daemon_context is not None:
|
489
|
-
daemon_context.close()
|
490
|
-
|
491
|
-
_close_pools()
|
492
|
-
|
493
|
-
import threading
|
494
|
-
for thread in threading.enumerate():
|
495
|
-
if thread.name == 'MainThread':
|
496
|
-
continue
|
497
|
-
try:
|
498
|
-
if thread.is_alive():
|
499
|
-
stack = traceback.format_stack(sys._current_frames()[thread.ident])
|
500
|
-
thread.join()
|
501
|
-
except Exception as e:
|
502
|
-
warn(traceback.format_exc())
|
503
|
-
raise KeyboardInterrupt()
|
504
|
-
|
505
|
-
|
506
476
|
def _handle_sigterm(self, signal_number: int, stack_frame: 'frame') -> None:
|
507
477
|
"""
|
508
478
|
Handle `SIGTERM` within the `Daemon` context.
|
@@ -522,7 +492,6 @@ class Daemon:
|
|
522
492
|
_close_pools()
|
523
493
|
raise SystemExit(0)
|
524
494
|
|
525
|
-
|
526
495
|
def _send_signal(
|
527
496
|
self,
|
528
497
|
signal_to_send,
|
@@ -578,7 +547,6 @@ class Daemon:
|
|
578
547
|
+ ('s' if timeout != 1 else '') + '.'
|
579
548
|
)
|
580
549
|
|
581
|
-
|
582
550
|
def mkdir_if_not_exists(self, allow_dirty_run: bool = False):
|
583
551
|
"""Create the Daemon's directory.
|
584
552
|
If `allow_dirty_run` is `False` and the directory already exists,
|
@@ -618,7 +586,6 @@ class Daemon:
|
|
618
586
|
return None
|
619
587
|
return self._process
|
620
588
|
|
621
|
-
|
622
589
|
@property
|
623
590
|
def status(self) -> str:
|
624
591
|
"""
|
@@ -633,7 +600,7 @@ class Daemon:
|
|
633
600
|
return 'paused'
|
634
601
|
if self.process.status() == 'zombie':
|
635
602
|
raise psutil.NoSuchProcess(self.process.pid)
|
636
|
-
except psutil.NoSuchProcess:
|
603
|
+
except (psutil.NoSuchProcess, AttributeError):
|
637
604
|
if self.pid_path.exists():
|
638
605
|
try:
|
639
606
|
self.pid_path.unlink()
|
@@ -643,7 +610,6 @@ class Daemon:
|
|
643
610
|
|
644
611
|
return 'running'
|
645
612
|
|
646
|
-
|
647
613
|
@classmethod
|
648
614
|
def _get_path_from_daemon_id(cls, daemon_id: str) -> pathlib.Path:
|
649
615
|
"""
|
@@ -651,7 +617,6 @@ class Daemon:
|
|
651
617
|
"""
|
652
618
|
return DAEMON_RESOURCES_PATH / daemon_id
|
653
619
|
|
654
|
-
|
655
620
|
@property
|
656
621
|
def path(self) -> pathlib.Path:
|
657
622
|
"""
|
@@ -659,7 +624,6 @@ class Daemon:
|
|
659
624
|
"""
|
660
625
|
return self._get_path_from_daemon_id(self.daemon_id)
|
661
626
|
|
662
|
-
|
663
627
|
@classmethod
|
664
628
|
def _get_properties_path_from_daemon_id(cls, daemon_id: str) -> pathlib.Path:
|
665
629
|
"""
|
@@ -667,7 +631,6 @@ class Daemon:
|
|
667
631
|
"""
|
668
632
|
return cls._get_path_from_daemon_id(daemon_id) / 'properties.json'
|
669
633
|
|
670
|
-
|
671
634
|
@property
|
672
635
|
def properties_path(self) -> pathlib.Path:
|
673
636
|
"""
|
@@ -675,7 +638,6 @@ class Daemon:
|
|
675
638
|
"""
|
676
639
|
return self._get_properties_path_from_daemon_id(self.daemon_id)
|
677
640
|
|
678
|
-
|
679
641
|
@property
|
680
642
|
def log_path(self) -> pathlib.Path:
|
681
643
|
"""
|
@@ -683,7 +645,6 @@ class Daemon:
|
|
683
645
|
"""
|
684
646
|
return LOGS_RESOURCES_PATH / (self.daemon_id + '.log')
|
685
647
|
|
686
|
-
|
687
648
|
@property
|
688
649
|
def log_offset_path(self) -> pathlib.Path:
|
689
650
|
"""
|
@@ -691,7 +652,6 @@ class Daemon:
|
|
691
652
|
"""
|
692
653
|
return LOGS_RESOURCES_PATH / ('.' + self.daemon_id + '.log.offset')
|
693
654
|
|
694
|
-
|
695
655
|
@property
|
696
656
|
def rotating_log(self) -> RotatingFile:
|
697
657
|
if '_rotating_log' in self.__dict__:
|
@@ -705,7 +665,6 @@ class Daemon:
|
|
705
665
|
)
|
706
666
|
return self._rotating_log
|
707
667
|
|
708
|
-
|
709
668
|
@property
|
710
669
|
def log_text(self) -> Optional[str]:
|
711
670
|
"""Read the log files and return their contents.
|
@@ -720,7 +679,6 @@ class Daemon:
|
|
720
679
|
)
|
721
680
|
return new_rotating_log.read()
|
722
681
|
|
723
|
-
|
724
682
|
def readlines(self) -> List[str]:
|
725
683
|
"""
|
726
684
|
Read the next log lines, persisting the cursor for later use.
|
@@ -731,7 +689,6 @@ class Daemon:
|
|
731
689
|
self._write_log_offset()
|
732
690
|
return lines
|
733
691
|
|
734
|
-
|
735
692
|
def _read_log_offset(self) -> Tuple[int, int]:
|
736
693
|
"""
|
737
694
|
Return the current log offset cursor.
|
@@ -749,7 +706,6 @@ class Daemon:
|
|
749
706
|
subfile_index, subfile_position = int(cursor_parts[0]), int(cursor_parts[1])
|
750
707
|
return subfile_index, subfile_position
|
751
708
|
|
752
|
-
|
753
709
|
def _write_log_offset(self) -> None:
|
754
710
|
"""
|
755
711
|
Write the current log offset file.
|
@@ -759,7 +715,6 @@ class Daemon:
|
|
759
715
|
subfile_position = self.rotating_log._cursor[1]
|
760
716
|
f.write(f"{subfile_index} {subfile_position}")
|
761
717
|
|
762
|
-
|
763
718
|
@property
|
764
719
|
def pid(self) -> Union[int, None]:
|
765
720
|
"""Read the PID file and return its contents.
|
@@ -777,7 +732,6 @@ class Daemon:
|
|
777
732
|
pid = None
|
778
733
|
return pid
|
779
734
|
|
780
|
-
|
781
735
|
@property
|
782
736
|
def pid_path(self) -> pathlib.Path:
|
783
737
|
"""
|
@@ -785,7 +739,6 @@ class Daemon:
|
|
785
739
|
"""
|
786
740
|
return self.path / 'process.pid'
|
787
741
|
|
788
|
-
|
789
742
|
@property
|
790
743
|
def pid_lock(self) -> 'fasteners.InterProcessLock':
|
791
744
|
"""
|
@@ -798,7 +751,6 @@ class Daemon:
|
|
798
751
|
self._pid_lock = fasteners.InterProcessLock(self.pid_path)
|
799
752
|
return self._pid_lock
|
800
753
|
|
801
|
-
|
802
754
|
@property
|
803
755
|
def pickle_path(self) -> pathlib.Path:
|
804
756
|
"""
|
@@ -806,7 +758,6 @@ class Daemon:
|
|
806
758
|
"""
|
807
759
|
return self.path / 'pickle.pkl'
|
808
760
|
|
809
|
-
|
810
761
|
def read_properties(self) -> Optional[Dict[str, Any]]:
|
811
762
|
"""Read the properties JSON file and return the dictionary."""
|
812
763
|
if not self.properties_path.exists():
|
@@ -817,7 +768,6 @@ class Daemon:
|
|
817
768
|
except Exception as e:
|
818
769
|
return {}
|
819
770
|
|
820
|
-
|
821
771
|
def read_pickle(self) -> Daemon:
|
822
772
|
"""Read a Daemon's pickle file and return the `Daemon`."""
|
823
773
|
import pickle, traceback
|
@@ -837,21 +787,30 @@ class Daemon:
|
|
837
787
|
error(msg)
|
838
788
|
return daemon
|
839
789
|
|
840
|
-
|
841
790
|
@property
|
842
791
|
def properties(self) -> Dict[str, Any]:
|
843
792
|
"""
|
844
793
|
Return the contents of the properties JSON file.
|
845
794
|
"""
|
846
|
-
|
795
|
+
try:
|
796
|
+
_file_properties = self.read_properties()
|
797
|
+
except Exception:
|
798
|
+
traceback.print_exc()
|
799
|
+
_file_properties = {}
|
800
|
+
|
847
801
|
if not self._properties:
|
848
802
|
self._properties = _file_properties
|
803
|
+
|
849
804
|
if self._properties is None:
|
850
805
|
self._properties = {}
|
806
|
+
|
851
807
|
if _file_properties is not None:
|
852
|
-
self._properties = apply_patch_to_config(
|
853
|
-
|
808
|
+
self._properties = apply_patch_to_config(
|
809
|
+
_file_properties,
|
810
|
+
self._properties,
|
811
|
+
)
|
854
812
|
|
813
|
+
return self._properties
|
855
814
|
|
856
815
|
@property
|
857
816
|
def hidden(self) -> bool:
|
@@ -860,12 +819,14 @@ class Daemon:
|
|
860
819
|
"""
|
861
820
|
return self.daemon_id.startswith('_') or self.daemon_id.startswith('.')
|
862
821
|
|
863
|
-
|
864
822
|
def write_properties(self) -> SuccessTuple:
|
865
823
|
"""Write the properties dictionary to the properties JSON file
|
866
824
|
(only if self.properties exists).
|
867
825
|
"""
|
868
|
-
success, msg =
|
826
|
+
success, msg = (
|
827
|
+
False,
|
828
|
+
f"No properties to write for daemon '{self.daemon_id}'."
|
829
|
+
)
|
869
830
|
if self.properties is not None:
|
870
831
|
try:
|
871
832
|
self.path.mkdir(parents=True, exist_ok=True)
|
@@ -876,7 +837,6 @@ class Daemon:
|
|
876
837
|
success, msg = False, str(e)
|
877
838
|
return success, msg
|
878
839
|
|
879
|
-
|
880
840
|
def write_pickle(self) -> SuccessTuple:
|
881
841
|
"""Write the pickle file for the daemon."""
|
882
842
|
import pickle, traceback
|
@@ -892,9 +852,9 @@ class Daemon:
|
|
892
852
|
|
893
853
|
|
894
854
|
def _setup(
|
895
|
-
|
896
|
-
|
897
|
-
|
855
|
+
self,
|
856
|
+
allow_dirty_run: bool = False,
|
857
|
+
) -> None:
|
898
858
|
"""
|
899
859
|
Update properties before starting the Daemon.
|
900
860
|
"""
|
@@ -918,7 +878,6 @@ class Daemon:
|
|
918
878
|
if not _write_pickle_success_tuple[0]:
|
919
879
|
error(_write_pickle_success_tuple[1])
|
920
880
|
|
921
|
-
|
922
881
|
def cleanup(self, keep_logs: bool = False) -> SuccessTuple:
|
923
882
|
"""
|
924
883
|
Remove a daemon's directory after execution.
|
@@ -9,10 +9,12 @@ Intercept OS-level file descriptors.
|
|
9
9
|
import os
|
10
10
|
import select
|
11
11
|
import traceback
|
12
|
+
import errno
|
12
13
|
from threading import Event
|
13
14
|
from datetime import datetime
|
14
15
|
from meerschaum.utils.typing import Callable
|
15
16
|
from meerschaum.utils.warnings import warn
|
17
|
+
from meerschaum.config.paths import DAEMON_ERROR_LOG_PATH
|
16
18
|
|
17
19
|
FD_CLOSED: int = 9
|
18
20
|
STOP_READING_FD_EVENT: Event = Event()
|
@@ -65,8 +67,13 @@ class FileDescriptorInterceptor:
|
|
65
67
|
except BlockingIOError:
|
66
68
|
continue
|
67
69
|
except OSError as e:
|
68
|
-
|
69
|
-
|
70
|
+
if e.errno == errno.EBADF:
|
71
|
+
### File descriptor is closed.
|
72
|
+
pass
|
73
|
+
elif e.errno == errno.EINTR:
|
74
|
+
continue # Interrupted system call, just try again
|
75
|
+
else:
|
76
|
+
warn(f"OSError in FileDescriptorInterceptor: {e}")
|
70
77
|
break
|
71
78
|
|
72
79
|
try:
|
@@ -86,9 +93,11 @@ class FileDescriptorInterceptor:
|
|
86
93
|
else data.replace(b'\n', b'\n' + injected_bytes)
|
87
94
|
)
|
88
95
|
os.write(self.new_file_descriptor, modified_data)
|
89
|
-
except
|
90
|
-
|
91
|
-
|
96
|
+
except (BrokenPipeError, OSError):
|
97
|
+
break
|
98
|
+
except Exception:
|
99
|
+
with open(DAEMON_ERROR_LOG_PATH, 'a+', encoding='utf-8') as f:
|
100
|
+
f.write(traceback.format_exc())
|
92
101
|
break
|
93
102
|
|
94
103
|
|
@@ -371,6 +371,9 @@ class RotatingFile(io.IOBase):
|
|
371
371
|
self._current_file_obj.write(data)
|
372
372
|
if suffix_str:
|
373
373
|
self._current_file_obj.write(suffix_str)
|
374
|
+
except BrokenPipeError:
|
375
|
+
warn("BrokenPipeError encountered. The daemon may have been terminated.")
|
376
|
+
return
|
374
377
|
except Exception as e:
|
375
378
|
warn(f"Failed to write to subfile:\n{traceback.format_exc()}")
|
376
379
|
self.flush()
|
@@ -581,10 +584,14 @@ class RotatingFile(io.IOBase):
|
|
581
584
|
if self.redirect_streams:
|
582
585
|
try:
|
583
586
|
sys.stdout.flush()
|
587
|
+
except BrokenPipeError:
|
588
|
+
pass
|
584
589
|
except Exception as e:
|
585
590
|
warn(f"Failed to flush STDOUT:\n{traceback.format_exc()}")
|
586
591
|
try:
|
587
592
|
sys.stderr.flush()
|
593
|
+
except BrokenPipeError:
|
594
|
+
pass
|
588
595
|
except Exception as e:
|
589
596
|
warn(f"Failed to flush STDERR:\n{traceback.format_exc()}")
|
590
597
|
|
@@ -596,7 +603,6 @@ class RotatingFile(io.IOBase):
|
|
596
603
|
if not self.write_timestamps:
|
597
604
|
return
|
598
605
|
|
599
|
-
threads = self.__dict__.get('_interceptor_threads', [])
|
600
606
|
self._stdout_interceptor = FileDescriptorInterceptor(
|
601
607
|
sys.stdout.fileno(),
|
602
608
|
self.get_timestamp_prefix_str,
|
@@ -639,6 +645,7 @@ class RotatingFile(io.IOBase):
|
|
639
645
|
"""
|
640
646
|
if not self.write_timestamps:
|
641
647
|
return
|
648
|
+
|
642
649
|
interceptors = self.__dict__.get('_interceptors', [])
|
643
650
|
interceptor_threads = self.__dict__.get('_interceptor_threads', [])
|
644
651
|
|