meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. meerschaum/__init__.py +6 -1
  2. meerschaum/__main__.py +9 -9
  3. meerschaum/_internal/arguments/__init__.py +1 -1
  4. meerschaum/_internal/arguments/_parse_arguments.py +72 -6
  5. meerschaum/_internal/arguments/_parser.py +45 -15
  6. meerschaum/_internal/docs/index.py +265 -8
  7. meerschaum/_internal/entry.py +167 -37
  8. meerschaum/_internal/shell/Shell.py +290 -99
  9. meerschaum/_internal/shell/updates.py +175 -0
  10. meerschaum/actions/__init__.py +29 -17
  11. meerschaum/actions/api.py +12 -12
  12. meerschaum/actions/attach.py +113 -0
  13. meerschaum/actions/copy.py +68 -41
  14. meerschaum/actions/delete.py +112 -50
  15. meerschaum/actions/edit.py +3 -3
  16. meerschaum/actions/install.py +40 -32
  17. meerschaum/actions/pause.py +44 -27
  18. meerschaum/actions/register.py +19 -5
  19. meerschaum/actions/restart.py +107 -0
  20. meerschaum/actions/show.py +130 -159
  21. meerschaum/actions/start.py +161 -100
  22. meerschaum/actions/stop.py +78 -42
  23. meerschaum/actions/sync.py +3 -3
  24. meerschaum/actions/upgrade.py +28 -36
  25. meerschaum/api/_events.py +25 -1
  26. meerschaum/api/_oauth2.py +2 -0
  27. meerschaum/api/_websockets.py +2 -2
  28. meerschaum/api/dash/callbacks/jobs.py +36 -44
  29. meerschaum/api/dash/jobs.py +89 -78
  30. meerschaum/api/routes/__init__.py +1 -0
  31. meerschaum/api/routes/_actions.py +148 -17
  32. meerschaum/api/routes/_jobs.py +407 -0
  33. meerschaum/api/routes/_pipes.py +25 -25
  34. meerschaum/config/_default.py +1 -0
  35. meerschaum/config/_formatting.py +1 -0
  36. meerschaum/config/_jobs.py +1 -1
  37. meerschaum/config/_paths.py +11 -0
  38. meerschaum/config/_shell.py +84 -67
  39. meerschaum/config/_version.py +1 -1
  40. meerschaum/config/static/__init__.py +18 -0
  41. meerschaum/connectors/Connector.py +13 -7
  42. meerschaum/connectors/__init__.py +28 -15
  43. meerschaum/connectors/api/APIConnector.py +27 -1
  44. meerschaum/connectors/api/_actions.py +71 -6
  45. meerschaum/connectors/api/_jobs.py +368 -0
  46. meerschaum/connectors/api/_misc.py +1 -1
  47. meerschaum/connectors/api/_pipes.py +85 -84
  48. meerschaum/connectors/api/_request.py +13 -9
  49. meerschaum/connectors/parse.py +27 -15
  50. meerschaum/core/Pipe/_bootstrap.py +16 -8
  51. meerschaum/core/Pipe/_sync.py +3 -0
  52. meerschaum/jobs/_Executor.py +69 -0
  53. meerschaum/jobs/_Job.py +899 -0
  54. meerschaum/jobs/__init__.py +396 -0
  55. meerschaum/jobs/systemd.py +694 -0
  56. meerschaum/plugins/__init__.py +97 -12
  57. meerschaum/utils/daemon/Daemon.py +352 -147
  58. meerschaum/utils/daemon/FileDescriptorInterceptor.py +19 -10
  59. meerschaum/utils/daemon/RotatingFile.py +22 -8
  60. meerschaum/utils/daemon/StdinFile.py +121 -0
  61. meerschaum/utils/daemon/__init__.py +42 -27
  62. meerschaum/utils/daemon/_names.py +15 -13
  63. meerschaum/utils/formatting/__init__.py +83 -37
  64. meerschaum/utils/formatting/_jobs.py +146 -55
  65. meerschaum/utils/formatting/_shell.py +6 -0
  66. meerschaum/utils/misc.py +41 -22
  67. meerschaum/utils/packages/__init__.py +21 -15
  68. meerschaum/utils/packages/_packages.py +9 -6
  69. meerschaum/utils/process.py +9 -9
  70. meerschaum/utils/prompt.py +20 -7
  71. meerschaum/utils/schedule.py +21 -15
  72. meerschaum/utils/venv/__init__.py +2 -2
  73. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/METADATA +22 -25
  74. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/RECORD +80 -70
  75. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/WHEEL +1 -1
  76. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/LICENSE +0 -0
  77. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/NOTICE +0 -0
  78. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/entry_points.txt +0 -0
  79. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/top_level.txt +0 -0
  80. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/zip-safe +0 -0
@@ -8,6 +8,7 @@ Manage running daemons via the Daemon class.
8
8
 
9
9
  from __future__ import annotations
10
10
  import os
11
+ import importlib
11
12
  import pathlib
12
13
  import json
13
14
  import shutil
@@ -17,22 +18,59 @@ import time
17
18
  import traceback
18
19
  from functools import partial
19
20
  from datetime import datetime, timezone
20
- from meerschaum.utils.typing import Optional, Dict, Any, SuccessTuple, Callable, List, Union
21
+
22
+ import meerschaum as mrsm
23
+ from meerschaum.utils.typing import (
24
+ Optional, Dict, Any, SuccessTuple, Callable, List, Union,
25
+ is_success_tuple, Tuple,
26
+ )
21
27
  from meerschaum.config import get_config
22
28
  from meerschaum.config.static import STATIC_CONFIG
23
- from meerschaum.config._paths import DAEMON_RESOURCES_PATH, LOGS_RESOURCES_PATH
29
+ from meerschaum.config._paths import (
30
+ DAEMON_RESOURCES_PATH, LOGS_RESOURCES_PATH, DAEMON_ERROR_LOG_PATH,
31
+ )
24
32
  from meerschaum.config._patch import apply_patch_to_config
25
33
  from meerschaum.utils.warnings import warn, error
26
34
  from meerschaum.utils.packages import attempt_import
27
35
  from meerschaum.utils.venv import venv_exec
28
36
  from meerschaum.utils.daemon._names import get_new_daemon_name
29
37
  from meerschaum.utils.daemon.RotatingFile import RotatingFile
38
+ from meerschaum.utils.daemon.StdinFile import StdinFile
30
39
  from meerschaum.utils.threading import RepeatTimer
31
40
  from meerschaum.__main__ import _close_pools
32
41
 
42
+ _daemons = []
43
+ _results = {}
44
+
33
45
  class Daemon:
34
46
  """
35
47
  Daemonize Python functions into background processes.
48
+
49
+ Examples
50
+ --------
51
+ >>> import meerschaum as mrsm
52
+ >>> from meerschaum.utils.daemons import Daemon
53
+ >>> daemon = Daemon(print, ('hi',))
54
+ >>> success, msg = daemon.run()
55
+ >>> print(daemon.log_text)
56
+
57
+ 2024-07-29 18:03 | hi
58
+ 2024-07-29 18:03 |
59
+ >>> daemon.run(allow_dirty_run=True)
60
+ >>> print(daemon.log_text)
61
+
62
+ 2024-07-29 18:03 | hi
63
+ 2024-07-29 18:03 |
64
+ 2024-07-29 18:05 | hi
65
+ 2024-07-29 18:05 |
66
+ >>> mrsm.pprint(daemon.properties)
67
+ {
68
+ 'label': 'print',
69
+ 'target': {'name': 'print', 'module': 'builtins', 'args': ['hi'], 'kw': {}},
70
+ 'result': None,
71
+ 'process': {'ended': '2024-07-29T18:03:33.752806'}
72
+ }
73
+
36
74
  """
37
75
 
38
76
  def __new__(
@@ -51,10 +89,57 @@ class Daemon:
51
89
  instance = instance.read_pickle()
52
90
  return instance
53
91
 
92
+ @classmethod
93
+ def from_properties_file(cls, daemon_id: str) -> Daemon:
94
+ """
95
+ Return a Daemon from a properties dictionary.
96
+ """
97
+ properties_path = cls._get_properties_path_from_daemon_id(daemon_id)
98
+ if not properties_path.exists():
99
+ raise OSError(f"Properties file '{properties_path}' does not exist.")
100
+
101
+ try:
102
+ with open(properties_path, 'r', encoding='utf-8') as f:
103
+ properties = json.load(f)
104
+ except Exception:
105
+ properties = {}
106
+
107
+ if not properties:
108
+ raise ValueError(f"No properties could be read for daemon '{daemon_id}'.")
109
+
110
+ daemon_id = properties_path.parent.name
111
+ target_cf = properties.get('target', {})
112
+ target_module_name = target_cf.get('module', None)
113
+ target_function_name = target_cf.get('name', None)
114
+ target_args = target_cf.get('args', None)
115
+ target_kw = target_cf.get('kw', None)
116
+ label = properties.get('label', None)
117
+
118
+ if None in [
119
+ target_module_name,
120
+ target_function_name,
121
+ target_args,
122
+ target_kw,
123
+ ]:
124
+ raise ValueError("Missing target function information.")
125
+
126
+ target_module = importlib.import_module(target_module_name)
127
+ target_function = getattr(target_module, target_function_name)
128
+
129
+ return Daemon(
130
+ daemon_id=daemon_id,
131
+ target=target_function,
132
+ target_args=target_args,
133
+ target_kw=target_kw,
134
+ properties=properties,
135
+ label=label,
136
+ )
137
+
138
+
54
139
  def __init__(
55
140
  self,
56
141
  target: Optional[Callable[[Any], Any]] = None,
57
- target_args: Optional[List[str]] = None,
142
+ target_args: Union[List[Any], Tuple[Any], None] = None,
58
143
  target_kw: Optional[Dict[str, Any]] = None,
59
144
  daemon_id: Optional[str] = None,
60
145
  label: Optional[str] = None,
@@ -66,7 +151,7 @@ class Daemon:
66
151
  target: Optional[Callable[[Any], Any]], default None,
67
152
  The function to execute in a child process.
68
153
 
69
- target_args: Optional[List[str]], default None
154
+ target_args: Union[List[Any], Tuple[Any], None], default None
70
155
  Positional arguments to pass to the target function.
71
156
 
72
157
  target_kw: Optional[Dict[str, Any]], default None
@@ -81,25 +166,54 @@ class Daemon:
81
166
  Label string to help identifiy a daemon.
82
167
  If `None`, use the function name instead.
83
168
 
84
- propterties: Optional[Dict[str, Any]], default None
169
+ properties: Optional[Dict[str, Any]], default None
85
170
  Override reading from the properties JSON by providing an existing dictionary.
86
171
  """
87
172
  _pickle = self.__dict__.get('_pickle', False)
88
173
  if daemon_id is not None:
89
174
  self.daemon_id = daemon_id
90
175
  if not self.pickle_path.exists() and not target and ('target' not in self.__dict__):
91
- raise Exception(
92
- f"Daemon '{self.daemon_id}' does not exist. "
93
- + "Pass a target to create a new Daemon."
94
- )
176
+
177
+ if not self.properties_path.exists():
178
+ raise Exception(
179
+ f"Daemon '{self.daemon_id}' does not exist. "
180
+ + "Pass a target to create a new Daemon."
181
+ )
182
+
183
+ try:
184
+ new_daemon = self.from_properties_file(daemon_id)
185
+ except Exception:
186
+ new_daemon = None
187
+
188
+ if new_daemon is not None:
189
+ new_daemon.write_pickle()
190
+ target = new_daemon.target
191
+ target_args = new_daemon.target_args
192
+ target_kw = new_daemon.target_kw
193
+ label = new_daemon.label
194
+ self._properties = new_daemon.properties
195
+ else:
196
+ try:
197
+ self.properties_path.unlink()
198
+ except Exception:
199
+ pass
200
+
201
+ raise Exception(
202
+ f"Could not recover daemon '{self.daemon_id}' "
203
+ + "from its properties file."
204
+ )
205
+
95
206
  if 'target' not in self.__dict__:
96
207
  if target is None:
97
- error(f"Cannot create a Daemon without a target.")
208
+ error("Cannot create a Daemon without a target.")
98
209
  self.target = target
99
- if 'target_args' not in self.__dict__:
100
- self.target_args = target_args if target_args is not None else []
101
- if 'target_kw' not in self.__dict__:
102
- self.target_kw = target_kw if target_kw is not None else {}
210
+
211
+ ### NOTE: We have to check self.__dict__ in case we un-pickling.
212
+ if '_target_args' not in self.__dict__:
213
+ self._target_args = target_args
214
+ if '_target_kw' not in self.__dict__:
215
+ self._target_kw = target_kw
216
+
103
217
  if 'label' not in self.__dict__:
104
218
  if label is None:
105
219
  label = (
@@ -113,16 +227,16 @@ class Daemon:
113
227
  self._properties = properties
114
228
  if self._properties is None:
115
229
  self._properties = {}
116
- self._properties.update({'label' : self.label})
230
+ self._properties.update({'label': self.label})
117
231
  ### Instantiate the process and if it doesn't exist, make sure the PID is removed.
118
232
  _ = self.process
119
233
 
120
234
 
121
235
  def _run_exit(
122
- self,
123
- keep_daemon_output: bool = True,
124
- allow_dirty_run: bool = False,
125
- ) -> Any:
236
+ self,
237
+ keep_daemon_output: bool = True,
238
+ allow_dirty_run: bool = False,
239
+ ) -> Any:
126
240
  """Run the daemon's target function.
127
241
  NOTE: This WILL EXIT the parent process!
128
242
 
@@ -130,7 +244,7 @@ class Daemon:
130
244
  ----------
131
245
  keep_daemon_output: bool, default True
132
246
  If `False`, delete the daemon's output directory upon exiting.
133
-
247
+
134
248
  allow_dirty_run, bool, default False:
135
249
  If `True`, run the daemon, even if the `daemon_id` directory exists.
136
250
  This option is dangerous because if the same `daemon_id` runs twice,
@@ -141,31 +255,34 @@ class Daemon:
141
255
  Nothing — this will exit the parent process.
142
256
  """
143
257
  import platform, sys, os, traceback
144
- from meerschaum.config._paths import DAEMON_ERROR_LOG_PATH
145
258
  from meerschaum.utils.warnings import warn
146
259
  from meerschaum.config import get_config
147
260
  daemon = attempt_import('daemon')
148
261
  lines = get_config('jobs', 'terminal', 'lines')
149
- columns = get_config('jobs','terminal', 'columns')
262
+ columns = get_config('jobs', 'terminal', 'columns')
150
263
 
151
264
  if platform.system() == 'Windows':
152
265
  return False, "Windows is no longer supported."
153
266
 
154
267
  self._setup(allow_dirty_run)
155
268
 
269
+ ### NOTE: The SIGINT handler has been removed so that child processes may handle
270
+ ### KeyboardInterrupts themselves.
271
+ ### The previous aggressive approach was redundant because of the SIGTERM handler.
156
272
  self._daemon_context = daemon.DaemonContext(
157
- pidfile = self.pid_lock,
158
- stdout = self.rotating_log,
159
- stderr = self.rotating_log,
160
- working_directory = os.getcwd(),
161
- detach_process = True,
162
- files_preserve = list(self.rotating_log.subfile_objects.values()),
163
- signal_map = {
164
- signal.SIGINT: self._handle_interrupt,
273
+ pidfile=self.pid_lock,
274
+ stdout=self.rotating_log,
275
+ stderr=self.rotating_log,
276
+ working_directory=os.getcwd(),
277
+ detach_process=True,
278
+ files_preserve=list(self.rotating_log.subfile_objects.values()),
279
+ signal_map={
165
280
  signal.SIGTERM: self._handle_sigterm,
166
281
  },
167
282
  )
168
283
 
284
+ _daemons.append(self)
285
+
169
286
  log_refresh_seconds = get_config('jobs', 'logs', 'refresh_files_seconds')
170
287
  self._log_refresh_timer = RepeatTimer(
171
288
  log_refresh_seconds,
@@ -175,55 +292,56 @@ class Daemon:
175
292
  try:
176
293
  os.environ['LINES'], os.environ['COLUMNS'] = str(int(lines)), str(int(columns))
177
294
  with self._daemon_context:
295
+ sys.stdin = self.stdin_file
178
296
  os.environ[STATIC_CONFIG['environment']['daemon_id']] = self.daemon_id
297
+ os.environ['PYTHONUNBUFFERED'] = '1'
179
298
  self.rotating_log.refresh_files(start_interception=True)
299
+ result = None
180
300
  try:
181
301
  with open(self.pid_path, 'w+', encoding='utf-8') as f:
182
302
  f.write(str(os.getpid()))
183
303
 
184
304
  self._log_refresh_timer.start()
305
+ self.properties['result'] = None
306
+ self._capture_process_timestamp('began')
185
307
  result = self.target(*self.target_args, **self.target_kw)
186
308
  self.properties['result'] = result
309
+ except (BrokenPipeError, KeyboardInterrupt, SystemExit):
310
+ pass
187
311
  except Exception as e:
188
- warn(f"Exception in daemon target function: {e}", stacklevel=3)
312
+ warn(
313
+ f"Exception in daemon target function: {traceback.format_exc()}",
314
+ )
189
315
  result = e
190
316
  finally:
317
+ _results[self.daemon_id] = result
318
+
319
+ if keep_daemon_output:
320
+ self._capture_process_timestamp('ended')
321
+ else:
322
+ self.cleanup()
323
+
191
324
  self._log_refresh_timer.cancel()
192
- self.rotating_log.close()
193
325
  if self.pid is None and self.pid_path.exists():
194
326
  self.pid_path.unlink()
195
327
 
196
- if keep_daemon_output:
197
- self._capture_process_timestamp('ended')
198
- else:
199
- self.cleanup()
328
+ if is_success_tuple(result):
329
+ try:
330
+ mrsm.pprint(result)
331
+ except BrokenPipeError:
332
+ pass
200
333
 
201
- return result
202
334
  except Exception as e:
203
335
  daemon_error = traceback.format_exc()
204
336
  with open(DAEMON_ERROR_LOG_PATH, 'a+', encoding='utf-8') as f:
205
337
  f.write(daemon_error)
206
338
  warn(f"Encountered an error while running the daemon '{self}':\n{daemon_error}")
207
- finally:
208
- self._cleanup_on_exit()
209
-
210
- def _cleanup_on_exit(self):
211
- """Perform cleanup operations when the daemon exits."""
212
- try:
213
- self._log_refresh_timer.cancel()
214
- self.rotating_log.close()
215
- if self.pid is None and self.pid_path.exists():
216
- self.pid_path.unlink()
217
- self._capture_process_timestamp('ended')
218
- except Exception as e:
219
- warn(f"Error during daemon cleanup: {e}")
220
-
221
339
 
222
340
  def _capture_process_timestamp(
223
- self,
224
- process_key: str,
225
- write_properties: bool = True,
226
- ) -> None:
341
+ self,
342
+ process_key: str,
343
+ write_properties: bool = True,
344
+ ) -> None:
227
345
  """
228
346
  Record the current timestamp to the parameters `process:<process_key>`.
229
347
 
@@ -238,7 +356,7 @@ class Daemon:
238
356
  if 'process' not in self.properties:
239
357
  self.properties['process'] = {}
240
358
 
241
- if process_key not in ('began', 'ended', 'paused'):
359
+ if process_key not in ('began', 'ended', 'paused', 'stopped'):
242
360
  raise ValueError(f"Invalid key '{process_key}'.")
243
361
 
244
362
  self.properties['process'][process_key] = (
@@ -247,13 +365,12 @@ class Daemon:
247
365
  if write_properties:
248
366
  self.write_properties()
249
367
 
250
-
251
368
  def run(
252
- self,
253
- keep_daemon_output: bool = True,
254
- allow_dirty_run: bool = False,
255
- debug: bool = False,
256
- ) -> SuccessTuple:
369
+ self,
370
+ keep_daemon_output: bool = True,
371
+ allow_dirty_run: bool = False,
372
+ debug: bool = False,
373
+ ) -> SuccessTuple:
257
374
  """Run the daemon as a child process and continue executing the parent.
258
375
 
259
376
  Parameters
@@ -279,6 +396,10 @@ class Daemon:
279
396
  if self.status == 'paused':
280
397
  return self.resume()
281
398
 
399
+ self._remove_stop_file()
400
+ if self.status == 'running':
401
+ return True, f"Daemon '{self}' is already running."
402
+
282
403
  self.mkdir_if_not_exists(allow_dirty_run)
283
404
  _write_pickle_success_tuple = self.write_pickle()
284
405
  if not _write_pickle_success_tuple[0]:
@@ -288,7 +409,7 @@ class Daemon:
288
409
  "from meerschaum.utils.daemon import Daemon; "
289
410
  + f"daemon = Daemon(daemon_id='{self.daemon_id}'); "
290
411
  + f"daemon._run_exit(keep_daemon_output={keep_daemon_output}, "
291
- + f"allow_dirty_run=True)"
412
+ + "allow_dirty_run=True)"
292
413
  )
293
414
  env = dict(os.environ)
294
415
  env['MRSM_NOASK'] = 'true'
@@ -298,12 +419,11 @@ class Daemon:
298
419
  if _launch_success_bool
299
420
  else f"Failed to start daemon '{self.daemon_id}'."
300
421
  )
301
- self._capture_process_timestamp('began')
302
422
  return _launch_success_bool, msg
303
423
 
304
-
305
- def kill(self, timeout: Optional[int] = 8) -> SuccessTuple:
306
- """Forcibly terminate a running daemon.
424
+ def kill(self, timeout: Union[int, float, None] = 8) -> SuccessTuple:
425
+ """
426
+ Forcibly terminate a running daemon.
307
427
  Sends a SIGTERM signal to the process.
308
428
 
309
429
  Parameters
@@ -318,10 +438,11 @@ class Daemon:
318
438
  if self.status != 'paused':
319
439
  success, msg = self._send_signal(signal.SIGTERM, timeout=timeout)
320
440
  if success:
321
- self._capture_process_timestamp('ended')
441
+ self._write_stop_file('kill')
322
442
  return success, msg
323
443
 
324
444
  if self.status == 'stopped':
445
+ self._write_stop_file('kill')
325
446
  return True, "Process has already stopped."
326
447
 
327
448
  process = self.process
@@ -332,30 +453,30 @@ class Daemon:
332
453
  except Exception as e:
333
454
  return False, f"Failed to kill job {self} with exception: {e}"
334
455
 
335
- self._capture_process_timestamp('ended')
336
456
  if self.pid_path.exists():
337
457
  try:
338
458
  self.pid_path.unlink()
339
459
  except Exception as e:
340
460
  pass
341
- return True, "Success"
342
461
 
462
+ self._write_stop_file('kill')
463
+ return True, "Success"
343
464
 
344
465
  def quit(self, timeout: Union[int, float, None] = None) -> SuccessTuple:
345
466
  """Gracefully quit a running daemon."""
346
467
  if self.status == 'paused':
347
468
  return self.kill(timeout)
469
+
348
470
  signal_success, signal_msg = self._send_signal(signal.SIGINT, timeout=timeout)
349
471
  if signal_success:
350
- self._capture_process_timestamp('ended')
472
+ self._write_stop_file('quit')
351
473
  return signal_success, signal_msg
352
474
 
353
-
354
475
  def pause(
355
- self,
356
- timeout: Union[int, float, None] = None,
357
- check_timeout_interval: Union[float, int, None] = None,
358
- ) -> SuccessTuple:
476
+ self,
477
+ timeout: Union[int, float, None] = None,
478
+ check_timeout_interval: Union[float, int, None] = None,
479
+ ) -> SuccessTuple:
359
480
  """
360
481
  Pause the daemon if it is running.
361
482
 
@@ -377,6 +498,7 @@ class Daemon:
377
498
  if self.status == 'paused':
378
499
  return True, f"Daemon '{self.daemon_id}' is already paused."
379
500
 
501
+ self._write_stop_file('pause')
380
502
  try:
381
503
  self.process.suspend()
382
504
  except Exception as e:
@@ -414,12 +536,11 @@ class Daemon:
414
536
  + ('s' if timeout != 1 else '') + '.'
415
537
  )
416
538
 
417
-
418
539
  def resume(
419
- self,
420
- timeout: Union[int, float, None] = None,
421
- check_timeout_interval: Union[float, int, None] = None,
422
- ) -> SuccessTuple:
540
+ self,
541
+ timeout: Union[int, float, None] = None,
542
+ check_timeout_interval: Union[float, int, None] = None,
543
+ ) -> SuccessTuple:
423
544
  """
424
545
  Resume the daemon if it is paused.
425
546
 
@@ -441,6 +562,7 @@ class Daemon:
441
562
  if self.status == 'stopped':
442
563
  return False, f"Daemon '{self.daemon_id}' is stopped and cannot be resumed."
443
564
 
565
+ self._remove_stop_file()
444
566
  try:
445
567
  self.process.resume()
446
568
  except Exception as e:
@@ -470,38 +592,50 @@ class Daemon:
470
592
  + ('s' if timeout != 1 else '') + '.'
471
593
  )
472
594
 
595
+ def _write_stop_file(self, action: str) -> SuccessTuple:
596
+ """Write the stop file timestamp and action."""
597
+ if action not in ('quit', 'kill', 'pause'):
598
+ return False, f"Unsupported action '{action}'."
599
+
600
+ if not self.stop_path.parent.exists():
601
+ self.stop_path.parent.mkdir(parents=True, exist_ok=True)
602
+
603
+ with open(self.stop_path, 'w+', encoding='utf-8') as f:
604
+ json.dump(
605
+ {
606
+ 'stop_time': datetime.now(timezone.utc).isoformat(),
607
+ 'action': action,
608
+ },
609
+ f
610
+ )
473
611
 
474
- def _handle_interrupt(self, signal_number: int, stack_frame: 'frame') -> None:
475
- """
476
- Handle `SIGINT` within the Daemon context.
477
- This method is injected into the `DaemonContext`.
478
- """
479
- from meerschaum.utils.process import signal_handler
480
- signal_handler(signal_number, stack_frame)
612
+ return True, "Success"
481
613
 
482
- self.rotating_log.stop_log_fd_interception(unused_only=False)
483
- timer = self.__dict__.get('_log_refresh_timer', None)
484
- if timer is not None:
485
- timer.cancel()
614
+ def _remove_stop_file(self) -> SuccessTuple:
615
+ """Remove the stop file"""
616
+ if not self.stop_path.exists():
617
+ return True, "Stop file does not exist."
486
618
 
487
- daemon_context = self.__dict__.get('_daemon_context', None)
488
- if daemon_context is not None:
489
- daemon_context.close()
619
+ try:
620
+ self.stop_path.unlink()
621
+ except Exception as e:
622
+ return False, f"Failed to remove stop file:\n{e}"
490
623
 
491
- _close_pools()
624
+ return True, "Success"
492
625
 
493
- import threading
494
- for thread in threading.enumerate():
495
- if thread.name == 'MainThread':
496
- continue
497
- try:
498
- if thread.is_alive():
499
- stack = traceback.format_stack(sys._current_frames()[thread.ident])
500
- thread.join()
501
- except Exception as e:
502
- warn(traceback.format_exc())
503
- raise KeyboardInterrupt()
626
+ def _read_stop_file(self) -> Dict[str, Any]:
627
+ """
628
+ Read the stop file if it exists.
629
+ """
630
+ if not self.stop_path.exists():
631
+ return {}
504
632
 
633
+ try:
634
+ with open(self.stop_path, 'r', encoding='utf-8') as f:
635
+ data = json.load(f)
636
+ return data
637
+ except Exception:
638
+ return {}
505
639
 
506
640
  def _handle_sigterm(self, signal_number: int, stack_frame: 'frame') -> None:
507
641
  """
@@ -522,7 +656,6 @@ class Daemon:
522
656
  _close_pools()
523
657
  raise SystemExit(0)
524
658
 
525
-
526
659
  def _send_signal(
527
660
  self,
528
661
  signal_to_send,
@@ -578,7 +711,6 @@ class Daemon:
578
711
  + ('s' if timeout != 1 else '') + '.'
579
712
  )
580
713
 
581
-
582
714
  def mkdir_if_not_exists(self, allow_dirty_run: bool = False):
583
715
  """Create the Daemon's directory.
584
716
  If `allow_dirty_run` is `False` and the directory already exists,
@@ -618,7 +750,6 @@ class Daemon:
618
750
  return None
619
751
  return self._process
620
752
 
621
-
622
753
  @property
623
754
  def status(self) -> str:
624
755
  """
@@ -633,7 +764,7 @@ class Daemon:
633
764
  return 'paused'
634
765
  if self.process.status() == 'zombie':
635
766
  raise psutil.NoSuchProcess(self.process.pid)
636
- except psutil.NoSuchProcess:
767
+ except (psutil.NoSuchProcess, AttributeError):
637
768
  if self.pid_path.exists():
638
769
  try:
639
770
  self.pid_path.unlink()
@@ -643,7 +774,6 @@ class Daemon:
643
774
 
644
775
  return 'running'
645
776
 
646
-
647
777
  @classmethod
648
778
  def _get_path_from_daemon_id(cls, daemon_id: str) -> pathlib.Path:
649
779
  """
@@ -651,7 +781,6 @@ class Daemon:
651
781
  """
652
782
  return DAEMON_RESOURCES_PATH / daemon_id
653
783
 
654
-
655
784
  @property
656
785
  def path(self) -> pathlib.Path:
657
786
  """
@@ -659,7 +788,6 @@ class Daemon:
659
788
  """
660
789
  return self._get_path_from_daemon_id(self.daemon_id)
661
790
 
662
-
663
791
  @classmethod
664
792
  def _get_properties_path_from_daemon_id(cls, daemon_id: str) -> pathlib.Path:
665
793
  """
@@ -667,7 +795,6 @@ class Daemon:
667
795
  """
668
796
  return cls._get_path_from_daemon_id(daemon_id) / 'properties.json'
669
797
 
670
-
671
798
  @property
672
799
  def properties_path(self) -> pathlib.Path:
673
800
  """
@@ -675,6 +802,12 @@ class Daemon:
675
802
  """
676
803
  return self._get_properties_path_from_daemon_id(self.daemon_id)
677
804
 
805
+ @property
806
+ def stop_path(self) -> pathlib.Path:
807
+ """
808
+ Return the path for the stop file (created when manually stopped).
809
+ """
810
+ return self.path / '.stop.json'
678
811
 
679
812
  @property
680
813
  def log_path(self) -> pathlib.Path:
@@ -683,6 +816,22 @@ class Daemon:
683
816
  """
684
817
  return LOGS_RESOURCES_PATH / (self.daemon_id + '.log')
685
818
 
819
+ @property
820
+ def stdin_file_path(self) -> pathlib.Path:
821
+ """
822
+ Return the stdin file path.
823
+ """
824
+ return self.path / 'input.stdin'
825
+
826
+ @property
827
+ def blocking_stdin_file_path(self) -> pathlib.Path:
828
+ """
829
+ Return the stdin file path.
830
+ """
831
+ if '_blocking_stdin_file_path' in self.__dict__:
832
+ return self._blocking_stdin_file_path
833
+
834
+ return self.path / 'input.stdin.block'
686
835
 
687
836
  @property
688
837
  def log_offset_path(self) -> pathlib.Path:
@@ -691,24 +840,46 @@ class Daemon:
691
840
  """
692
841
  return LOGS_RESOURCES_PATH / ('.' + self.daemon_id + '.log.offset')
693
842
 
694
-
695
843
  @property
696
844
  def rotating_log(self) -> RotatingFile:
845
+ """
846
+ The rotating log file for the daemon's output.
847
+ """
697
848
  if '_rotating_log' in self.__dict__:
698
849
  return self._rotating_log
699
850
 
851
+ write_timestamps = (
852
+ self.properties.get('logs', {}).get('write_timestamps', None)
853
+ )
854
+ if write_timestamps is None:
855
+ write_timestamps = get_config('jobs', 'logs', 'timestamps', 'enabled')
856
+
700
857
  self._rotating_log = RotatingFile(
701
858
  self.log_path,
702
- redirect_streams = True,
703
- write_timestamps = get_config('jobs', 'logs', 'timestamps', 'enabled'),
704
- timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format'),
859
+ redirect_streams=True,
860
+ write_timestamps=write_timestamps,
861
+ timestamp_format=get_config('jobs', 'logs', 'timestamps', 'format'),
705
862
  )
706
863
  return self._rotating_log
707
864
 
865
+ @property
866
+ def stdin_file(self):
867
+ """
868
+ Return the file handler for the stdin file.
869
+ """
870
+ if '_stdin_file' in self.__dict__:
871
+ return self._stdin_file
872
+
873
+ self._stdin_file = StdinFile(
874
+ self.stdin_file_path,
875
+ lock_file_path=self.blocking_stdin_file_path,
876
+ )
877
+ return self._stdin_file
708
878
 
709
879
  @property
710
880
  def log_text(self) -> Optional[str]:
711
- """Read the log files and return their contents.
881
+ """
882
+ Read the log files and return their contents.
712
883
  Returns `None` if the log file does not exist.
713
884
  """
714
885
  new_rotating_log = RotatingFile(
@@ -720,7 +891,6 @@ class Daemon:
720
891
  )
721
892
  return new_rotating_log.read()
722
893
 
723
-
724
894
  def readlines(self) -> List[str]:
725
895
  """
726
896
  Read the next log lines, persisting the cursor for later use.
@@ -731,7 +901,6 @@ class Daemon:
731
901
  self._write_log_offset()
732
902
  return lines
733
903
 
734
-
735
904
  def _read_log_offset(self) -> Tuple[int, int]:
736
905
  """
737
906
  Return the current log offset cursor.
@@ -749,7 +918,6 @@ class Daemon:
749
918
  subfile_index, subfile_position = int(cursor_parts[0]), int(cursor_parts[1])
750
919
  return subfile_index, subfile_position
751
920
 
752
-
753
921
  def _write_log_offset(self) -> None:
754
922
  """
755
923
  Write the current log offset file.
@@ -759,10 +927,10 @@ class Daemon:
759
927
  subfile_position = self.rotating_log._cursor[1]
760
928
  f.write(f"{subfile_index} {subfile_position}")
761
929
 
762
-
763
930
  @property
764
931
  def pid(self) -> Union[int, None]:
765
- """Read the PID file and return its contents.
932
+ """
933
+ Read the PID file and return its contents.
766
934
  Returns `None` if the PID file does not exist.
767
935
  """
768
936
  if not self.pid_path.exists():
@@ -770,6 +938,8 @@ class Daemon:
770
938
  try:
771
939
  with open(self.pid_path, 'r', encoding='utf-8') as f:
772
940
  text = f.read()
941
+ if len(text) == 0:
942
+ return None
773
943
  pid = int(text.rstrip())
774
944
  except Exception as e:
775
945
  warn(e)
@@ -777,7 +947,6 @@ class Daemon:
777
947
  pid = None
778
948
  return pid
779
949
 
780
-
781
950
  @property
782
951
  def pid_path(self) -> pathlib.Path:
783
952
  """
@@ -785,7 +954,6 @@ class Daemon:
785
954
  """
786
955
  return self.path / 'process.pid'
787
956
 
788
-
789
957
  @property
790
958
  def pid_lock(self) -> 'fasteners.InterProcessLock':
791
959
  """
@@ -798,7 +966,6 @@ class Daemon:
798
966
  self._pid_lock = fasteners.InterProcessLock(self.pid_path)
799
967
  return self._pid_lock
800
968
 
801
-
802
969
  @property
803
970
  def pickle_path(self) -> pathlib.Path:
804
971
  """
@@ -806,25 +973,27 @@ class Daemon:
806
973
  """
807
974
  return self.path / 'pickle.pkl'
808
975
 
809
-
810
976
  def read_properties(self) -> Optional[Dict[str, Any]]:
811
977
  """Read the properties JSON file and return the dictionary."""
812
978
  if not self.properties_path.exists():
813
979
  return None
814
980
  try:
815
981
  with open(self.properties_path, 'r', encoding='utf-8') as file:
816
- return json.load(file)
982
+ properties = json.load(file)
817
983
  except Exception as e:
818
- return {}
819
-
984
+ properties = {}
985
+
986
+ return properties
820
987
 
821
988
  def read_pickle(self) -> Daemon:
822
989
  """Read a Daemon's pickle file and return the `Daemon`."""
823
990
  import pickle, traceback
824
991
  if not self.pickle_path.exists():
825
992
  error(f"Pickle file does not exist for daemon '{self.daemon_id}'.")
993
+
826
994
  if self.pickle_path.stat().st_size == 0:
827
995
  error(f"Pickle was empty for daemon '{self.daemon_id}'.")
996
+
828
997
  try:
829
998
  with open(self.pickle_path, 'rb') as pickle_file:
830
999
  daemon = pickle.load(pickle_file)
@@ -837,21 +1006,30 @@ class Daemon:
837
1006
  error(msg)
838
1007
  return daemon
839
1008
 
840
-
841
1009
  @property
842
1010
  def properties(self) -> Dict[str, Any]:
843
1011
  """
844
1012
  Return the contents of the properties JSON file.
845
1013
  """
846
- _file_properties = self.read_properties()
1014
+ try:
1015
+ _file_properties = self.read_properties()
1016
+ except Exception:
1017
+ traceback.print_exc()
1018
+ _file_properties = {}
1019
+
847
1020
  if not self._properties:
848
1021
  self._properties = _file_properties
1022
+
849
1023
  if self._properties is None:
850
1024
  self._properties = {}
1025
+
851
1026
  if _file_properties is not None:
852
- self._properties = apply_patch_to_config(_file_properties, self._properties)
853
- return self._properties
1027
+ self._properties = apply_patch_to_config(
1028
+ _file_properties,
1029
+ self._properties,
1030
+ )
854
1031
 
1032
+ return self._properties
855
1033
 
856
1034
  @property
857
1035
  def hidden(self) -> bool:
@@ -860,12 +1038,14 @@ class Daemon:
860
1038
  """
861
1039
  return self.daemon_id.startswith('_') or self.daemon_id.startswith('.')
862
1040
 
863
-
864
1041
  def write_properties(self) -> SuccessTuple:
865
1042
  """Write the properties dictionary to the properties JSON file
866
1043
  (only if self.properties exists).
867
1044
  """
868
- success, msg = False, f"No properties to write for daemon '{self.daemon_id}'."
1045
+ success, msg = (
1046
+ False,
1047
+ f"No properties to write for daemon '{self.daemon_id}'."
1048
+ )
869
1049
  if self.properties is not None:
870
1050
  try:
871
1051
  self.path.mkdir(parents=True, exist_ok=True)
@@ -876,7 +1056,6 @@ class Daemon:
876
1056
  success, msg = False, str(e)
877
1057
  return success, msg
878
1058
 
879
-
880
1059
  def write_pickle(self) -> SuccessTuple:
881
1060
  """Write the pickle file for the daemon."""
882
1061
  import pickle, traceback
@@ -892,9 +1071,9 @@ class Daemon:
892
1071
 
893
1072
 
894
1073
  def _setup(
895
- self,
896
- allow_dirty_run: bool = False,
897
- ) -> None:
1074
+ self,
1075
+ allow_dirty_run: bool = False,
1076
+ ) -> None:
898
1077
  """
899
1078
  Update properties before starting the Daemon.
900
1079
  """
@@ -918,7 +1097,6 @@ class Daemon:
918
1097
  if not _write_pickle_success_tuple[0]:
919
1098
  error(_write_pickle_success_tuple[1])
920
1099
 
921
-
922
1100
  def cleanup(self, keep_logs: bool = False) -> SuccessTuple:
923
1101
  """
924
1102
  Remove a daemon's directory after execution.
@@ -962,9 +1140,9 @@ class Daemon:
962
1140
 
963
1141
 
964
1142
  def get_check_timeout_interval_seconds(
965
- self,
966
- check_timeout_interval: Union[int, float, None] = None,
967
- ) -> Union[int, float]:
1143
+ self,
1144
+ check_timeout_interval: Union[int, float, None] = None,
1145
+ ) -> Union[int, float]:
968
1146
  """
969
1147
  Return the interval value to check the status of timeouts.
970
1148
  """
@@ -972,6 +1150,33 @@ class Daemon:
972
1150
  return check_timeout_interval
973
1151
  return get_config('jobs', 'check_timeout_interval_seconds')
974
1152
 
1153
+ @property
1154
+ def target_args(self) -> Union[Tuple[Any], None]:
1155
+ """
1156
+ Return the positional arguments to pass to the target function.
1157
+ """
1158
+ target_args = (
1159
+ self.__dict__.get('_target_args', None)
1160
+ or self.properties.get('target', {}).get('args', None)
1161
+ )
1162
+ if target_args is None:
1163
+ return tuple([])
1164
+
1165
+ return tuple(target_args)
1166
+
1167
+ @property
1168
+ def target_kw(self) -> Union[Dict[str, Any], None]:
1169
+ """
1170
+ Return the keyword arguments to pass to the target function.
1171
+ """
1172
+ target_kw = (
1173
+ self.__dict__.get('_target_kw', None)
1174
+ or self.properties.get('target', {}).get('kw', None)
1175
+ )
1176
+ if target_kw is None:
1177
+ return {}
1178
+
1179
+ return {key: val for key, val in target_kw.items()}
975
1180
 
976
1181
  def __getstate__(self):
977
1182
  """
@@ -1022,4 +1227,4 @@ class Daemon:
1022
1227
  return self.daemon_id == other.daemon_id
1023
1228
 
1024
1229
  def __hash__(self):
1025
- return hash(self.daemon_id)
1230
+ return hash(self.daemon_id)