meerschaum 2.3.0.dev1__py3-none-any.whl → 2.3.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. meerschaum/__init__.py +5 -2
  2. meerschaum/__main__.py +0 -5
  3. meerschaum/_internal/arguments/_parse_arguments.py +10 -3
  4. meerschaum/_internal/arguments/_parser.py +6 -2
  5. meerschaum/_internal/entry.py +36 -6
  6. meerschaum/_internal/shell/Shell.py +32 -20
  7. meerschaum/actions/__init__.py +8 -6
  8. meerschaum/actions/attach.py +31 -13
  9. meerschaum/actions/copy.py +68 -41
  10. meerschaum/actions/delete.py +64 -21
  11. meerschaum/actions/edit.py +3 -3
  12. meerschaum/actions/install.py +40 -32
  13. meerschaum/actions/pause.py +44 -27
  14. meerschaum/actions/restart.py +107 -0
  15. meerschaum/actions/show.py +8 -8
  16. meerschaum/actions/start.py +26 -41
  17. meerschaum/actions/stop.py +11 -4
  18. meerschaum/api/_events.py +10 -3
  19. meerschaum/api/dash/jobs.py +69 -70
  20. meerschaum/api/routes/_actions.py +8 -3
  21. meerschaum/api/routes/_jobs.py +86 -37
  22. meerschaum/config/_default.py +1 -1
  23. meerschaum/config/_paths.py +5 -0
  24. meerschaum/config/_shell.py +1 -1
  25. meerschaum/config/_version.py +1 -1
  26. meerschaum/config/static/__init__.py +6 -1
  27. meerschaum/connectors/Connector.py +13 -7
  28. meerschaum/connectors/__init__.py +21 -5
  29. meerschaum/connectors/api/APIConnector.py +3 -0
  30. meerschaum/connectors/api/_jobs.py +108 -11
  31. meerschaum/connectors/parse.py +10 -13
  32. meerschaum/core/Pipe/_bootstrap.py +16 -8
  33. meerschaum/jobs/_Executor.py +69 -0
  34. meerschaum/{utils/jobs → jobs}/_Job.py +206 -40
  35. meerschaum/jobs/_LocalExecutor.py +88 -0
  36. meerschaum/jobs/_SystemdExecutor.py +608 -0
  37. meerschaum/jobs/__init__.py +365 -0
  38. meerschaum/plugins/__init__.py +6 -6
  39. meerschaum/utils/daemon/Daemon.py +7 -0
  40. meerschaum/utils/daemon/RotatingFile.py +5 -2
  41. meerschaum/utils/daemon/StdinFile.py +12 -2
  42. meerschaum/utils/daemon/__init__.py +2 -0
  43. meerschaum/utils/formatting/_jobs.py +52 -16
  44. meerschaum/utils/misc.py +23 -5
  45. meerschaum/utils/packages/_packages.py +7 -4
  46. meerschaum/utils/process.py +9 -9
  47. meerschaum/utils/venv/__init__.py +2 -2
  48. {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/METADATA +14 -17
  49. {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/RECORD +55 -51
  50. meerschaum/utils/jobs/__init__.py +0 -245
  51. {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/LICENSE +0 -0
  52. {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/NOTICE +0 -0
  53. {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/WHEEL +0 -0
  54. {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/entry_points.txt +0 -0
  55. {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/top_level.txt +0 -0
  56. {meerschaum-2.3.0.dev1.dist-info → meerschaum-2.3.0rc1.dist-info}/zip-safe +0 -0
@@ -78,7 +78,7 @@ default_shell_config = {
78
78
  },
79
79
  'executor' : {
80
80
  'rich' : {
81
- 'style' : 'gold1',
81
+ 'style' : 'yellow',
82
82
  },
83
83
  },
84
84
  'username' : {
@@ -2,4 +2,4 @@
2
2
  Specify the Meerschaum release version.
3
3
  """
4
4
 
5
- __version__ = "2.3.0.dev1"
5
+ __version__ = "2.3.0rc1"
@@ -43,7 +43,9 @@ STATIC_CONFIG: Dict[str, Any] = {
43
43
  'webterm_job_name': '_webterm',
44
44
  'default_timeout': 600,
45
45
  'jobs': {
46
- 'stdin_message': 'MRSM_STDIN'
46
+ 'stdin_message': 'MRSM_STDIN',
47
+ 'stop_message': 'MRSM_STOP',
48
+ 'metadata_cache_seconds': 5,
47
49
  },
48
50
  },
49
51
  'sql': {
@@ -68,6 +70,9 @@ STATIC_CONFIG: Dict[str, Any] = {
68
70
  'noask': 'MRSM_NOASK',
69
71
  'id': 'MRSM_SERVER_ID',
70
72
  'daemon_id': 'MRSM_DAEMON_ID',
73
+ 'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
74
+ 'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
75
+ 'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
71
76
  'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
72
77
  'prefix': 'MRSM_',
73
78
  },
@@ -21,11 +21,11 @@ class Connector(metaclass=abc.ABCMeta):
21
21
  The base connector class to hold connection attributes.
22
22
  """
23
23
  def __init__(
24
- self,
25
- type: Optional[str] = None,
26
- label: Optional[str] = None,
27
- **kw: Any
28
- ):
24
+ self,
25
+ type: Optional[str] = None,
26
+ label: Optional[str] = None,
27
+ **kw: Any
28
+ ):
29
29
  """
30
30
  Set the given keyword arguments as attributes.
31
31
 
@@ -101,7 +101,7 @@ class Connector(metaclass=abc.ABCMeta):
101
101
 
102
102
  ### load user config into self._attributes
103
103
  if self.type in conn_configs and self.label in conn_configs[self.type]:
104
- self._attributes.update(conn_configs[self.type][self.label])
104
+ self._attributes.update(conn_configs[self.type][self.label] or {})
105
105
 
106
106
  ### load system config into self._sys_config
107
107
  ### (deep copy so future Connectors don't inherit changes)
@@ -200,7 +200,13 @@ class Connector(metaclass=abc.ABCMeta):
200
200
  _type = self.__dict__.get('type', None)
201
201
  if _type is None:
202
202
  import re
203
- _type = re.sub(r'connector$', '', self.__class__.__name__.lower())
203
+ is_executor = self.__class__.__name__.lower().endswith('executor')
204
+ suffix_regex = (
205
+ r'connector$'
206
+ if not is_executor
207
+ else r'executor$'
208
+ )
209
+ _type = re.sub(suffix_regex, '', self.__class__.__name__.lower())
204
210
  self.__dict__['type'] = _type
205
211
  return _type
206
212
 
@@ -36,9 +36,9 @@ __all__ = (
36
36
  ### store connectors partitioned by
37
37
  ### type, label for reuse
38
38
  connectors: Dict[str, Dict[str, Connector]] = {
39
- 'api' : {},
40
- 'sql' : {},
41
- 'plugin': {},
39
+ 'api' : {},
40
+ 'sql' : {},
41
+ 'plugin' : {},
42
42
  }
43
43
  instance_types: List[str] = ['sql', 'api']
44
44
  _locks: Dict[str, RLock] = {
@@ -127,10 +127,13 @@ def get_connector(
127
127
  global _loaded_plugin_connectors
128
128
  if isinstance(type, str) and not label and ':' in type:
129
129
  type, label = type.split(':', maxsplit=1)
130
+
130
131
  with _locks['_loaded_plugin_connectors']:
131
132
  if not _loaded_plugin_connectors:
132
133
  load_plugin_connectors()
134
+ _load_builtin_custom_connectors()
133
135
  _loaded_plugin_connectors = True
136
+
134
137
  if type is None and label is None:
135
138
  default_instance_keys = get_config('meerschaum', 'instance', patch=True)
136
139
  ### recursive call to get_connector
@@ -274,7 +277,7 @@ def is_connected(keys: str, **kw) -> bool:
274
277
  return False
275
278
 
276
279
 
277
- def make_connector(cls):
280
+ def make_connector(cls, _is_executor: bool = False):
278
281
  """
279
282
  Register a class as a `Connector`.
280
283
  The `type` will be the lower case of the class name, without the suffix `connector`.
@@ -300,7 +303,12 @@ def make_connector(cls):
300
303
  >>>
301
304
  """
302
305
  import re
303
- typ = re.sub(r'connector$', '', cls.__name__.lower())
306
+ suffix_regex = (
307
+ r'connector$'
308
+ if not _is_executor
309
+ else r'executor$'
310
+ )
311
+ typ = re.sub(suffix_regex, '', cls.__name__.lower())
304
312
  with _locks['types']:
305
313
  types[typ] = cls
306
314
  with _locks['custom_types']:
@@ -363,3 +371,11 @@ def get_connector_plugin(
363
371
  )
364
372
  plugin = mrsm.Plugin(plugin_name)
365
373
  return plugin if plugin.is_installed() else None
374
+
375
+
376
+ def _load_builtin_custom_connectors():
377
+ """
378
+ Import custom connectors decorated with `@make_connector` or `@make_executor`.
379
+ """
380
+ import meerschaum.jobs._SystemdExecutor
381
+ # import meerschaum.jobs._LocalExecutor
@@ -88,6 +88,9 @@ class APIConnector(Connector):
88
88
  monitor_logs,
89
89
  monitor_logs_async,
90
90
  get_job_is_blocking_on_stdin,
91
+ get_job_began,
92
+ get_job_ended,
93
+ get_job_status,
91
94
  )
92
95
 
93
96
  def __init__(
@@ -7,17 +7,21 @@ Manage jobs via the Meerschaum API.
7
7
  """
8
8
 
9
9
  import asyncio
10
+ import time
11
+ import json
10
12
  from datetime import datetime
11
13
 
12
14
  import meerschaum as mrsm
13
15
  from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable
14
- from meerschaum.utils.jobs import Job
16
+ from meerschaum.jobs import Job
15
17
  from meerschaum.config.static import STATIC_CONFIG
16
- from meerschaum.utils.warnings import warn
18
+ from meerschaum.utils.warnings import warn, dprint
17
19
 
18
20
  JOBS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['jobs']
19
21
  LOGS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['logs']
20
22
  JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
23
+ JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
24
+ JOB_METADATA_CACHE_SECONDS: int = STATIC_CONFIG['api']['jobs']['metadata_cache_seconds']
21
25
 
22
26
 
23
27
  def get_jobs(self, debug: bool = False) -> Dict[str, Job]:
@@ -59,6 +63,20 @@ def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
59
63
  """
60
64
  Return the metadata for a single job.
61
65
  """
66
+ now = time.perf_counter()
67
+ _job_metadata_cache = self.__dict__.get('_job_metadata_cache', None)
68
+ _job_metadata_timestamp = (
69
+ _job_metadata_cache.get(name, {}).get('timestamp', None)
70
+ ) if _job_metadata_cache is not None else None
71
+
72
+ if (
73
+ _job_metadata_timestamp is not None
74
+ and (now - _job_metadata_timestamp) < JOB_METADATA_CACHE_SECONDS
75
+ ):
76
+ if debug:
77
+ dprint(f"Returning cached metadata for job '{name}'.")
78
+ return _job_metadata_cache[name]['metadata']
79
+
62
80
  response = self.get(JOBS_ENDPOINT + f"/{name}", debug=debug)
63
81
  if not response:
64
82
  if debug:
@@ -70,8 +88,15 @@ def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
70
88
  warn(f"Failed to get metadata for job '{name}':\n{msg}")
71
89
  return {}
72
90
 
73
- return response.json()
91
+ metadata = response.json()
92
+ if _job_metadata_cache is None:
93
+ self._job_metadata_cache = {}
74
94
 
95
+ self._job_metadata_cache[name] = {
96
+ 'timestamp': now,
97
+ 'metadata': metadata,
98
+ }
99
+ return metadata
75
100
 
76
101
  def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
77
102
  """
@@ -80,6 +105,34 @@ def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
80
105
  metadata = self.get_job_metadata(name, debug=debug)
81
106
  return metadata.get('daemon', {}).get('properties', {})
82
107
 
108
+ def get_job_status(self, name: str, debug: bool = False) -> str:
109
+ """
110
+ Return the job's status.
111
+ """
112
+ metadata = self.get_job_metadata(name, debug=debug)
113
+ return metadata.get('status', 'stopped')
114
+
115
+ def get_job_began(self, name: str, debug: bool = False) -> str:
116
+ """
117
+ Return a job's `began` timestamp, if it exists.
118
+ """
119
+ properties = self.get_job_properties(name, debug=debug)
120
+ began_str = properties.get('daemon', {}).get('began', None)
121
+ if began_str is None:
122
+ return None
123
+
124
+ return began_str
125
+
126
+ def get_job_ended(self, name: str, debug: bool = False) -> str:
127
+ """
128
+ Return a job's `ended` timestamp, if it exists.
129
+ """
130
+ properties = self.get_job_properties(name, debug=debug)
131
+ ended_str = properties.get('daemon', {}).get('ended', None)
132
+ if ended_str is None:
133
+ return None
134
+
135
+ return ended_str
83
136
 
84
137
  def get_job_exists(self, name: str, debug: bool = False) -> bool:
85
138
  """
@@ -191,17 +244,55 @@ async def monitor_logs_async(
191
244
  name: str,
192
245
  callback_function: Callable[[Any], Any],
193
246
  input_callback_function: Callable[[], str],
247
+ stop_callback_function: Callable[[SuccessTuple], str],
248
+ stop_on_exit: bool = False,
249
+ strip_timestamps: bool = False,
194
250
  accept_input: bool = True,
195
251
  debug: bool = False,
196
252
  ):
197
253
  """
198
254
  Monitor a job's log files and await a callback with the changes.
199
255
  """
256
+ from meerschaum.jobs import StopMonitoringLogs
257
+ from meerschaum.utils.formatting._jobs import strip_timestamp_from_line
258
+
200
259
  websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
201
260
  protocol = 'ws' if self.URI.startswith('http://') else 'wss'
202
261
  port = self.port if 'port' in self.__dict__ else ''
203
262
  uri = f"{protocol}://{self.host}:{port}{LOGS_ENDPOINT}/{name}/ws"
204
263
 
264
+ async def _stdin_callback(client):
265
+ if input_callback_function is None:
266
+ return
267
+
268
+ if asyncio.iscoroutinefunction(input_callback_function):
269
+ data = await input_callback_function()
270
+ else:
271
+ data = input_callback_function()
272
+
273
+ await client.send(data)
274
+
275
+ async def _stop_callback(client):
276
+ try:
277
+ result = tuple(json.loads(await client.recv()))
278
+ except Exception as e:
279
+ warn(traceback.format_exc())
280
+ result = False, str(e)
281
+
282
+ if stop_callback_function is not None:
283
+ if asyncio.iscoroutinefunction(stop_callback_function):
284
+ await stop_callback_function(result)
285
+ else:
286
+ stop_callback_function(result)
287
+
288
+ if stop_on_exit:
289
+ raise StopMonitoringLogs
290
+
291
+ message_callbacks = {
292
+ JOBS_STDIN_MESSAGE: _stdin_callback,
293
+ JOBS_STOP_MESSAGE: _stop_callback,
294
+ }
295
+
205
296
  async with websockets.connect(uri) as websocket:
206
297
  try:
207
298
  await websocket.send(self.token or 'no-login')
@@ -211,28 +302,31 @@ async def monitor_logs_async(
211
302
  while True:
212
303
  try:
213
304
  response = await websocket.recv()
214
- if response == JOBS_STDIN_MESSAGE:
215
- if asyncio.iscoroutinefunction(input_callback_function):
216
- data = await input_callback_function()
217
- else:
218
- data = input_callback_function()
219
-
220
- await websocket.send(data)
305
+ callback = message_callbacks.get(response, None)
306
+ if callback is not None:
307
+ await callback(websocket)
221
308
  continue
222
309
 
310
+ if strip_timestamps:
311
+ response = strip_timestamp_from_line(response)
312
+
223
313
  if asyncio.iscoroutinefunction(callback_function):
224
314
  await callback_function(response)
225
315
  else:
226
316
  callback_function(response)
227
- except KeyboardInterrupt:
317
+ except (KeyboardInterrupt, StopMonitoringLogs):
228
318
  await websocket.close()
229
319
  break
230
320
 
321
+
231
322
  def monitor_logs(
232
323
  self,
233
324
  name: str,
234
325
  callback_function: Callable[[Any], Any],
235
326
  input_callback_function: Callable[[None], str],
327
+ stop_callback_function: Callable[[None], str],
328
+ stop_on_exit: bool = False,
329
+ strip_timestamps: bool = False,
236
330
  accept_input: bool = True,
237
331
  debug: bool = False,
238
332
  ):
@@ -244,6 +338,9 @@ def monitor_logs(
244
338
  name,
245
339
  callback_function,
246
340
  input_callback_function=input_callback_function,
341
+ stop_callback_function=stop_callback_function,
342
+ stop_on_exit=stop_on_exit,
343
+ strip_timestamps=strip_timestamps,
247
344
  accept_input=accept_input,
248
345
  debug=debug
249
346
  )
@@ -87,19 +87,17 @@ def parse_connector_keys(
87
87
 
88
88
 
89
89
  def parse_instance_keys(
90
- keys: Optional[str],
91
- construct: bool = True,
92
- as_tuple: bool = False,
93
- **kw
94
- ):
90
+ keys: Optional[str],
91
+ construct: bool = True,
92
+ as_tuple: bool = False,
93
+ **kw
94
+ ):
95
95
  """
96
96
  Parse the Meerschaum instance value into a Connector object.
97
97
  """
98
98
  from meerschaum.utils.warnings import warn
99
99
  from meerschaum.config import get_config
100
100
 
101
- ### TODO Check for valid types? Not sure how to do that if construct = False.
102
-
103
101
  if keys is None:
104
102
  keys = get_config('meerschaum', 'instance')
105
103
  keys = str(keys)
@@ -120,25 +118,24 @@ def parse_repo_keys(keys: Optional[str] = None, **kw):
120
118
 
121
119
 
122
120
  def parse_executor_keys(keys: Optional[str] = None, **kw):
123
- """Parse the executor keys into an APIConnector or None."""
121
+ """Parse the executor keys into an APIConnector or string."""
122
+ from meerschaum.jobs import get_executor_keys_from_context
124
123
  from meerschaum.config import get_config
125
124
  if keys is None:
126
- keys = get_config('meerschaum', 'default_executor')
125
+ keys = get_executor_keys_from_context()
127
126
 
128
127
  if keys is None or keys == 'local':
129
128
  return 'local'
130
129
 
131
130
  keys = str(keys)
132
- if ':' not in keys:
133
- keys = 'api:' + keys
134
-
135
131
  return parse_connector_keys(keys, **kw)
136
132
 
137
133
 
138
134
  def is_valid_connector_keys(
139
135
  keys: str
140
136
  ) -> bool:
141
- """Verify a connector_keys string references a valid connector.
137
+ """
138
+ Verify a connector_keys string references a valid connector.
142
139
  """
143
140
  try:
144
141
  success = parse_connector_keys(keys, construct=False) is not None
@@ -88,21 +88,29 @@ def bootstrap(
88
88
 
89
89
  try:
90
90
  if yes_no(
91
- f"Would you like to edit the definition for {self}?", yes=yes, noask=noask
91
+ f"Would you like to edit the definition for {self}?",
92
+ yes=yes,
93
+ noask=noask,
94
+ default='n',
92
95
  ):
93
96
  edit_tuple = self.edit_definition(debug=debug)
94
97
  if not edit_tuple[0]:
95
98
  return edit_tuple
96
99
 
97
- if yes_no(f"Would you like to try syncing {self} now?", yes=yes, noask=noask):
100
+ if yes_no(
101
+ f"Would you like to try syncing {self} now?",
102
+ yes=yes,
103
+ noask=noask,
104
+ default='n',
105
+ ):
98
106
  sync_tuple = actions['sync'](
99
107
  ['pipes'],
100
- connector_keys = [self.connector_keys],
101
- metric_keys = [self.metric_key],
102
- location_keys = [self.location_key],
103
- mrsm_instance = str(self.instance_connector),
104
- debug = debug,
105
- shell = shell,
108
+ connector_keys=[self.connector_keys],
109
+ metric_keys=[self.metric_key],
110
+ location_keys=[self.location_key],
111
+ mrsm_instance=str(self.instance_connector),
112
+ debug=debug,
113
+ shell=shell,
106
114
  )
107
115
  if not sync_tuple[0]:
108
116
  return sync_tuple
@@ -0,0 +1,69 @@
1
+ #! /usr/bin/env python3
2
+ # vim:fenc=utf-8
3
+
4
+ """
5
+ Define the base class for a Job executor.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from abc import abstractmethod
11
+
12
+ from meerschaum.connectors import Connector
13
+ from meerschaum.utils.typing import List, Dict, SuccessTuple, TYPE_CHECKING
14
+
15
+ if TYPE_CHECKING:
16
+ from meerschaum.jobs import Job
17
+
18
+ class Executor(Connector):
19
+ """
20
+ Define the methods for managing jobs.
21
+ """
22
+
23
+ @abstractmethod
24
+ def get_job_exists(self, name: str, debug: bool = False) -> bool:
25
+ """
26
+ Return whether a job exists.
27
+ """
28
+
29
+ @abstractmethod
30
+ def get_jobs(self) -> Dict[str, Job]:
31
+ """
32
+ Return a dictionary of names -> Jobs.
33
+ """
34
+
35
+ @abstractmethod
36
+ def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
37
+ """
38
+ Create a new job.
39
+ """
40
+
41
+ @abstractmethod
42
+ def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
43
+ """
44
+ Start a job.
45
+ """
46
+
47
+ @abstractmethod
48
+ def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
49
+ """
50
+ Stop a job.
51
+ """
52
+
53
+ @abstractmethod
54
+ def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
55
+ """
56
+ Pause a job.
57
+ """
58
+
59
+ @abstractmethod
60
+ def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
61
+ """
62
+ Delete a job.
63
+ """
64
+
65
+ @abstractmethod
66
+ def get_logs(self, name: str, debug: bool = False) -> str:
67
+ """
68
+ Return a job's log output.
69
+ """