meerschaum 2.3.0.dev3__py3-none-any.whl → 2.3.0rc1__py3-none-any.whl

meerschaum/api/dash/jobs.py

@@ -16,13 +16,12 @@ from meerschaum.api import CHECK_UPDATE
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 html, dcc = import_html(), import_dcc()
 dateutil_parser = attempt_import('dateutil.parser', check_update=CHECK_UPDATE)
-from meerschaum.utils.jobs import get_jobs, Job
-from meerschaum.utils.daemon import (
-    get_daemons,
-    get_running_daemons,
-    get_paused_daemons,
-    get_stopped_daemons,
-    Daemon,
+from meerschaum.jobs import (
+    get_jobs,
+    get_running_jobs,
+    get_paused_jobs,
+    get_stopped_jobs,
+    Job,
 )
 from meerschaum.config import get_config
 from meerschaum.utils.misc import sorted_dict
@@ -38,27 +37,25 @@ def get_jobs_cards(state: WebState):
     """
     Build cards and alerts lists for jobs.
     """
-    daemons = get_daemons()
-    jobs = get_jobs()
+    jobs = get_jobs(include_hidden=False)
     session_id = state['session-store.data'].get('session-id', None)
     is_authenticated = is_session_authenticated(session_id)
 
-    alert = alert_from_success_tuple(daemons)
     cards = []
 
     for name, job in jobs.items():
         d = job.daemon
         footer_children = html.Div(
             build_process_timestamps_children(d),
-            id = {'type': 'process-timestamps-div', 'index': d.daemon_id},
+            id = {'type': 'process-timestamps-div', 'index': name},
         )
         follow_logs_button = dbc.DropdownMenuItem(
             "Follow logs",
-            id = {'type': 'follow-logs-button', 'index': d.daemon_id},
+            id = {'type': 'follow-logs-button', 'index': name},
         )
         download_logs_button = dbc.DropdownMenuItem(
             "Download logs",
-            id = {'type': 'job-download-logs-button', 'index': d.daemon_id},
+            id = {'type': 'job-download-logs-button', 'index': name},
         )
         logs_menu_children = (
             ([follow_logs_button] if is_authenticated else []) + [download_logs_button]
@@ -66,19 +63,19 @@ def get_jobs_cards(state: WebState):
         header_children = [
             html.Div(
                 build_status_children(d),
-                id = {'type': 'manage-job-status-div', 'index': d.daemon_id},
-                style = {'float': 'left'},
+                id={'type': 'manage-job-status-div', 'index': name},
+                style={'float': 'left'},
             ),
             html.Div(
                 dbc.DropdownMenu(
                     logs_menu_children,
-                    label = "Logs",
-                    size = "sm",
-                    align_end = True,
-                    color = "secondary",
-                    menu_variant = 'dark',
+                    label="Logs",
+                    size="sm",
+                    align_end=True,
+                    color="secondary",
+                    menu_variant='dark',
                 ),
-                style = {'float': 'right'},
+                style={'float': 'right'},
             ),
         ]
 
@@ -87,10 +84,10 @@ def get_jobs_cards(state: WebState):
             html.Div(
                 html.P(
                     d.label,
-                    className = "card-text job-card-text", 
-                    style = {"word-wrap": "break-word"}
+                    className="card-text job-card-text",
+                    style={"word-wrap": "break-word"},
                 ),
-                style = {"white-space": "pre-wrap"},
+                style={"white-space": "pre-wrap"},
             ),
             html.Div(
                 (
@@ -98,9 +95,9 @@ def get_jobs_cards(state: WebState):
                     if is_authenticated
                     else []
                 ),
-                id = {'type': 'manage-job-buttons-div', 'index': d.daemon_id},
+                id={'type': 'manage-job-buttons-div', 'index': name},
             ),
-            html.Div(id={'type': 'manage-job-alert-div', 'index': d.daemon_id}),
+            html.Div(id={'type': 'manage-job-alert-div', 'index': name}),
         ]
 
         cards.append(
@@ -114,11 +111,11 @@ def get_jobs_cards(state: WebState):
     return cards, []
 
 
-def build_manage_job_buttons_div_children(daemon: Daemon):
+def build_manage_job_buttons_div_children(job: Job):
     """
     Return the children for the manage job buttons div.
     """
-    buttons = build_manage_job_buttons(daemon)
+    buttons = build_manage_job_buttons(job)
     if not buttons:
         return []
     return [
@@ -130,96 +127,98 @@ def build_manage_job_buttons_div_children(daemon: Daemon):
     ]
 
 
-def build_manage_job_buttons(daemon: Daemon):
+def build_manage_job_buttons(job: Job):
     """
-    Return the currently available job management buttons for a given Daemon.
+    Return the currently available job management buttons for a given Job.
     """
-    if daemon is None:
+    if job is None:
         return []
+
     start_button = dbc.Button(
         'Start',
-        size = 'sm',
-        color = 'success',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='success',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'start',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     pause_button = dbc.Button(
         'Pause',
-        size = 'sm',
-        color = 'warning',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='warning',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'pause',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     stop_button = dbc.Button(
         'Stop',
-        size = 'sm',
-        color = 'danger',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='danger',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'stop',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     delete_button = dbc.Button(
         'Delete',
-        size = 'sm',
-        color = 'danger',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='danger',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'delete',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     buttons = []
-    if daemon.status in ('stopped', 'paused'):
+    if job.status in ('stopped', 'paused'):
         buttons.append(start_button)
-    if daemon.status == 'stopped':
+    if job.status == 'stopped':
         buttons.append(delete_button)
-    if daemon.status in ('running',):
+    if job.status in ('running',):
         buttons.append(pause_button)
-    if daemon.status in ('running', 'paused'):
+    if job.status in ('running', 'paused'):
         buttons.append(stop_button)
 
     return buttons
 
 
-def build_status_children(daemon: Daemon) -> List[html.P]:
+def build_status_children(job: Job) -> List[html.P]:
     """
     Return the status HTML component for this daemon.
     """
-    if daemon is None:
+    if job is None:
         return STATUS_EMOJI['dne']
 
     status_str = (
-        STATUS_EMOJI.get(daemon.status, STATUS_EMOJI['stopped'])
+        STATUS_EMOJI.get(job.status, STATUS_EMOJI['stopped'])
         + ' '
-        + daemon.status.capitalize()
+        + job.status.capitalize()
     )
     return html.P(
         html.B(status_str),
-        className = f"{daemon.status}-job",
+        className=f"{job.status}-job",
     )
 
 
-def build_process_timestamps_children(daemon: Daemon) -> List[dbc.Row]:
+def build_process_timestamps_children(job: Job) -> List[dbc.Row]:
     """
     Return the children to the process timestamps in the footer of the job card.
     """
-    if daemon is None:
+    if job is None:
         return []
+
     children = []
     for timestamp_key, timestamp_val in sorted_dict(
-        daemon.properties.get('process', {})
+        job.daemon.properties.get('process', {})
     ).items():
         timestamp = dateutil_parser.parse(timestamp_val)
         timestamp_str = timestamp.strftime('%Y-%m-%d %H:%M UTC')
@@ -229,21 +228,21 @@ def build_process_timestamps_children(daemon: Daemon) -> List[dbc.Row]:
                     dbc.Col(
                         html.P(
                             timestamp_key.capitalize(),
-                            style = {'font-size': 'small'},
-                            className = 'text-muted mb-0',
+                            style={'font-size': 'small'},
+                            className='text-muted mb-0',
                         ),
-                        width = 4,
+                        width=4,
                     ),
                     dbc.Col(
                         html.P(
                             timestamp_str,
-                            style = {'font-size': 'small', 'text-align': 'right'},
-                            className = 'text-muted mb-0',
+                            style={'font-size': 'small', 'text-align': 'right'},
+                            className='text-muted mb-0',
                         ),
-                        width = 8,
+                        width=8,
                     ),
                 ],
-                justify = 'between', 
+                justify='between',
             )
         )
     return children

meerschaum/api/routes/_actions.py

@@ -16,7 +16,7 @@ from datetime import datetime, timezone
 from fastapi import WebSocket, WebSocketDisconnect
 
 from meerschaum.utils.misc import generate_password
-from meerschaum.utils.jobs import Job
+from meerschaum.jobs import Job
 from meerschaum.utils.warnings import warn
 from meerschaum.utils.typing import SuccessTuple, Union, List, Dict
 from meerschaum.api import (
@@ -92,6 +92,7 @@ async def do_action_websocket(websocket: WebSocket):
         )
 
     job = None
+    job_name = '.' + generate_password(12)
     try:
         token = await websocket.receive_text()
         user = await manager.get_current_user(token) if not no_auth else None
@@ -101,7 +102,11 @@ async def do_action_websocket(websocket: WebSocket):
                 detail="Invalid credentials.",
             )
 
-        auth_success, auth_msg = is_user_allowed_to_execute(user)
+        auth_success, auth_msg = (
+            is_user_allowed_to_execute(user)
+            if not no_auth
+            else (True, "Success")
+        )
         auth_payload = {
             'is_authenticated': auth_success,
             'timestamp': datetime.now(timezone.utc).isoformat(),
@@ -116,10 +121,10 @@ async def do_action_websocket(websocket: WebSocket):
         _ = kwargs.pop('shell', None)
         sysargs = parse_dict_to_sysargs(kwargs)
 
-        job_name = '.' + generate_password(12)
         job = Job(
             job_name,
             sysargs,
+            executor_keys='local',
             _properties={
                 'logs': {
                     'write_timestamps': False,

meerschaum/api/routes/_jobs.py

@@ -19,7 +19,12 @@ from functools import partial
 from fastapi import WebSocket, WebSocketDisconnect
 
 from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
-from meerschaum.utils.jobs import get_jobs as _get_jobs, Job, StopMonitoringLogs
+from meerschaum.jobs import (
+    get_jobs as _get_jobs,
+    Job,
+    StopMonitoringLogs,
+    get_executor_keys_from_context,
+)
 from meerschaum.utils.warnings import warn
 
 from meerschaum.api import (
@@ -35,6 +40,7 @@ from meerschaum.config.static import STATIC_CONFIG
 
 JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
 JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
+EXECUTOR_KEYS: str = get_executor_keys_from_context()
 
 
 @app.get(endpoints['jobs'], tags=['Jobs'])
@@ -46,15 +52,21 @@ def get_jobs(
     """
     Return metadata about the current jobs.
     """
-    jobs = _get_jobs()
+    jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=False)
     return {
         name: {
             'sysargs': job.sysargs,
             'result': job.result,
+            'restart': job.restart,
+            'status': job.status,
             'daemon': {
-                'status': job.daemon.status,
-                'pid': job.daemon.pid,
-                'properties': job.daemon.properties,
+                'status': job.daemon.status if job.executor_keys is None else job.status,
+                'pid': job.pid,
+                'properties': (
+                    job.daemon.properties
+                    if job.executor is None
+                    else job.executor.get_job_properties(name)
+                ),
             },
         }
         for name, job in jobs.items()
@@ -71,7 +83,7 @@ def get_job(
     """
     Return metadata for a single job.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -81,10 +93,16 @@ def get_job(
     return {
         'sysargs': job.sysargs,
         'result': job.result,
+        'restart': job.restart,
+        'status': job.status,
         'daemon': {
-            'status': job.daemon.status,
-            'pid': job.daemon.pid,
-            'properties': job.daemon.properties,
+            'status': job.daemon.status if job.executor_keys is None else job.status,
+            'pid': job.pid,
+            'properties': (
+                job.daemon.properties
+                if job.executor is None
+                else job.executor.get_job_properties(job.name)
+            ),
         },
     }
 
@@ -100,7 +118,7 @@ def create_job(
     """
     Create and start a new job.
     """
-    job = Job(name, sysargs)
+    job = Job(name, sysargs, executor_keys=EXECUTOR_KEYS)
     if job.exists():
         raise fastapi.HTTPException(
             status_code=409,
@@ -120,7 +138,7 @@ def delete_job(
     """
     Delete a job.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.delete()
 
 
@@ -134,7 +152,7 @@ def get_job_exists(
     """
     Return whether a job exists.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.exists()
 
 
@@ -149,7 +167,7 @@ def get_logs(
     Return a job's log text.
     To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -169,7 +187,7 @@ def start_job(
     """
     Start a job if stopped.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -188,7 +206,7 @@ def stop_job(
     """
     Stop a job if running.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -207,7 +225,7 @@ def pause_job(
     """
     Pause a job if running.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -226,7 +244,7 @@ def get_stop_time(
     """
     Get the timestamp when the job was manually stopped.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.stop_time
 
 
@@ -240,7 +258,7 @@ def get_is_blocking_on_stdin(
     """
     Return whether a job is blocking on stdin.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.is_blocking_on_stdin()
 
 
@@ -314,7 +332,7 @@ async def logs_websocket(name: str, websocket: WebSocket):
     Stream logs from a job over a websocket.
     """
     await websocket.accept()
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     _job_clients[name].append(websocket)
 
     async def monitor_logs():

meerschaum/config/_default.py

@@ -16,7 +16,7 @@ default_meerschaum_config = {
     'api_instance': 'MRSM{meerschaum:instance}',
     'web_instance': 'MRSM{meerschaum:instance}',
     'default_repository': 'api:mrsm',
-    'default_executor': 'local',
+    #  'default_executor': 'local',
     'connectors': {
         'sql': {
             'default': {},

meerschaum/config/_paths.py

@@ -181,6 +181,11 @@ paths = {
     'LOGS_RESOURCES_PATH'            : ('{ROOT_DIR_PATH}', 'logs'),
     'DAEMON_ERROR_LOG_PATH'          : ('{ROOT_DIR_PATH}', 'daemon_errors.log'),
     'CHECK_JOBS_LOCK_PATH'           : ('{INTERNAL_RESOURCES_PATH}', 'check-jobs.lock'),
+    
+    'SYSTEMD_RESOURCES_PATH'         : ('{DOT_CONFIG_DIR_PATH}', 'systemd'),
+    'SYSTEMD_USER_RESOURCES_PATH'    : ('{SYSTEMD_RESOURCES_PATH}', 'user'),
+    'SYSTEMD_ROOT_RESOURCES_PATH'    : ('{ROOT_DIR_PATH}', 'systemd'),
+    'SYSTEMD_LOGS_RESOURCES_PATH'    : ('{SYSTEMD_ROOT_RESOURCES_PATH}', 'logs'),
 }
 
 def set_root(root: Union[Path, str]):

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.3.0.dev3"
+__version__ = "2.3.0rc1"

meerschaum/config/static/__init__.py

@@ -70,6 +70,9 @@ STATIC_CONFIG: Dict[str, Any] = {
         'noask': 'MRSM_NOASK',
         'id': 'MRSM_SERVER_ID',
         'daemon_id': 'MRSM_DAEMON_ID',
+        'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
+        'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
+        'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },

meerschaum/connectors/Connector.py

@@ -21,11 +21,11 @@ class Connector(metaclass=abc.ABCMeta):
     The base connector class to hold connection attributes.
     """
     def __init__(
-            self,
-            type: Optional[str] = None,
-            label: Optional[str] = None,
-            **kw: Any
-        ):
+        self,
+        type: Optional[str] = None,
+        label: Optional[str] = None,
+        **kw: Any
+    ):
         """
         Set the given keyword arguments as attributes.
 
@@ -101,7 +101,7 @@ class Connector(metaclass=abc.ABCMeta):
 
         ### load user config into self._attributes
         if self.type in conn_configs and self.label in conn_configs[self.type]:
-            self._attributes.update(conn_configs[self.type][self.label])
+            self._attributes.update(conn_configs[self.type][self.label] or {})
 
         ### load system config into self._sys_config
         ### (deep copy so future Connectors don't inherit changes)
@@ -200,7 +200,13 @@ class Connector(metaclass=abc.ABCMeta):
         _type = self.__dict__.get('type', None)
         if _type is None:
             import re
-            _type = re.sub(r'connector$', '', self.__class__.__name__.lower())
+            is_executor = self.__class__.__name__.lower().endswith('executor')
+            suffix_regex = (
+                r'connector$'
+                if not is_executor
+                else r'executor$'
+            )
+            _type = re.sub(suffix_regex, '', self.__class__.__name__.lower())
             self.__dict__['type'] = _type
         return _type
 

meerschaum/connectors/__init__.py

@@ -36,9 +36,9 @@ __all__ = (
 ### store connectors partitioned by
 ### type, label for reuse
 connectors: Dict[str, Dict[str, Connector]] = {
-    'api'   : {},
-    'sql'   : {},
-    'plugin': {},
+    'api'    : {},
+    'sql'    : {},
+    'plugin' : {},
 }
 instance_types: List[str] = ['sql', 'api']
 _locks: Dict[str, RLock] = {
@@ -127,10 +127,13 @@ def get_connector(
     global _loaded_plugin_connectors
     if isinstance(type, str) and not label and ':' in type:
         type, label = type.split(':', maxsplit=1)
+
     with _locks['_loaded_plugin_connectors']:
         if not _loaded_plugin_connectors:
             load_plugin_connectors()
+            _load_builtin_custom_connectors()
             _loaded_plugin_connectors = True
+
     if type is None and label is None:
         default_instance_keys = get_config('meerschaum', 'instance', patch=True)
         ### recursive call to get_connector
@@ -274,7 +277,7 @@ def is_connected(keys: str, **kw) -> bool:
         return False
 
 
-def make_connector(cls):
+def make_connector(cls, _is_executor: bool = False):
     """
     Register a class as a `Connector`.
     The `type` will be the lower case of the class name, without the suffix `connector`.
@@ -300,7 +303,12 @@ def make_connector(cls):
     >>> 
     """
     import re
-    typ = re.sub(r'connector$', '', cls.__name__.lower())
+    suffix_regex = (
+        r'connector$'
+        if not _is_executor
+        else r'executor$'
+    )
+    typ = re.sub(suffix_regex, '', cls.__name__.lower())
     with _locks['types']:
         types[typ] = cls
     with _locks['custom_types']:
@@ -363,3 +371,11 @@ def get_connector_plugin(
     )
     plugin = mrsm.Plugin(plugin_name)
     return plugin if plugin.is_installed() else None
+
+
+def _load_builtin_custom_connectors():
+    """
+    Import custom connectors decorated with `@make_connector` or `@make_executor`.
+    """
+    import meerschaum.jobs._SystemdExecutor
+    #  import meerschaum.jobs._LocalExecutor

meerschaum/connectors/api/APIConnector.py

@@ -88,6 +88,9 @@ class APIConnector(Connector):
         monitor_logs,
         monitor_logs_async,
         get_job_is_blocking_on_stdin,
+        get_job_began,
+        get_job_ended,
+        get_job_status,
     )
 
     def __init__(