meerschaum 2.3.0.dev1__py3-none-any.whl → 2.3.0rc1__py3-none-any.whl

meerschaum/actions/stop.py

@@ -27,13 +27,20 @@ def _complete_stop(
     """
     Override the default Meerschaum `complete_` function.
     """
-    from meerschaum.actions.start import _complete_start_jobs
+    from meerschaum.actions.delete import _complete_delete_jobs
+    from functools import partial
+
     if action is None:
         action = []
 
+    _complete_stop_jobs = partial(
+        _complete_delete_jobs,
+        _get_job_method=('running', 'paused', 'restart'),
+    )
+
     options = {
-        'job' : _complete_start_jobs,
-        'jobs' : _complete_start_jobs,
+        'job' : _complete_stop_jobs,
+        'jobs' : _complete_stop_jobs,
     }
 
     if (
@@ -64,7 +71,7 @@ def _stop_jobs(
     
     To see running processes, run `show jobs`.
     """
-    from meerschaum.utils.jobs import (
+    from meerschaum.jobs import (
         get_filtered_jobs,
         get_running_jobs,
         get_paused_jobs,

meerschaum/api/_events.py

@@ -19,7 +19,11 @@ from meerschaum.utils.debug import dprint
 from meerschaum.connectors.poll import retry_connect
 from meerschaum.utils.warnings import warn
 from meerschaum._internal.term.tools import is_webterm_running
-from meerschaum.utils.jobs import start_check_jobs_thread, stop_check_jobs_thread
+from meerschaum.jobs import (
+    start_check_jobs_thread,
+    stop_check_jobs_thread,
+    get_executor_keys_from_context,
+)
 
 _check_jobs_thread = None
 
@@ -47,7 +51,8 @@ async def startup():
         await shutdown()
         os._exit(1)
 
-    start_check_jobs_thread()
+    if get_executor_keys_from_context() == 'local':
+        start_check_jobs_thread()
 
 
 @app.on_event("shutdown")
@@ -60,7 +65,9 @@ async def shutdown():
     if get_api_connector().type == 'sql':
         get_api_connector().engine.dispose()
 
-    stop_check_jobs_thread()
+    if get_executor_keys_from_context() == 'local':
+        stop_check_jobs_thread()
+
     from meerschaum.api.routes._actions import _temp_jobs
     for name, job in _temp_jobs.items():
         job.delete()

meerschaum/api/dash/jobs.py

@@ -16,13 +16,12 @@ from meerschaum.api import CHECK_UPDATE
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 html, dcc = import_html(), import_dcc()
 dateutil_parser = attempt_import('dateutil.parser', check_update=CHECK_UPDATE)
-from meerschaum.utils.jobs import get_jobs, Job
-from meerschaum.utils.daemon import (
-    get_daemons,
-    get_running_daemons,
-    get_paused_daemons,
-    get_stopped_daemons,
-    Daemon,
+from meerschaum.jobs import (
+    get_jobs,
+    get_running_jobs,
+    get_paused_jobs,
+    get_stopped_jobs,
+    Job,
 )
 from meerschaum.config import get_config
 from meerschaum.utils.misc import sorted_dict
@@ -38,27 +37,25 @@ def get_jobs_cards(state: WebState):
     """
     Build cards and alerts lists for jobs.
     """
-    daemons = get_daemons()
-    jobs = get_jobs()
+    jobs = get_jobs(include_hidden=False)
     session_id = state['session-store.data'].get('session-id', None)
     is_authenticated = is_session_authenticated(session_id)
 
-    alert = alert_from_success_tuple(daemons)
     cards = []
 
     for name, job in jobs.items():
         d = job.daemon
         footer_children = html.Div(
             build_process_timestamps_children(d),
-            id = {'type': 'process-timestamps-div', 'index': d.daemon_id},
+            id = {'type': 'process-timestamps-div', 'index': name},
         )
         follow_logs_button = dbc.DropdownMenuItem(
             "Follow logs",
-            id = {'type': 'follow-logs-button', 'index': d.daemon_id},
+            id = {'type': 'follow-logs-button', 'index': name},
         )
         download_logs_button = dbc.DropdownMenuItem(
             "Download logs",
-            id = {'type': 'job-download-logs-button', 'index': d.daemon_id},
+            id = {'type': 'job-download-logs-button', 'index': name},
         )
         logs_menu_children = (
             ([follow_logs_button] if is_authenticated else []) + [download_logs_button]
@@ -66,19 +63,19 @@ def get_jobs_cards(state: WebState):
         header_children = [
             html.Div(
                 build_status_children(d),
-                id = {'type': 'manage-job-status-div', 'index': d.daemon_id},
-                style = {'float': 'left'},
+                id={'type': 'manage-job-status-div', 'index': name},
+                style={'float': 'left'},
             ),
             html.Div(
                 dbc.DropdownMenu(
                     logs_menu_children,
-                    label = "Logs",
-                    size = "sm",
-                    align_end = True,
-                    color = "secondary",
-                    menu_variant = 'dark',
+                    label="Logs",
+                    size="sm",
+                    align_end=True,
+                    color="secondary",
+                    menu_variant='dark',
                 ),
-                style = {'float': 'right'},
+                style={'float': 'right'},
             ),
         ]
 
@@ -87,10 +84,10 @@ def get_jobs_cards(state: WebState):
             html.Div(
                 html.P(
                     d.label,
-                    className = "card-text job-card-text", 
-                    style = {"word-wrap": "break-word"}
+                    className="card-text job-card-text",
+                    style={"word-wrap": "break-word"},
                 ),
-                style = {"white-space": "pre-wrap"},
+                style={"white-space": "pre-wrap"},
             ),
             html.Div(
                 (
@@ -98,9 +95,9 @@ def get_jobs_cards(state: WebState):
                     if is_authenticated
                     else []
                 ),
-                id = {'type': 'manage-job-buttons-div', 'index': d.daemon_id},
+                id={'type': 'manage-job-buttons-div', 'index': name},
             ),
-            html.Div(id={'type': 'manage-job-alert-div', 'index': d.daemon_id}),
+            html.Div(id={'type': 'manage-job-alert-div', 'index': name}),
         ]
 
         cards.append(
@@ -114,11 +111,11 @@ def get_jobs_cards(state: WebState):
     return cards, []
 
 
-def build_manage_job_buttons_div_children(daemon: Daemon):
+def build_manage_job_buttons_div_children(job: Job):
     """
     Return the children for the manage job buttons div.
     """
-    buttons = build_manage_job_buttons(daemon)
+    buttons = build_manage_job_buttons(job)
     if not buttons:
         return []
     return [
@@ -130,96 +127,98 @@ def build_manage_job_buttons_div_children(daemon: Daemon):
     ]
 
 
-def build_manage_job_buttons(daemon: Daemon):
+def build_manage_job_buttons(job: Job):
     """
-    Return the currently available job management buttons for a given Daemon.
+    Return the currently available job management buttons for a given Job.
     """
-    if daemon is None:
+    if job is None:
         return []
+
     start_button = dbc.Button(
         'Start',
-        size = 'sm',
-        color = 'success',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='success',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'start',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     pause_button = dbc.Button(
         'Pause',
-        size = 'sm',
-        color = 'warning',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='warning',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'pause',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     stop_button = dbc.Button(
         'Stop',
-        size = 'sm',
-        color = 'danger',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='danger',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'stop',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     delete_button = dbc.Button(
         'Delete',
-        size = 'sm',
-        color = 'danger',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='danger',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'delete',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     buttons = []
-    if daemon.status in ('stopped', 'paused'):
+    if job.status in ('stopped', 'paused'):
         buttons.append(start_button)
-    if daemon.status == 'stopped':
+    if job.status == 'stopped':
         buttons.append(delete_button)
-    if daemon.status in ('running',):
+    if job.status in ('running',):
         buttons.append(pause_button)
-    if daemon.status in ('running', 'paused'):
+    if job.status in ('running', 'paused'):
         buttons.append(stop_button)
 
     return buttons
 
 
-def build_status_children(daemon: Daemon) -> List[html.P]:
+def build_status_children(job: Job) -> List[html.P]:
     """
     Return the status HTML component for this daemon.
     """
-    if daemon is None:
+    if job is None:
         return STATUS_EMOJI['dne']
 
     status_str = (
-        STATUS_EMOJI.get(daemon.status, STATUS_EMOJI['stopped'])
+        STATUS_EMOJI.get(job.status, STATUS_EMOJI['stopped'])
         + ' '
-        + daemon.status.capitalize()
+        + job.status.capitalize()
     )
     return html.P(
         html.B(status_str),
-        className = f"{daemon.status}-job",
+        className=f"{job.status}-job",
     )
 
 
-def build_process_timestamps_children(daemon: Daemon) -> List[dbc.Row]:
+def build_process_timestamps_children(job: Job) -> List[dbc.Row]:
     """
     Return the children to the process timestamps in the footer of the job card.
     """
-    if daemon is None:
+    if job is None:
         return []
+
     children = []
     for timestamp_key, timestamp_val in sorted_dict(
-        daemon.properties.get('process', {})
+        job.daemon.properties.get('process', {})
     ).items():
         timestamp = dateutil_parser.parse(timestamp_val)
         timestamp_str = timestamp.strftime('%Y-%m-%d %H:%M UTC')
@@ -229,21 +228,21 @@ def build_process_timestamps_children(daemon: Daemon) -> List[dbc.Row]:
                     dbc.Col(
                         html.P(
                             timestamp_key.capitalize(),
-                            style = {'font-size': 'small'},
-                            className = 'text-muted mb-0',
+                            style={'font-size': 'small'},
+                            className='text-muted mb-0',
                         ),
-                        width = 4,
+                        width=4,
                     ),
                     dbc.Col(
                         html.P(
                             timestamp_str,
-                            style = {'font-size': 'small', 'text-align': 'right'},
-                            className = 'text-muted mb-0',
+                            style={'font-size': 'small', 'text-align': 'right'},
+                            className='text-muted mb-0',
                         ),
-                        width = 8,
+                        width=8,
                     ),
                 ],
-                justify = 'between', 
+                justify='between',
             )
         )
     return children

meerschaum/api/routes/_actions.py

@@ -16,7 +16,7 @@ from datetime import datetime, timezone
 from fastapi import WebSocket, WebSocketDisconnect
 
 from meerschaum.utils.misc import generate_password
-from meerschaum.utils.jobs import Job
+from meerschaum.jobs import Job
 from meerschaum.utils.warnings import warn
 from meerschaum.utils.typing import SuccessTuple, Union, List, Dict
 from meerschaum.api import (
@@ -92,6 +92,7 @@ async def do_action_websocket(websocket: WebSocket):
         )
 
     job = None
+    job_name = '.' + generate_password(12)
     try:
         token = await websocket.receive_text()
         user = await manager.get_current_user(token) if not no_auth else None
@@ -101,7 +102,11 @@ async def do_action_websocket(websocket: WebSocket):
                 detail="Invalid credentials.",
             )
 
-        auth_success, auth_msg = is_user_allowed_to_execute(user)
+        auth_success, auth_msg = (
+            is_user_allowed_to_execute(user)
+            if not no_auth
+            else (True, "Success")
+        )
         auth_payload = {
             'is_authenticated': auth_success,
             'timestamp': datetime.now(timezone.utc).isoformat(),
@@ -116,10 +121,10 @@ async def do_action_websocket(websocket: WebSocket):
         _ = kwargs.pop('shell', None)
         sysargs = parse_dict_to_sysargs(kwargs)
 
-        job_name = '.' + generate_password(12)
         job = Job(
             job_name,
             sysargs,
+            executor_keys='local',
             _properties={
                 'logs': {
                     'write_timestamps': False,

meerschaum/api/routes/_jobs.py

@@ -19,7 +19,12 @@ from functools import partial
 from fastapi import WebSocket, WebSocketDisconnect
 
 from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
-from meerschaum.utils.jobs import get_jobs as _get_jobs, Job, StopMonitoringLogs
+from meerschaum.jobs import (
+    get_jobs as _get_jobs,
+    Job,
+    StopMonitoringLogs,
+    get_executor_keys_from_context,
+)
 from meerschaum.utils.warnings import warn
 
 from meerschaum.api import (
@@ -34,6 +39,8 @@ from meerschaum.api import (
 from meerschaum.config.static import STATIC_CONFIG
 
 JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
+JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
+EXECUTOR_KEYS: str = get_executor_keys_from_context()
 
 
 @app.get(endpoints['jobs'], tags=['Jobs'])
@@ -45,15 +52,21 @@ def get_jobs(
     """
     Return metadata about the current jobs.
     """
-    jobs = _get_jobs()
+    jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=False)
     return {
         name: {
             'sysargs': job.sysargs,
             'result': job.result,
+            'restart': job.restart,
+            'status': job.status,
             'daemon': {
-                'status': job.daemon.status,
-                'pid': job.daemon.pid,
-                'properties': job.daemon.properties,
+                'status': job.daemon.status if job.executor_keys is None else job.status,
+                'pid': job.pid,
+                'properties': (
+                    job.daemon.properties
+                    if job.executor is None
+                    else job.executor.get_job_properties(name)
+                ),
             },
         }
         for name, job in jobs.items()
@@ -70,7 +83,7 @@ def get_job(
     """
     Return metadata for a single job.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -80,10 +93,16 @@ def get_job(
     return {
         'sysargs': job.sysargs,
         'result': job.result,
+        'restart': job.restart,
+        'status': job.status,
         'daemon': {
-            'status': job.daemon.status,
-            'pid': job.daemon.pid,
-            'properties': job.daemon.properties,
+            'status': job.daemon.status if job.executor_keys is None else job.status,
+            'pid': job.pid,
+            'properties': (
+                job.daemon.properties
+                if job.executor is None
+                else job.executor.get_job_properties(job.name)
+            ),
         },
     }
 
@@ -99,7 +118,7 @@ def create_job(
     """
     Create and start a new job.
     """
-    job = Job(name, sysargs)
+    job = Job(name, sysargs, executor_keys=EXECUTOR_KEYS)
     if job.exists():
         raise fastapi.HTTPException(
             status_code=409,
@@ -119,7 +138,7 @@ def delete_job(
     """
     Delete a job.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.delete()
 
 
@@ -133,7 +152,7 @@ def get_job_exists(
     """
     Return whether a job exists.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.exists()
 
 
@@ -148,7 +167,7 @@ def get_logs(
     Return a job's log text.
     To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -168,7 +187,7 @@ def start_job(
     """
     Start a job if stopped.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -187,7 +206,7 @@ def stop_job(
     """
     Stop a job if running.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -206,7 +225,7 @@ def pause_job(
     """
     Pause a job if running.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -225,7 +244,7 @@ def get_stop_time(
     """
     Get the timestamp when the job was manually stopped.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.stop_time
 
 
@@ -239,19 +258,16 @@ def get_is_blocking_on_stdin(
     """
     Return whether a job is blocking on stdin.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.is_blocking_on_stdin()
 
 
 _job_clients = defaultdict(lambda: [])
 _job_stop_events = defaultdict(lambda: asyncio.Event())
-async def notify_clients(name: str, content: str):
+async def notify_clients(name: str, websocket: WebSocket, content: str):
     """
     Write the given content to all connected clients.
     """
-    if not _job_clients[name]:
-        _job_stop_events[name].set()
-
     async def _notify_client(client):
         try:
             await client.send_text(content)
@@ -261,20 +277,14 @@ async def notify_clients(name: str, content: str):
         except Exception:
             pass
 
-    notify_tasks = [
-        asyncio.create_task(_notify_client(client))
-        for client in _job_clients[name]
-    ]
-    await asyncio.wait(notify_tasks)
+    await _notify_client(websocket)
 
 
-async def get_input_from_clients(name):
+async def get_input_from_clients(name: str, websocket: WebSocket) -> str:
     """
     When a job is blocking on input, return input from the first client which provides it.
     """
-    print('GET INPUT FROM CLIENTS')
     if not _job_clients[name]:
-        print('NO CLIENTS')
         return ''
 
     async def _read_client(client):
@@ -299,21 +309,59 @@ async def get_input_from_clients(name):
         return task.result()
 
 
+async def send_stop_message(name: str, client: WebSocket, result: SuccessTuple):
+    """
+    Send a stop message to clients when the job stops.
+    """
+    try:
+        await client.send_text(JOBS_STOP_MESSAGE)
+        await client.send_json(result)
+    except WebSocketDisconnect:
+        _job_stop_events[name].set()
+        if client in _job_clients[name]:
+            _job_clients[name].remove(client)
+    except RuntimeError:
+        pass
+    except Exception:
+        warn(traceback.format_exc())
+
+
 @app.websocket(endpoints['logs'] + '/{name}/ws')
 async def logs_websocket(name: str, websocket: WebSocket):
     """
     Stream logs from a job over a websocket.
     """
     await websocket.accept()
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     _job_clients[name].append(websocket)
 
     async def monitor_logs():
-        await job.monitor_logs_async(
-            partial(notify_clients, name),
-            input_callback_function=partial(get_input_from_clients, name),
-            stop_event=_job_stop_events[name],
-        )
+        try:
+            callback_function = partial(
+                notify_clients,
+                name,
+                websocket,
+            )
+            input_callback_function = partial(
+                get_input_from_clients,
+                name,
+                websocket,
+            )
+            stop_callback_function = partial(
+                send_stop_message,
+                name,
+                websocket,
+            )
+            await job.monitor_logs_async(
+                callback_function=callback_function,
+                input_callback_function=input_callback_function,
+                stop_callback_function=stop_callback_function,
+                stop_event=_job_stop_events[name],
+                stop_on_exit=True,
+                accept_input=True,
+            )
+        except Exception:
+            warn(traceback.format_exc())
 
     try:
         token = await websocket.receive_text()
@@ -329,7 +377,8 @@ async def logs_websocket(name: str, websocket: WebSocket):
         await websocket.send_text("Invalid credentials.")
         await websocket.close()
     except WebSocketDisconnect:
-        pass
+        _job_stop_events[name].set()
+        monitor_task.cancel()
     except asyncio.CancelledError:
         pass
     except Exception:

meerschaum/config/_default.py

@@ -16,7 +16,7 @@ default_meerschaum_config = {
     'api_instance': 'MRSM{meerschaum:instance}',
     'web_instance': 'MRSM{meerschaum:instance}',
     'default_repository': 'api:mrsm',
-    'default_executor': 'local',
+    #  'default_executor': 'local',
     'connectors': {
         'sql': {
             'default': {},

meerschaum/config/_paths.py

@@ -181,6 +181,11 @@ paths = {
     'LOGS_RESOURCES_PATH'            : ('{ROOT_DIR_PATH}', 'logs'),
     'DAEMON_ERROR_LOG_PATH'          : ('{ROOT_DIR_PATH}', 'daemon_errors.log'),
     'CHECK_JOBS_LOCK_PATH'           : ('{INTERNAL_RESOURCES_PATH}', 'check-jobs.lock'),
+    
+    'SYSTEMD_RESOURCES_PATH'         : ('{DOT_CONFIG_DIR_PATH}', 'systemd'),
+    'SYSTEMD_USER_RESOURCES_PATH'    : ('{SYSTEMD_RESOURCES_PATH}', 'user'),
+    'SYSTEMD_ROOT_RESOURCES_PATH'    : ('{ROOT_DIR_PATH}', 'systemd'),
+    'SYSTEMD_LOGS_RESOURCES_PATH'    : ('{SYSTEMD_ROOT_RESOURCES_PATH}', 'logs'),
 }
 
 def set_root(root: Union[Path, str]):