meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl

meerschaum/api/_events.py

@@ -19,14 +19,24 @@ from meerschaum.utils.debug import dprint
 from meerschaum.connectors.poll import retry_connect
 from meerschaum.utils.warnings import warn
 from meerschaum._internal.term.tools import is_webterm_running
+from meerschaum.jobs import (
+    start_check_jobs_thread,
+    stop_check_jobs_thread,
+    get_executor_keys_from_context,
+)
+
+_check_jobs_thread = None
 
 @app.on_event("startup")
 async def startup():
-    conn = get_api_connector()
+    """
+    Connect to the instance database and begin monitoring jobs.
+    """
     try:
         if not no_dash:
             from meerschaum.api.dash.webterm import start_webterm
             start_webterm()
+
         connected = retry_connect(
             get_api_connector(),
             workers = get_uvicorn_config().get('workers', None),
@@ -36,18 +46,32 @@ async def startup():
         import traceback
         traceback.print_exc()
         connected = False
+
     if not connected:
         await shutdown()
         os._exit(1)
 
+    if get_executor_keys_from_context() == 'local':
+        start_check_jobs_thread()
+
 
 @app.on_event("shutdown")
 async def shutdown():
+    """
+    Close the database connection and stop monitoring jobs.
+    """
     if debug:
         dprint("Closing connection...")
     if get_api_connector().type == 'sql':
         get_api_connector().engine.dispose()
 
+    if get_executor_keys_from_context() == 'local':
+        stop_check_jobs_thread()
+
+    from meerschaum.api.routes._actions import _temp_jobs
+    for name, job in _temp_jobs.items():
+        job.delete()
+
     ### Terminate any running jobs left over.
     if 'meerschaum.api.dash' in sys.modules:
         from meerschaum.api.dash.actions import running_jobs, stop_action

meerschaum/api/_oauth2.py

@@ -30,6 +30,7 @@ class CustomOAuth2PasswordRequestForm:
         self.client_id = client_id
         self.client_secret = client_secret
 
+
 LoginManager = fastapi_login.LoginManager
 def generate_secret_key() -> str:
     """
@@ -46,5 +47,6 @@ def generate_secret_key() -> str:
 
     return secret_key.encode('utf-8')
 
+
 SECRET = generate_secret_key()
 manager = LoginManager(SECRET, token_url=endpoints['login'])

meerschaum/api/_websockets.py

@@ -23,8 +23,8 @@ sessions = {}
 @app.websocket('/dash/ws')
 @app.websocket(_websocket_endpoint)
 async def websocket_endpoint(
-        websocket: fastapi.WebSocket,
-    ):
+    websocket: fastapi.WebSocket,
+):
     """
     Communicate with the Web Interface over a websocket.
     """

meerschaum/api/dash/callbacks/jobs.py

@@ -24,13 +24,13 @@ from dash import Patch
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 import dash_bootstrap_components as dbc
 from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid
-from meerschaum.utils.daemon import Daemon
 from dash.exceptions import PreventUpdate
 from meerschaum.api.dash.jobs import (
     build_manage_job_buttons_div_children,
     build_status_children,
     build_process_timestamps_children,
 )
+from meerschaum.jobs import Job
 from meerschaum.api.dash.users import is_session_authenticated
 
 @dash_app.callback(
@@ -53,15 +53,11 @@ def download_job_logs(n_clicks):
         raise PreventUpdate
 
     component_dict = json.loads(ctx[0]['prop_id'].split('.' + 'n_clicks')[0])
-    daemon_id = component_dict['index']
-    daemon = Daemon(daemon_id=daemon_id)
+    job_name = component_dict['index']
     now = datetime.now(timezone.utc)
-    filename = (
-        daemon.rotating_log.file_path.name[:(-1 * len('.log'))]
-        + '_' + str(int(now.timestamp())) + '.log'
-    )
+    filename = job_name + '_' + str(int(now.timestamp())) + '.log'
     return {
-        'content': daemon.log_text,
+        'content': job.get_logs(),
         'filename': filename,
     }
 
@@ -73,12 +69,14 @@ def download_job_logs(n_clicks):
     Output({'type': 'process-timestamps-div', 'index': MATCH}, 'children'),
     Input({'type': 'manage-job-button', 'action': ALL, 'index': MATCH}, 'n_clicks'),
     State('session-store', 'data'),
+    State({'type': 'job-label-p', 'index': MATCH}, 'children'),
     prevent_initial_call = True,
 )
 def manage_job_button_click(
-        n_clicks: Optional[int] = None,
-        session_data: Optional[Dict[str, Any]] = None,
-    ):
+    n_clicks: Optional[int] = None,
+    session_data: Optional[Dict[str, Any]] = None,
+    job_label: Optional[str] = None,
+):
     """
     Start, stop, pause, or delete the given job.
     """
@@ -102,20 +100,20 @@ def manage_job_button_click(
         raise PreventUpdate
 
     component_dict = json.loads(ctx[0]['prop_id'].split('.' + 'n_clicks')[0])
-    daemon_id = component_dict['index']
+    job_name = component_dict['index']
     manage_job_action = component_dict['action']
     try:
-        daemon = Daemon(daemon_id=daemon_id)
+        job = Job(job_name, job_label.replace('\n', ' ') if job_label else None)
     except Exception as e:
-        daemon = None
-    if daemon is None:
+        job = None
+    if job is None:
         raise PreventUpdate
 
     manage_functions = {
-        'start': functools.partial(daemon.run, allow_dirty_run=True),
-        'stop': daemon.quit,
-        'pause': daemon.pause,
-        'delete': daemon.cleanup,
+        'start': job.start,
+        'stop': job.stop,
+        'pause': job.pause,
+        'delete': job.delete,
     }
     if manage_job_action not in manage_functions:
         return (
@@ -125,7 +123,7 @@ def manage_job_button_click(
             dash.no_update,
         )
 
-    old_status = daemon.status
+    old_status = job.status
     try:
         success, msg = manage_functions[manage_job_action]()
     except Exception as e:
@@ -136,15 +134,15 @@ def manage_job_button_click(
     check_interval_seconds = 0.01
     begin = time.perf_counter()
     while (time.perf_counter() - begin) < timeout_seconds:
-        if daemon.status != old_status:
+        if job.status != old_status:
             break
         time.sleep(check_interval_seconds)
 
     return (
         alert_from_success_tuple((success, msg)),
-        build_manage_job_buttons_div_children(daemon),
-        build_status_children(daemon),
-        build_process_timestamps_children(daemon),
+        build_manage_job_buttons_div_children(job),
+        build_status_children(job),
+        build_process_timestamps_children(job),
     )
 
 dash_app.clientside_callback(
@@ -165,7 +163,7 @@ dash_app.clientside_callback(
         }
 
         const triggered_id = dash_clientside.callback_context.triggered_id;
-        const job_daemon_id = triggered_id["index"];
+        const job_name = triggered_id["index"];
 
         iframe = document.getElementById('webterm-iframe');
         if (!iframe){ return dash_clientside.no_update; }
@@ -174,7 +172,7 @@ dash_app.clientside_callback(
             {
                action: "show",
                subaction: "logs",
-               subaction_text: job_daemon_id,
+               subaction_text: job_name,
             },
             url
         );
@@ -197,44 +195,38 @@ dash_app.clientside_callback(
     prevent_initial_call = True,
 )
 def refresh_jobs_on_interval(
-        n_intervals: Optional[int] = None,
-        session_data: Optional[Dict[str, Any]] = None,
-    ):
+    n_intervals: Optional[int] = None,
+    session_data: Optional[Dict[str, Any]] = None,
+):
     """
     When the jobs refresh interval fires, rebuild the jobs' onscreen components.
     """
     session_id = session_data.get('session-id', None)
     is_authenticated = is_session_authenticated(session_id)
 
-    daemon_ids = [
+    job_names = [
         component_dict['id']['index']
         for component_dict in dash.callback_context.outputs_grouping[0]
     ]
 
-    ### NOTE: The daemon may have been deleted, but the card may still exist.
-    daemons = []
-    for daemon_id in daemon_ids:
-        try:
-            daemon = Daemon(daemon_id=daemon_id)
-        except Exception as e:
-            daemon = None
-        daemons.append(daemon)
+    ### NOTE: The job may have been deleted, but the card may still exist.
+    jobs = [Job(name) for name in job_names]
 
     return (
         [
             (
-                build_manage_job_buttons_div_children(daemon)
+                build_manage_job_buttons_div_children(job)
                 if is_authenticated
                 else []
             )
-            for daemon in daemons
+            for job in jobs
         ],
         [
-            build_status_children(daemon)
-            for daemon in daemons
+            build_status_children(job)
+            for job in jobs
         ],
         [
-            build_process_timestamps_children(daemon)
-            for daemon in daemons
+            build_process_timestamps_children(job)
+            for job in jobs
         ],
     )

meerschaum/api/dash/jobs.py

@@ -15,13 +15,13 @@ from meerschaum.api.dash.users import is_session_authenticated
 from meerschaum.api import CHECK_UPDATE
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 html, dcc = import_html(), import_dcc()
-dateutil_parser = attempt_import('dateutil.parser', check_update=CHECK_UPDATE)
-from meerschaum.utils.daemon import (
-    get_daemons,
-    get_running_daemons,
-    get_paused_daemons,
-    get_stopped_daemons,
-    Daemon,
+from meerschaum.jobs import (
+    get_jobs,
+    get_running_jobs,
+    get_paused_jobs,
+    get_stopped_jobs,
+    get_executor_keys_from_context,
+    Job,
 )
 from meerschaum.config import get_config
 from meerschaum.utils.misc import sorted_dict
@@ -33,71 +33,73 @@ STATUS_EMOJI: Dict[str, str] = {
     'dne': get_config('formatting', 'emoji', 'failure')
 }
 
+EXECUTOR_KEYS: str = get_executor_keys_from_context()
+
 def get_jobs_cards(state: WebState):
     """
     Build cards and alerts lists for jobs.
     """
-    daemons = get_daemons()
+    jobs = get_jobs(executor_keys=EXECUTOR_KEYS, include_hidden=False)
     session_id = state['session-store.data'].get('session-id', None)
     is_authenticated = is_session_authenticated(session_id)
 
-    alert = alert_from_success_tuple(daemons)
     cards = []
 
-    for d in daemons:
+    for name, job in jobs.items():
         footer_children = html.Div(
-            build_process_timestamps_children(d),
-            id = {'type': 'process-timestamps-div', 'index': d.daemon_id},
+            build_process_timestamps_children(job),
+            id = {'type': 'process-timestamps-div', 'index': name},
         )
         follow_logs_button = dbc.DropdownMenuItem(
             "Follow logs",
-            id = {'type': 'follow-logs-button', 'index': d.daemon_id},
+            id = {'type': 'follow-logs-button', 'index': name},
         )
         download_logs_button = dbc.DropdownMenuItem(
             "Download logs",
-            id = {'type': 'job-download-logs-button', 'index': d.daemon_id},
+            id = {'type': 'job-download-logs-button', 'index': name},
         )
         logs_menu_children = (
             ([follow_logs_button] if is_authenticated else []) + [download_logs_button]
         )
         header_children = [
             html.Div(
-                build_status_children(d),
-                id = {'type': 'manage-job-status-div', 'index': d.daemon_id},
-                style = {'float': 'left'},
+                build_status_children(job),
+                id={'type': 'manage-job-status-div', 'index': name},
+                style={'float': 'left'},
             ),
             html.Div(
                 dbc.DropdownMenu(
                     logs_menu_children,
-                    label = "Logs",
-                    size = "sm",
-                    align_end = True,
-                    color = "secondary",
-                    menu_variant = 'dark',
+                    label="Logs",
+                    size="sm",
+                    align_end=True,
+                    color="secondary",
+                    menu_variant='dark',
                 ),
-                style = {'float': 'right'},
+                style={'float': 'right'},
             ),
         ]
 
         body_children = [
-            html.H4(html.B(d.daemon_id), className="card-title"),
+            html.H4(html.B(name), className="card-title"),
             html.Div(
                 html.P(
-                    d.label,
-                    className = "card-text job-card-text", 
-                    style = {"word-wrap": "break-word"}
+                    job.label,
+                    className="card-text job-card-text",
+                    style={"word-wrap": "break-word"},
+                    id={'type': 'job-label-p', 'index': name},
                 ),
-                style = {"white-space": "pre-wrap"},
+                style={"white-space": "pre-wrap"},
             ),
             html.Div(
                 (
-                    build_manage_job_buttons_div_children(d)
+                    build_manage_job_buttons_div_children(job)
                     if is_authenticated
                     else []
                 ),
-                id = {'type': 'manage-job-buttons-div', 'index': d.daemon_id},
+                id={'type': 'manage-job-buttons-div', 'index': name},
             ),
-            html.Div(id={'type': 'manage-job-alert-div', 'index': d.daemon_id}),
+            html.Div(id={'type': 'manage-job-alert-div', 'index': name}),
         ]
 
         cards.append(
@@ -111,11 +113,11 @@ def get_jobs_cards(state: WebState):
     return cards, []
 
 
-def build_manage_job_buttons_div_children(daemon: Daemon):
+def build_manage_job_buttons_div_children(job: Job):
     """
     Return the children for the manage job buttons div.
     """
-    buttons = build_manage_job_buttons(daemon)
+    buttons = build_manage_job_buttons(job)
     if not buttons:
         return []
     return [
@@ -127,98 +129,107 @@ def build_manage_job_buttons_div_children(daemon: Daemon):
     ]
 
 
-def build_manage_job_buttons(daemon: Daemon):
+def build_manage_job_buttons(job: Job):
     """
-    Return the currently available job management buttons for a given Daemon.
+    Return the currently available job management buttons for a given Job.
     """
-    if daemon is None:
+    if job is None:
         return []
+
     start_button = dbc.Button(
         'Start',
-        size = 'sm',
-        color = 'success',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='success',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'start',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     pause_button = dbc.Button(
         'Pause',
-        size = 'sm',
-        color = 'warning',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='warning',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'pause',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     stop_button = dbc.Button(
         'Stop',
-        size = 'sm',
-        color = 'danger',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='danger',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'stop',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     delete_button = dbc.Button(
         'Delete',
-        size = 'sm',
-        color = 'danger',
-        style = {'width': '100%'},
-        id = {
+        size='sm',
+        color='danger',
+        style={'width': '100%'},
+        id={
             'type': 'manage-job-button',
             'action': 'delete',
-            'index': daemon.daemon_id,
+            'index': job.name,
         },
     )
     buttons = []
-    if daemon.status in ('stopped', 'paused'):
+    status = job.status
+    if status in ('stopped', 'paused'):
         buttons.append(start_button)
-    if daemon.status == 'stopped':
+    if status == 'stopped':
         buttons.append(delete_button)
-    if daemon.status in ('running',):
+    if status in ('running',):
         buttons.append(pause_button)
-    if daemon.status in ('running', 'paused'):
+    if status in ('running', 'paused'):
         buttons.append(stop_button)
 
     return buttons
 
 
-def build_status_children(daemon: Daemon) -> List[html.P]:
+def build_status_children(job: Job) -> List[html.P]:
     """
-    Return the status HTML component for this daemon.
+    Return the status HTML component for this Job.
     """
-    if daemon is None:
+    if job is None:
         return STATUS_EMOJI['dne']
 
     status_str = (
-        STATUS_EMOJI.get(daemon.status, STATUS_EMOJI['stopped'])
+        STATUS_EMOJI.get(job.status, STATUS_EMOJI['stopped'])
         + ' '
-        + daemon.status.capitalize()
+        + job.status.capitalize()
     )
     return html.P(
         html.B(status_str),
-        className = f"{daemon.status}-job",
+        className=f"{job.status}-job",
     )
 
 
-def build_process_timestamps_children(daemon: Daemon) -> List[dbc.Row]:
+def build_process_timestamps_children(job: Job) -> List[dbc.Row]:
     """
     Return the children to the process timestamps in the footer of the job card.
     """
-    if daemon is None:
+    if job is None:
         return []
+
     children = []
-    for timestamp_key, timestamp_val in sorted_dict(
-        daemon.properties.get('process', {})
+    for timestamp_key, timestamp in sorted_dict(
+        {
+            'began': job.began,
+            'paused': job.paused,
+            'ended': job.ended,
+        }
     ).items():
-        timestamp = dateutil_parser.parse(timestamp_val)
+        if timestamp is None:
+            continue
+
         timestamp_str = timestamp.strftime('%Y-%m-%d %H:%M UTC')
         children.append(
             dbc.Row(
@@ -226,21 +237,21 @@ def build_process_timestamps_children(daemon: Daemon) -> List[dbc.Row]:
                     dbc.Col(
                         html.P(
                             timestamp_key.capitalize(),
-                            style = {'font-size': 'small'},
-                            className = 'text-muted mb-0',
+                            style={'font-size': 'small'},
+                            className='text-muted mb-0',
                         ),
-                        width = 4,
+                        width=4,
                     ),
                     dbc.Col(
                         html.P(
                             timestamp_str,
-                            style = {'font-size': 'small', 'text-align': 'right'},
-                            className = 'text-muted mb-0',
+                            style={'font-size': 'small', 'text-align': 'right'},
+                            className='text-muted mb-0',
                         ),
-                        width = 8,
+                        width=8,
                     ),
                 ],
-                justify = 'between', 
+                justify='between',
             )
         )
     return children

meerschaum/api/routes/__init__.py

@@ -8,6 +8,7 @@ Import all routes from other modules in package
 
 from meerschaum.api.routes._login import *
 from meerschaum.api.routes._actions import *
+from meerschaum.api.routes._jobs import *
 from meerschaum.api.routes._connectors import *
 from meerschaum.api.routes._index import *
 from meerschaum.api.routes._misc import *