meerschaum 2.2.7__py3-none-any.whl → 2.3.0.dev1__py3-none-any.whl

meerschaum/actions/start.py

@@ -78,6 +78,9 @@ def _start_api(action: Optional[List[str]] = None, **kw):
 def _start_jobs(
         action: Optional[List[str]] = None,
         name: Optional[str] = None,
+        sysargs: Optional[List[str]] = None,
+        executor_keys: Optional[str] = None,
+        debug: bool = False,
         **kw
     ) -> SuccessTuple:
     """
@@ -111,11 +114,16 @@ def _start_jobs(
     """
     import textwrap
     from meerschaum.utils.warnings import warn, info
-    from meerschaum.utils.daemon import (
-        daemon_action, Daemon, get_daemon_ids, get_daemons, get_filtered_daemons,
-        get_stopped_daemons, get_running_daemons, get_paused_daemons,
-    )
     from meerschaum.utils.daemon._names import get_new_daemon_name
+    from meerschaum.utils.jobs import (
+        Job,
+        get_jobs,
+        get_filtered_jobs,
+        get_stopped_jobs,
+        get_running_jobs,
+        get_paused_jobs,
+        get_restart_jobs,
+    )
     from meerschaum._internal.arguments._parse_arguments import parse_arguments
     from meerschaum.actions import actions
     from meerschaum.utils.prompt import yes_no
@@ -125,7 +133,7 @@ def _start_jobs(
     from meerschaum.utils.misc import items_str
 
     names = []
-    daemon_ids = get_daemon_ids()
+    jobs = get_filtered_jobs(executor_keys, debug=debug)
 
     new_job = len(list(action)) > 0
     _potential_jobs = {'known': [], 'unknown': []}
@@ -134,7 +142,7 @@ def _start_jobs(
         for a in action:
             _potential_jobs[(
                 'known'
-                if a in daemon_ids
+                if a in jobs
                 else 'unknown'
             )].append(a)
 
@@ -182,88 +190,83 @@ def _start_jobs(
 
     ### No action or --name was provided. Ask to start all stopped jobs.
     else:
-        _running_daemons = get_running_daemons()
-        _paused_daemons = get_paused_daemons()
-        _stopped_daemons = get_stopped_daemons()
-        if not _stopped_daemons and not _paused_daemons:
-            if not _running_daemons:
-                return False, "No jobs to start."
+        running_jobs = get_running_jobs(executor_keys, jobs, debug=debug)
+        paused_jobs = get_paused_jobs(executor_keys, jobs, debug=debug)
+        stopped_jobs = get_stopped_jobs(executor_keys, jobs, debug=debug)
+
+        if not stopped_jobs and not paused_jobs:
+            if not running_jobs:
+                return False, "No jobs to start"
             return True, "All jobs are running."
 
-        names = [d.daemon_id for d in _stopped_daemons + _paused_daemons]
+        names = [
+            name
+            for name in list(stopped_jobs) + list(paused_jobs)
+        ]
 
     def _run_new_job(name: Optional[str] = None):
-        kw['action'] = action
         name = name or get_new_daemon_name()
-        kw['name'] = name
-        _action_success_tuple = daemon_action(daemon_id=name, **kw)
-        return _action_success_tuple, name
-
-    def _run_existing_job(name: Optional[str] = None):
-        daemon = Daemon(daemon_id=name)
-        if daemon.process is not None:
-            if daemon.status == 'paused':
-                return daemon.resume(), daemon.daemon_id
-            return (True, f"Job '{name}' is already running."), daemon.daemon_id
-
-        if not daemon.path.exists():
-            if not kw.get('nopretty', False):
-                warn(f"There isn't a job with the name '{name}'.", stack=False)
-                print(
-                    f"You can start a new job named '{name}' with `start job "
-                    + "{options}" + f" --name {name}`"
-                )
-            return (False, f"Job '{name}' does not exist."), daemon.daemon_id
+        job = Job(name, sysargs, executor_keys=executor_keys)
+        return job.start(debug=debug), name
 
-        return daemon.run(allow_dirty_run=True), daemon.daemon_id
+    def _run_existing_job(name: str):
+        job = Job(name, executor_keys=executor_keys)
+        return job.start(debug=debug), name
 
     if not names:
         return False, "No jobs to start."
 
     ### Get user permission to clear logs.
-    _filtered_daemons = get_filtered_daemons(names)
-    if not kw.get('force', False) and _filtered_daemons:
-        _filtered_running_daemons = get_running_daemons(_filtered_daemons)
-        _skipped_daemons = []
-        if _filtered_running_daemons:
-            pprint_jobs(_filtered_running_daemons)
+    _filtered_jobs = get_filtered_jobs(executor_keys, names, debug=debug)
+    if not kw.get('force', False) and _filtered_jobs:
+        _filtered_running_jobs = get_running_jobs(executor_keys, _filtered_jobs, debug=debug)
+        _skipped_jobs = []
+        if _filtered_running_jobs:
+            pprint_jobs(_filtered_running_jobs)
             if yes_no(
                 "Do you want to first stop these jobs?",
-                default = 'n',
-                yes = kw.get('yes', False),
-                noask = kw.get('noask', False)
+                default='n',
+                yes=kw.get('yes', False),
+                noask=kw.get('noask', False)
             ):
                 stop_success_tuple = actions['stop'](
-                    action = ['jobs'] + [d.daemon_id for d in _filtered_running_daemons],
-                    force = True,
+                    action=['jobs'] + [_name for _name in _filtered_running_jobs],
+                    force=True,
+                    mrsm_instance=mrsm_instance,
+                    debug=debug,
                 )
                 if not stop_success_tuple[0]:
                     warn(
-                        "Failed to stop job" + ("s" if len(_filtered_running_daemons) != 1 else '')
-                        + items_str([d.daemon_id for d in _filtered_running_daemons])
-                        + ".",
-                        stack = False
+                        (
+                            "Failed to stop job"
+                            + ("s" if len(_filtered_running_jobs) != 1 else '')
+                            + items_str([_name for _name in _filtered_running_jobs])
+                            + "."
+                        ),
+                        stack=False
                     )
-                    for d in _filtered_running_daemons:
-                        names.remove(d.daemon_id)
-                        _filtered_daemons.remove(d)
+                    for _name in _filtered_running_jobs:
+                        names.remove(_name)
+                        _filtered_jobs.pop(_name)
             else:
                 info(
                     "Skipping already running job"
-                    + ("s" if len(_filtered_running_daemons) != 1 else '') + ' '
-                    + items_str([d.daemon_id for d in _filtered_running_daemons]) + '.'
+                    + ("s" if len(_filtered_running_jobs) != 1 else '')
+                    + ' '
+                    + items_str([_name for _name in _filtered_running_jobs])
+                    + '.'
                 )
-                for d in _filtered_running_daemons:
-                    names.remove(d.daemon_id)
-                    _filtered_daemons.remove(d)
-                    _skipped_daemons.append(d)
+                for _name in _filtered_running_jobs:
+                    names.remove(_name)
+                    _filtered_jobs.pop(_name)
+                    _skipped_jobs.append(_name)
 
-        if not _filtered_daemons:
-            return len(_skipped_daemons) > 0, "No jobs to start."
+        if not _filtered_jobs:
+            return len(_skipped_jobs) > 0, "No jobs to start."
 
-        pprint_jobs(_filtered_daemons, nopretty=kw.get('nopretty', False))
+        pprint_jobs(_filtered_jobs, nopretty=kw.get('nopretty', False))
         info(
-            f"Starting the job"
+            "Starting the job"
             + ("s" if len(names) != 1 else "")
             + " " + items_str(names)
             + "..."
@@ -278,7 +281,11 @@ def _start_jobs(
         )
         if not kw.get('nopretty', False):
             print_tuple(success_tuple)
-        _successes.append(_name) if success_tuple[0] else _failures.append(_name)
+
+        if success_tuple[0]:
+            _successes.append(_name)
+        else:
+            _failures.append(_name)
 
     msg = (
         (("Successfully started job" + ("s" if len(_successes) != 1 else '')
@@ -289,25 +296,28 @@ def _start_jobs(
     )
     return len(_failures) == 0, msg
 
+
 def _complete_start_jobs(
-        action: Optional[List[str]] = None,
-        line: str = '',
-        **kw
-    ) -> List[str]:
-    from meerschaum.utils.daemon import get_daemon_ids
-    daemon_ids = get_daemon_ids()
+    action: Optional[List[str]] = None,
+    executor_keys: Optional[str] = None,
+    line: str = '',
+    **kw
+) -> List[str]:
+    from meerschaum.utils.jobs import get_filtered_jobs
+    jobs = get_filtered_jobs(executor_keys, action)
     if not action:
-        return daemon_ids
+        return list(jobs)
+
     possibilities = []
     _line_end = line.split(' ')[-1]
-    for daemon_id in daemon_ids:
-        if daemon_id in action:
+    for name in jobs:
+        if name in action:
             continue
         if _line_end == '':
-            possibilities.append(daemon_id)
+            possibilities.append(name)
             continue
-        if daemon_id.startswith(action[-1]):
-            possibilities.append(daemon_id)
+        if name.startswith(action[-1]):
+            possibilities.append(name)
     return possibilities
 
 

meerschaum/actions/stop.py

@@ -50,11 +50,13 @@ def _complete_stop(
 
 def _stop_jobs(
     action: Optional[List[str]] = None,
+    executor_keys: Optional[str] = None,
     timeout_seconds: Optional[int] = None,
     noask: bool = False,
     force: bool = False,
     yes: bool = False,
     nopretty: bool = False,
+    debug: bool = False,
     **kw
 ) -> SuccessTuple:
     """
@@ -62,66 +64,93 @@ def _stop_jobs(
     
     To see running processes, run `show jobs`.
     """
+    from meerschaum.utils.jobs import (
+        get_filtered_jobs,
+        get_running_jobs,
+        get_paused_jobs,
+        get_stopped_jobs,
+        get_restart_jobs,
+    )
     from meerschaum.utils.formatting._jobs import pprint_jobs
     from meerschaum.utils.daemon import (
         get_filtered_daemons, get_running_daemons, get_stopped_daemons, get_paused_daemons,
     )
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.prompt import yes_no
-    daemons = get_filtered_daemons(action, warn=(not nopretty))
-    _running_daemons = get_running_daemons(daemons)
-    _paused_daemons = get_paused_daemons(daemons)
-    _stopped_daemons = get_stopped_daemons(daemons)
-    if action and _stopped_daemons and not nopretty:
+
+    jobs = get_filtered_jobs(executor_keys, action, warn=(not nopretty))
+    running_jobs = get_running_jobs(executor_keys, jobs)
+    paused_jobs = get_paused_jobs(executor_keys, jobs)
+    restart_jobs = get_restart_jobs(executor_keys, jobs)
+    stopped_jobs = {
+        name: job
+        for name, job in get_stopped_jobs(executor_keys, jobs).items()
+        if name not in restart_jobs
+    }
+
+    jobs_to_stop = {
+        **running_jobs,
+        **paused_jobs,
+        **restart_jobs,
+    }
+
+    if action and stopped_jobs and not nopretty:
         warn(
-            f"Skipping stopped job" + ("s" if len(_stopped_daemons) > 1 else '') + " '" +
-                ("', '".join(d.daemon_id for d in _stopped_daemons)) + "'.",
-            stack = False
+            "Skipping stopped job"
+            + ("s" if len(stopped_jobs) != 1 else '')
+            + " '"
+            + ("', '".join(name for name in stopped_jobs)) + "'.",
+            stack=False,
         )
 
-    daemons_to_stop = _running_daemons + _paused_daemons
-    if not daemons_to_stop:
-        return False, "No running or paused jobs to stop."
+    if not jobs_to_stop:
+        return False, "No running, paused or restarting jobs to stop."
 
     if not action:
         if not force:
-            pprint_jobs(daemons_to_stop)
+            pprint_jobs(jobs_to_stop)
             if not yes_no(
                 "Stop the above jobs?",
                 noask=noask, yes=yes, default='n'
             ):
                 return False, "No jobs were stopped."
 
-    _quit_daemons, _kill_daemons = [], []
-    for d in daemons_to_stop:
-        quit_success, quit_msg = d.quit(timeout=timeout_seconds)
-        if quit_success:
-            _quit_daemons.append(d)
-            continue
-        else:
-            warn(
-                f"Failed to gracefully quit job '{d.daemon_id}', attempting to terminate:\n   "
-                + f"{quit_msg}",
-                stack = False,
-            )
-
-        kill_success, kill_msg = d.kill(timeout=timeout_seconds)
-        if kill_success:
-            _kill_daemons.append(d)
-            continue
-        if not nopretty:
-            warn(f"Failed to kill job '{d.daemon_id}' (PID {d.pid}):\n{kill_msg}", stack=False)
+    job_success_tuples = {}
+    for name, job in jobs_to_stop.items():
+        stop_success, stop_msg = job.stop(
+            timeout_seconds=timeout_seconds,
+            debug=debug,
+        )
+        job_success_tuples[name] = (stop_success, stop_msg)
 
+    num_success = sum(
+        (
+            1
+            for name, (stop_success, stop_msg) in job_success_tuples.items()
+            if stop_success
+        )
+    )
+    num_fail = sum(
+        (
+            1
+            for name, (stop_success, stop_msg) in job_success_tuples.items()
+            if not stop_success
+        )
+    )
+    success = num_success > 0
     msg = (
-        (("Stopped job" + ("s" if len(_quit_daemons) != 1 else '') +
-            " '" + "', '".join([d.daemon_id for d in _quit_daemons]) + "'.")
-            if _quit_daemons else '')
-        + (("\n" if _quit_daemons else "")
-           + ("Killed job" + ("s" if len(_kill_daemons) != 1 else '') +
-            " '" + "', '".join([d.daemon_id for d in _kill_daemons]) + "'.")
-            if _kill_daemons else '')
+        f"Stopped {num_success} job"
+        + ('s' if num_success != 1 else '')
+        + '.'
     )
-    return (len(_quit_daemons + _kill_daemons) > 0), msg
+    if num_fail > 0:
+        msg += (
+            f"\nFailed to stop {num_fail} job"
+            + ('s' if num_fail != 1 else '')
+            + '.'
+        )
+
+    return success, msg
 
 
 ### NOTE: This must be the final statement of the module.

meerschaum/api/_events.py

@@ -19,14 +19,20 @@ from meerschaum.utils.debug import dprint
 from meerschaum.connectors.poll import retry_connect
 from meerschaum.utils.warnings import warn
 from meerschaum._internal.term.tools import is_webterm_running
+from meerschaum.utils.jobs import start_check_jobs_thread, stop_check_jobs_thread
+
+_check_jobs_thread = None
 
 @app.on_event("startup")
 async def startup():
-    conn = get_api_connector()
+    """
+    Connect to the instance database and begin monitoring jobs.
+    """
     try:
         if not no_dash:
             from meerschaum.api.dash.webterm import start_webterm
             start_webterm()
+
         connected = retry_connect(
             get_api_connector(),
             workers = get_uvicorn_config().get('workers', None),
@@ -36,18 +42,29 @@ async def startup():
         import traceback
         traceback.print_exc()
         connected = False
+
     if not connected:
         await shutdown()
         os._exit(1)
 
+    start_check_jobs_thread()
+
 
 @app.on_event("shutdown")
 async def shutdown():
+    """
+    Close the database connection and stop monitoring jobs.
+    """
     if debug:
         dprint("Closing connection...")
     if get_api_connector().type == 'sql':
         get_api_connector().engine.dispose()
 
+    stop_check_jobs_thread()
+    from meerschaum.api.routes._actions import _temp_jobs
+    for name, job in _temp_jobs.items():
+        job.delete()
+
     ### Terminate any running jobs left over.
     if 'meerschaum.api.dash' in sys.modules:
         from meerschaum.api.dash.actions import running_jobs, stop_action

meerschaum/api/_oauth2.py

@@ -30,6 +30,7 @@ class CustomOAuth2PasswordRequestForm:
         self.client_id = client_id
         self.client_secret = client_secret
 
+
 LoginManager = fastapi_login.LoginManager
 def generate_secret_key() -> str:
     """
@@ -46,5 +47,6 @@ def generate_secret_key() -> str:
 
     return secret_key.encode('utf-8')
 
+
 SECRET = generate_secret_key()
 manager = LoginManager(SECRET, token_url=endpoints['login'])

meerschaum/api/_websockets.py

@@ -23,8 +23,8 @@ sessions = {}
 @app.websocket('/dash/ws')
 @app.websocket(_websocket_endpoint)
 async def websocket_endpoint(
-        websocket: fastapi.WebSocket,
-    ):
+    websocket: fastapi.WebSocket,
+):
     """
     Communicate with the Web Interface over a websocket.
     """

meerschaum/api/dash/jobs.py

@@ -16,6 +16,7 @@ from meerschaum.api import CHECK_UPDATE
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 html, dcc = import_html(), import_dcc()
 dateutil_parser = attempt_import('dateutil.parser', check_update=CHECK_UPDATE)
+from meerschaum.utils.jobs import get_jobs, Job
 from meerschaum.utils.daemon import (
     get_daemons,
     get_running_daemons,
@@ -38,13 +39,15 @@ def get_jobs_cards(state: WebState):
     Build cards and alerts lists for jobs.
     """
     daemons = get_daemons()
+    jobs = get_jobs()
     session_id = state['session-store.data'].get('session-id', None)
     is_authenticated = is_session_authenticated(session_id)
 
     alert = alert_from_success_tuple(daemons)
     cards = []
 
-    for d in daemons:
+    for name, job in jobs.items():
+        d = job.daemon
         footer_children = html.Div(
             build_process_timestamps_children(d),
             id = {'type': 'process-timestamps-div', 'index': d.daemon_id},
@@ -80,7 +83,7 @@ def get_jobs_cards(state: WebState):
         ]
 
         body_children = [
-            html.H4(html.B(d.daemon_id), className="card-title"),
+            html.H4(html.B(name), className="card-title"),
             html.Div(
                 html.P(
                     d.label,

meerschaum/api/routes/__init__.py

@@ -8,6 +8,7 @@ Import all routes from other modules in package
 
 from meerschaum.api.routes._login import *
 from meerschaum.api.routes._actions import *
+from meerschaum.api.routes._jobs import *
 from meerschaum.api.routes._connectors import *
 from meerschaum.api.routes._index import *
 from meerschaum.api.routes._misc import *