meerschaum 2.2.6__py3-none-any.whl → 2.3.0.dev1__py3-none-any.whl

meerschaum/actions/show.py

@@ -11,9 +11,9 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Union, Sequence, Any, Optional, List, Dict, Tuple
 
 def show(
-        action: Optional[List[str]] = None,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> SuccessTuple:
     """Show elements of a certain type.
     
     Command:
@@ -552,16 +552,19 @@ def _complete_show_packages(
 
 def _show_jobs(
     action: Optional[List[str]] = None,
+    executor_keys: Optional[str] = None,
     nopretty: bool = False,
+    debug: bool = False,
     **kw: Any
 ) -> SuccessTuple:
     """
     Show the currently running and stopped jobs.
     """
-    from meerschaum.utils.daemon import get_filtered_daemons
+    from meerschaum.utils.jobs import get_filtered_jobs
     from meerschaum.utils.formatting._jobs import pprint_jobs
-    daemons = get_filtered_daemons(action)
-    if not daemons:
+
+    jobs = get_filtered_jobs(executor_keys, action, debug=debug)
+    if not jobs:
         if not action and not nopretty:
             from meerschaum.utils.warnings import info
             info('No running or stopped jobs.')
@@ -573,12 +576,14 @@ def _show_jobs(
                 "      - start job sync pipes --loop"
             )
         return False, "No jobs to show."
-    pprint_jobs(daemons, nopretty=nopretty)
+
+    pprint_jobs(jobs, nopretty=nopretty)
     return True, "Success"
 
 
 def _show_logs(
     action: Optional[List[str]] = None,
+    executor_keys: Optional[str] = None,
     nopretty: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -594,179 +599,145 @@ def _show_logs(
         `show logs myjob myotherjob`
     """
     import os, pathlib, random, asyncio
+    from functools import partial
     from datetime import datetime, timezone
     from meerschaum.utils.packages import attempt_import, import_rich
-    from meerschaum.utils.daemon import get_filtered_daemons, Daemon
+    from meerschaum.utils.jobs import get_filtered_jobs, Job
     from meerschaum.utils.warnings import warn, info
     from meerschaum.utils.formatting import get_console, ANSI, UNICODE
     from meerschaum.utils.misc import tail
     from meerschaum.config._paths import LOGS_RESOURCES_PATH
     from meerschaum.config import get_config
+    rich = import_rich()
+    rich_text = attempt_import('rich.text')
+
     if not ANSI:
         info = print
     colors = get_config('jobs', 'logs', 'colors')
     timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format')
     follow_timestamp_format = get_config('jobs', 'logs', 'timestamps', 'follow_format')
-    daemons = get_filtered_daemons(action)
+
+    jobs = get_filtered_jobs(executor_keys, action)
     now = datetime.now(timezone.utc)
     now_str = now.strftime(timestamp_format)
     now_follow_str = now.strftime(follow_timestamp_format)
 
-    def build_buffer_spaces(daemons) -> Dict[str, str]:
+    def build_buffer_spaces(_jobs) -> Dict[str, str]:
         max_len_id = (
-            max(len(d.daemon_id) for d in daemons) + 1
-        ) if daemons else 0
+            max(len(name) for name in _jobs) + 1
+        ) if _jobs else 0
         buffer_len = max(
             get_config('jobs', 'logs', 'min_buffer_len'),
             max_len_id
         )
         return {
-            d.daemon_id: ''.join([' '] * (buffer_len - len(d.daemon_id)))
-            for d in daemons
+            name: ' ' * (buffer_len - len(name))
+            for name in _jobs
         }
 
-    def build_job_colors(daemons, _old_job_colors = None) -> Dict[str, str]:
-        return {d.daemon_id: colors[i % len(colors)] for i, d in enumerate(daemons)}
-
-    buffer_spaces = build_buffer_spaces(daemons)
-    job_colors = build_job_colors(daemons)
-
-    def get_buffer_spaces(daemon_id):
-        nonlocal buffer_spaces, daemons
-        if daemon_id not in buffer_spaces:
-            d = Daemon(daemon_id=daemon_id)
-            if d not in daemons:
-                daemons = get_filtered_daemons(action)
-            buffer_spaces = build_buffer_spaces(daemons)
-        return buffer_spaces[daemon_id] or ' '
-
-    def get_job_colors(daemon_id):
-        nonlocal job_colors, daemons
-        if daemon_id not in job_colors:
-            d = Daemon(daemon_id=daemon_id)
-            if d not in daemons:
-                daemons = get_filtered_daemons(action)
-            job_colors = build_job_colors(daemons)
-        return job_colors[daemon_id]
-
-    def follow_pretty_print():
-        watchfiles = attempt_import('watchfiles')
-        rich = import_rich()
-        rich_text = attempt_import('rich.text')
-        watch_daemon_ids = {d.daemon_id: d for d in daemons}
-        info("Watching log files...")
-
-        previous_line_timestamp = None
-        def print_job_line(daemon, line):
-            nonlocal previous_line_timestamp
-            date_prefix_str = line[:len(now_str)]
-            try:
-                line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
-                previous_line_timestamp = line_timestamp
-            except Exception as e:
-                line_timestamp = None
-            if line_timestamp:
-                line = line[(len(now_str) + 3):]
-            else:
-                line_timestamp = previous_line_timestamp
-
-            if len(line) == 0 or line == '\n':
-                return
-
-            text = rich_text.Text(daemon.daemon_id)
-            line_prefix = (
-                get_buffer_spaces(daemon.daemon_id)
-                + (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
-                + ' | '
-            )
-            text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
-            if ANSI:
-                text.stylize(
-                    get_job_colors(daemon.daemon_id),
-                    0,
-                    len(daemon.daemon_id) + len(line_prefix)
-                )
-            get_console().print(text)
-
-
-        def print_log_lines(daemon):
-            for line in daemon.readlines():
-                print_job_line(daemon, line)
-
-        def seek_back_offset(d) -> bool:
-            if d.log_offset_path.exists():
-                d.log_offset_path.unlink()
-
-            latest_subfile_path = d.rotating_log.get_latest_subfile_path()
-            latest_subfile_index = d.rotating_log.get_latest_subfile_index()
-
-            ### Sometimes the latest file is empty.
-            if os.stat(latest_subfile_path).st_size == 0 and latest_subfile_index > 0:
-                latest_subfile_index -= 1
-                latest_subfile_path = d.rotating_log.get_subfile_path_from_index(
-                    latest_subfile_index
-                )
-
-            with open(latest_subfile_path, 'r', encoding='utf-8') as f:
-                latest_lines = f.readlines()
+    def build_job_colors(_jobs, _old_job_colors=None) -> Dict[str, str]:
+        return {name: colors[i % len(colors)] for i, name in enumerate(_jobs)}
+
+    buffer_spaces = build_buffer_spaces(jobs)
+    job_colors = build_job_colors(jobs)
+
+    def get_buffer_spaces(name):
+        nonlocal buffer_spaces, jobs
+        if name not in buffer_spaces:
+            if name not in jobs:
+                jobs = get_filtered_jobs(executor_keys, action)
+            buffer_spaces = build_buffer_spaces(jobs)
+        return buffer_spaces[name] or ' '
+
+    def get_job_colors(name):
+        nonlocal job_colors, jobs
+        if name not in job_colors:
+            if name not in jobs:
+                jobs = get_filtered_jobs(executor_keys, action)
+            job_colors = build_job_colors(jobs)
+        return job_colors[name]
+
+    previous_line_timestamp = None
+    def print_job_line(job, line):
+        nonlocal previous_line_timestamp
+        date_prefix_str = line[:len(now_str)]
+        try:
+            line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
+            previous_line_timestamp = line_timestamp
+        except Exception as e:
+            line_timestamp = None
+        if line_timestamp:
+            line = line[(len(now_str) + 3):]
+        else:
+            line_timestamp = previous_line_timestamp
 
-            lines_to_show = get_config('jobs', 'logs', 'lines_to_show')
-            positions_to_rewind = len(''.join(latest_lines[(-1 * lines_to_show):]))
-            backup_index = len(''.join(latest_lines)) - positions_to_rewind
+        if len(line) == 0 or line == '\n':
+            return
 
-            d.rotating_log._cursor = (
-                latest_subfile_index,
-                max(0, backup_index)
+        text = rich_text.Text(job.name)
+        line_prefix = (
+            get_buffer_spaces(job.name)
+            + (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
+            + ' | '
+        )
+        text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
+        if ANSI:
+            text.stylize(
+                get_job_colors(job.name),
+                0,
+                len(job.name) + len(line_prefix)
             )
-            d._write_log_offset()
-            return True
+        get_console().print(text)
+
+    stop_event = asyncio.Event()
+    job_tasks = {}
+    job_stop_events = {}
+
+    async def refresh_job_tasks():
+        nonlocal job_tasks, jobs
+        while not stop_event.is_set():
+            jobs = get_filtered_jobs(executor_keys, action)
+            for name, job in jobs.items():
+                if name not in job_tasks:
+                    job_stop_events[name] = asyncio.Event()
+                    job_tasks[name] = asyncio.create_task(
+                        job.monitor_logs_async(
+                            partial(print_job_line, job),
+                            stop_event=job_stop_events[name],
+                            accept_input=False,
+                            stop_on_exit=False,
+                        )
+                    )
+
+            for name, task in [(k, v) for k, v in job_tasks.items()]:
+                if name not in jobs:
+                    job_stop_events[name].set()
+                    task.cancel()
 
-        daemons_being_watched = {
-            d.daemon_id: d
-            for d in daemons
-        }
-        for d in daemons:
-            seek_back_offset(d)
-            print_log_lines(d)
-
-        _quit = False
-        def watch_logs():
-            for changes in watchfiles.watch(LOGS_RESOURCES_PATH):
-                if _quit:
-                    return
-                for change in changes:
-                    file_path_str = change[1]
-                    if '.log' not in file_path_str or '.log.offset' in file_path_str:
-                        continue
-                    file_path = pathlib.Path(file_path_str)
-                    if not file_path.exists():
-                        continue
-                    daemon_id = file_path.name.split('.log')[0]
-                    if daemon_id not in watch_daemon_ids and action:
-                        continue
                     try:
-                        daemon = Daemon(daemon_id=daemon_id)
-                    except Exception as e:
-                        daemon = None
-                        warn(f"Seeing new logs for non-existent job '{daemon_id}'.", stack=False)
-
-                    if daemon is not None:
-                        print_log_lines(daemon)
-
-        try:
-            watch_logs()
-        except KeyboardInterrupt as ki:
-            _quit = True
+                        await task
+                    except asyncio.CancelledError:
+                        pass
+                    finally:
+                        _ = job_tasks.pop(name, None)
+                        _ = job_stop_events.pop(name, None)
 
-    def print_nopretty_log_text():
-        for d in daemons:
-            log_text = d.log_text
-            print(d.daemon_id)
-            print(log_text)
+            await asyncio.sleep(1)
 
-    print_log_text = follow_pretty_print if not nopretty else print_nopretty_log_text
-    print_log_text()
+    async def gather_tasks():
+        tasks = [refresh_job_tasks()] + list(job_tasks.values())
+        await asyncio.gather(*tasks)
 
+    if not nopretty:
+        info("Watching logs...")
+        try:
+            asyncio.run(gather_tasks())
+        except KeyboardInterrupt:
+            pass
+    else:
+        for name, job in jobs.items():
+            print(f'\n-*-\nMRSM_JOB: {name}\n-*-')
+            print(job.get_logs())
     return True, "Success"
 
 

meerschaum/actions/start.py

@@ -78,6 +78,9 @@ def _start_api(action: Optional[List[str]] = None, **kw):
 def _start_jobs(
         action: Optional[List[str]] = None,
         name: Optional[str] = None,
+        sysargs: Optional[List[str]] = None,
+        executor_keys: Optional[str] = None,
+        debug: bool = False,
         **kw
     ) -> SuccessTuple:
     """
@@ -111,11 +114,16 @@ def _start_jobs(
     """
     import textwrap
     from meerschaum.utils.warnings import warn, info
-    from meerschaum.utils.daemon import (
-        daemon_action, Daemon, get_daemon_ids, get_daemons, get_filtered_daemons,
-        get_stopped_daemons, get_running_daemons, get_paused_daemons,
-    )
     from meerschaum.utils.daemon._names import get_new_daemon_name
+    from meerschaum.utils.jobs import (
+        Job,
+        get_jobs,
+        get_filtered_jobs,
+        get_stopped_jobs,
+        get_running_jobs,
+        get_paused_jobs,
+        get_restart_jobs,
+    )
     from meerschaum._internal.arguments._parse_arguments import parse_arguments
     from meerschaum.actions import actions
     from meerschaum.utils.prompt import yes_no
@@ -125,7 +133,7 @@ def _start_jobs(
     from meerschaum.utils.misc import items_str
 
     names = []
-    daemon_ids = get_daemon_ids()
+    jobs = get_filtered_jobs(executor_keys, debug=debug)
 
     new_job = len(list(action)) > 0
     _potential_jobs = {'known': [], 'unknown': []}
@@ -134,7 +142,7 @@ def _start_jobs(
         for a in action:
             _potential_jobs[(
                 'known'
-                if a in daemon_ids
+                if a in jobs
                 else 'unknown'
             )].append(a)
 
@@ -182,88 +190,83 @@ def _start_jobs(
 
     ### No action or --name was provided. Ask to start all stopped jobs.
     else:
-        _running_daemons = get_running_daemons()
-        _paused_daemons = get_paused_daemons()
-        _stopped_daemons = get_stopped_daemons()
-        if not _stopped_daemons and not _paused_daemons:
-            if not _running_daemons:
-                return False, "No jobs to start."
+        running_jobs = get_running_jobs(executor_keys, jobs, debug=debug)
+        paused_jobs = get_paused_jobs(executor_keys, jobs, debug=debug)
+        stopped_jobs = get_stopped_jobs(executor_keys, jobs, debug=debug)
+
+        if not stopped_jobs and not paused_jobs:
+            if not running_jobs:
+                return False, "No jobs to start"
             return True, "All jobs are running."
 
-        names = [d.daemon_id for d in _stopped_daemons + _paused_daemons]
+        names = [
+            name
+            for name in list(stopped_jobs) + list(paused_jobs)
+        ]
 
     def _run_new_job(name: Optional[str] = None):
-        kw['action'] = action
         name = name or get_new_daemon_name()
-        kw['name'] = name
-        _action_success_tuple = daemon_action(daemon_id=name, **kw)
-        return _action_success_tuple, name
-
-    def _run_existing_job(name: Optional[str] = None):
-        daemon = Daemon(daemon_id=name)
-        if daemon.process is not None:
-            if daemon.status == 'paused':
-                return daemon.resume(), daemon.daemon_id
-            return (True, f"Job '{name}' is already running."), daemon.daemon_id
-
-        if not daemon.path.exists():
-            if not kw.get('nopretty', False):
-                warn(f"There isn't a job with the name '{name}'.", stack=False)
-                print(
-                    f"You can start a new job named '{name}' with `start job "
-                    + "{options}" + f" --name {name}`"
-                )
-            return (False, f"Job '{name}' does not exist."), daemon.daemon_id
+        job = Job(name, sysargs, executor_keys=executor_keys)
+        return job.start(debug=debug), name
 
-        return daemon.run(allow_dirty_run=True), daemon.daemon_id
+    def _run_existing_job(name: str):
+        job = Job(name, executor_keys=executor_keys)
+        return job.start(debug=debug), name
 
     if not names:
         return False, "No jobs to start."
 
     ### Get user permission to clear logs.
-    _filtered_daemons = get_filtered_daemons(names)
-    if not kw.get('force', False) and _filtered_daemons:
-        _filtered_running_daemons = get_running_daemons(_filtered_daemons)
-        _skipped_daemons = []
-        if _filtered_running_daemons:
-            pprint_jobs(_filtered_running_daemons)
+    _filtered_jobs = get_filtered_jobs(executor_keys, names, debug=debug)
+    if not kw.get('force', False) and _filtered_jobs:
+        _filtered_running_jobs = get_running_jobs(executor_keys, _filtered_jobs, debug=debug)
+        _skipped_jobs = []
+        if _filtered_running_jobs:
+            pprint_jobs(_filtered_running_jobs)
             if yes_no(
                 "Do you want to first stop these jobs?",
-                default = 'n',
-                yes = kw.get('yes', False),
-                noask = kw.get('noask', False)
+                default='n',
+                yes=kw.get('yes', False),
+                noask=kw.get('noask', False)
             ):
                 stop_success_tuple = actions['stop'](
-                    action = ['jobs'] + [d.daemon_id for d in _filtered_running_daemons],
-                    force = True,
+                    action=['jobs'] + [_name for _name in _filtered_running_jobs],
+                    force=True,
+                    mrsm_instance=mrsm_instance,
+                    debug=debug,
                 )
                 if not stop_success_tuple[0]:
                     warn(
-                        "Failed to stop job" + ("s" if len(_filtered_running_daemons) != 1 else '')
-                        + items_str([d.daemon_id for d in _filtered_running_daemons])
-                        + ".",
-                        stack = False
+                        (
+                            "Failed to stop job"
+                            + ("s" if len(_filtered_running_jobs) != 1 else '')
+                            + items_str([_name for _name in _filtered_running_jobs])
+                            + "."
+                        ),
+                        stack=False
                     )
-                    for d in _filtered_running_daemons:
-                        names.remove(d.daemon_id)
-                        _filtered_daemons.remove(d)
+                    for _name in _filtered_running_jobs:
+                        names.remove(_name)
+                        _filtered_jobs.pop(_name)
             else:
                 info(
                     "Skipping already running job"
-                    + ("s" if len(_filtered_running_daemons) != 1 else '') + ' '
-                    + items_str([d.daemon_id for d in _filtered_running_daemons]) + '.'
+                    + ("s" if len(_filtered_running_jobs) != 1 else '')
+                    + ' '
+                    + items_str([_name for _name in _filtered_running_jobs])
+                    + '.'
                 )
-                for d in _filtered_running_daemons:
-                    names.remove(d.daemon_id)
-                    _filtered_daemons.remove(d)
-                    _skipped_daemons.append(d)
+                for _name in _filtered_running_jobs:
+                    names.remove(_name)
+                    _filtered_jobs.pop(_name)
+                    _skipped_jobs.append(_name)
 
-        if not _filtered_daemons:
-            return len(_skipped_daemons) > 0, "No jobs to start."
+        if not _filtered_jobs:
+            return len(_skipped_jobs) > 0, "No jobs to start."
 
-        pprint_jobs(_filtered_daemons, nopretty=kw.get('nopretty', False))
+        pprint_jobs(_filtered_jobs, nopretty=kw.get('nopretty', False))
         info(
-            f"Starting the job"
+            "Starting the job"
             + ("s" if len(names) != 1 else "")
             + " " + items_str(names)
             + "..."
@@ -278,7 +281,11 @@ def _start_jobs(
         )
         if not kw.get('nopretty', False):
             print_tuple(success_tuple)
-        _successes.append(_name) if success_tuple[0] else _failures.append(_name)
+
+        if success_tuple[0]:
+            _successes.append(_name)
+        else:
+            _failures.append(_name)
 
     msg = (
         (("Successfully started job" + ("s" if len(_successes) != 1 else '')
@@ -289,25 +296,28 @@ def _start_jobs(
     )
     return len(_failures) == 0, msg
 
+
 def _complete_start_jobs(
-        action: Optional[List[str]] = None,
-        line: str = '',
-        **kw
-    ) -> List[str]:
-    from meerschaum.utils.daemon import get_daemon_ids
-    daemon_ids = get_daemon_ids()
+    action: Optional[List[str]] = None,
+    executor_keys: Optional[str] = None,
+    line: str = '',
+    **kw
+) -> List[str]:
+    from meerschaum.utils.jobs import get_filtered_jobs
+    jobs = get_filtered_jobs(executor_keys, action)
     if not action:
-        return daemon_ids
+        return list(jobs)
+
     possibilities = []
     _line_end = line.split(' ')[-1]
-    for daemon_id in daemon_ids:
-        if daemon_id in action:
+    for name in jobs:
+        if name in action:
             continue
         if _line_end == '':
-            possibilities.append(daemon_id)
+            possibilities.append(name)
             continue
-        if daemon_id.startswith(action[-1]):
-            possibilities.append(daemon_id)
+        if name.startswith(action[-1]):
+            possibilities.append(name)
     return possibilities