meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl

meerschaum/utils/formatting/__init__.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 import platform
 import os
 import sys
+import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Union, Any, Dict
 from meerschaum.utils.formatting._shell import make_header
 from meerschaum.utils.formatting._pprint import pprint
@@ -27,12 +28,14 @@ _attrs = {
     'ANSI': None,
     'UNICODE': None,
     'CHARSET': None,
+    'RESET': '\033[0m',
 }
 __all__ = sorted([
-    'ANSI', 'CHARSET', 'UNICODE',
+    'ANSI', 'CHARSET', 'UNICODE', 'RESET',
     'colored',
     'translate_rich_to_termcolor',
     'get_console',
+    'format_success_tuple',
     'print_tuple',
     'print_options',
     'fill_ansi',
@@ -222,16 +225,17 @@ def get_console():
 
 
 def print_tuple(
-        tup: tuple,
-        skip_common: bool = True,
-        common_only: bool = False,
-        upper_padding: int = 0,
-        lower_padding: int = 0,
-        calm: bool = False,
-        _progress: Optional['rich.progress.Progress'] = None,
-    ) -> None:
+    tup: mrsm.SuccessTuple,
+    skip_common: bool = True,
+    common_only: bool = False,
+    upper_padding: int = 0,
+    lower_padding: int = 0,
+    left_padding: int = 1,
+    calm: bool = False,
+    _progress: Optional['rich.progress.Progress'] = None,
+) -> None:
     """
-    Print `meerschaum.utils.typing.SuccessTuple`.
+    Format `meerschaum.utils.typing.SuccessTuple`.
 
     Parameters
     ----------
@@ -247,24 +251,18 @@ def print_tuple(
     lower_padding: int, default 0
         How many newlines to append to the message.
 
+    left_padding: int, default 1
+        How mant spaces to preprend to the message.
+
     calm: bool, default False
         If `True`, use the default emoji and color scheme.
+
     """
     from meerschaum.config.static import STATIC_CONFIG
-    _init()
-    try:
-        status = 'success' if tup[0] else 'failure'
-    except TypeError:
-        status = 'failure'
-        tup = None, None
-
-    if calm:
-        status += '_calm'
+    do_print = True
 
     omit_messages = STATIC_CONFIG['system']['success']['ignore']
 
-    do_print = True
-
     if common_only:
         skip_common = False
         do_print = tup[1] in omit_messages
@@ -272,22 +270,70 @@ def print_tuple(
     if skip_common:
         do_print = tup[1] not in omit_messages
 
-    if do_print:
-        ANSI, CHARSET = __getattr__('ANSI'), __getattr__('CHARSET')
-        from meerschaum.config import get_config
-        status_config = get_config('formatting', status, patch=True)
-
-        msg = ' ' + status_config[CHARSET]['icon'] + ' ' + str(tup[1])
-        lines = msg.split('\n')
-        lines = [lines[0]] + [
-            (('    ' + line if not line.startswith(' ') else line))
-            for line in lines[1:]
-        ]
-        if ANSI:
-            lines[0] = fill_ansi(highlight_pipes(lines[0]), **status_config['ansi']['rich'])
-        msg = '\n'.join(lines)
-        msg = ('\n' * upper_padding) + msg + ('\n' * lower_padding)
-        print(msg)
+    if not do_print:
+        return
+
+    print(format_success_tuple(
+        tup,
+        upper_padding=upper_padding,
+        lower_padding=lower_padding,
+        calm=calm,
+        _progress=_progress,
+    ))
+
+
+def format_success_tuple(
+    tup: mrsm.SuccessTuple,
+    upper_padding: int = 0,
+    lower_padding: int = 0,
+    left_padding: int = 1,
+    calm: bool = False,
+    _progress: Optional['rich.progress.Progress'] = None,
+) -> str:
+    """
+    Format `meerschaum.utils.typing.SuccessTuple`.
+
+    Parameters
+    ----------
+    upper_padding: int, default 0
+        How many newlines to prepend to the message.
+
+    lower_padding: int, default 0
+        How many newlines to append to the message.
+
+    left_padding: int, default 1
+        How mant spaces to preprend to the message.
+
+    calm: bool, default False
+        If `True`, use the default emoji and color scheme.
+    """
+    from meerschaum.config.static import STATIC_CONFIG
+    _init()
+    try:
+        status = 'success' if tup[0] else 'failure'
+    except TypeError:
+        status = 'failure'
+        tup = None, None
+
+    if calm:
+        status += '_calm'
+
+    ANSI, CHARSET = __getattr__('ANSI'), __getattr__('CHARSET')
+    from meerschaum.config import get_config
+    status_config = get_config('formatting', status, patch=True)
+
+    msg = (' ' * left_padding) + status_config[CHARSET]['icon'] + ' ' + str(tup[1])
+    lines = msg.split('\n')
+    lines = [lines[0]] + [
+        (('    ' + line if not line.startswith(' ') else line))
+        for line in lines[1:]
+    ]
+    if ANSI:
+        lines[0] = fill_ansi(highlight_pipes(lines[0]), **status_config['ansi']['rich'])
+
+    msg = '\n'.join(lines)
+    msg = ('\n' * upper_padding) + msg + ('\n' * lower_padding)
+    return msg
 
 
 def print_options(

meerschaum/utils/formatting/_jobs.py

@@ -7,86 +7,160 @@ Print jobs information.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import List, Optional, Any
-from meerschaum.utils.daemon import (
-    Daemon,
-    get_daemons,
-    get_running_daemons,
-    get_stopped_daemons,
-    get_paused_daemons,
+
+from datetime import datetime, timezone
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import List, Optional, Any, is_success_tuple, Dict
+from meerschaum.jobs import (
+    Job,
+    get_jobs,
+    get_running_jobs,
+    get_stopped_jobs,
+    get_paused_jobs,
+    get_executor_keys_from_context,
 )
+from meerschaum.config import get_config
+
 
 def pprint_jobs(
-        daemons: List[Daemon],
-        nopretty: bool = False,
-    ):
+    jobs: Dict[str, Job],
+    nopretty: bool = False,
+):
     """Pretty-print a list of Daemons."""
     from meerschaum.utils.formatting import make_header
+    from meerschaum.utils.misc import items_str
     
-    running_daemons = get_running_daemons(daemons)
-    paused_daemons = get_paused_daemons(daemons)
-    stopped_daemons = get_stopped_daemons(daemons)
+    running_jobs = get_running_jobs(jobs=jobs)
+    paused_jobs = get_paused_jobs(jobs=jobs)
+    stopped_jobs = get_stopped_jobs(jobs=jobs)
+    executor_keys_list = list(set(
+        [
+            job.executor_keys.replace('systemd:main', 'systemd')
+            for job in jobs.values()
+        ]
+    )) if jobs else [get_executor_keys_from_context()]
 
     def _nopretty_print():
         from meerschaum.utils.misc import print_options
-        if running_daemons:
+        if running_jobs:
             if not nopretty:
                 print('\n' + make_header('Running jobs'))
-            for d in running_daemons:
-                pprint_job(d, nopretty=nopretty)
+            for name, job in running_jobs.items():
+                pprint_job(job, nopretty=nopretty)
 
-        if paused_daemons:
+        if paused_jobs:
             if not nopretty:
                 print('\n' + make_header('Paused jobs'))
-            for d in paused_daemons:
-                pprint_job(d, nopretty=nopretty)
+            for name, job in paused_jobs.items():
+                pprint_job(job, nopretty=nopretty)
 
-        if stopped_daemons:
+        if stopped_jobs:
             if not nopretty:
                 print('\n' + make_header('Stopped jobs'))
-            for d in stopped_daemons:
-                pprint_job(d, nopretty=nopretty)
+            for name, job in stopped_jobs.items():
+                pprint_job(job, nopretty=nopretty)
 
     def _pretty_print():
-        from meerschaum.utils.formatting import get_console, UNICODE, ANSI
+        from meerschaum.utils.formatting import get_console, UNICODE, ANSI, format_success_tuple
         from meerschaum.utils.packages import import_rich, attempt_import
         rich = import_rich()
-        rich_table, rich_text, rich_box = attempt_import('rich.table', 'rich.text', 'rich.box')
+        rich_table, rich_text, rich_box, rich_json, rich_panel, rich_console = attempt_import(
+            'rich.table', 'rich.text', 'rich.box', 'rich.json', 'rich.panel', 'rich.console',
+        )
         table = rich_table.Table(
-            title = rich_text.Text('Jobs'),
-            box = (rich_box.ROUNDED if UNICODE else rich_box.ASCII),
-            show_lines = True,
-            show_header = ANSI,
+            title=rich_text.Text(
+                f"\nJobs on Executor"
+                + ('s' if len(executor_keys_list) != 1 else '')
+                + f" {items_str(executor_keys_list)}"
+            ),
+            box=(rich_box.ROUNDED if UNICODE else rich_box.ASCII),
+            show_lines=True,
+            show_header=ANSI,
         )
         table.add_column("Name", justify='right', style=('magenta' if ANSI else ''))
+        table.add_column(
+            "Executor",
+            style=(get_config('shell', 'ansi', 'executor', 'rich', 'style') if ANSI else ''),
+        )
         table.add_column("Command")
         table.add_column("Status")
 
-        for d in running_daemons:
-            if d.hidden:
-                continue
+        def get_success_text(job):
+            success_tuple = job.result
+            if not is_success_tuple(success_tuple):
+                return rich_text.Text('')
+
+            success = success_tuple[0]
+            msg = success_tuple[1]
+            lines = msg.split('\n')
+            msg = '\n'.join(line.lstrip().rstrip() for line in lines)
+            success_tuple = success, msg
+            success_tuple_str = (
+                format_success_tuple(success_tuple, left_padding=1)
+                if success_tuple is not None
+                else None
+            )
+            success_tuple_text = (
+                rich_text.Text.from_ansi(success_tuple_str)
+            ) if success_tuple_str is not None else None
+
+            if success_tuple_text is None:
+                return rich_text.Text('')
+
+            return rich_text.Text('\n') + success_tuple_text
+
+
+        for name, job in running_jobs.items():
+            status_group = [
+                (
+                    rich_text.Text(job.status, style=('green' if ANSI else ''))
+                    if not job.is_blocking_on_stdin()
+                    else rich_text.Text('waiting for input', style=('yellow' if ANSI else ''))
+                ),
+            ] + ([rich_text.Text(f"PID: {pid}")] if (pid := job.pid) else [])
+
+            if job.restart:
+                status_group.append(rich_text.Text('(restarts)'))
+
             table.add_row(
-                d.daemon_id,
-                d.label,
-                rich_text.Text(d.status, style=('green' if ANSI else ''))
+                job.name,
+                job.executor_keys.replace('systemd:main', 'systemd'),
+                job.label,
+                rich_console.Group(*status_group),
             )
 
-        for d in paused_daemons:
-            if d.hidden:
-                continue
+        for name, job in paused_jobs.items():
+            status_group = [
+                rich_text.Text(job.status, style=('yellow' if ANSI else '')),
+            ]
+            if job.restart:
+                status_group.append(rich_text.Text('(restarts)'))
+
             table.add_row(
-                d.daemon_id,
-                d.label,
-                rich_text.Text(d.status, style=('yellow' if ANSI else ''))
+                job.name,
+                job.executor_keys.replace('systemd:main', 'systemd'),
+                job.label,
+                rich_console.Group(*status_group),
             )
 
-        for d in stopped_daemons:
-            if d.hidden:
-                continue
+        for name, job in stopped_jobs.items():
+            status_group = [
+                rich_text.Text(job.status, style=('red' if ANSI else '')),
+            ]
+            if job.restart:
+                if job.stop_time is None:
+                    status_group.append(rich_text.Text('(restarts)'))
+                else:
+                    status_group.append(rich_text.Text('(start to resume restarts)'))
+
+            status_group.append(get_success_text(job))
+
             table.add_row(
-                d.daemon_id,
-                d.label,
-                rich_text.Text(d.status, style=('red' if ANSI else ''))
+                job.name,
+                job.executor_keys.replace('systemd:main', 'systemd'),
+                job.label,
+                rich_console.Group(*status_group),
             )
         get_console().print(table)
 
@@ -95,15 +169,32 @@ def pprint_jobs(
 
 
 def pprint_job(
-        daemon: Daemon,
-        nopretty: bool = False,
-    ):
-    """Pretty-print a single Daemon."""
-    if daemon.hidden:
-        return
+    job: Job,
+    nopretty: bool = False,
+):
+    """Pretty-print a single `Job`."""
     from meerschaum.utils.warnings import info
     if not nopretty:
-        info(f"Command for job '{daemon.daemon_id}':")
-        print('\n' + daemon.label + '\n')
+        info(f"Command for job '{job.name}':")
+        print('\n' + job.label + '\n')
     else:
-        print(daemon.daemon_id)
+        print(job.name)
+
+
+def strip_timestamp_from_line(line: str) -> str:
+    """
+    Remove the leading timestamp from a job's line (if present).
+    """
+    now = datetime.now(timezone.utc)
+    timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format')
+    now_str = now.strftime(timestamp_format)
+
+    date_prefix_str = line[:len(now_str)]
+    try:
+        line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
+    except Exception:
+        line_timestamp = None
+    if line_timestamp:
+        line = line[(len(now_str) + 3):]
+
+    return line

meerschaum/utils/formatting/_shell.py

@@ -47,9 +47,15 @@ def clear_screen(debug: bool = False) -> bool:
     from meerschaum.utils.formatting import ANSI, get_console
     from meerschaum.utils.debug import dprint
     from meerschaum.config import get_config
+    from meerschaum.utils.daemon import running_in_daemon
     global _tried_clear_command
+
+    if running_in_daemon():
+        return True
+
     if not get_config('shell', 'clear_screen'):
         return True
+
     print("", end="", flush=True)
     if debug:
         dprint("Skipping screen clear.")

meerschaum/utils/misc.py

@@ -875,7 +875,7 @@ def dict_from_od(od: collections.OrderedDict) -> Dict[Any, Any]:
             _d[k] = dict_from_od(v)
     return _d
 
-def remove_ansi(s : str) -> str:
+def remove_ansi(s: str) -> str:
     """
     Remove ANSI escape characters from a string.
 
@@ -899,10 +899,11 @@ def remove_ansi(s : str) -> str:
 
 
 def get_connector_labels(
-        *types: str,
-        search_term: str = '',
-        ignore_exact_match = True,
-    ) -> List[str]:
+    *types: str,
+    search_term: str = '',
+    ignore_exact_match = True,
+    _additional_options: Optional[List[str]] = None,
+) -> List[str]:
     """
     Read connector labels from the configuration dictionary.
 
@@ -941,12 +942,16 @@ def get_connector_labels(
             continue
         conns += [ f'{t}:{label}' for label in connectors.get(t, {}) if label != 'default' ]
 
+    if _additional_options:
+        conns += _additional_options
+
     possibilities = [
-        c for c in conns
-            if c.startswith(search_term)
-                and c != (
-                    search_term if ignore_exact_match else ''
-                )
+        c
+        for c in conns
+        if c.startswith(search_term)
+            and c != (
+                search_term if ignore_exact_match else ''
+            )
     ]
     return sorted(possibilities)
 
@@ -1063,7 +1068,7 @@ def async_wrap(func):
             loop = asyncio.get_event_loop()
         pfunc = partial(func, *args, **kwargs)
         return await loop.run_in_executor(executor, pfunc)
-    return run 
+    return run
 
 
 def debug_trace(browser: bool = True):
@@ -1077,17 +1082,17 @@ def debug_trace(browser: bool = True):
 
 
 def items_str(
-        items: List[Any],
-        quotes: bool = True,
-        quote_str: str = "'",
-        commas: bool = True,
-        comma_str: str = ',',
-        and_: bool = True,
-        and_str: str = 'and',
-        oxford_comma: bool = True,
-        spaces: bool = True,
-        space_str = ' ',
-    ) -> str:
+    items: List[Any],
+    quotes: bool = True,
+    quote_str: str = "'",
+    commas: bool = True,
+    comma_str: str = ',',
+    and_: bool = True,
+    and_str: str = 'and',
+    oxford_comma: bool = True,
+    spaces: bool = True,
+    space_str = ' ',
+) -> str:
     """
     Return a formatted string if list items separated by commas.
 
@@ -1218,6 +1223,19 @@ def is_bcp_available() -> bool:
     return has_bcp
 
 
+def is_systemd_available() -> bool:
+    """Check if running on systemd."""
+    import subprocess
+    try:
+        has_systemctl = subprocess.call(
+            ['systemctl', '-h'],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.STDOUT,
+        ) == 0
+    except Exception:
+        has_systemctl = False
+    return has_systemctl
+
 def get_last_n_lines(file_name: str, N: int):
     """
     https://thispointer.com/python-get-last-n-lines-of-a-text-file-like-tail-command/
@@ -1522,6 +1540,7 @@ def safely_extract_tar(tarf: 'file', output_dir: Union[str, 'pathlib.Path']) ->
 
     return safe_extract(tarf, output_dir)
 
+
 ##################
 # Legacy imports #
 ##################

meerschaum/utils/packages/__init__.py

@@ -816,6 +816,7 @@ def pip_install(
     """
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
     from meerschaum.config import get_config
+    from meerschaum.config.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.misc import is_android
     if args is None:
@@ -827,6 +828,11 @@ def pip_install(
     if check_wheel:
         have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
 
+    daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
+    inside_daemon = daemon_env_var in os.environ
+    if inside_daemon:
+        silent = True
+
     _args = list(args)
     have_pip = venv_contains_package('pip', venv=None, debug=debug)
     try:
@@ -844,16 +850,16 @@ def pip_install(
     if have_pip and not have_uv_pip and _install_uv_pip and not is_android():
         if not pip_install(
             'uv',
-            venv = None,
-            debug = debug,
-            _install_uv_pip = False,
-            check_update = False,
-            check_pypi = False,
-            check_wheel = False,
-        ):
+            venv=None,
+            debug=debug,
+            _install_uv_pip=False,
+            check_update=False,
+            check_pypi=False,
+            check_wheel=False,
+        ) and not silent:
             warn(
                 f"Failed to install `uv` for virtual environment '{venv}'.",
-                color = False,
+                color=False,
             )
 
     use_uv_pip = (
@@ -909,13 +915,13 @@ def pip_install(
                     check_wheel = False,
                     debug = debug,
                     _install_uv_pip = False,
-                ):
+                ) and not silent:
                     warn(
                         (
                             "Failed to install `setuptools`, `wheel`, and `uv` for virtual "
                             + f"environment '{venv}'."
                         ),
-                        color = False,
+                        color=False,
                     )
 
         if requirements_file_path is not None:
@@ -975,7 +981,7 @@ def pip_install(
                 if not completely_uninstall_package(
                     _install_no_version,
                     venv=venv, debug=debug,
-                ):
+                ) and not silent:
                     warn(
                         f"Failed to clean up package '{_install_no_version}'.",
                     )
@@ -989,9 +995,9 @@ def pip_install(
         rc = run_python_package(
             ('pip' if not use_uv_pip else 'uv'),
             _args + _packages,
-            venv = None,
-            env = _get_pip_os_env(color=color),
-            debug = debug,
+            venv=None,
+            env=_get_pip_os_env(color=color),
+            debug=debug,
         )
         if debug:
             print(f"{rc=}")
@@ -1003,7 +1009,7 @@ def pip_install(
     )
     if not silent:
         print(msg)
-    if debug:
+    if debug and not silent:
         print('pip ' + ('un' if _uninstall else '') + 'install returned:', success)
     return success
 

meerschaum/utils/packages/_packages.py

@@ -47,18 +47,20 @@ packages: Dict[str, Dict[str, str]] = {
         'packaging'                  : 'packaging>=21.3.0',
         'prompt_toolkit'             : 'prompt-toolkit>=3.0.39',
         'more_itertools'             : 'more-itertools>=8.7.0',
-        'daemon'                     : 'python-daemon>=0.2.3',
-        'fasteners'                  : 'fasteners>=0.18.0',
-        'psutil'                     : 'psutil>=5.8.0',
-        'watchfiles'                 : 'watchfiles>=0.21.0',
-        'dill'                       : 'dill>=0.3.3',
+        'fasteners'                  : 'fasteners>=0.19.0',
         'virtualenv'                 : 'virtualenv>=20.1.0',
         'apscheduler'                : 'APScheduler>=4.0.0a5',
         'uv'                         : 'uv>=0.2.11',
     },
+    'jobs': {
+        'dill'                       : 'dill>=0.3.3',
+        'daemon'                     : 'python-daemon>=0.2.3',
+        'watchfiles'                 : 'watchfiles>=0.21.0',
+        'psutil'                     : 'psutil>=5.8.0',
+    },
     'drivers': {
         'cryptography'               : 'cryptography>=38.0.1',
-        'psycopg'                    : 'psycopg[binary]>=3.1.18',
+        'psycopg'                    : 'psycopg[binary]>=3.2.1',
         'pymysql'                    : 'PyMySQL>=0.9.0',
         'aiomysql'                   : 'aiomysql>=0.0.21',
         'sqlalchemy_cockroachdb'     : 'sqlalchemy-cockroachdb>=2.0.0',
@@ -156,6 +158,7 @@ packages['api'] = {
 packages['api'].update(packages['sql'])
 packages['api'].update(packages['formatting'])
 packages['api'].update(packages['dash'])
+packages['api'].update(packages['jobs'])
 
 all_packages = {}
 for group, import_names in packages.items():