meerschaum 2.3.0.dev3__py3-none-any.whl → 2.3.0rc2__py3-none-any.whl

meerschaum/__init__.py

@@ -24,7 +24,7 @@ from meerschaum.utils.packages import attempt_import
 from meerschaum.core.Pipe import Pipe
 from meerschaum.plugins import Plugin
 from meerschaum.utils.venv import Venv
-from meerschaum.utils.jobs import Job
+from meerschaum.jobs import Job, Executor, make_executor
 from meerschaum.connectors import get_connector, Connector, make_connector
 from meerschaum.utils import get_pipes
 from meerschaum.utils.formatting import pprint
@@ -45,7 +45,8 @@ __all__ = (
     "Venv",
     "Plugin",
     "Job",
-    "Daemon",
+    "Executor",
+    "make_executor",
     "pprint",
     "attempt_import",
     "actions",

meerschaum/__main__.py

@@ -45,11 +45,6 @@ def main(sysargs: Optional[List[str]] = None) -> None:
         parse_version(sysargs)
         return _exit(old_cwd=old_cwd)
 
-    if ('-d' in sysargs or '--daemon' in sysargs) and ('stack' not in sysargs):
-        from meerschaum.utils.daemon import daemon_entry
-        _print_tuple(daemon_entry(sysargs), upper_padding=1)
-        return _exit(old_cwd=old_cwd)
-
     from meerschaum._internal.entry import entry, get_shell
 
     ### Try to launch a shell if --shell is provided.

meerschaum/_internal/arguments/__init__.py

@@ -8,7 +8,7 @@ This package includes argument parsing utilities.
 
 from meerschaum._internal.arguments._parse_arguments import (
     parse_arguments, parse_line, remove_leading_action,
-    parse_dict_to_sysargs,
+    parse_dict_to_sysargs, split_chained_sysargs,
 )
 from meerschaum._internal.arguments._parser import parser
 from meerschaum.plugins import add_plugin_argument

meerschaum/_internal/arguments/_parse_arguments.py

@@ -19,6 +19,44 @@ _locks = {
 _loaded_plugins_args: bool = False
 
 
+def split_chained_sysargs(sysargs: List[str]) -> List[List[str]]:
+    """
+    Split a `sysargs` list containing "and" keys (`&&`)
+    into a list of individual `sysargs`.
+    """
+    from meerschaum.config.static import STATIC_CONFIG
+    and_key = STATIC_CONFIG['system']['arguments']['and_key']
+
+    if not sysargs or and_key not in sysargs:
+        return [sysargs]
+
+    ### Coalesce and consecutive joiners into one.
+    coalesce_args = []
+    previous_arg = None
+    for arg in [_arg for _arg in sysargs]:
+        if arg == and_key and previous_arg == and_key:
+            continue
+        coalesce_args.append(arg)
+        previous_arg = arg
+
+    ### Remove any joiners from the ends.
+    if coalesce_args[0] == and_key:
+        coalesce_args = coalesce_args[1:]
+    if coalesce_args[-1] == and_key:
+        coalesce_args = coalesce_args[:-1]
+
+    chained_sysargs = []
+    current_sysargs = []
+    for arg in coalesce_args:
+        if arg != and_key:
+            current_sysargs.append(arg)
+        else:
+            chained_sysargs.append(current_sysargs)
+            current_sysargs = []
+    chained_sysargs.append(current_sysargs)
+    return chained_sysargs
+
+
 def parse_arguments(sysargs: List[str]) -> Dict[str, Any]:
     """
     Parse a list of arguments into standard Meerschaum arguments.
@@ -206,9 +244,24 @@ def parse_dict_to_sysargs(
     args_dict: Dict[str, Any]
 ) -> List[str]:
     """Revert an arguments dictionary back to a command line list."""
+    import shlex
     from meerschaum._internal.arguments._parser import get_arguments_triggers
-    sysargs = []
-    sysargs += args_dict.get('action', [])
+    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum.utils.warnings import warn
+
+    and_key = STATIC_CONFIG['system']['arguments']['and_key']
+    if (line := args_dict.get('line', None)):
+        return shlex.split(line)
+
+    if (_sysargs := args_dict.get('sysargs', None)):
+        return _sysargs
+
+    action = args_dict.get('action', None)
+    if action and and_key in action:
+        warn(f"Cannot determine flags from chained actions:\n{args_dict}")
+
+    sysargs: List[str] = []
+    sysargs.extend(action or [])
     allow_none_args = {'location_keys'}
 
     triggers = get_arguments_triggers()
@@ -216,6 +269,7 @@ def parse_dict_to_sysargs(
     for a, t in triggers.items():
         if a == 'action' or a not in args_dict:
             continue
+
         ### Add boolean flags
         if isinstance(args_dict[a], bool):
             if args_dict[a] is True:

meerschaum/_internal/arguments/_parser.py

@@ -66,7 +66,7 @@ def parse_executor_keys(executor_keys_str: str) -> Union[str, None]:
     """
     Ensure that only API keys are provided for executor_keys.
     """
-    if executor_keys_str == 'local':
+    if executor_keys_str in ('local', 'systemd'):
         return executor_keys_str
 
     if executor_keys_str.lower() == 'none':
@@ -214,10 +214,14 @@ groups['jobs'].add_argument(
     '--restart', action='store_true',
     help=("Restart a job if not stopped manually."),
 )
+groups['jobs'].add_argument(
+    '--systemd', action='store_true',
+    help=("Create a job via systemd. Shorthand for `-e systemd`."),
+)
 groups['jobs'].add_argument(
     '--executor-keys', '--executor', '-e', type=parse_executor_keys,
     help=(
-        "Remotely execute jobs on an API instance."
+        "Execute jobs on an API instance or via systemd."
     ),
 )
 groups['jobs'].add_argument(

meerschaum/_internal/entry.py

@@ -8,7 +8,29 @@ The entry point for launching Meerschaum actions.
 """
 
 from __future__ import annotations
+
+import os
+import sys
 from meerschaum.utils.typing import SuccessTuple, List, Optional, Dict, Callable, Any
+from meerschaum.config.static import STATIC_CONFIG as _STATIC_CONFIG
+
+_systemd_result_path = None
+if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
+    from meerschaum.utils.daemon import RotatingFile as _RotatingFile, StdinFile as _StdinFile
+    from meerschaum.config import get_config as _get_config
+
+    _systemd_result_path = os.environ[_STATIC_CONFIG['environment']['systemd_result_path']]
+    _systemd_log_path = os.environ[_STATIC_CONFIG['environment']['systemd_log_path']]
+    _systemd_log = _RotatingFile(
+        _systemd_log_path,
+        write_timestamps=True,
+        timestamp_format=_get_config('jobs', 'logs', 'timestamps', 'format'),
+    )
+    sys.stdout = _systemd_log
+    sys.stderr = _systemd_log
+    _systemd_stdin_path = os.environ.get(_STATIC_CONFIG['environment']['systemd_stdin_path'], None)
+    if _systemd_stdin_path:
+        sys.stdin = _StdinFile(_systemd_stdin_path)
 
 def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     """
@@ -18,31 +40,68 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     -------
     A `SuccessTuple` indicating success.
     """
-    from meerschaum._internal.arguments import parse_arguments
+    import shlex
+    from meerschaum.utils.formatting import make_header
+    from meerschaum._internal.arguments import parse_arguments, split_chained_sysargs
     from meerschaum.config.static import STATIC_CONFIG
     if sysargs is None:
         sysargs = []
     if not isinstance(sysargs, list):
-        import shlex
         sysargs = shlex.split(sysargs)
-    args = parse_arguments(sysargs)
-    argparse_exception = args.get(
-        STATIC_CONFIG['system']['arguments']['failure_key'],
-        None,
+
+    has_daemon = '-d' in sysargs or '--daemon' in sysargs
+    has_start_job = sysargs[:2] == ['start', 'job']
+    chained_sysargs = (
+        [sysargs]
+        if has_daemon or has_start_job
+        else split_chained_sysargs(sysargs)
     )
-    if argparse_exception is not None:
-        args_text = args.get('text', '')
-        if not args_text.startswith('show arguments'):
-            return (
-                False,
+    results: List[SuccessTuple] = []
+
+    for _sysargs in chained_sysargs:
+        args = parse_arguments(_sysargs)
+        argparse_exception = args.get(
+            STATIC_CONFIG['system']['arguments']['failure_key'],
+            None,
+        )
+        if argparse_exception is not None:
+            args_text = args.get('text', '')
+            if not args_text.startswith('show arguments'):
+                return (
+                    False,
+                    (
+                        "Invalid arguments:"
+                        + (f"\n{args_text}" if args_text else '')
+                        + f"\n    {argparse_exception}"
+                    )
+                )
+
+        entry_success, entry_msg = entry_with_args(**args)
+        if not entry_success:
+            return entry_success, entry_msg
+
+        results.append((entry_success, entry_msg))
+
+    success = all(_success for _success, _ in results)
+    msg = (
+        results[0][1]
+        if len(results) == 1
+        else 'Successfully completed steps:\n\n' + '\n'.join(
+            [
                 (
-                    "Invalid arguments:"
-                    + (f"\n{args_text}" if args_text else '')
-                    + f"\n    {argparse_exception}"
+                    make_header(shlex.join(_sysargs))
+                    + '\n    ' + _msg + '\n'
                 )
-            )
+                for i, ((_, _msg), _sysargs) in enumerate(zip(results, chained_sysargs))
+            ]
+        )
+    )
+    if _systemd_result_path:
+        import json
+        with open(_systemd_result_path, 'w+', encoding='utf-8') as f:
+            json.dump((success, msg), f)
 
-    return entry_with_args(**args)
+    return success, msg
 
 
 def entry_with_args(
@@ -57,7 +116,16 @@ def entry_with_args(
     import inspect
     from meerschaum.actions import get_action, get_main_action_name
     from meerschaum._internal.arguments import remove_leading_action
-    from meerschaum.utils.venv import Venv, active_venvs, deactivate_venv
+    from meerschaum.utils.venv import active_venvs, deactivate_venv
+    from meerschaum.config.static import STATIC_CONFIG
+
+    and_key = STATIC_CONFIG['system']['arguments']['and_key']
+    escaped_and_key = STATIC_CONFIG['system']['arguments']['escaped_and_key']
+    if and_key in (sysargs := kw.get('sysargs', [])):
+        if '-d' in sysargs or '--daemon' in sysargs:
+            sysargs = [(arg if arg != and_key else escaped_and_key) for arg in sysargs]
+        return entry(sysargs)
+
     if kw.get('trace', None):
         from meerschaum.utils.misc import debug_trace
         debug_trace()
@@ -87,6 +155,10 @@ def entry_with_args(
             kw['action'].insert(1, api_label)
             skip_schedule = True
 
+    ### If the `--daemon` flag is present, prepend 'start job'.
+    if kw.get('daemon', False) and kw['action'][0] != 'stack':
+        kw['action'] = ['start', 'jobs'] + kw['action']
+
     action_function = get_action(kw['action'], _actions=_actions)
 
     ### If action does not exist, execute in a subshell.
@@ -117,7 +189,6 @@ def entry_with_args(
         **kw
     )
 
-
     if kw.get('schedule', None) and not skip_schedule:
         from meerschaum.utils.schedule import schedule_function
         from meerschaum.utils.misc import interval_str
@@ -160,16 +231,16 @@ def _do_action_wrapper(action_function, plugin_name, **kw):
                 )
             )
         except KeyboardInterrupt:
-            result = False, f"Cancelled action `{action_name}`."
+            result = False, f"Cancelled action `{action_name.lstrip()}`."
     return result
 
 _shells = []
 _shell = None
 def get_shell(
-        sysargs: Optional[List[str]] = None,
-        reload: bool = False,
-        debug: bool = False
-    ):
+    sysargs: Optional[List[str]] = None,
+    reload: bool = False,
+    debug: bool = False
+):
     """Initialize and return the Meerschaum shell object."""
     global _shell
     from meerschaum.utils.debug import dprint

meerschaum/_internal/shell/Shell.py

@@ -8,6 +8,8 @@ This module is the entry point for the interactive shell.
 from __future__ import annotations
 import os
 from copy import deepcopy
+from itertools import chain
+
 from meerschaum.utils.typing import Union, SuccessTuple, Any, Callable, Optional, List, Dict
 from meerschaum.utils.packages import attempt_import
 from meerschaum.config import __doc__, __version__ as version, get_config
@@ -29,6 +31,13 @@ from meerschaum._internal.shell.ValidAutoSuggest import ValidAutoSuggest
 from meerschaum._internal.shell.ShellCompleter import ShellCompleter
 _clear_screen = get_config('shell', 'clear_screen', patch=True)
 from meerschaum.utils.misc import string_width, remove_ansi
+from meerschaum.jobs import get_executor_keys_from_context
+from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.arguments._parse_arguments import (
+    split_chained_sysargs,
+    parse_arguments,
+    parse_line,
+)
 
 patch = True
 ### remove default cmd2 commands
@@ -62,6 +71,8 @@ reserved_completers = {
 ### To handle dynamic reloading, store shell attributes externally.
 ### This is because the shell object address gets lost upon reloads.
 shell_attrs = {}
+AND_KEY: str = STATIC_CONFIG['system']['arguments']['and_key']
+ESCAPED_AND_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_and_key']
 
 def _insert_shell_actions(
         _shell: Optional['Shell'] = None,
@@ -110,7 +121,6 @@ def _completer_wrapper(
         if _check_keys is not None:
             return _check_keys
 
-        from meerschaum._internal.arguments._parse_arguments import parse_line
         args = parse_line(line)
         if target.__name__ != 'default_action_completer':
             if len(args['action']) > 0:
@@ -148,7 +158,6 @@ def default_action_completer(
 
 def _check_complete_keys(line: str) -> Optional[List[str]]:
     from meerschaum._internal.arguments._parser import parser, get_arguments_triggers
-    from meerschaum._internal.arguments._parse_arguments import parse_line
 
     ### TODO Add all triggers
     trigger_args = {
@@ -339,10 +348,7 @@ class Shell(cmd.Cmd):
         if shell_attrs.get('repo_keys', None) is None:
             shell_attrs['repo_keys'] = get_config('meerschaum', 'default_repository', patch=patch)
         if shell_attrs.get('executor_keys', None) is None:
-            shell_attrs['executor_keys'] = get_config(
-                'meerschaum', 'default_executor',
-                patch=patch,
-            ) or 'local'
+            shell_attrs['executor_keys'] = get_executor_keys_from_context()
 
         ### this will be updated later in update_prompt ONLY IF {username} is in the prompt
         shell_attrs['username'] = ''
@@ -548,27 +554,74 @@ class Shell(cmd.Cmd):
             else:
                 main_action_name = args['action'][0]
 
+        def _add_flag_to_sysargs(
+                chained_sysargs, key, value, flag, shell_key=None,
+            ):
+            shell_key = shell_key or key
+            shell_value = remove_ansi(shell_attrs.get(shell_key) or '')
+            if key == 'mrsm_instance':
+                default_value = get_config('meerschaum', 'instance')
+            elif key == 'repository':
+                default_value = get_config('meerschaum', 'default_repository')
+            elif key == 'executor_keys':
+                default_value = get_executor_keys_from_context()
+            else:
+                default_value = None
+
+            if shell_value == default_value:
+                return
+
+            for sysargs in chained_sysargs:
+                kwargs = parse_arguments(sysargs)
+                if key in kwargs:
+                    continue
+                sysargs.extend([flag, value])
+
+            args['sysargs'] = list(
+                chain.from_iterable(
+                    sub_sysargs + [AND_KEY]
+                    for sub_sysargs in chained_sysargs
+                )
+            )[:-1]
+
+
         ### if no instance is provided, use current shell default,
         ### but not for the 'api' command (to avoid recursion)
-        if 'mrsm_instance' not in args and main_action_name != 'api':
-            args['mrsm_instance'] = str(shell_attrs['instance_keys'])
+        if main_action_name != 'api':
+            chained_sysargs = split_chained_sysargs(args['sysargs'])
+            chained_filtered_sysargs = split_chained_sysargs(args['filtered_sysargs'])
+            mrsm_instance = shell_attrs['instance_keys']
+            args['mrsm_instance'] = mrsm_instance
+            _add_flag_to_sysargs(
+                chained_sysargs, 'mrsm_instance', mrsm_instance, '-i', 'instance_keys',
+            )
+            _add_flag_to_sysargs(
+                chained_filtered_sysargs, 'mrsm_instance', mrsm_instance, '-i', 'instance_keys',
+            )
 
-        if 'repository' not in args and main_action_name != 'api':
-            args['repository'] = str(shell_attrs['repo_keys'])
+            repo_keys = str(shell_attrs['repo_keys'])
+            args['repository'] = repo_keys
+            _add_flag_to_sysargs(
+                chained_sysargs, 'repository', repo_keys, '-r', 'repo_keys',
+            )
+            _add_flag_to_sysargs(
+                chained_filtered_sysargs, 'repository', repo_keys, '-r', 'repo_keys',
+            )
 
-        if 'executor_keys' not in args:
-            args['executor_keys'] = remove_ansi(str(shell_attrs['executor_keys']))
+            executor_keys = remove_ansi(str(shell_attrs['executor_keys']))
+            args['executor_keys'] = remove_ansi(executor_keys)
+            _add_flag_to_sysargs(
+                chained_sysargs, 'executor_keys', executor_keys, '-e',
+            )
+            _add_flag_to_sysargs(
+                chained_filtered_sysargs, 'executor_keys', executor_keys, '-e',
+            )
 
         ### parse out empty strings
         if args['action'][0].strip("\"'") == '':
             self.emptyline()
             return ""
 
-        ### If the `--daemon` flag is present, prepend 'start job'.
-        if args.get('daemon', False) and 'stack' not in args['action']:
-            args['action'] = ['start', 'jobs'] + args['action']
-            main_action_name = 'start'
-
         positional_only = (main_action_name not in shell_attrs['_actions'])
         if positional_only:
             return original_line
@@ -577,7 +630,6 @@ class Shell(cmd.Cmd):
 
         try:
             success_tuple = entry_with_args(_actions=shell_attrs['_actions'], **args)
-            #  success_tuple = entry_with_args(**args)
         except Exception as e:
             success_tuple = False, str(e)
 
@@ -585,9 +637,9 @@ class Shell(cmd.Cmd):
         if isinstance(success_tuple, tuple):
             print_tuple(
                 success_tuple,
-                skip_common = (not shell_attrs['debug']),
-                upper_padding = 1,
-                lower_padding = 0,
+                skip_common=(not shell_attrs['debug']),
+                upper_padding=1,
+                lower_padding=0,
             )
 
         ### Restore the old working directory.
@@ -638,12 +690,12 @@ class Shell(cmd.Cmd):
         info(f"Debug mode is {'on' if shell_attrs['debug'] else 'off'}.")
 
     def do_instance(
-            self,
-            action: Optional[List[str]] = None,
-            executor_keys=None,
-            debug: bool = False,
-            **kw: Any
-        ) -> SuccessTuple:
+        self,
+        action: Optional[List[str]] = None,
+        executor_keys=None,
+        debug: bool = False,
+        **kw: Any
+    ) -> SuccessTuple:
         """
         Temporarily set a default Meerschaum instance for the duration of the shell.
         The default instance is loaded from the Meerschaum configuraton file
@@ -700,14 +752,33 @@ class Shell(cmd.Cmd):
         return True, "Success"
 
 
-    def complete_instance(self, text: str, line: str, begin_index: int, end_index: int):
+    def complete_instance(
+        self,
+        text: str,
+        line: str,
+        begin_index: int,
+        end_index: int,
+        _executor: bool = False,
+        _additional_options: Optional[List[str]] = None,
+    ):
         from meerschaum.utils.misc import get_connector_labels
         from meerschaum._internal.arguments._parse_arguments import parse_line
-        from meerschaum.connectors import instance_types
+        from meerschaum.connectors import instance_types, _load_builtin_custom_connectors
+        if _executor:
+            _load_builtin_custom_connectors()
+            from meerschaum.jobs import executor_types
+
+        conn_types = instance_types if not _executor else executor_types
+
         args = parse_line(line)
         action = args['action']
         _text = action[1] if len(action) > 1 else ""
-        return get_connector_labels(*instance_types, search_term=_text, ignore_exact_match=True)
+        return get_connector_labels(
+            *conn_types,
+            search_term=_text,
+            ignore_exact_match=True,
+            _additional_options=_additional_options,
+        )
 
 
     def do_repo(
@@ -793,6 +864,7 @@ class Shell(cmd.Cmd):
         from meerschaum import get_connector
         from meerschaum.connectors.parse import parse_executor_keys
         from meerschaum.utils.warnings import warn, info
+        from meerschaum.jobs import get_executor_keys_from_context
 
         if action is None:
             action = []
@@ -802,18 +874,23 @@ class Shell(cmd.Cmd):
         except (IndexError, AttributeError):
             executor_keys = ''
         if executor_keys == '':
-            executor_keys = get_config('meerschaum', 'default_executor') or 'local'
+            executor_keys = get_executor_keys_from_context()
+
+        if executor_keys == 'systemd' and get_executor_keys_from_context() != 'systemd':
+            warn(f"Cannot execute via `systemd`, falling back to `local`...", stack=False)
+            executor_keys = 'local'
         
         conn = parse_executor_keys(executor_keys, debug=debug)
 
-        shell_attrs['executor_keys'] = str(conn)
+        shell_attrs['executor_keys'] = str(conn).replace('systemd:main', 'systemd')
 
         info(f"Default executor for the current shell: {executor_keys}")
         return True, "Success"
 
     def complete_executor(self, *args) -> List[str]:
-        results = self.complete_instance(*args)
-        return ['local'] + [result for result in results if result.startswith('api:')]
+        from meerschaum.jobs import executor_types
+        results = self.complete_instance(*args, _executor=True, _additional_options=['local'])
+        return [result for result in results if result.split(':')[0] in executor_types]
 
     def do_help(self, line: str, executor_keys=None) -> List[str]:
         """
@@ -975,7 +1052,7 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None):
         )
 
         try:
-            typ, label = shell_attrs['instance_keys'].split(':')
+            typ, label = shell_attrs['instance_keys'].split(':', maxsplit=1)
             connected = typ in connectors and label in connectors[typ]
         except Exception as e:
             connected = False

meerschaum/actions/attach.py

@@ -15,7 +15,7 @@ def attach(
     **kwargs: Any
 ) -> SuccessTuple:
     """
-    Attach to a job.
+    Attach to a job, and prompt the user when blocking on input.
     """
     from meerschaum.actions import choose_subaction
     attach_options = {
@@ -32,16 +32,16 @@ def _complete_attach(
     """
     Override the default Meerschaum `complete_` function.
     """
-    from meerschaum.actions.start import _complete_start_jobs
+    from meerschaum.actions.delete import _complete_delete_jobs
 
     if action is None:
         action = []
 
     options = {
-        'job': _complete_start_jobs,
-        'jobs': _complete_start_jobs,
-        'log': _complete_start_jobs,
-        'logs': _complete_start_jobs,
+        'job': _complete_delete_jobs,
+        'jobs': _complete_delete_jobs,
+        'log': _complete_delete_jobs,
+        'logs': _complete_delete_jobs,
     }
 
     if (
@@ -71,8 +71,13 @@ def _attach_jobs(
 
     name = name or action[0]
     job = mrsm.Job(name, executor_keys=executor_keys)
+    other_executor_keys = 'systemd' if executor_keys in (None, 'local') else 'local'
     if not job.exists():
-        return False, f"Job '{job.name}' does not exist."
+        other_job = mrsm.Job(name, executor_keys=other_executor_keys)
+        if not other_job.exists():
+            return False, f"Job '{job.name}' does not exist."
+
+        job = other_job
 
     success, message = True, "Success"