meerschaum 2.3.0rc2__py3-none-any.whl → 2.3.0rc5__py3-none-any.whl

meerschaum/_internal/arguments/__init__.py

@@ -8,7 +8,7 @@ This package includes argument parsing utilities.
 
 from meerschaum._internal.arguments._parse_arguments import (
     parse_arguments, parse_line, remove_leading_action,
-    parse_dict_to_sysargs, split_chained_sysargs,
+    parse_dict_to_sysargs, split_chained_sysargs, split_pipeline_sysargs,
 )
 from meerschaum._internal.arguments._parser import parser
 from meerschaum.plugins import add_plugin_argument

meerschaum/_internal/arguments/_parse_arguments.py

@@ -10,7 +10,7 @@ This module contains functions for parsing arguments
 from __future__ import annotations
 import json
 from datetime import timedelta
-from meerschaum.utils.typing import List, Dict, Any, Optional, Callable, SuccessTuple
+from meerschaum.utils.typing import List, Dict, Any, Optional, Callable, SuccessTuple, Tuple
 from meerschaum.utils.threading import RLock
 
 _locks = {
@@ -18,10 +18,25 @@ _locks = {
 }
 _loaded_plugins_args: bool = False
 
+def split_pipeline_sysargs(sysargs: List[str]) -> Tuple[List[str], List[str]]:
+    """
+    Split `sysargs` into the main pipeline and the flags following the pipeline separator (`:`).
+    """
+    from meerschaum.config.static import STATIC_CONFIG
+    pipeline_key = STATIC_CONFIG['system']['arguments']['pipeline_key']
+    if pipeline_key not in sysargs:
+        return sysargs, []
+
+    ### Find the index of the last occurrence of `:`.
+    pipeline_ix = len(sysargs) - 1 - sysargs[::-1].index(pipeline_key)
+    sysargs_after_pipeline_key = sysargs[pipeline_ix+1:]
+    sysargs = [arg for arg in sysargs[:pipeline_ix] if arg != pipeline_key]
+    return sysargs, sysargs_after_pipeline_key
+
 
 def split_chained_sysargs(sysargs: List[str]) -> List[List[str]]:
     """
-    Split a `sysargs` list containing "and" keys (`&&`)
+    Split a `sysargs` list containing "and" keys (`+`)
     into a list of individual `sysargs`.
     """
     from meerschaum.config.static import STATIC_CONFIG
@@ -249,17 +264,7 @@ def parse_dict_to_sysargs(
     from meerschaum.config.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
 
-    and_key = STATIC_CONFIG['system']['arguments']['and_key']
-    if (line := args_dict.get('line', None)):
-        return shlex.split(line)
-
-    if (_sysargs := args_dict.get('sysargs', None)):
-        return _sysargs
-
     action = args_dict.get('action', None)
-    if action and and_key in action:
-        warn(f"Cannot determine flags from chained actions:\n{args_dict}")
-
     sysargs: List[str] = []
     sysargs.extend(action or [])
     allow_none_args = {'location_keys'}

meerschaum/_internal/arguments/_parser.py

@@ -215,13 +215,10 @@ groups['jobs'].add_argument(
     help=("Restart a job if not stopped manually."),
 )
 groups['jobs'].add_argument(
-    '--systemd', action='store_true',
-    help=("Create a job via systemd. Shorthand for `-e systemd`."),
-)
-groups['jobs'].add_argument(
-    '--executor-keys', '--executor', '-e', type=parse_executor_keys,
+    '-e', '--executor-keys', type=parse_executor_keys,
     help=(
-        "Execute jobs on an API instance or via systemd."
+        "Execute jobs locally or remotely. "
+        "Supported values are 'local', 'systemd', and 'api:{label}'."
     ),
 )
 groups['jobs'].add_argument(

meerschaum/_internal/entry.py

@@ -32,7 +32,10 @@ if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
     if _systemd_stdin_path:
         sys.stdin = _StdinFile(_systemd_stdin_path)
 
-def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
+def entry(
+    sysargs: Optional[List[str]] = None,
+    _patch_args: Optional[Dict[str, Any]] = None,
+) -> SuccessTuple:
     """
     Parse arguments and launch a Meerschaum action.
 
@@ -41,14 +44,23 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     A `SuccessTuple` indicating success.
     """
     import shlex
+    import json
     from meerschaum.utils.formatting import make_header
-    from meerschaum._internal.arguments import parse_arguments, split_chained_sysargs
+    from meerschaum._internal.arguments import (
+        parse_arguments,
+        split_chained_sysargs,
+        split_pipeline_sysargs,
+    )
     from meerschaum.config.static import STATIC_CONFIG
     if sysargs is None:
         sysargs = []
     if not isinstance(sysargs, list):
         sysargs = shlex.split(sysargs)
 
+    pipeline_key = STATIC_CONFIG['system']['arguments']['pipeline_key']
+    escaped_pipeline_key = STATIC_CONFIG['system']['arguments']['escaped_pipeline_key']
+    sysargs, pipeline_args = split_pipeline_sysargs(sysargs)
+
     has_daemon = '-d' in sysargs or '--daemon' in sysargs
     has_start_job = sysargs[:2] == ['start', 'job']
     chained_sysargs = (
@@ -56,10 +68,32 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
         if has_daemon or has_start_job
         else split_chained_sysargs(sysargs)
     )
+    if pipeline_args:
+        chained_sysargs = [
+            ['start', 'pipeline']
+            + [str(arg) for arg in pipeline_args]
+            + (
+                ['--params', json.dumps(_patch_args)]
+                if _patch_args
+                else []
+            )
+            + ['--sub-args', shlex.join(sysargs)]
+        ]
+
     results: List[SuccessTuple] = []
 
     for _sysargs in chained_sysargs:
+        if escaped_pipeline_key in _sysargs:
+            _sysargs = [
+                pipeline_key
+                if _arg == escaped_pipeline_key
+                else _arg
+                for _arg in _sysargs
+            ]
+
         args = parse_arguments(_sysargs)
+        if _patch_args:
+            args.update(_patch_args)
         argparse_exception = args.get(
             STATIC_CONFIG['system']['arguments']['failure_key'],
             None,
@@ -76,7 +110,7 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
                     )
                 )
 
-        entry_success, entry_msg = entry_with_args(**args)
+        entry_success, entry_msg = entry_with_args(_patch_args=_patch_args, **args)
         if not entry_success:
             return entry_success, entry_msg
 
@@ -106,6 +140,7 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
 
 def entry_with_args(
     _actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
+    _patch_args: Optional[Dict[str, Any]] = None,
     **kw
 ) -> SuccessTuple:
     """Execute a Meerschaum action with keyword arguments.
@@ -119,12 +154,15 @@ def entry_with_args(
     from meerschaum.utils.venv import active_venvs, deactivate_venv
     from meerschaum.config.static import STATIC_CONFIG
 
+    if _patch_args:
+        kw.update(_patch_args)
+
     and_key = STATIC_CONFIG['system']['arguments']['and_key']
     escaped_and_key = STATIC_CONFIG['system']['arguments']['escaped_and_key']
     if and_key in (sysargs := kw.get('sysargs', [])):
         if '-d' in sysargs or '--daemon' in sysargs:
             sysargs = [(arg if arg != and_key else escaped_and_key) for arg in sysargs]
-        return entry(sysargs)
+        return entry(sysargs, _patch_args=_patch_args)
 
     if kw.get('trace', None):
         from meerschaum.utils.misc import debug_trace

meerschaum/_internal/shell/Shell.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 import os
 from copy import deepcopy
 from itertools import chain
+import shlex
 
 from meerschaum.utils.typing import Union, SuccessTuple, Any, Callable, Optional, List, Dict
 from meerschaum.utils.packages import attempt_import
@@ -31,12 +32,15 @@ from meerschaum._internal.shell.ValidAutoSuggest import ValidAutoSuggest
 from meerschaum._internal.shell.ShellCompleter import ShellCompleter
 _clear_screen = get_config('shell', 'clear_screen', patch=True)
 from meerschaum.utils.misc import string_width, remove_ansi
+from meerschaum.utils.warnings import warn
 from meerschaum.jobs import get_executor_keys_from_context
 from meerschaum.config.static import STATIC_CONFIG
 from meerschaum._internal.arguments._parse_arguments import (
     split_chained_sysargs,
+    split_pipeline_sysargs,
     parse_arguments,
     parse_line,
+    parse_dict_to_sysargs,
 )
 
 patch = True
@@ -73,6 +77,8 @@ reserved_completers = {
 shell_attrs = {}
 AND_KEY: str = STATIC_CONFIG['system']['arguments']['and_key']
 ESCAPED_AND_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_and_key']
+PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['pipeline_key']
+ESCAPED_PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_pipeline_key']
 
 def _insert_shell_actions(
         _shell: Optional['Shell'] = None,
@@ -173,7 +179,6 @@ def _check_complete_keys(line: str) -> Optional[List[str]]:
 
     ### TODO Find out arg possibilities
     possibilities = []
-    #  last_word = line.split(' ')[-1]
     last_word = line.rstrip(' ').split(' ')[-1]
 
     if last_word.startswith('-'):
@@ -485,6 +490,9 @@ class Shell(cmd.Cmd):
         ### make a backup of line for later
         original_line = deepcopy(line)
 
+        ### Escape backslashes to allow for multi-line input.
+        line = line.replace('\\\n', ' ')
+
         ### cmd2 support: check if command exists
         try:
             command = line.command
@@ -515,48 +523,71 @@ class Shell(cmd.Cmd):
             return "help " + line[len(help_token):]
 
         from meerschaum._internal.arguments import parse_line
-        args = parse_line(line)
-        if args.get('help', False):
+        try:
+            sysargs = shlex.split(line)
+        except ValueError as e:
+            warn(e, stack=False)
+            return ""
+
+        sysargs, pipeline_args = split_pipeline_sysargs(sysargs)
+        chained_sysargs = split_chained_sysargs(sysargs)
+        chained_kwargs = [
+            parse_arguments(_sysargs)
+            for _sysargs in chained_sysargs
+        ]
+
+        if '--help' in sysargs or '-h' in sysargs:
             from meerschaum._internal.arguments._parser import parse_help
-            parse_help(args)
+            parse_help(sysargs)
             return ""
 
+        patch_args: Dict[str, Any] = {}
+
         ### NOTE: pass `shell` flag in case actions need to distinguish between
         ###       being run on the command line and being run in the shell
-        args['shell'] = True
-        args['line'] = line
+        patch_args.update({
+            'shell': True,
+            'line': line,
+        })
+        patches: List[Dict[str, Any]] = [{} for _ in chained_kwargs]
 
         ### if debug is not set on the command line,
         ### default to shell setting
-        if not args.get('debug', False):
-            args['debug'] = shell_attrs['debug']
+        for kwargs in chained_kwargs:
+            if not kwargs.get('debug', False):
+                kwargs['debug'] = shell_attrs['debug']
 
         ### Make sure an action was provided.
-        if not args.get('action', None):
+        if (
+            not chained_kwargs
+            or not chained_kwargs[0].get('action', None)
+        ):
             self.emptyline()
             return ''
 
         ### Strip a leading 'mrsm' if it's provided.
-        if args['action'][0] == 'mrsm':
-            args['action'] = args['action'][1:]
-            if not args['action']:
-                self.emptyline()
-                return ''
+        for kwargs in chained_kwargs:
+            if kwargs['action'][0] == 'mrsm':
+                kwargs['action'] = kwargs['action'][1:]
+                if not kwargs['action']:
+                    self.emptyline()
+                    return ''
 
         ### If we don't recognize the action,
         ### make it a shell action.
-        from meerschaum.actions import get_main_action_name
-        main_action_name = get_main_action_name(args['action'])
-        if main_action_name is None:
-            if not hasattr(self, 'do_' + args['action'][0]):
-                args['action'].insert(0, 'sh')
-                main_action_name = 'sh'
-            else:
-                main_action_name = args['action'][0]
-
-        def _add_flag_to_sysargs(
-                chained_sysargs, key, value, flag, shell_key=None,
-            ):
+        ### TODO: make this work for chained actions
+        def _get_main_action_name(kwargs):
+            from meerschaum.actions import get_main_action_name
+            main_action_name = get_main_action_name(kwargs['action'])
+            if main_action_name is None:
+                if not hasattr(self, 'do_' + kwargs['action'][0]):
+                    kwargs['action'].insert(0, 'sh')
+                    main_action_name = 'sh'
+                else:
+                    main_action_name = kwargs['action'][0]
+            return main_action_name
+
+        def _add_flag_to_kwargs(kwargs, i, key, shell_key=None):
             shell_key = shell_key or key
             shell_value = remove_ansi(shell_attrs.get(shell_key) or '')
             if key == 'mrsm_instance':
@@ -568,68 +599,49 @@ class Shell(cmd.Cmd):
             else:
                 default_value = None
 
-            if shell_value == default_value:
+            if key in kwargs or shell_value == default_value:
                 return
 
-            for sysargs in chained_sysargs:
-                kwargs = parse_arguments(sysargs)
-                if key in kwargs:
-                    continue
-                sysargs.extend([flag, value])
-
-            args['sysargs'] = list(
-                chain.from_iterable(
-                    sub_sysargs + [AND_KEY]
-                    for sub_sysargs in chained_sysargs
-                )
-            )[:-1]
-
+            patches[i][key] = shell_value
 
         ### if no instance is provided, use current shell default,
         ### but not for the 'api' command (to avoid recursion)
-        if main_action_name != 'api':
-            chained_sysargs = split_chained_sysargs(args['sysargs'])
-            chained_filtered_sysargs = split_chained_sysargs(args['filtered_sysargs'])
-            mrsm_instance = shell_attrs['instance_keys']
-            args['mrsm_instance'] = mrsm_instance
-            _add_flag_to_sysargs(
-                chained_sysargs, 'mrsm_instance', mrsm_instance, '-i', 'instance_keys',
-            )
-            _add_flag_to_sysargs(
-                chained_filtered_sysargs, 'mrsm_instance', mrsm_instance, '-i', 'instance_keys',
-            )
-
-            repo_keys = str(shell_attrs['repo_keys'])
-            args['repository'] = repo_keys
-            _add_flag_to_sysargs(
-                chained_sysargs, 'repository', repo_keys, '-r', 'repo_keys',
-            )
-            _add_flag_to_sysargs(
-                chained_filtered_sysargs, 'repository', repo_keys, '-r', 'repo_keys',
-            )
+        for i, kwargs in enumerate(chained_kwargs):
+            main_action_name = _get_main_action_name(kwargs)
+            if main_action_name == 'api':
+                continue
 
-            executor_keys = remove_ansi(str(shell_attrs['executor_keys']))
-            args['executor_keys'] = remove_ansi(executor_keys)
-            _add_flag_to_sysargs(
-                chained_sysargs, 'executor_keys', executor_keys, '-e',
-            )
-            _add_flag_to_sysargs(
-                chained_filtered_sysargs, 'executor_keys', executor_keys, '-e',
-            )
+            _add_flag_to_kwargs(kwargs, i, 'mrsm_instance', shell_key='instance_keys')
+            _add_flag_to_kwargs(kwargs, i, 'repository', shell_key='repo_keys')
+            _add_flag_to_kwargs(kwargs, i, 'executor_keys')
 
         ### parse out empty strings
-        if args['action'][0].strip("\"'") == '':
+        if chained_kwargs[0]['action'][0].strip("\"'") == '':
             self.emptyline()
             return ""
 
-        positional_only = (main_action_name not in shell_attrs['_actions'])
+        positional_only = (_get_main_action_name(chained_kwargs[0]) not in shell_attrs['_actions'])
         if positional_only:
             return original_line
 
-        from meerschaum._internal.entry import entry_with_args
+        ### Apply patch to all kwargs.
+        for i, kwargs in enumerate([_ for _ in chained_kwargs]):
+            kwargs.update(patches[i])
+
+        from meerschaum._internal.entry import entry_with_args, entry
+        sysargs_to_execute = []
+        for i, kwargs in enumerate(chained_kwargs):
+            step_kwargs = {k: v for k, v in kwargs.items() if k != 'line'}
+            step_sysargs = parse_dict_to_sysargs(step_kwargs)
+            sysargs_to_execute.extend(step_sysargs)
+            sysargs_to_execute.append(AND_KEY)
+        
+        sysargs_to_execute = sysargs_to_execute[:-1] + (
+            ([':'] + pipeline_args) if pipeline_args else []
+        )
 
         try:
-            success_tuple = entry_with_args(_actions=shell_attrs['_actions'], **args)
+            success_tuple = entry(sysargs_to_execute, _patch_args=patch_args)
         except Exception as e:
             success_tuple = False, str(e)
 

meerschaum/actions/delete.py

@@ -512,6 +512,7 @@ def _complete_delete_jobs(
         get_executor_keys_from_context,
     )
     from meerschaum.utils.misc import remove_ansi
+    from meerschaum.connectors.parse import parse_executor_keys
 
     executor_keys = (
         executor_keys
@@ -520,6 +521,9 @@ def _complete_delete_jobs(
         )
     )
 
+    if parse_executor_keys(executor_keys, construct=False) is None:
+        return []
+
     jobs = get_jobs(executor_keys, include_hidden=False)
     if _get_job_method:
         method_keys = [_get_job_method] if isinstance(_get_job_method, str) else _get_job_method

meerschaum/actions/show.py

@@ -155,10 +155,10 @@ def _complete_show_config(action: Optional[List[str]] = None, **kw : Any):
 
 
 def _show_pipes(
-        nopretty: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    nopretty: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Print a stylized tree of available Meerschaum pipes.
     Respects global ANSI and UNICODE settings.
@@ -170,7 +170,7 @@ def _show_pipes(
     pipes = get_pipes(debug=debug, **kw)
 
     if len(pipes) == 0:
-        return False, "No pipes to show."
+        return True, "No pipes to show."
 
     if len(flatten_pipes_dict(pipes)) == 1:
         return flatten_pipes_dict(pipes)[0].show(debug=debug, nopretty=nopretty, **kw)

meerschaum/actions/start.py

@@ -7,7 +7,7 @@ Start subsystems (API server, logging daemon, etc.).
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Optional, List, Any
+from meerschaum.utils.typing import SuccessTuple, Optional, List, Any, Union
 
 def start(
     action: Optional[List[str]] = None,
@@ -23,6 +23,7 @@ def start(
         'gui': _start_gui,
         'webterm': _start_webterm,
         'connectors': _start_connectors,
+        'pipeline': _start_pipeline,
     }
     return choose_subaction(action, options, **kw)
 
@@ -537,6 +538,75 @@ def _complete_start_connectors(**kw) -> List[str]:
     return _complete_show_connectors(**kw)
 
 
+def _start_pipeline(
+    action: Optional[List[str]] = None,
+    sub_args: Optional[List[str]] = None,
+    loop: bool = False,
+    min_seconds: Union[float, int, None] = 1.0,
+    params: Optional[Dict[str, Any]] = None,
+    **kwargs
+) -> SuccessTuple:
+    """
+    Run a series of Meerschaum commands as a single action.
+
+    Add `:` to the end of chained arguments to apply additional flags to the pipeline.
+
+    Examples
+    --------
+
+    `sync pipes -i sql:local + sync pipes -i sql:main :: -s 'daily'`
+
+    `show version + show arguments :: --loop`
+
+    """
+    import time
+    from meerschaum._internal.entry import entry
+    from meerschaum.utils.warnings import info, warn
+    from meerschaum.utils.misc import is_int
+
+    do_n_times = (
+        int(action[0].lstrip('x'))
+        if action and is_int(action[0].lstrip('x'))
+        else 1
+    )
+
+    if not sub_args:
+        return False, "Nothing to do."
+
+    if min_seconds is None:
+        min_seconds = 1.0
+
+    ran_n_times = 0
+    success, msg = False, "Did not run pipeline."
+    def run_loop():
+        nonlocal ran_n_times, success, msg
+        while True:
+            success, msg = entry(sub_args, _patch_args=params)
+            ran_n_times += 1
+
+            if not loop and do_n_times == 1:
+                break
+
+            if min_seconds != 0 and ran_n_times != do_n_times:
+                info(f"Sleeping for {min_seconds} seconds...")
+                time.sleep(min_seconds)
+
+            if loop:
+                continue
+
+            if ran_n_times >= do_n_times:
+                break
+
+    try:
+        run_loop()
+    except KeyboardInterrupt:
+        warn("Cancelled pipeline.", stack=False)
+
+    if do_n_times != 1:
+        info(f"Ran pipeline {ran_n_times} time" + ('s' if ran_n_times != 1 else '') + '.')
+    return success, msg
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.