meerschaum 2.3.0rc3__py3-none-any.whl → 2.3.1__py3-none-any.whl

meerschaum/_internal/shell/Shell.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 import os
 from copy import deepcopy
 from itertools import chain
+import shlex
 
 from meerschaum.utils.typing import Union, SuccessTuple, Any, Callable, Optional, List, Dict
 from meerschaum.utils.packages import attempt_import
@@ -31,12 +32,15 @@ from meerschaum._internal.shell.ValidAutoSuggest import ValidAutoSuggest
 from meerschaum._internal.shell.ShellCompleter import ShellCompleter
 _clear_screen = get_config('shell', 'clear_screen', patch=True)
 from meerschaum.utils.misc import string_width, remove_ansi
+from meerschaum.utils.warnings import warn
 from meerschaum.jobs import get_executor_keys_from_context
 from meerschaum.config.static import STATIC_CONFIG
 from meerschaum._internal.arguments._parse_arguments import (
     split_chained_sysargs,
+    split_pipeline_sysargs,
     parse_arguments,
     parse_line,
+    parse_dict_to_sysargs,
 )
 
 patch = True
@@ -73,6 +77,8 @@ reserved_completers = {
 shell_attrs = {}
 AND_KEY: str = STATIC_CONFIG['system']['arguments']['and_key']
 ESCAPED_AND_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_and_key']
+PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['pipeline_key']
+ESCAPED_PIPELINE_KEY: str = STATIC_CONFIG['system']['arguments']['escaped_pipeline_key']
 
 def _insert_shell_actions(
         _shell: Optional['Shell'] = None,
@@ -173,7 +179,6 @@ def _check_complete_keys(line: str) -> Optional[List[str]]:
 
     ### TODO Find out arg possibilities
     possibilities = []
-    #  last_word = line.split(' ')[-1]
     last_word = line.rstrip(' ').split(' ')[-1]
 
     if last_word.startswith('-'):
@@ -485,6 +490,9 @@ class Shell(cmd.Cmd):
         ### make a backup of line for later
         original_line = deepcopy(line)
 
+        ### Escape backslashes to allow for multi-line input.
+        line = line.replace('\\\n', ' ')
+
         ### cmd2 support: check if command exists
         try:
             command = line.command
@@ -515,48 +523,71 @@ class Shell(cmd.Cmd):
             return "help " + line[len(help_token):]
 
         from meerschaum._internal.arguments import parse_line
-        args = parse_line(line)
-        if args.get('help', False):
+        try:
+            sysargs = shlex.split(line)
+        except ValueError as e:
+            warn(e, stack=False)
+            return ""
+
+        sysargs, pipeline_args = split_pipeline_sysargs(sysargs)
+        chained_sysargs = split_chained_sysargs(sysargs)
+        chained_kwargs = [
+            parse_arguments(_sysargs)
+            for _sysargs in chained_sysargs
+        ]
+
+        if '--help' in sysargs or '-h' in sysargs:
             from meerschaum._internal.arguments._parser import parse_help
-            parse_help(args)
+            parse_help(sysargs)
             return ""
 
+        patch_args: Dict[str, Any] = {}
+
         ### NOTE: pass `shell` flag in case actions need to distinguish between
         ###       being run on the command line and being run in the shell
-        args['shell'] = True
-        args['line'] = line
+        patch_args.update({
+            'shell': True,
+            'line': line,
+        })
+        patches: List[Dict[str, Any]] = [{} for _ in chained_kwargs]
 
         ### if debug is not set on the command line,
         ### default to shell setting
-        if not args.get('debug', False):
-            args['debug'] = shell_attrs['debug']
+        for kwargs in chained_kwargs:
+            if not kwargs.get('debug', False):
+                kwargs['debug'] = shell_attrs['debug']
 
         ### Make sure an action was provided.
-        if not args.get('action', None):
+        if (
+            not chained_kwargs
+            or not chained_kwargs[0].get('action', None)
+        ):
             self.emptyline()
             return ''
 
         ### Strip a leading 'mrsm' if it's provided.
-        if args['action'][0] == 'mrsm':
-            args['action'] = args['action'][1:]
-            if not args['action']:
-                self.emptyline()
-                return ''
+        for kwargs in chained_kwargs:
+            if kwargs['action'][0] == 'mrsm':
+                kwargs['action'] = kwargs['action'][1:]
+                if not kwargs['action']:
+                    self.emptyline()
+                    return ''
 
         ### If we don't recognize the action,
         ### make it a shell action.
-        from meerschaum.actions import get_main_action_name
-        main_action_name = get_main_action_name(args['action'])
-        if main_action_name is None:
-            if not hasattr(self, 'do_' + args['action'][0]):
-                args['action'].insert(0, 'sh')
-                main_action_name = 'sh'
-            else:
-                main_action_name = args['action'][0]
-
-        def _add_flag_to_sysargs(
-                chained_sysargs, key, value, flag, shell_key=None,
-            ):
+        ### TODO: make this work for chained actions
+        def _get_main_action_name(kwargs):
+            from meerschaum.actions import get_main_action_name
+            main_action_name = get_main_action_name(kwargs['action'])
+            if main_action_name is None:
+                if not hasattr(self, 'do_' + kwargs['action'][0]):
+                    kwargs['action'].insert(0, 'sh')
+                    main_action_name = 'sh'
+                else:
+                    main_action_name = kwargs['action'][0]
+            return main_action_name
+
+        def _add_flag_to_kwargs(kwargs, i, key, shell_key=None):
             shell_key = shell_key or key
             shell_value = remove_ansi(shell_attrs.get(shell_key) or '')
             if key == 'mrsm_instance':
@@ -568,68 +599,49 @@ class Shell(cmd.Cmd):
             else:
                 default_value = None
 
-            if shell_value == default_value:
+            if key in kwargs or shell_value == default_value:
                 return
 
-            for sysargs in chained_sysargs:
-                kwargs = parse_arguments(sysargs)
-                if key in kwargs:
-                    continue
-                sysargs.extend([flag, value])
-
-            args['sysargs'] = list(
-                chain.from_iterable(
-                    sub_sysargs + [AND_KEY]
-                    for sub_sysargs in chained_sysargs
-                )
-            )[:-1]
-
+            patches[i][key] = shell_value
 
         ### if no instance is provided, use current shell default,
         ### but not for the 'api' command (to avoid recursion)
-        if main_action_name != 'api':
-            chained_sysargs = split_chained_sysargs(args['sysargs'])
-            chained_filtered_sysargs = split_chained_sysargs(args['filtered_sysargs'])
-            mrsm_instance = shell_attrs['instance_keys']
-            args['mrsm_instance'] = mrsm_instance
-            _add_flag_to_sysargs(
-                chained_sysargs, 'mrsm_instance', mrsm_instance, '-i', 'instance_keys',
-            )
-            _add_flag_to_sysargs(
-                chained_filtered_sysargs, 'mrsm_instance', mrsm_instance, '-i', 'instance_keys',
-            )
-
-            repo_keys = str(shell_attrs['repo_keys'])
-            args['repository'] = repo_keys
-            _add_flag_to_sysargs(
-                chained_sysargs, 'repository', repo_keys, '-r', 'repo_keys',
-            )
-            _add_flag_to_sysargs(
-                chained_filtered_sysargs, 'repository', repo_keys, '-r', 'repo_keys',
-            )
+        for i, kwargs in enumerate(chained_kwargs):
+            main_action_name = _get_main_action_name(kwargs)
+            if main_action_name == 'api':
+                continue
 
-            executor_keys = remove_ansi(str(shell_attrs['executor_keys']))
-            args['executor_keys'] = remove_ansi(executor_keys)
-            _add_flag_to_sysargs(
-                chained_sysargs, 'executor_keys', executor_keys, '-e',
-            )
-            _add_flag_to_sysargs(
-                chained_filtered_sysargs, 'executor_keys', executor_keys, '-e',
-            )
+            _add_flag_to_kwargs(kwargs, i, 'mrsm_instance', shell_key='instance_keys')
+            _add_flag_to_kwargs(kwargs, i, 'repository', shell_key='repo_keys')
+            _add_flag_to_kwargs(kwargs, i, 'executor_keys')
 
         ### parse out empty strings
-        if args['action'][0].strip("\"'") == '':
+        if chained_kwargs[0]['action'][0].strip("\"'") == '':
             self.emptyline()
             return ""
 
-        positional_only = (main_action_name not in shell_attrs['_actions'])
+        positional_only = (_get_main_action_name(chained_kwargs[0]) not in shell_attrs['_actions'])
         if positional_only:
             return original_line
 
-        from meerschaum._internal.entry import entry_with_args
+        ### Apply patch to all kwargs.
+        for i, kwargs in enumerate([_ for _ in chained_kwargs]):
+            kwargs.update(patches[i])
+
+        from meerschaum._internal.entry import entry_with_args, entry
+        sysargs_to_execute = []
+        for i, kwargs in enumerate(chained_kwargs):
+            step_kwargs = {k: v for k, v in kwargs.items() if k != 'line'}
+            step_sysargs = parse_dict_to_sysargs(step_kwargs)
+            sysargs_to_execute.extend(step_sysargs)
+            sysargs_to_execute.append(AND_KEY)
+        
+        sysargs_to_execute = sysargs_to_execute[:-1] + (
+            ([':'] + pipeline_args) if pipeline_args else []
+        )
 
         try:
-            success_tuple = entry_with_args(_actions=shell_attrs['_actions'], **args)
+            success_tuple = entry(sysargs_to_execute, _patch_args=patch_args)
         except Exception as e:
             success_tuple = False, str(e)
 

meerschaum/actions/__init__.py

@@ -11,6 +11,16 @@ from meerschaum.utils.typing import Callable, Any, Optional, Union, List, Dict,
 from meerschaum.utils.packages import get_modules_from_package
 _custom_actions = []
 
+__all__ = (
+    'get_action',
+    'get_subactions',
+    'make_action',
+    'pre_sync_hook',
+    'post_sync_hook',
+    'get_main_action_name',
+    'get_completer',
+)
+
 def get_subactions(
     action: Union[str, List[str]],
     _actions: Optional[Dict[str, Callable[[Any], Any]]] = None,
@@ -101,9 +111,9 @@ def get_action(
 
 
 def get_main_action_name(
-        action: Union[List[str], str],
-        _actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
-    ) -> Union[str, None]:
+    action: Union[List[str], str],
+    _actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
+) -> Union[str, None]:
     """
     Given an action list, return the name of the main function.
     For subactions, this will return the root function.
@@ -140,9 +150,9 @@ def get_main_action_name(
 
 
 def get_completer(
-        action: Union[List[str], str],
-        _actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
-    ) -> Union[
+    action: Union[List[str], str],
+    _actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
+) -> Union[
         Callable[['meerschaum._internal.shell.Shell', str, str, int, int], List[str]], None
     ]:
     """Search for a custom completer function for an action."""
@@ -179,10 +189,10 @@ def get_completer(
 
 
 def choose_subaction(
-        action: Optional[List[str]] = None,
-        options: Optional[Dict[str, Any]] = None,
-        **kw
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    options: Optional[Dict[str, Any]] = None,
+    **kw
+) -> SuccessTuple:
     """
     Given a dictionary of options and the standard Meerschaum actions list,
     check if choice is valid and execute chosen function, else show available
@@ -247,7 +257,7 @@ def _get_subaction_names(action: str, globs: dict = None) -> List[str]:
     return subactions
 
 
-def choices_docstring(action: str, globs : Optional[Dict[str, Any]] = None) -> str:
+def choices_docstring(action: str, globs: Optional[Dict[str, Any]] = None) -> str:
     """
     Append the an action's available options to the module docstring.
     This function is to be placed at the bottom of each action module.

meerschaum/actions/show.py

@@ -155,10 +155,10 @@ def _complete_show_config(action: Optional[List[str]] = None, **kw : Any):
 
 
 def _show_pipes(
-        nopretty: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    nopretty: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Print a stylized tree of available Meerschaum pipes.
     Respects global ANSI and UNICODE settings.
@@ -170,7 +170,7 @@ def _show_pipes(
     pipes = get_pipes(debug=debug, **kw)
 
     if len(pipes) == 0:
-        return False, "No pipes to show."
+        return True, "No pipes to show."
 
     if len(flatten_pipes_dict(pipes)) == 1:
         return flatten_pipes_dict(pipes)[0].show(debug=debug, nopretty=nopretty, **kw)

meerschaum/actions/start.py

@@ -7,7 +7,7 @@ Start subsystems (API server, logging daemon, etc.).
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Optional, List, Any
+from meerschaum.utils.typing import SuccessTuple, Optional, List, Any, Union
 
 def start(
     action: Optional[List[str]] = None,
@@ -23,6 +23,7 @@ def start(
         'gui': _start_gui,
         'webterm': _start_webterm,
         'connectors': _start_connectors,
+        'pipeline': _start_pipeline,
     }
     return choose_subaction(action, options, **kw)
 
@@ -537,6 +538,75 @@ def _complete_start_connectors(**kw) -> List[str]:
     return _complete_show_connectors(**kw)
 
 
+def _start_pipeline(
+    action: Optional[List[str]] = None,
+    sub_args: Optional[List[str]] = None,
+    loop: bool = False,
+    min_seconds: Union[float, int, None] = 1.0,
+    params: Optional[Dict[str, Any]] = None,
+    **kwargs
+) -> SuccessTuple:
+    """
+    Run a series of Meerschaum commands as a single action.
+
+    Add `:` to the end of chained arguments to apply additional flags to the pipeline.
+
+    Examples
+    --------
+
+    `sync pipes -i sql:local + sync pipes -i sql:main :: -s 'daily'`
+
+    `show version + show arguments :: --loop`
+
+    """
+    import time
+    from meerschaum._internal.entry import entry
+    from meerschaum.utils.warnings import info, warn
+    from meerschaum.utils.misc import is_int
+
+    do_n_times = (
+        int(action[0].lstrip('x'))
+        if action and is_int(action[0].lstrip('x'))
+        else 1
+    )
+
+    if not sub_args:
+        return False, "Nothing to do."
+
+    if min_seconds is None:
+        min_seconds = 1.0
+
+    ran_n_times = 0
+    success, msg = False, "Did not run pipeline."
+    def run_loop():
+        nonlocal ran_n_times, success, msg
+        while True:
+            success, msg = entry(sub_args, _patch_args=params)
+            ran_n_times += 1
+
+            if not loop and do_n_times == 1:
+                break
+
+            if min_seconds != 0 and ran_n_times != do_n_times:
+                info(f"Sleeping for {min_seconds} seconds...")
+                time.sleep(min_seconds)
+
+            if loop:
+                continue
+
+            if ran_n_times >= do_n_times:
+                break
+
+    try:
+        run_loop()
+    except KeyboardInterrupt:
+        warn("Cancelled pipeline.", stack=False)
+
+    if do_n_times != 1:
+        info(f"Ran pipeline {ran_n_times} time" + ('s' if ran_n_times != 1 else '') + '.')
+    return success, msg
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.

meerschaum/api/routes/_actions.py

@@ -18,7 +18,7 @@ from fastapi import WebSocket, WebSocketDisconnect
 from meerschaum.utils.misc import generate_password
 from meerschaum.jobs import Job
 from meerschaum.utils.warnings import warn
-from meerschaum.utils.typing import SuccessTuple, Union, List, Dict
+from meerschaum.utils.typing import SuccessTuple, Union, List, Dict, Any
 from meerschaum.api import (
     fastapi, app, endpoints, get_api_connector, debug, manager, private, no_auth
 )
@@ -153,3 +153,50 @@ async def do_action_websocket(websocket: WebSocket):
             job.delete()
         _ = _temp_jobs.pop(job_name, None)
         stop_event.set()
+
+
+@app.post(actions_endpoint + "/{action}", tags=['Actions'])
+def do_action_legacy(
+    action: str,
+    keywords: Dict[str, Any] = fastapi.Body(...),
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Perform a Meerschaum action (if permissions allow).
+
+    Parameters
+    ----------
+    action: str
+        The action to perform.
+        
+    keywords: Dict[str, Any]
+        The keywords dictionary to pass to the action.
+
+    Returns
+    -------
+    A `SuccessTuple`.
+    """
+    if curr_user is not None and curr_user.type != 'admin':
+        from meerschaum.config import get_config
+        allow_non_admin = get_config(
+            'system', 'api', 'permissions', 'actions', 'non_admin', patch=True
+        )
+        if not allow_non_admin:
+            return False, (
+                "The administrator for this server has not allowed users to perform actions.\n\n"
+                + "Please contact the system administrator, or if you are running this server, "
+                + "open the configuration file with `edit config system` "
+                + "and search for 'permissions'. "
+                + "\nUnder the keys 'api:permissions:actions', "
+                + "you can allow non-admin users to perform actions."
+        )
+
+    if action not in actions:
+        return False, f"Invalid action '{action}'."
+
+    keywords['mrsm_instance'] = keywords.get('mrsm_instance', str(get_api_connector()))
+    _debug = keywords.get('debug', debug)
+    keywords.pop('debug', None)
+    return actions[action](debug=_debug, **keywords)

meerschaum/config/_paths.py

@@ -185,6 +185,7 @@ paths = {
     'SYSTEMD_RESOURCES_PATH'         : ('{DOT_CONFIG_DIR_PATH}', 'systemd'),
     'SYSTEMD_USER_RESOURCES_PATH'    : ('{SYSTEMD_RESOURCES_PATH}', 'user'),
     'SYSTEMD_ROOT_RESOURCES_PATH'    : ('{ROOT_DIR_PATH}', 'systemd'),
+    'SYSTEMD_JOBS_RESOURCES_PATH'    : ('{SYSTEMD_ROOT_RESOURCES_PATH}', 'services'),
     'SYSTEMD_LOGS_RESOURCES_PATH'    : ('{SYSTEMD_ROOT_RESOURCES_PATH}', 'logs'),
 }
 

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.3.0rc3"
+__version__ = "2.3.1"

meerschaum/config/static/__init__.py

@@ -90,6 +90,8 @@ STATIC_CONFIG: Dict[str, Any] = {
             'failure_key': '_argparse_exception',
             'and_key': '+',
             'escaped_and_key': '++',
+            'pipeline_key': ':',
+            'escaped_pipeline_key': '::',
         },
         'urls': {
             'get-pip.py': 'https://bootstrap.pypa.io/get-pip.py',

meerschaum/connectors/__init__.py

@@ -377,5 +377,4 @@ def _load_builtin_custom_connectors():
     """
     Import custom connectors decorated with `@make_connector` or `@make_executor`.
     """
-    import meerschaum.jobs._SystemdExecutor
-    #  import meerschaum.jobs._LocalExecutor
+    import meerschaum.jobs.systemd

meerschaum/connectors/api/APIConnector.py

@@ -33,7 +33,12 @@ class APIConnector(Connector):
         delete,
         wget,
     )
-    from ._actions import get_actions, do_action, do_action_async
+    from ._actions import (
+        get_actions,
+        do_action,
+        do_action_async,
+        do_action_legacy,
+    )
     from ._misc import get_mrsm_version, get_chaining_status
     from ._pipes import (
         register_pipe,

meerschaum/connectors/api/_actions.py

@@ -13,7 +13,7 @@ import asyncio
 from functools import partial
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import SuccessTuple, List, Callable
+from meerschaum.utils.typing import SuccessTuple, List, Callable, Optional
 from meerschaum.config.static import STATIC_CONFIG
 
 ACTIONS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['actions']
@@ -74,3 +74,79 @@ async def do_action_async(
                 break
 
     return True, "Success"
+
+
+def do_action_legacy(
+    self,
+    action: Optional[List[str]] = None,
+    sysargs: Optional[List[str]] = None,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
+    """
+    NOTE: This method is deprecated.
+    Please use `do_action()` or `do_action_async()`.
+
+    Execute a Meerschaum action remotely.
+
+    If `sysargs` are provided, parse those instead.
+    Otherwise infer everything from keyword arguments.
+
+    Examples
+    --------
+    >>> conn = mrsm.get_connector('api:main')
+    >>> conn.do_action(['show', 'pipes'])
+    (True, "Success")
+    >>> conn.do_action(['show', 'arguments'], name='test')
+    (True, "Success")
+    """
+    import sys, json
+    from meerschaum.utils.debug import dprint
+    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum.utils.misc import json_serialize_datetime
+    if action is None:
+        action = []
+
+    if sysargs is not None and action and action[0] == '':
+        from meerschaum._internal.arguments import parse_arguments
+        if debug:
+            dprint(f"Parsing sysargs:\n{sysargs}")
+        json_dict = parse_arguments(sysargs)
+    else:
+        json_dict = kw
+        json_dict['action'] = action
+        if 'noask' not in kw:
+            json_dict['noask'] = True
+        if 'yes' not in kw:
+            json_dict['yes'] = True
+        if debug:
+            json_dict['debug'] = debug
+
+    root_action = json_dict['action'][0]
+    del json_dict['action'][0]
+    r_url = f"{STATIC_CONFIG['api']['endpoints']['actions']}/{root_action}"
+    
+    if debug:
+        from meerschaum.utils.formatting import pprint
+        dprint(f"Sending data to '{self.url + r_url}':")
+        pprint(json_dict, stream=sys.stderr)
+
+    response = self.post(
+        r_url,
+        data = json.dumps(json_dict, default=json_serialize_datetime),
+        debug = debug,
+    )
+    try:
+        response_list = json.loads(response.text)
+        if isinstance(response_list, dict) and 'detail' in response_list:
+            return False, response_list['detail']
+    except Exception as e:
+        print(f"Invalid response: {response}")
+        print(e)
+        return False, response.text
+    if debug:
+        dprint(response)
+    try:
+        return response_list[0], response_list[1]
+    except Exception as e:
+        return False, f"Failed to parse result from action '{root_action}'"