meerschaum 2.2.6__py3-none-any.whl → 2.3.0.dev1__py3-none-any.whl

meerschaum/__init__.py

@@ -20,15 +20,16 @@ limitations under the License.
 
 import atexit
 from meerschaum.utils.typing import SuccessTuple
+from meerschaum.utils.packages import attempt_import
 from meerschaum.core.Pipe import Pipe
 from meerschaum.plugins import Plugin
 from meerschaum.utils.venv import Venv
+from meerschaum.utils.jobs import Job
 from meerschaum.connectors import get_connector, Connector, make_connector
 from meerschaum.utils import get_pipes
 from meerschaum.utils.formatting import pprint
 from meerschaum._internal.docs import index as __doc__
 from meerschaum.config import __version__, get_config
-from meerschaum.utils.packages import attempt_import
 from meerschaum.__main__ import _close_pools
 
 atexit.register(_close_pools)
@@ -42,6 +43,8 @@ __all__ = (
     "Plugin",
     "Venv",
     "Plugin",
+    "Job",
+    "Daemon",
     "pprint",
     "attempt_import",
     "actions",

meerschaum/__main__.py

@@ -19,9 +19,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-import sys, os, copy
+import sys
+import os
+import copy
 
-def main(sysargs: list = None) -> None:
+from meerschaum.utils.typing import List, Optional
+from meerschaum.utils.formatting import print_tuple as _print_tuple
+
+
+def main(sysargs: Optional[List[str]] = None) -> None:
     """Main CLI entry point."""
     if sysargs is None:
         sysargs = copy.deepcopy(sys.argv[1:])
@@ -41,7 +47,7 @@ def main(sysargs: list = None) -> None:
 
     if ('-d' in sysargs or '--daemon' in sysargs) and ('stack' not in sysargs):
         from meerschaum.utils.daemon import daemon_entry
-        daemon_entry(sysargs)
+        _print_tuple(daemon_entry(sysargs), upper_padding=1)
         return _exit(old_cwd=old_cwd)
 
     from meerschaum._internal.entry import entry, get_shell
@@ -57,8 +63,7 @@ def main(sysargs: list = None) -> None:
     return_tuple = entry(sysargs)
     rc = 0
     if isinstance(return_tuple, tuple) and '--nopretty' not in sysargs:
-        from meerschaum.utils.formatting import print_tuple
-        print_tuple(return_tuple, upper_padding=1)
+        _print_tuple(return_tuple, upper_padding=1)
         rc = 0 if (return_tuple[0] is True) else 1
 
     return _exit(rc, old_cwd=old_cwd)

meerschaum/_internal/arguments/_parser.py

@@ -62,7 +62,24 @@ def parse_datetime(dt_str: str) -> Union[datetime, int, str]:
     return dt
 
 
-def parse_help(sysargs : Union[List[str], Dict[str, Any]]) -> None:
+def parse_executor_keys(executor_keys_str: str) -> Union[str, None]:
+    """
+    Ensure that only API keys are provided for executor_keys.
+    """
+    if executor_keys_str == 'local':
+        return executor_keys_str
+
+    if executor_keys_str.lower() == 'none':
+        return 'local'
+
+    if not executor_keys_str.startswith('api:'):
+        from meerschaum.utils.warnings import error
+        error(f"Invalid exectutor keys '{executor_keys_str}'.", stack=False)
+
+    return executor_keys_str
+
+
+def parse_help(sysargs: Union[List[str], Dict[str, Any]]) -> None:
     """Parse the `--help` flag to determine which help message to print."""
     from meerschaum._internal.arguments._parse_arguments import parse_arguments, parse_line
     from meerschaum.actions import actions, get_subactions
@@ -135,6 +152,7 @@ _seen_plugin_args = {}
 
 groups = {}
 groups['actions'] = parser.add_argument_group(title='Actions options')
+groups['jobs'] = parser.add_argument_group(title='Jobs options')
 groups['pipes'] = parser.add_argument_group(title='Pipes options')
 groups['sync'] = parser.add_argument_group(title='Sync options')
 groups['api'] = parser.add_argument_group(title='API options')
@@ -166,18 +184,25 @@ groups['actions'].add_argument(
     help="Automatically choose the defaults answers to questions. Does not result in data loss.",
 )
 groups['actions'].add_argument(
-    '-d', '--daemon', action='store_true',
-    help = "Run an action as a background daemon."
+    '-A', '--sub-args', nargs=argparse.REMAINDER,
+    help = (
+        "Provide a list of arguments for subprocesses. " +
+        "You can also type sub-arguments in [] instead." +
+        " E.g. `stack -A='--version'`, `ls [-lh]`, `echo -A these are sub-arguments`"
+    )
 )
-groups['actions'].add_argument(
-    '--rm', action='store_true', help="Delete a job once it has finished executing."
+
+### Jobs options
+groups['jobs'].add_argument(
+    '-d', '--daemon', action='store_true',
+    help = "Run an action as a background daemon to create a job."
 )
-groups['actions'].add_argument(
+groups['jobs'].add_argument(
     '--name', '--job-name', type=parse_name, help=(
         "Assign a name to a job. If no name is provided, a random name will be assigned."
     ),
 )
-groups['actions'].add_argument(
+groups['jobs'].add_argument(
     '-s', '--schedule', '--cron', type=str,
     help = (
         "Continue executing the action according to a schedule (e.g. 'every 1 seconds'). \n"
@@ -185,15 +210,19 @@ groups['actions'].add_argument(
         + "https://red-engine.readthedocs.io/en/stable/condition_syntax/index.html"
     )
 )
-groups['actions'].add_argument(
-    '-A', '--sub-args', nargs=argparse.REMAINDER,
-    help = (
-        "Provide a list of arguments for subprocesses. " +
-        "You can also type sub-arguments in [] instead." +
-        " E.g. `stack -A='--version'`, `ls [-lh]`, `echo -A these are sub-arguments`"
-    )
+groups['jobs'].add_argument(
+    '--restart', action='store_true',
+    help=("Restart a job if not stopped manually."),
+)
+groups['jobs'].add_argument(
+    '--executor-keys', '--executor', '-e', type=parse_executor_keys,
+    help=(
+        "Remotely execute jobs on an API instance."
+    ),
+)
+groups['jobs'].add_argument(
+    '--rm', action='store_true', help="Delete a job once it has finished executing."
 )
-
 ### Pipes options
 groups['pipes'].add_argument(
     '-c', '-C', '--connector-keys', nargs='+',

meerschaum/_internal/entry.py

@@ -8,20 +8,15 @@ The entry point for launching Meerschaum actions.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, List, Optional, Dict
+from meerschaum.utils.typing import SuccessTuple, List, Optional, Dict, Callable, Any
 
 def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
-    """Parse arguments and launch a Meerschaum action.
-    The `action` list removes the first element.
-    
-    Examples of action:
-        'show actions' -> ['actions']
-        'show' -> []
+    """
+    Parse arguments and launch a Meerschaum action.
 
     Returns
     -------
-    A `SuccessTuple` indicating success. If `schedule` is provided, this will never return.
-
+    A `SuccessTuple` indicating success.
     """
     from meerschaum._internal.arguments import parse_arguments
     from meerschaum.config.static import STATIC_CONFIG
@@ -51,14 +46,15 @@ def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
 
 
 def entry_with_args(
-        _actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
-        **kw
-    ) -> SuccessTuple:
+    _actions: Optional[Dict[str, Callable[[Any], SuccessTuple]]] = None,
+    **kw
+) -> SuccessTuple:
     """Execute a Meerschaum action with keyword arguments.
     Use `_entry()` for parsing sysargs before executing.
     """
     import sys
     import functools
+    import inspect
     from meerschaum.actions import get_action, get_main_action_name
     from meerschaum._internal.arguments import remove_leading_action
     from meerschaum.utils.venv import Venv, active_venvs, deactivate_venv
@@ -72,6 +68,25 @@ def entry_with_args(
     ):
         return get_shell().cmdloop()
 
+    skip_schedule = False
+
+    executor_keys = kw.get('executor_keys', None)
+    if executor_keys is None:
+        executor_keys = 'local'
+
+    if executor_keys.startswith('api:'):
+        intended_action_function = get_action(kw['action'], _actions=_actions)
+        function_accepts_executor_keys = (
+            'executor_keys' in inspect.signature(intended_action_function).parameters
+            if intended_action_function is not None
+            else False
+        )
+        if not function_accepts_executor_keys:
+            api_label = executor_keys.split(':')[-1]
+            kw['action'].insert(0, 'api')
+            kw['action'].insert(1, api_label)
+            skip_schedule = True
+
     action_function = get_action(kw['action'], _actions=_actions)
 
     ### If action does not exist, execute in a subshell.
@@ -86,7 +101,6 @@ def entry_with_args(
         ) else None
     )
 
-    skip_schedule = False
     if (
         kw['action']
         and kw['action'][0] == 'start'
@@ -96,7 +110,14 @@ def entry_with_args(
 
     kw['action'] = remove_leading_action(kw['action'], _actions=_actions)
 
-    do_action = functools.partial(_do_action_wrapper, action_function, plugin_name, **kw)
+    do_action = functools.partial(
+        _do_action_wrapper,
+        action_function,
+        plugin_name,
+        **kw
+    )
+
+
     if kw.get('schedule', None) and not skip_schedule:
         from meerschaum.utils.schedule import schedule_function
         from meerschaum.utils.misc import interval_str

meerschaum/_internal/shell/Shell.py

@@ -28,7 +28,7 @@ prompt_toolkit = attempt_import('prompt_toolkit', lazy=False, warn=False, instal
 from meerschaum._internal.shell.ValidAutoSuggest import ValidAutoSuggest
 from meerschaum._internal.shell.ShellCompleter import ShellCompleter
 _clear_screen = get_config('shell', 'clear_screen', patch=True)
-from meerschaum.utils.misc import string_width
+from meerschaum.utils.misc import string_width, remove_ansi
 
 patch = True
 ### remove default cmd2 commands
@@ -56,7 +56,7 @@ hidden_commands = {
     'ipy',
 }
 reserved_completers = {
-    'instance', 'repo'
+    'instance', 'repo', 'executor',
 }
 
 ### To handle dynamic reloading, store shell attributes externally.
@@ -95,8 +95,8 @@ def _insert_shell_actions(
         setattr(_shell_class, 'complete_' + a, completer)
 
 def _completer_wrapper(
-        target: Callable[[Any], List[str]]
-    ) -> Callable[['meerschaum._internal.shell.Shell', str, str, int, int], Any]:
+    target: Callable[[Any], List[str]]
+) -> Callable[['meerschaum._internal.shell.Shell', str, str, int, int], Any]:
     """
     Wrapper for `complete_` functions so they can instead use Meerschaum arguments.
     """
@@ -125,13 +125,13 @@ def _completer_wrapper(
 
 
 def default_action_completer(
-        text: Optional[str] = None,
-        line: Optional[str] = None,
-        begin_index: Optional[int] = None,
-        end_index: Optional[int] = None,
-        action: Optional[List[str]] = None,
-        **kw: Any
-    ) -> List[str]:
+    text: Optional[str] = None,
+    line: Optional[str] = None,
+    begin_index: Optional[int] = None,
+    end_index: Optional[int] = None,
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> List[str]:
     """
     Search for subactions by default. This may be overridden by each action.
     """
@@ -206,13 +206,11 @@ def get_shell_intro(with_color: bool = True) -> str:
     """
     from meerschaum.utils.formatting import CHARSET, ANSI, colored
     intro = get_config('shell', CHARSET, 'intro', patch=patch)
-    intro += '\n' + ''.join(
-        [' '
-            for i in range(
-                string_width(intro) - len('v' + version)
-            )
-        ]
-    ) + 'v' + version
+    intro += (
+        '\n'
+        + (' ' * (string_width(intro) - len('v' + version)))
+        + f'v{version}'
+    )
 
     if not with_color or not ANSI:
         return intro
@@ -225,10 +223,10 @@ def get_shell_intro(with_color: bool = True) -> str:
 
 class Shell(cmd.Cmd):
     def __init__(
-            self,
-            actions: Optional[Dict[str, Any]] = None,
-            sysargs: Optional[List[str]] = None
-        ):
+        self,
+        actions: Optional[Dict[str, Any]] = None,
+        sysargs: Optional[List[str]] = None
+    ):
         """
         Customize the CLI from configuration
         """
@@ -249,11 +247,11 @@ class Shell(cmd.Cmd):
 
         from meerschaum.config._paths import SHELL_HISTORY_PATH
         shell_attrs['session'] = prompt_toolkit_shortcuts.PromptSession(
-            history = prompt_toolkit_history.FileHistory(str(SHELL_HISTORY_PATH)),
-            auto_suggest = ValidAutoSuggest(),
-            completer = ShellCompleter(),
-            complete_while_typing = True,
-            reserve_space_for_menu = False,
+            history=prompt_toolkit_history.FileHistory(SHELL_HISTORY_PATH.as_posix()),
+            auto_suggest=ValidAutoSuggest(),
+            completer=ShellCompleter(),
+            complete_while_typing=True,
+            reserve_space_for_menu=False,
         )
 
         try: ### try cmd2 arguments first
@@ -283,6 +281,7 @@ class Shell(cmd.Cmd):
         shell_attrs['_sysargs'] = sysargs
         shell_attrs['_actions']['instance'] = self.do_instance
         shell_attrs['_actions']['repo'] = self.do_repo
+        shell_attrs['_actions']['executor'] = self.do_executor
         shell_attrs['_actions']['debug'] = self.do_debug
         shell_attrs['_update_bottom_toolbar'] = True
         shell_attrs['_old_bottom_toolbar'] = ''
@@ -297,19 +296,25 @@ class Shell(cmd.Cmd):
         except Exception as e:
             pass
 
+        ### Finally, spawn the version update thread.
+        from meerschaum._internal.shell.updates import run_version_check_thread
+        self._update_thread = run_version_check_thread(debug=shell_attrs.get('debug', False))
+
+
     def load_config(self, instance: Optional[str] = None):
         """
         Set attributes from the shell configuration.
         """
         from meerschaum.utils.misc import remove_ansi
         from meerschaum.utils.formatting import CHARSET, ANSI, colored
-        
+        from meerschaum._internal.shell.updates import get_update_message
+
         if shell_attrs.get('intro', None) != '':
             self.intro = (
                 get_shell_intro(with_color=False)
                 if shell_attrs.get('intro', None) != ''
                 else ""
-            )
+            ) + get_update_message()
 
         shell_attrs['intro'] = self.intro
         shell_attrs['_prompt'] = get_config('shell', CHARSET, 'prompt', patch=patch)
@@ -333,6 +338,12 @@ class Shell(cmd.Cmd):
             shell_attrs['instance_keys'] = remove_ansi(str(instance))
         if shell_attrs.get('repo_keys', None) is None:
             shell_attrs['repo_keys'] = get_config('meerschaum', 'default_repository', patch=patch)
+        if shell_attrs.get('executor_keys', None) is None:
+            shell_attrs['executor_keys'] = get_config(
+                'meerschaum', 'default_executor',
+                patch=patch,
+            ) or 'local'
+
         ### this will be updated later in update_prompt ONLY IF {username} is in the prompt
         shell_attrs['username'] = ''
 
@@ -358,14 +369,19 @@ class Shell(cmd.Cmd):
     def insert_actions(self):
         from meerschaum.actions import actions
 
-    def update_prompt(self, instance: Optional[str] = None, username: Optional[str] = None):
+    def update_prompt(
+        self,
+        instance: Optional[str] = None,
+        username: Optional[str] = None,
+        executor_keys: Optional[str] = None,
+    ):
         from meerschaum.utils.formatting import ANSI, colored
         from meerschaum._internal.entry import _shell, get_shell
 
         cmd.__builtins__['input'] = input_with_sigint(
             _old_input,
             shell_attrs['session'],
-            shell = self,
+            shell=self,
         )
         prompt = shell_attrs['_prompt']
         mask = prompt
@@ -391,7 +407,8 @@ class Shell(cmd.Cmd):
                 from meerschaum.connectors.sql import SQLConnector
                 try:
                     conn_attrs = parse_instance_keys(
-                        remove_ansi(shell_attrs['instance_keys']), construct=False
+                        remove_ansi(shell_attrs['instance_keys']),
+                        construct=False,
                     )
                     if 'username' not in conn_attrs:
                         if 'uri' in conn_attrs:
@@ -405,12 +422,27 @@ class Shell(cmd.Cmd):
                 if username is None:
                    username = '(no username)'
             shell_attrs['username'] = (
-                username if not ANSI else
-                colored(username, **get_config('shell', 'ansi', 'username', 'rich'))
+                username
+                if not ANSI
+                else colored(username, **get_config('shell', 'ansi', 'username', 'rich'))
             )
             prompt = prompt.replace('{username}', shell_attrs['username'])
             mask = mask.replace('{username}', ''.join(['\0' for c in '{username}']))
 
+        if '{executor_keys}' in shell_attrs['_prompt']:
+            if executor_keys is None:
+                executor_keys = shell_attrs.get('executor_keys', None) or 'local'
+            shell_attrs['executor_keys'] = (
+                executor_keys
+                if not ANSI
+                else colored(
+                    remove_ansi(executor_keys),
+                    **get_config('shell', 'ansi', 'executor', 'rich')
+                )
+            )
+            prompt = prompt.replace('{executor_keys}', shell_attrs['executor_keys'])
+            mask = mask.replace('{executor_keys}', ''.join(['\0' for c in '{executor_keys}']))
+
         remainder_prompt = list(shell_attrs['_prompt'])
         for i, c in enumerate(mask):
             if c != '\0':
@@ -418,10 +450,13 @@ class Shell(cmd.Cmd):
                 if ANSI:
                     _c = colored(_c, **get_config('shell', 'ansi', 'prompt', 'rich'))
                 remainder_prompt[i] = _c
+
         self.prompt = ''.join(remainder_prompt).replace(
             '{username}', shell_attrs['username']
         ).replace(
             '{instance}', shell_attrs['instance']
+        ).replace(
+            '{executor_keys}', shell_attrs['executor_keys']
         )
         shell_attrs['prompt'] = self.prompt
         ### flush stdout
@@ -521,6 +556,9 @@ class Shell(cmd.Cmd):
         if 'repository' not in args and main_action_name != 'api':
             args['repository'] = str(shell_attrs['repo_keys'])
 
+        if 'executor_keys' not in args:
+            args['executor_keys'] = remove_ansi(str(shell_attrs['executor_keys']))
+
         ### parse out empty strings
         if args['action'][0].strip("\"'") == '':
             self.emptyline()
@@ -565,12 +603,12 @@ class Shell(cmd.Cmd):
         if stop:
             return True
 
-    def do_pass(self, line):
+    def do_pass(self, line, executor_keys=None):
         """
         Do nothing.
         """
 
-    def do_debug(self, action: Optional[List[str]] = None, **kw):
+    def do_debug(self, action: Optional[List[str]] = None, executor_keys=None, **kw):
         """
         Toggle the shell's debug mode.
         If debug = on, append `--debug` to all commands.
@@ -601,9 +639,10 @@ class Shell(cmd.Cmd):
 
     def do_instance(
             self,
-            action : Optional[List[str]] = None,
-            debug : bool = False,
-            **kw : Any
+            action: Optional[List[str]] = None,
+            executor_keys=None,
+            debug: bool = False,
+            **kw: Any
         ) -> SuccessTuple:
         """
         Temporarily set a default Meerschaum instance for the duration of the shell.
@@ -672,11 +711,12 @@ class Shell(cmd.Cmd):
 
 
     def do_repo(
-            self,
-            action: Optional[List[str]] = None,
-            debug: bool = False,
-            **kw: Any
-        ) -> SuccessTuple:
+        self,
+        action: Optional[List[str]] = None,
+        executor_keys=None,
+        debug: bool = False,
+        **kw: Any
+    ) -> SuccessTuple:
         """
         Temporarily set a default Meerschaum repository for the duration of the shell.
         The default repository (mrsm.io) is loaded from the Meerschaum configuraton file
@@ -723,9 +763,59 @@ class Shell(cmd.Cmd):
         return True, "Success"
 
     def complete_repo(self, *args) -> List[str]:
-        return self.complete_instance(*args)
+        results = self.complete_instance(*args)
+        return [result for result in results if result.startswith('api:')]
+
+    def do_executor(
+        self,
+        action: Optional[List[str]] = None,
+        executor_keys=None,
+        debug: bool = False,
+        **kw: Any
+    ) -> SuccessTuple:
+        """
+        Temporarily set a default Meerschaum executor for the duration of the shell.
+        
+        You can change the default repository with `edit config`.
+        
+        Usage:
+            executor {API label}
+        
+        Examples:
+            ### reset to default executor
+            executor
+        
+            ### set the executor to 'api:main'
+            executor api:main
+        
+        Note that executors are API instances.
+        """
+        from meerschaum import get_connector
+        from meerschaum.connectors.parse import parse_executor_keys
+        from meerschaum.utils.warnings import warn, info
+
+        if action is None:
+            action = []
+
+        try:
+            executor_keys = action[0]
+        except (IndexError, AttributeError):
+            executor_keys = ''
+        if executor_keys == '':
+            executor_keys = get_config('meerschaum', 'default_executor') or 'local'
+        
+        conn = parse_executor_keys(executor_keys, debug=debug)
+
+        shell_attrs['executor_keys'] = str(conn)
+
+        info(f"Default executor for the current shell: {executor_keys}")
+        return True, "Success"
 
-    def do_help(self, line: str) -> List[str]:
+    def complete_executor(self, *args) -> List[str]:
+        results = self.complete_instance(*args)
+        return ['local'] + [result for result in results if result.startswith('api:')]
+
+    def do_help(self, line: str, executor_keys=None) -> List[str]:
         """
         Show help for Meerschaum actions.
         
@@ -796,7 +886,7 @@ class Shell(cmd.Cmd):
                 possibilities.append(name.replace('do_', ''))
         return possibilities
 
-    def do_exit(self, params) -> True:
+    def do_exit(self, params, executor_keys=None) -> True:
         """
         Exit the Meerschaum shell.
         """
@@ -861,21 +951,29 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None):
 
         instance_colored = (
             colored(
-                shell_attrs['instance_keys'], 'on ' + get_config(
-                    'shell', 'ansi', 'instance', 'rich', 'style'
-                )
+                remove_ansi(shell_attrs['instance_keys']),
+                'on ' + get_config('shell', 'ansi', 'instance', 'rich', 'style')
             )
             if ANSI
             else colored(shell_attrs['instance_keys'], 'on white')
         )
         repo_colored = (
             colored(
-                shell_attrs['repo_keys'],
+                remove_ansi(shell_attrs['repo_keys']),
                 'on ' + get_config('shell', 'ansi', 'repo', 'rich', 'style')
             )
             if ANSI
             else colored(shell_attrs['repo_keys'], 'on white')
         )
+        executor_colored = (
+            colored(
+                remove_ansi(shell_attrs['executor_keys']),
+                'on ' + get_config('shell', 'ansi', 'executor', 'rich', 'style')
+            )
+            if ANSI
+            else colored(remove_ansi(shell_attrs['executor_keys']), 'on white')
+        )
+
         try:
             typ, label = shell_attrs['instance_keys'].split(':')
             connected = typ in connectors and label in connectors[typ]
@@ -894,8 +992,12 @@ def input_with_sigint(_input, session, shell: Optional[Shell] = None):
         )
 
         left = (
-            colored(' Instance: ', 'on white') + instance_colored
-            + colored('   Repo: ', 'on white') + repo_colored
+            ' '
+            + instance_colored
+            + colored(' | ', 'on white')
+            + executor_colored
+            + colored(' | ', 'on white')
+            + repo_colored
         )
         right = connection_text
         buffer_size = (