meerschaum 2.3.0.dev1__py3-none-any.whl → 2.3.0rc1__py3-none-any.whl

meerschaum/__init__.py

@@ -24,12 +24,13 @@ from meerschaum.utils.packages import attempt_import
 from meerschaum.core.Pipe import Pipe
 from meerschaum.plugins import Plugin
 from meerschaum.utils.venv import Venv
-from meerschaum.utils.jobs import Job
+from meerschaum.jobs import Job, Executor, make_executor
 from meerschaum.connectors import get_connector, Connector, make_connector
 from meerschaum.utils import get_pipes
 from meerschaum.utils.formatting import pprint
 from meerschaum._internal.docs import index as __doc__
 from meerschaum.config import __version__, get_config
+from meerschaum._internal.entry import entry
 from meerschaum.__main__ import _close_pools
 
 atexit.register(_close_pools)
@@ -44,7 +45,8 @@ __all__ = (
     "Venv",
     "Plugin",
     "Job",
-    "Daemon",
+    "Executor",
+    "make_executor",
     "pprint",
     "attempt_import",
     "actions",
@@ -55,4 +57,5 @@ __all__ = (
     "SuccessTuple",
     "Connector",
     "make_connector",
+    "entry",
 )

meerschaum/__main__.py

@@ -45,11 +45,6 @@ def main(sysargs: Optional[List[str]] = None) -> None:
         parse_version(sysargs)
         return _exit(old_cwd=old_cwd)
 
-    if ('-d' in sysargs or '--daemon' in sysargs) and ('stack' not in sysargs):
-        from meerschaum.utils.daemon import daemon_entry
-        _print_tuple(daemon_entry(sysargs), upper_padding=1)
-        return _exit(old_cwd=old_cwd)
-
     from meerschaum._internal.entry import entry, get_shell
 
     ### Try to launch a shell if --shell is provided.

meerschaum/_internal/arguments/_parse_arguments.py

@@ -288,9 +288,6 @@ def remove_leading_action(
     for a in action:
         _action.append(a.replace('_', UNDERSCORE_STANDIN))
 
-    ### e.g. 'show_pipes_baz'
-    action_str = '_'.join(_action)
-
     ### e.g. 'show_pipes'
     action_name = action_function.__name__.lstrip('_')
 
@@ -300,6 +297,16 @@ def remove_leading_action(
     ### Strip away any leading prefices.
     action_name = action_name[main_action_index:]
 
+    subaction_parts = action_name.replace(main_action_name, '').lstrip('_').split('_')
+    subaction_name = subaction_parts[0] if subaction_parts else None
+
+    ### e.g. 'pipe' -> 'pipes'
+    if subaction_name and subaction_name.endswith('s') and not action[1].endswith('s'):
+        _action[1] += 's'
+
+    ### e.g. 'show_pipes_baz'
+    action_str = '_'.join(_action)
+
     if not action_str.replace(UNDERSCORE_STANDIN, '_').startswith(action_name):
         warn(f"Unable to parse '{action_str}' for action '{action_name}'.")
         return action

meerschaum/_internal/arguments/_parser.py

@@ -66,7 +66,7 @@ def parse_executor_keys(executor_keys_str: str) -> Union[str, None]:
     """
     Ensure that only API keys are provided for executor_keys.
     """
-    if executor_keys_str == 'local':
+    if executor_keys_str in ('local', 'systemd'):
         return executor_keys_str
 
     if executor_keys_str.lower() == 'none':
@@ -214,10 +214,14 @@ groups['jobs'].add_argument(
     '--restart', action='store_true',
     help=("Restart a job if not stopped manually."),
 )
+groups['jobs'].add_argument(
+    '--systemd', action='store_true',
+    help=("Create a job via systemd. Shorthand for `-e systemd`."),
+)
 groups['jobs'].add_argument(
     '--executor-keys', '--executor', '-e', type=parse_executor_keys,
     help=(
-        "Remotely execute jobs on an API instance."
+        "Execute jobs on an API instance or via systemd."
     ),
 )
 groups['jobs'].add_argument(

meerschaum/_internal/entry.py

@@ -8,7 +8,29 @@ The entry point for launching Meerschaum actions.
 """
 
 from __future__ import annotations
+
+import os
+import sys
 from meerschaum.utils.typing import SuccessTuple, List, Optional, Dict, Callable, Any
+from meerschaum.config.static import STATIC_CONFIG as _STATIC_CONFIG
+
+_systemd_result_path = None
+if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
+    from meerschaum.utils.daemon import RotatingFile as _RotatingFile, StdinFile as _StdinFile
+    from meerschaum.config import get_config as _get_config
+
+    _systemd_result_path = os.environ[_STATIC_CONFIG['environment']['systemd_result_path']]
+    _systemd_log_path = os.environ[_STATIC_CONFIG['environment']['systemd_log_path']]
+    _systemd_log = _RotatingFile(
+        _systemd_log_path,
+        write_timestamps=True,
+        timestamp_format=_get_config('jobs', 'logs', 'timestamps', 'format'),
+    )
+    sys.stdout = _systemd_log
+    sys.stderr = _systemd_log
+    _systemd_stdin_path = os.environ.get(_STATIC_CONFIG['environment']['systemd_stdin_path'], None)
+    if _systemd_stdin_path:
+        sys.stdin = _StdinFile(_systemd_stdin_path)
 
 def entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     """
@@ -87,6 +109,10 @@ def entry_with_args(
             kw['action'].insert(1, api_label)
             skip_schedule = True
 
+    ### If the `--daemon` flag is present, prepend 'start job'.
+    if kw.get('daemon', False) and kw['action'][0] != 'stack':
+        kw['action'] = ['start', 'jobs'] + kw['action']
+
     action_function = get_action(kw['action'], _actions=_actions)
 
     ### If action does not exist, execute in a subshell.
@@ -117,7 +143,6 @@ def entry_with_args(
         **kw
     )
 
-
     if kw.get('schedule', None) and not skip_schedule:
         from meerschaum.utils.schedule import schedule_function
         from meerschaum.utils.misc import interval_str
@@ -134,6 +159,11 @@ def entry_with_args(
     for venv in [venv for venv in active_venvs]:
         deactivate_venv(venv, debug=kw.get('debug', False), force=True)
 
+    if _systemd_result_path:
+        import json
+        with open(_systemd_result_path, 'w+', encoding='utf-8') as f:
+            json.dump(result, f)
+
     return result
 
 
@@ -160,16 +190,16 @@ def _do_action_wrapper(action_function, plugin_name, **kw):
                 )
             )
         except KeyboardInterrupt:
-            result = False, f"Cancelled action `{action_name}`."
+            result = False, f"Cancelled action `{action_name.lstrip()}`."
     return result
 
 _shells = []
 _shell = None
 def get_shell(
-        sysargs: Optional[List[str]] = None,
-        reload: bool = False,
-        debug: bool = False
-    ):
+    sysargs: Optional[List[str]] = None,
+    reload: bool = False,
+    debug: bool = False
+):
     """Initialize and return the Meerschaum shell object."""
     global _shell
     from meerschaum.utils.debug import dprint

meerschaum/_internal/shell/Shell.py

@@ -29,6 +29,7 @@ from meerschaum._internal.shell.ValidAutoSuggest import ValidAutoSuggest
 from meerschaum._internal.shell.ShellCompleter import ShellCompleter
 _clear_screen = get_config('shell', 'clear_screen', patch=True)
 from meerschaum.utils.misc import string_width, remove_ansi
+from meerschaum.jobs import get_executor_keys_from_context
 
 patch = True
 ### remove default cmd2 commands
@@ -339,10 +340,7 @@ class Shell(cmd.Cmd):
         if shell_attrs.get('repo_keys', None) is None:
             shell_attrs['repo_keys'] = get_config('meerschaum', 'default_repository', patch=patch)
         if shell_attrs.get('executor_keys', None) is None:
-            shell_attrs['executor_keys'] = get_config(
-                'meerschaum', 'default_executor',
-                patch=patch,
-            ) or 'local'
+            shell_attrs['executor_keys'] = get_executor_keys_from_context()
 
         ### this will be updated later in update_prompt ONLY IF {username} is in the prompt
         shell_attrs['username'] = ''
@@ -564,11 +562,6 @@ class Shell(cmd.Cmd):
             self.emptyline()
             return ""
 
-        ### If the `--daemon` flag is present, prepend 'start job'.
-        if args.get('daemon', False) and 'stack' not in args['action']:
-            args['action'] = ['start', 'jobs'] + args['action']
-            main_action_name = 'start'
-
         positional_only = (main_action_name not in shell_attrs['_actions'])
         if positional_only:
             return original_line
@@ -577,7 +570,6 @@ class Shell(cmd.Cmd):
 
         try:
             success_tuple = entry_with_args(_actions=shell_attrs['_actions'], **args)
-            #  success_tuple = entry_with_args(**args)
         except Exception as e:
             success_tuple = False, str(e)
 
@@ -585,9 +577,9 @@ class Shell(cmd.Cmd):
         if isinstance(success_tuple, tuple):
             print_tuple(
                 success_tuple,
-                skip_common = (not shell_attrs['debug']),
-                upper_padding = 1,
-                lower_padding = 0,
+                skip_common=(not shell_attrs['debug']),
+                upper_padding=1,
+                lower_padding=0,
             )
 
         ### Restore the old working directory.
@@ -700,14 +692,33 @@ class Shell(cmd.Cmd):
         return True, "Success"
 
 
-    def complete_instance(self, text: str, line: str, begin_index: int, end_index: int):
+    def complete_instance(
+        self,
+        text: str,
+        line: str,
+        begin_index: int,
+        end_index: int,
+        _executor: bool = False,
+        _additional_options: Optional[List[str]] = None,
+    ):
         from meerschaum.utils.misc import get_connector_labels
         from meerschaum._internal.arguments._parse_arguments import parse_line
-        from meerschaum.connectors import instance_types
+        from meerschaum.connectors import instance_types, _load_builtin_custom_connectors
+        if _executor:
+            _load_builtin_custom_connectors()
+            from meerschaum.jobs import executor_types
+
+        conn_types = instance_types if not _executor else executor_types
+
         args = parse_line(line)
         action = args['action']
         _text = action[1] if len(action) > 1 else ""
-        return get_connector_labels(*instance_types, search_term=_text, ignore_exact_match=True)
+        return get_connector_labels(
+            *conn_types,
+            search_term=_text,
+            ignore_exact_match=True,
+            _additional_options=_additional_options,
+        )
 
 
     def do_repo(
@@ -802,18 +813,19 @@ class Shell(cmd.Cmd):
         except (IndexError, AttributeError):
             executor_keys = ''
         if executor_keys == '':
-            executor_keys = get_config('meerschaum', 'default_executor') or 'local'
+            executor_keys = get_executor_keys_from_context()
         
         conn = parse_executor_keys(executor_keys, debug=debug)
 
-        shell_attrs['executor_keys'] = str(conn)
+        shell_attrs['executor_keys'] = str(conn).replace('systemd:main', 'systemd')
 
         info(f"Default executor for the current shell: {executor_keys}")
         return True, "Success"
 
     def complete_executor(self, *args) -> List[str]:
-        results = self.complete_instance(*args)
-        return ['local'] + [result for result in results if result.startswith('api:')]
+        from meerschaum.jobs import executor_types
+        results = self.complete_instance(*args, _executor=True, _additional_options=['local'])
+        return [result for result in results if result.split(':')[0] in executor_types]
 
     def do_help(self, line: str, executor_keys=None) -> List[str]:
         """

meerschaum/actions/__init__.py

@@ -12,9 +12,9 @@ from meerschaum.utils.packages import get_modules_from_package
 _custom_actions = []
 
 def get_subactions(
-        action: Union[str, List[str]],
-        _actions: Optional[Dict[str, Callable[[Any], Any]]] = None,
-    ) -> Dict[str, Callable[[Any], Any]]:
+    action: Union[str, List[str]],
+    _actions: Optional[Dict[str, Callable[[Any], Any]]] = None,
+) -> Dict[str, Callable[[Any], Any]]:
     """
     Return a dictionary of an action's sub-action functions.
 
@@ -52,9 +52,9 @@ def get_subactions(
 
 
 def get_action(
-        action: Union[str, List[str]],
-        _actions: Optional[Dict[str, Callable[[Any], Any]]] = None,
-    ) -> Union[Callable[[Any], Any], None]:
+    action: Union[str, List[str]],
+    _actions: Optional[Dict[str, Callable[[Any], Any]]] = None,
+) -> Union[Callable[[Any], Any], None]:
     """
     Return a function corresponding to the given action list.
     This may be a custom action with an underscore, in which case, allow for underscores.
@@ -92,6 +92,8 @@ def get_action(
     if action[0] in _actions:
         subactions = get_subactions([action[0]], _actions=_actions)
         if action[1] not in subactions:
+            if (action[1] + 's') in subactions:
+                return subactions[action[1] + 's']
             return _actions[action[0]]
         return subactions[action[1]]
 

meerschaum/actions/attach.py

@@ -11,17 +11,18 @@ from meerschaum.utils.typing import Optional, List, Any, SuccessTuple
 
 def attach(
     action: Optional[List[str]] = None,
+    executor_keys: Optional[str] = None,
     **kwargs: Any
 ) -> SuccessTuple:
     """
-    Attach to a job.
+    Attach to a job, and prompt the user when blocking on input.
     """
     from meerschaum.actions import choose_subaction
     attach_options = {
         'jobs': _attach_jobs,
         'logs': _attach_logs,
     }
-    return choose_subaction(action, attach_options, **kwargs)
+    return choose_subaction(action, attach_options, executor_keys=executor_keys, **kwargs)
 
 
 def _complete_attach(
@@ -31,16 +32,16 @@ def _complete_attach(
     """
     Override the default Meerschaum `complete_` function.
     """
-    from meerschaum.actions.start import _complete_start_jobs
+    from meerschaum.actions.delete import _complete_delete_jobs
 
     if action is None:
         action = []
 
     options = {
-        'job': _complete_start_jobs,
-        'jobs': _complete_start_jobs,
-        'log': _complete_start_jobs,
-        'logs': _complete_start_jobs,
+        'job': _complete_delete_jobs,
+        'jobs': _complete_delete_jobs,
+        'log': _complete_delete_jobs,
+        'logs': _complete_delete_jobs,
     }
 
     if (
@@ -64,20 +65,37 @@ def _attach_jobs(
     """
     Attach to a job, and prompt the user when blocking on input.
     """
+    action = action or []
     if not action and not name:
         return False, "Provide the name of the job to attach to."
 
     name = name or action[0]
     job = mrsm.Job(name, executor_keys=executor_keys)
+    other_executor_keys = 'systemd' if executor_keys in (None, 'local') else 'local'
     if not job.exists():
-        return False, f"Job '{job.name}' does not exist."
+        other_job = mrsm.Job(name, executor_keys=other_executor_keys)
+        if not other_job.exists():
+            return False, f"Job '{job.name}' does not exist."
+
+        job = other_job
+
+    success, message = True, "Success"
+
+    def _capture_result(result: SuccessTuple):
+        nonlocal success, message
+        success, message = result
 
-    job.monitor_logs(
-        stop_on_exit=True,
-        strip_timestamps=True,
-    )
+    try:
+        job.monitor_logs(
+            stop_callback_function=_capture_result,
+            accept_input=True,
+            stop_on_exit=True,
+            strip_timestamps=True,
+        )
+    except KeyboardInterrupt:
+        pass
 
-    return True, "Success"
+    return success, message
 
 
 def _attach_logs(*args, **kwargs) -> SuccessTuple:

meerschaum/actions/copy.py

@@ -10,9 +10,9 @@ from __future__ import annotations
 from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional, Tuple, List
 
 def copy(
-        action: Optional[List[str]] = None,
-        **kw : Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kw : Any
+) -> SuccessTuple:
     """
     Duplicate connectors or pipes.
     
@@ -32,17 +32,16 @@ def copy(
 
 
 def _complete_copy(
-        action : Optional[List[str]] = None,
-        **kw : Any
-    ) -> List[str]:
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> List[str]:
     """
     Override the default Meerschaum `complete_` function.
-
     """
-    from meerschaum.actions.start import _complete_start_jobs
     from meerschaum.actions.edit import _complete_edit_config
     if action is None:
         action = []
+
     options = {
         'connector': _complete_copy_connectors,
         'connectors': _complete_copy_connectors,
@@ -61,15 +60,14 @@ def _complete_copy(
 
 
 def _copy_pipes(
-        yes: bool = False,
-        noask: bool = False,
-        force: bool = False,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    yes: bool = False,
+    noask: bool = False,
+    force: bool = False,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Copy pipes' attributes and make new pipes.
-
     """
     from meerschaum import get_pipes, Pipe
     from meerschaum.utils.prompt import prompt, yes_no
@@ -132,16 +130,15 @@ def _copy_pipes(
 
     return successes > 0, msg
 
+
 def _copy_connectors(
-        action: Optional[List[str]] = None,
-        connector_keys: Optional[List[str]] = None,
-        nopretty: bool = False,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    connector_keys: Optional[List[str]] = None,
+    nopretty: bool = False,
+    force: bool = False,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Create a new connector from an existing one.
 
@@ -153,39 +150,69 @@ def _copy_connectors(
     from meerschaum.config._edit import write_config
     from meerschaum.utils.warnings import info, warn
     from meerschaum.utils.formatting import pprint
+    from meerschaum.actions import get_action
     cf = _config()
     if action is None:
         action = []
     if connector_keys is None:
         connector_keys = []
 
-    _keys = list(set(action + connector_keys))
+    _keys = (action or []) + connector_keys
 
     if not _keys:
         return False, "No connectors to copy."
 
-    for ck in _keys:
-        try:
-            conn = parse_connector_keys(ck)
-        except Exception as e:
-            warn(f"Unable to parse connector '{ck}'. Skipping...", stack=False)
-            continue
+    if len(_keys) < 1 or len(_keys) > 2:
+        return False, "Provide one set of connector keys."
 
-        attrs = get_config('meerschaum', 'connectors', conn.type, conn.label)
-        pprint(attrs, nopretty=nopretty)
+    ck = _keys[0]
 
-        asking = True
-        #  while asking:
-            #  new_ck = prompt("Please enter a new label for the new connector ():")
+    try:
+        conn = parse_connector_keys(ck)
+    except Exception as e:
+        return False, f"Unable to parse connector '{ck}'."
 
+    if len(_keys) == 2:
+        new_ck = _keys[1] if ':' in _keys[1] else None
+        new_label = _keys[1].split(':')[-1]
+    else:
+        new_ck = None
+        new_label = None
+
+    try:
+        if new_label is None:
+            new_label = prompt(f"Enter a label for the new '{conn.type}' connector:")
+    except KeyboardInterrupt:
+        return False, "Nothing was copied."
+
+    if new_ck is None:
+        new_ck = f"{conn.type}:{new_label}"
+
+    info(f"Registering connector '{new_ck}' from '{ck}'...")
+
+    attrs = get_config('meerschaum', 'connectors', conn.type, conn.label)
+    pprint(attrs, nopretty=nopretty)
+    if not force and not yes_no(
+        f"Register connector '{new_ck}' with the above attributes?",
+        default='n',
+        **kwargs
+    ):
+        return False, "Nothing was copied."
+
+    register_connector = get_action(['register', 'connector'])
+    register_success, register_msg = register_connector(
+        [new_ck],
+        params=attrs,
+        **kwargs
+    )
+    return register_success, register_msg
 
-    return False, "Not implemented."
 
 def _complete_copy_connectors(
-        action : Optional[List[str]] = None,
-        line : str = '',
-        **kw : Any
-    ) -> List[str]:
+    action: Optional[List[str]] = None,
+    line: str = '',
+    **kw: Any
+) -> List[str]:
     from meerschaum.config import get_config
     from meerschaum.utils.misc import get_connector_labels
     types = list(get_config('meerschaum', 'connectors').keys())