meerschaum 2.4.11__py3-none-any.whl → 2.4.13__py3-none-any.whl

meerschaum/_internal/arguments/_parse_arguments.py

@@ -256,7 +256,8 @@ def parse_synonyms(
 
 
 def parse_dict_to_sysargs(
-    args_dict: Dict[str, Any]
+    args_dict: Dict[str, Any],
+    coerce_dates: bool = True,
 ) -> List[str]:
     """Revert an arguments dictionary back to a command line list."""
     import shlex
@@ -304,6 +305,19 @@ def parse_dict_to_sysargs(
                 if len(args_dict[a]) > 0:
                     sysargs += [t[0], json.dumps(args_dict[a], separators=(',', ':'))]
 
+            ### Preserve the original datetime strings if possible
+            elif a in ('begin', 'end') and 'sysargs' in args_dict:
+                flag = t[0]
+                flag_ix = args_dict['sysargs'].index(flag)
+                if flag_ix < 0:
+                    continue
+                try:
+                    flag_val = args_dict['sysargs'][flag_ix + 1]
+                except IndexError:
+                    flag_val = str(args_dict[a])
+
+                sysargs += [flag, str(flag_val)]
+
             ### Account for None and other values
             elif (args_dict[a] is not None) or (args_dict[a] is None and a in allow_none_args):
                 sysargs += [t[0], str(args_dict[a])]

meerschaum/_internal/docs/index.py

@@ -742,6 +742,7 @@ def init_dash(dash_app):
       <li><code>meerschaum.utils.sql.get_db_version()</code></li>
       <li><code>meerschaum.utils.sql.get_rename_table_queries()</code></li>
       <li><code>meerschaum.utils.sql.get_create_table_query()</code></li>
+      <li><code>meerschaum.utils.sql.wrap_query_with_cte()</code></li>
       <li><code>meerschaum.utils.sql.format_cte_subquery()</code></li>
       <li><code>meerschaum.utils.sql.session_execute()</code></li>
     </ul>

meerschaum/_internal/entry.py

@@ -43,6 +43,7 @@ if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
     if _systemd_stdin_path:
         sys.stdin = _StdinFile(_systemd_stdin_path)
 
+
 def entry(
     sysargs: Optional[List[str]] = None,
     _patch_args: Optional[Dict[str, Any]] = None,

meerschaum/_internal/shell/Shell.py

@@ -235,6 +235,23 @@ def get_shell_intro(with_color: bool = True) -> str:
         **get_config('shell', 'ansi', 'intro', 'rich')
     )
 
+def get_shell_session():
+    """
+    Return the `prompt_toolkit` prompt session.
+    """
+    from meerschaum.config._paths import SHELL_HISTORY_PATH
+    if 'session' in shell_attrs:
+        return shell_attrs['session']
+
+    shell_attrs['session'] = prompt_toolkit_shortcuts.PromptSession(
+        history=prompt_toolkit_history.FileHistory(SHELL_HISTORY_PATH.as_posix()),
+        auto_suggest=ValidAutoSuggest(),
+        completer=ShellCompleter(),
+        complete_while_typing=True,
+        reserve_space_for_menu=False,
+    )
+    return shell_attrs['session']
+
 
 class Shell(cmd.Cmd):
     """
@@ -264,14 +281,7 @@ class Shell(cmd.Cmd):
         except AttributeError:
             pass
 
-        from meerschaum.config._paths import SHELL_HISTORY_PATH
-        shell_attrs['session'] = prompt_toolkit_shortcuts.PromptSession(
-            history=prompt_toolkit_history.FileHistory(SHELL_HISTORY_PATH.as_posix()),
-            auto_suggest=ValidAutoSuggest(),
-            completer=ShellCompleter(),
-            complete_while_typing=True,
-            reserve_space_for_menu=False,
-        )
+        _ = get_shell_session()
 
         super().__init__()
 
@@ -626,7 +636,7 @@ class Shell(cmd.Cmd):
             step_action_name = step_action[0] if step_action else None
             ### NOTE: For `stack`, revert argument parsing.
             step_sysargs = (
-                parse_dict_to_sysargs(step_kwargs)
+                parse_dict_to_sysargs(step_kwargs, coerce_dates=False)
                 if step_action_name != 'stack'
                 else chained_sysargs[i]
             )

meerschaum/_internal/shell/ShellCompleter.py

@@ -31,15 +31,18 @@ class ShellCompleter(Completer):
         ensure_readline()
         parts = document.text.split('-')
         ends_with_space = parts[0].endswith(' ')
-        part_0_subbed_spaces = parts[0].replace(' ', '_')
-        parsed_text = part_0_subbed_spaces + '-'.join(parts[1:])
+        last_action_line = parts[0].split('+')[-1]
+        part_0_subbed_spaces = last_action_line.replace(' ', '_')
+        parsed_text = (part_0_subbed_spaces + '-'.join(parts[1:]))
 
+        if not parsed_text:
+            return
 
         ### Index is the rank order (0 is closest match).
         ### Break when no results are returned.
         for i, a in enumerate(shell_actions):
             try:
-                poss = shell.complete(parsed_text, i)
+                poss = shell.complete(parsed_text.lstrip('_'), i)
                 if poss:
                     poss = poss.replace('_', ' ')
             ### Having issues with readline on portable Windows.
@@ -50,7 +53,9 @@ class ShellCompleter(Completer):
             yield Completion(poss, start_position=(-1 * len(poss)))
             yielded.append(poss)
 
-        args = parse_line(document.text)
+        line = document.text
+        current_action_line = line.split('+')[-1].lstrip()
+        args = parse_line(current_action_line)
         action_function = get_action(args['action'], _actions=shell_attrs.get('_actions', None))
         if action_function is None:
             return
@@ -74,8 +79,8 @@ class ShellCompleter(Completer):
                 shell,
                 complete_function_name
             )(
-                document.text.split(' ')[-1],
-                document.text,
+                current_action_line.split(' ')[-1],
+                current_action_line,
                 0,
                 0
             )

meerschaum/actions/bootstrap.py

@@ -8,15 +8,18 @@ Functions for bootstrapping elements
 """
 
 from __future__ import annotations
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional, Tuple, List
 
+
 def bootstrap(
-        action: Optional[List[str]] = None,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> SuccessTuple:
     """
     Launch an interactive wizard to bootstrap pipes or connectors.
-    
+
     Example:
         `bootstrap pipes`
 
@@ -26,23 +29,24 @@ def bootstrap(
         'pipes'      : _bootstrap_pipes,
         'connectors' : _bootstrap_connectors,
         'plugins'    : _bootstrap_plugins,
+        'jobs'       : _bootstrap_jobs,
     }
     return choose_subaction(action, options, **kw)
 
 
 def _bootstrap_pipes(
-        action: Optional[List[str]] = None,
-        connector_keys: Optional[List[str]] = None,
-        metric_keys: Optional[List[str]] = None,
-        location_keys: Optional[List[Optional[str]]] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        mrsm_instance: Optional[str] = None,
-        shell: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    connector_keys: Optional[List[str]] = None,
+    metric_keys: Optional[List[str]] = None,
+    location_keys: Optional[List[Optional[str]]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    mrsm_instance: Optional[str] = None,
+    shell: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Create a new pipe.
     If no keys are provided, guide the user through the steps required.
@@ -433,6 +437,105 @@ def _bootstrap_plugins(
     return True, "Success"
 
 
+def _bootstrap_jobs(
+    action: Optional[List[str]] = None,
+    executor_keys: Optional[str] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
+    """
+    Launch an interactive wizard to create new jobs.
+    """
+    import shlex
+    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum.actions import actions
+    from meerschaum.utils.formatting import print_options, make_header
+    from meerschaum.utils.formatting._shell import clear_screen
+    from meerschaum.utils.warnings import info
+    from meerschaum._internal.arguments import (
+        split_pipeline_sysargs,
+        split_chained_sysargs,
+    )
+    from meerschaum.utils.misc import items_str
+    from meerschaum._internal.shell.ShellCompleter import ShellCompleter
+
+    if not action:
+        action = [prompt("What is the name of the job you'd like to create?")]
+
+    for name in action:
+        clear_screen(debug=debug)
+        job = mrsm.Job(name, executor_keys=executor_keys)
+        if job.exists():
+            edit_success, edit_msg = actions['edit'](['job', name], **kwargs)
+            if not edit_success:
+                return edit_success, edit_msg
+            continue
+
+        info(
+            f"Editing arguments for job '{name}'.\n"
+            "    Press [Esc + Enter] to submit, [CTRL + C] to exit.\n\n"
+            "    Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
+            "    https://meerschaum.io/reference/actions/#chaining-actions\n"
+        )
+        try:
+            new_sysargs_str = prompt(
+                "",
+                multiline=True,
+                icon=False,
+                completer=ShellCompleter(),
+            )
+        except KeyboardInterrupt:
+            return True, "Nothing was changed."
+
+        new_sysargs = shlex.split(new_sysargs_str)
+        new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
+        chained_sysargs = split_chained_sysargs(new_sysargs)
+
+        clear_screen(debug=debug)
+        if len(chained_sysargs) > 1:
+            print_options(
+                [
+                    shlex.join(step_sysargs)
+                    for step_sysargs in chained_sysargs
+                ],
+                header=f"Steps in Job '{name}':",
+                number_options=True,
+                **kwargs
+            )
+        else:
+            print('\n' + make_header(f"Action for Job '{name}':"))
+            print(shlex.join(new_sysargs))
+
+        if pipeline_args:
+            print('\n' + make_header("Pipeline Arguments:"))
+            print(shlex.join(pipeline_args))
+            print()
+
+        if not yes_no(
+            (
+                f"Are you sure you want to create job '{name}' with the above arguments?\n"
+                + "    The job will be started if you continue."
+            ),
+            default='n',
+            **kwargs
+        ):
+            return True, "Nothing was changed."
+
+        new_job = mrsm.Job(name, new_sysargs_str, executor_keys=executor_keys)
+        start_success, start_msg = new_job.start()
+        if not start_success:
+            return start_success, start_msg
+
+    msg = (
+        "Successfully bootstrapped job"
+        + ('s' if len(action) != 1 else '')
+        + ' '
+        + items_str(action)
+        + '.'
+    )
+    return True, msg
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.

meerschaum/actions/clear.py

@@ -6,12 +6,16 @@ Functions for clearing pipes.
 """
 
 from __future__ import annotations
+
+from datetime import datetime
+import meerschaum as mrsm
 from meerschaum.utils.typing import List, SuccessTuple, Any, Optional
 
+
 def clear(
-        action: Optional[List[str]] = None,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> SuccessTuple:
     """
     Clear pipes of their data, or clear the screen.
 
@@ -34,24 +38,25 @@ def clear(
 
 
 def _clear_pipes(
-        action: Optional[List[str]] = None,
-        begin: Optional[datetime.datetime] = None,
-        end: Optional[datetime.datetime] = None,
-        connector_keys: Optional[List[str]] = None,
-        metric_keys: Optional[List[str]] = None,
-        mrsm_instance: Optional[str] = None,
-        location_keys: Optional[List[str]] = None,
-        force: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    connector_keys: Optional[List[str]] = None,
+    metric_keys: Optional[List[str]] = None,
+    mrsm_instance: Optional[str] = None,
+    location_keys: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Clear pipes' data without dropping any tables.
 
     """
     from meerschaum import get_pipes
     from meerschaum.utils.formatting import print_tuple
-    
+
     successes = {}
     fails = {}
 
@@ -61,13 +66,19 @@ def _clear_pipes(
     )
 
     if not force:
-        if not _ask_with_rowcounts(pipes, begin=begin, end=end, debug=debug, **kw):
+        if not _ask_with_rowcounts(pipes, begin=begin, end=end, debug=debug, yes=yes, **kw):
             return False, "No rows were deleted."
 
     for pipe in pipes:
-        success, msg = pipe.clear(begin=begin, end=end, debug=debug, **kw)
-        print_tuple((success, msg))
-        (successes if success else fails)[pipe] = msg
+        clear_success, clear_msg = pipe.clear(
+            begin=begin,
+            end=end,
+            yes=yes,
+            debug=debug,
+            **kw
+        )
+        print_tuple((clear_success, clear_msg))
+        (successes if clear_success else fails)[pipe] = clear_msg
 
     success = len(successes) > 0
     msg = (
@@ -79,15 +90,15 @@ def _clear_pipes(
 
 
 def _ask_with_rowcounts(
-        pipes: List[meerschaum.Pipe],
-        begin: Optional[datetime.datetime] = None,
-        end: Optional[datetime.datetime] = None,
-        yes: bool = False,
-        nopretty: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw
-    ) -> bool:
+    pipes: List[mrsm.Pipe],
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    yes: bool = False,
+    nopretty: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw
+) -> bool:
     """
     Count all of the pipes' rowcounts and confirm with the user that these rows need to be deleted.
 
@@ -109,13 +120,13 @@ def _ask_with_rowcounts(
                     warn(
                         f"No datetime could be determined for {pipe}!\n"
                         + "    THIS WILL DELETE THE ENTIRE TABLE!",
-                        stack = False
+                        stack=False
                     )
                 else:
                     warn(
                         f"A datetime wasn't specified for {pipe}.\n"
                         + f"    Using column \"{_dt}\" for datetime bounds...",
-                        stack = False
+                        stack=False
                     )
 
 

meerschaum/actions/delete.py

@@ -498,7 +498,7 @@ def _complete_delete_jobs(
     action: Optional[List[str]] = None,
     executor_keys: Optional[str] = None,
     line: str = '',
-    _get_job_method: Optional[str, List[str]] = None,
+    _get_job_method: Union[str, List[str], None] = None,
     **kw
 ) -> List[str]:
     from meerschaum._internal.shell.Shell import shell_attrs

meerschaum/actions/edit.py

@@ -7,9 +7,11 @@ Functions for editing elements belong here.
 """
 
 from __future__ import annotations
+
 import meerschaum as mrsm 
 from meerschaum.utils.typing import List, Any, SuccessTuple, Optional, Dict
 
+
 def edit(
     action: Optional[List[str]] = None,
     **kw: Any
@@ -24,6 +26,7 @@ def edit(
         'definition': _edit_definition,
         'users'     : _edit_users,
         'plugins'   : _edit_plugins,
+        'jobs'      : _edit_jobs,
     }
     return choose_subaction(action, options, **kw)
 
@@ -35,10 +38,17 @@ def _complete_edit(
     """
     Override the default Meerschaum `complete_` function.
     """
+    from meerschaum.actions.delete import _complete_delete_jobs
+
+    if action is None:
+        action = []
+
     options = {
         'config': _complete_edit_config,
         'plugin': _complete_edit_plugins,
         'plugins': _complete_edit_plugins,
+        'job': _complete_delete_jobs,
+        'jobs': _complete_delete_jobs,
     }
 
     if action is None:
@@ -130,7 +140,7 @@ def _edit_pipes(
 
     pipes = get_pipes(debug=debug, as_list=True, **kw)
     if pipes:
-        print_options(pipes, header=f'Pipes to be edited:')
+        print_options(pipes, header='Pipes to be edited:')
     else:
         return False, "No pipes to edit."
 
@@ -141,10 +151,10 @@ def _edit_pipes(
                     f"Press [Enter] to begin editing the above {len(pipes)} pipe"
                     + ("s" if len(pipes) != 1 else "")
                     + " or [CTRL-C] to cancel:",
-                    icon = False,
+                    icon=False,
                 )
         except KeyboardInterrupt:
-            return False, f"No pipes changed."
+            return False, "No pipes changed."
 
     interactive = (not bool(params))
     success, msg = True, ""
@@ -372,6 +382,111 @@ def _complete_edit_plugins(
     return possibilities
 
 
+def _edit_jobs(
+    action: Optional[List[str]] = None,
+    executor_keys: Optional[str] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> mrsm.SuccessTuple:
+    """
+    Edit existing jobs.
+    """
+    import shlex
+    from meerschaum.jobs import get_filtered_jobs
+    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum._internal.arguments import (
+        split_pipeline_sysargs,
+        split_chained_sysargs,
+    )
+    from meerschaum.utils.formatting import make_header, print_options
+    from meerschaum.utils.warnings import info
+    from meerschaum._internal.shell.ShellCompleter import ShellCompleter
+    from meerschaum.utils.misc import items_str
+    from meerschaum.utils.formatting._shell import clear_screen
+
+    jobs = get_filtered_jobs(executor_keys, action, debug=debug)
+    if not jobs:
+        return False, "No jobs to edit."
+
+    num_edited = 0
+    for name, job in jobs.items():
+        sysargs_str = shlex.join(job.sysargs)
+        clear_screen(debug=debug)
+        info(
+            f"Editing arguments for job '{name}'.\n"
+            "    Press [Esc + Enter] to submit, [CTRL + C] to exit.\n\n"
+            "    Tip: join actions with `+`, manage pipeline with `:`.\n"
+            "    https://meerschaum.io/reference/actions/#chaining-actions\n"
+        )
+
+        try:
+            new_sysargs_str = prompt(
+                "",
+                default_editable=sysargs_str.lstrip().rstrip().replace(' + ', '\n+ '),
+                multiline=True,
+                icon=False,
+                completer=ShellCompleter(),
+            )
+        except KeyboardInterrupt:
+            return True, "Nothing was changed."
+
+        if new_sysargs_str.strip() == sysargs_str.strip():
+            continue
+
+        new_sysargs = shlex.split(new_sysargs_str)
+        new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
+        chained_sysargs = split_chained_sysargs(new_sysargs)
+
+        clear_screen(debug=debug)
+        if len(chained_sysargs) > 1:
+            print_options(
+                [
+                    shlex.join(step_sysargs)
+                    for step_sysargs in chained_sysargs
+                ],
+                header=f"\nSteps in Job '{name}':",
+                number_options=True,
+                **kwargs
+            )
+        else:
+            print('\n' + make_header(f"Action for Job '{name}':"))
+            print(shlex.join(new_sysargs))
+
+        if pipeline_args:
+            print('\n' + make_header("Pipeline Arguments:"))
+            print(shlex.join(pipeline_args))
+            print()
+
+        if not yes_no(
+            (
+                f"Are you sure you want to recreate job '{name}' with the above arguments?\n"
+                + "    The job will be started if you continue."
+            ),
+            default='n',
+            **kwargs
+        ):
+            return True, "Nothing was changed."
+
+        delete_success, delete_msg = job.delete()
+        if not delete_success:
+            return delete_success, delete_msg
+
+        new_job = mrsm.Job(name, new_sysargs_str, executor_keys=executor_keys)
+        start_success, start_msg = new_job.start()
+        if not start_success:
+            return start_success, start_msg
+        num_edited += 1
+
+    msg = (
+        "Successfully edit job"
+        + ('s' if len(jobs) != 1 else '')
+        + ' '
+        + items_str(list(jobs.keys()))
+        + '.'
+        ) if num_edited > 0 else "Nothing was edited."
+    return True, msg
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.

meerschaum/actions/sh.py

@@ -9,14 +9,15 @@ NOTE: This action may be a huge security vulnerability
 
 from meerschaum.utils.typing import SuccessTuple, List, Any, Optional
 
+
 def sh(
-        action: Optional[List[str]] = None,
-        sub_args: Optional[List[str]] = None,
-        filtered_sysargs: Optional[List[str]] = None,
-        use_bash: bool = True,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    sub_args: Optional[List[str]] = None,
+    filtered_sysargs: Optional[List[str]] = None,
+    use_bash: bool = True,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Execute system commands.
     """
@@ -50,7 +51,7 @@ def sh(
         if len(action) != 0:
             try:
                 command_list += ["-c", shlex.join(cmd_list)]
-            except Exception as e:
+            except Exception:
                 command_list += ["-c", ' '.join(cmd_list)]
     else:
         if len(action) == 0:
@@ -64,8 +65,8 @@ def sh(
     try:
         process = subprocess.Popen(
             command_list,
-            shell = False,
-            env = os.environ,
+            shell=False,
+            env=os.environ,
         )
         exit_code = process.wait()
     except FileNotFoundError: