meerschaum 2.4.11__py3-none-any.whl → 2.4.12__py3-none-any.whl

meerschaum/_internal/arguments/_parse_arguments.py

@@ -256,7 +256,8 @@ def parse_synonyms(
 
 
 def parse_dict_to_sysargs(
-    args_dict: Dict[str, Any]
+    args_dict: Dict[str, Any],
+    coerce_dates: bool = True,
 ) -> List[str]:
     """Revert an arguments dictionary back to a command line list."""
     import shlex
@@ -304,6 +305,19 @@ def parse_dict_to_sysargs(
                 if len(args_dict[a]) > 0:
                     sysargs += [t[0], json.dumps(args_dict[a], separators=(',', ':'))]
 
+            ### Preserve the original datetime strings if possible
+            elif a in ('begin', 'end') and 'sysargs' in args_dict:
+                flag = t[0]
+                flag_ix = args_dict['sysargs'].index(flag)
+                if flag_ix < 0:
+                    continue
+                try:
+                    flag_val = args_dict['sysargs'][flag_ix + 1]
+                except IndexError:
+                    flag_val = str(args_dict[a])
+
+                sysargs += [flag, str(flag_val)]
+
             ### Account for None and other values
             elif (args_dict[a] is not None) or (args_dict[a] is None and a in allow_none_args):
                 sysargs += [t[0], str(args_dict[a])]

meerschaum/_internal/docs/index.py

@@ -742,6 +742,7 @@ def init_dash(dash_app):
       <li><code>meerschaum.utils.sql.get_db_version()</code></li>
       <li><code>meerschaum.utils.sql.get_rename_table_queries()</code></li>
       <li><code>meerschaum.utils.sql.get_create_table_query()</code></li>
+      <li><code>meerschaum.utils.sql.wrap_query_with_cte()</code></li>
       <li><code>meerschaum.utils.sql.format_cte_subquery()</code></li>
       <li><code>meerschaum.utils.sql.session_execute()</code></li>
     </ul>

meerschaum/_internal/shell/Shell.py

@@ -235,6 +235,23 @@ def get_shell_intro(with_color: bool = True) -> str:
         **get_config('shell', 'ansi', 'intro', 'rich')
     )
 
+def get_shell_session():
+    """
+    Return the `prompt_toolkit` prompt session.
+    """
+    from meerschaum.config._paths import SHELL_HISTORY_PATH
+    if 'session' in shell_attrs:
+        return shell_attrs['session']
+
+    shell_attrs['session'] = prompt_toolkit_shortcuts.PromptSession(
+        history=prompt_toolkit_history.FileHistory(SHELL_HISTORY_PATH.as_posix()),
+        auto_suggest=ValidAutoSuggest(),
+        completer=ShellCompleter(),
+        complete_while_typing=True,
+        reserve_space_for_menu=False,
+    )
+    return shell_attrs['session']
+
 
 class Shell(cmd.Cmd):
     """
@@ -264,14 +281,7 @@ class Shell(cmd.Cmd):
         except AttributeError:
             pass
 
-        from meerschaum.config._paths import SHELL_HISTORY_PATH
-        shell_attrs['session'] = prompt_toolkit_shortcuts.PromptSession(
-            history=prompt_toolkit_history.FileHistory(SHELL_HISTORY_PATH.as_posix()),
-            auto_suggest=ValidAutoSuggest(),
-            completer=ShellCompleter(),
-            complete_while_typing=True,
-            reserve_space_for_menu=False,
-        )
+        _ = get_shell_session()
 
         super().__init__()
 
@@ -626,7 +636,7 @@ class Shell(cmd.Cmd):
             step_action_name = step_action[0] if step_action else None
             ### NOTE: For `stack`, revert argument parsing.
             step_sysargs = (
-                parse_dict_to_sysargs(step_kwargs)
+                parse_dict_to_sysargs(step_kwargs, coerce_dates=False)
                 if step_action_name != 'stack'
                 else chained_sysargs[i]
             )

meerschaum/_internal/shell/ShellCompleter.py

@@ -31,15 +31,18 @@ class ShellCompleter(Completer):
         ensure_readline()
         parts = document.text.split('-')
         ends_with_space = parts[0].endswith(' ')
-        part_0_subbed_spaces = parts[0].replace(' ', '_')
-        parsed_text = part_0_subbed_spaces + '-'.join(parts[1:])
+        last_action_line = parts[0].split('+')[-1]
+        part_0_subbed_spaces = last_action_line.replace(' ', '_')
+        parsed_text = (part_0_subbed_spaces + '-'.join(parts[1:]))
 
+        if not parsed_text:
+            return
 
         ### Index is the rank order (0 is closest match).
         ### Break when no results are returned.
         for i, a in enumerate(shell_actions):
             try:
-                poss = shell.complete(parsed_text, i)
+                poss = shell.complete(parsed_text.lstrip('_'), i)
                 if poss:
                     poss = poss.replace('_', ' ')
             ### Having issues with readline on portable Windows.
@@ -50,7 +53,9 @@ class ShellCompleter(Completer):
             yield Completion(poss, start_position=(-1 * len(poss)))
             yielded.append(poss)
 
-        args = parse_line(document.text)
+        line = document.text
+        current_action_line = line.split('+')[-1].lstrip()
+        args = parse_line(current_action_line)
         action_function = get_action(args['action'], _actions=shell_attrs.get('_actions', None))
         if action_function is None:
             return
@@ -74,8 +79,8 @@ class ShellCompleter(Completer):
                 shell,
                 complete_function_name
             )(
-                document.text.split(' ')[-1],
-                document.text,
+                current_action_line.split(' ')[-1],
+                current_action_line,
                 0,
                 0
             )

meerschaum/actions/bootstrap.py

@@ -8,12 +8,15 @@ Functions for bootstrapping elements
 """
 
 from __future__ import annotations
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional, Tuple, List
 
+
 def bootstrap(
-        action: Optional[List[str]] = None,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> SuccessTuple:
     """
     Launch an interactive wizard to bootstrap pipes or connectors.
     
@@ -26,23 +29,24 @@ def bootstrap(
         'pipes'      : _bootstrap_pipes,
         'connectors' : _bootstrap_connectors,
         'plugins'    : _bootstrap_plugins,
+        'jobs'       : _bootstrap_jobs,
     }
     return choose_subaction(action, options, **kw)
 
 
 def _bootstrap_pipes(
-        action: Optional[List[str]] = None,
-        connector_keys: Optional[List[str]] = None,
-        metric_keys: Optional[List[str]] = None,
-        location_keys: Optional[List[Optional[str]]] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        mrsm_instance: Optional[str] = None,
-        shell: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    connector_keys: Optional[List[str]] = None,
+    metric_keys: Optional[List[str]] = None,
+    location_keys: Optional[List[Optional[str]]] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    mrsm_instance: Optional[str] = None,
+    shell: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Create a new pipe.
     If no keys are provided, guide the user through the steps required.
@@ -433,6 +437,107 @@ def _bootstrap_plugins(
     return True, "Success"
 
 
+def _bootstrap_jobs(
+    action: Optional[List[str]] = None,
+    executor_keys: Optional[str] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
+    """
+    Launch an interactive wizard to create new jobs.
+    """
+    import shlex
+    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum.actions import actions
+    from meerschaum.utils.formatting import print_options, make_header
+    from meerschaum.utils.formatting._shell import clear_screen
+    from meerschaum.utils.warnings import info
+    from meerschaum._internal.arguments import (
+        split_pipeline_sysargs,
+        split_chained_sysargs,
+    )
+    from meerschaum.utils.misc import items_str
+    from meerschaum._internal.shell.ShellCompleter import ShellCompleter
+
+    if not action:
+        action = [prompt("What is the name of the job you'd like to create?")]
+
+    new_jobs = {}
+    for name in action:
+        clear_screen(debug=debug)
+        job = mrsm.Job(name, executor_keys=executor_keys)
+        if job.exists():
+            edit_success, edit_msg = actions['edit'](['job', name], **kwargs)
+            if not edit_success:
+                return edit_success, edit_msg
+            continue
+
+        info(
+            "Press [Esc + Enter] to submit, [CTRL + C] to exit.\n"
+            "    Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
+            "    https://meerschaum.io/reference/actions/#chaining-actions\n"
+        )
+        try:
+            new_sysargs_str = prompt(
+                f"Arguments for job '{name}':",
+                multiline=True,
+                icon=False,
+                completer=ShellCompleter(),
+            )
+        except KeyboardInterrupt:
+            return True, "Nothing was changed."
+
+        new_sysargs = shlex.split(new_sysargs_str)
+        new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
+        chained_sysargs = split_chained_sysargs(new_sysargs)
+
+        if len(chained_sysargs) > 1:
+            print_options(
+                [
+                    shlex.join(step_sysargs)
+                    for step_sysargs in chained_sysargs
+                ],
+                header=f"Steps in Job '{name}':",
+                number_options=True,
+                **kwargs
+            )
+        else:
+            print('\n' + make_header(f"Action for Job '{name}':"))
+            print(shlex.join(new_sysargs))
+
+        if pipeline_args:
+            print('\n' + make_header("Pipeline Arguments:"))
+            print(shlex.join(pipeline_args))
+
+        if not yes_no(
+            (
+                f"Are you sure you want to create job '{name}' with the above arguments?\n"
+                + "    The job will be started if you continue."
+            ),
+            default='n',
+            **kwargs
+        ):
+            return True, "Nothing was changed."
+
+        new_job = mrsm.Job(name, new_sysargs_str, executor_keys=executor_keys)
+        start_success, start_msg = new_job.start()
+        if not start_success:
+            return start_success, start_msg
+
+        new_jobs[name] = new_job
+
+    if not new_jobs:
+        return False, "No new jobs were created."
+
+    msg = (
+        "Successfully bootstrapped job"
+        + ('s' if len(new_jobs) != 1 else '')
+        + items_str(list(new_jobs.keys()))
+        + '.'
+    )
+    return True, msg
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.

meerschaum/actions/clear.py

@@ -6,12 +6,16 @@ Functions for clearing pipes.
 """
 
 from __future__ import annotations
+
+from datetime import datetime
+import meerschaum as mrsm
 from meerschaum.utils.typing import List, SuccessTuple, Any, Optional
 
+
 def clear(
-        action: Optional[List[str]] = None,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> SuccessTuple:
     """
     Clear pipes of their data, or clear the screen.
 
@@ -34,24 +38,25 @@ def clear(
 
 
 def _clear_pipes(
-        action: Optional[List[str]] = None,
-        begin: Optional[datetime.datetime] = None,
-        end: Optional[datetime.datetime] = None,
-        connector_keys: Optional[List[str]] = None,
-        metric_keys: Optional[List[str]] = None,
-        mrsm_instance: Optional[str] = None,
-        location_keys: Optional[List[str]] = None,
-        force: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    connector_keys: Optional[List[str]] = None,
+    metric_keys: Optional[List[str]] = None,
+    mrsm_instance: Optional[str] = None,
+    location_keys: Optional[List[str]] = None,
+    yes: bool = False,
+    force: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Clear pipes' data without dropping any tables.
 
     """
     from meerschaum import get_pipes
     from meerschaum.utils.formatting import print_tuple
-    
+
     successes = {}
     fails = {}
 
@@ -61,13 +66,19 @@ def _clear_pipes(
     )
 
     if not force:
-        if not _ask_with_rowcounts(pipes, begin=begin, end=end, debug=debug, **kw):
+        if not _ask_with_rowcounts(pipes, begin=begin, end=end, debug=debug, yes=yes, **kw):
             return False, "No rows were deleted."
 
     for pipe in pipes:
-        success, msg = pipe.clear(begin=begin, end=end, debug=debug, **kw)
-        print_tuple((success, msg))
-        (successes if success else fails)[pipe] = msg
+        clear_success, clear_msg = pipe.clear(
+            begin=begin,
+            end=end,
+            yes=yes,
+            debug=debug,
+            **kw
+        )
+        print_tuple((clear_success, clear_msg))
+        (successes if clear_success else fails)[pipe] = clear_msg
 
     success = len(successes) > 0
     msg = (
@@ -79,15 +90,15 @@ def _clear_pipes(
 
 
 def _ask_with_rowcounts(
-        pipes: List[meerschaum.Pipe],
-        begin: Optional[datetime.datetime] = None,
-        end: Optional[datetime.datetime] = None,
-        yes: bool = False,
-        nopretty: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw
-    ) -> bool:
+    pipes: List[mrsm.Pipe],
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    yes: bool = False,
+    nopretty: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw
+) -> bool:
     """
     Count all of the pipes' rowcounts and confirm with the user that these rows need to be deleted.
 
@@ -109,13 +120,13 @@ def _ask_with_rowcounts(
                     warn(
                         f"No datetime could be determined for {pipe}!\n"
                         + "    THIS WILL DELETE THE ENTIRE TABLE!",
-                        stack = False
+                        stack=False
                     )
                 else:
                     warn(
                         f"A datetime wasn't specified for {pipe}.\n"
                         + f"    Using column \"{_dt}\" for datetime bounds...",
-                        stack = False
+                        stack=False
                     )
 
 

meerschaum/actions/edit.py

@@ -7,9 +7,11 @@ Functions for editing elements belong here.
 """
 
 from __future__ import annotations
+
 import meerschaum as mrsm 
 from meerschaum.utils.typing import List, Any, SuccessTuple, Optional, Dict
 
+
 def edit(
     action: Optional[List[str]] = None,
     **kw: Any
@@ -24,6 +26,7 @@ def edit(
         'definition': _edit_definition,
         'users'     : _edit_users,
         'plugins'   : _edit_plugins,
+        'jobs'      : _edit_jobs,
     }
     return choose_subaction(action, options, **kw)
 
@@ -372,6 +375,92 @@ def _complete_edit_plugins(
     return possibilities
 
 
+def _edit_jobs(
+    action: Optional[List[str]] = None,
+    executor_keys: Optional[str] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> mrsm.SuccessTuple:
+    """
+    Edit existing jobs.
+    """
+    import shlex
+    from meerschaum.jobs import get_filtered_jobs
+    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum._internal.arguments import (
+        split_pipeline_sysargs,
+        split_chained_sysargs,
+    )
+    from meerschaum.utils.formatting import make_header, print_options
+    from meerschaum.utils.warnings import info
+    from meerschaum.actions import actions
+    jobs = get_filtered_jobs(executor_keys, action, debug=debug)
+    if not jobs:
+        return False, "No jobs to edit."
+
+    info(
+        "Press [Esc + Enter] to submit, [CTRL + C] to exit.\n"
+        "    Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
+        "    https://meerschaum.io/reference/actions/#chaining-actions\n"
+    )
+
+    for name, job in jobs.items():
+        sysargs_str = shlex.join(job.sysargs)
+
+        try:
+            new_sysargs_str = prompt(
+                f"Arguments for job '{name}':",
+                default_editable=sysargs_str.lstrip().rstrip(),
+                multiline=True,
+                icon=False,
+            )
+        except KeyboardInterrupt:
+            return True, "Nothing was changed."
+
+        new_sysargs = shlex.split(new_sysargs_str)
+        new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
+        chained_sysargs = split_chained_sysargs(new_sysargs)
+
+        if len(chained_sysargs) > 1:
+            print_options(
+                [
+                    shlex.join(step_sysargs)
+                    for step_sysargs in chained_sysargs
+                ],
+                header=f"Steps in Job '{name}':",
+                number_options=True,
+                **kwargs
+            )
+        else:
+            print('\n' + make_header(f"Action for Job '{name}':"))
+            print(shlex.join(new_sysargs))
+
+        if pipeline_args:
+            print('\n' + make_header("Pipeline Arguments:"))
+            print(shlex.join(pipeline_args))
+
+        if not yes_no(
+            (
+                f"Are you sure you want to recreate job '{name}' with the above arguments?\n"
+                + "    The job will be started if you continue."
+            ),
+            default='n',
+            **kwargs
+        ):
+            return True, "Nothing was changed."
+
+        delete_success, delete_msg = job.delete()
+        if not delete_success:
+            return delete_success, delete_msg
+
+        new_job = mrsm.Job(name, new_sysargs_str, executor_keys=executor_keys)
+        start_success, start_msg = new_job.start()
+        if not start_success:
+            return start_success, start_msg
+
+    return True, "Success"
+
+
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not
 ###       be added to the `help` docstring.

meerschaum/actions/start.py

@@ -9,6 +9,7 @@ Start subsystems (API server, logging daemon, etc.).
 from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Optional, List, Any, Union, Dict
 
+
 def start(
     action: Optional[List[str]] = None,
     **kw: Any,
@@ -375,8 +376,8 @@ def _start_gui(
         webview.create_window(
             'Meerschaum Shell', 
             f'http://127.0.0.1:{port}',
-            height = 650,
-            width = 1000
+            height=650,
+            width=1000
         )
         webview.start(debug=debug)
     except Exception as e:

meerschaum/api/dash/callbacks/dashboard.py

@@ -98,6 +98,7 @@ _paths = {
     'plugins' : pages.plugins.layout,
     'register': pages.register.layout,
     'pipes'   : pages.pipes.layout,
+    'job'     : pages.job.layout,
 }
 _required_login = {''}
 
@@ -121,7 +122,7 @@ def update_page_layout_div(
     ----------
     pathname: str
         The path in the browser.
-        
+
     session_store_data: Dict[str, Any]:
         The stored session data.
 

meerschaum/api/dash/callbacks/jobs.py

@@ -12,6 +12,7 @@ import json
 import time
 import traceback
 from datetime import datetime, timezone
+
 from meerschaum.utils.typing import Optional, Dict, Any
 from meerschaum.api import CHECK_UPDATE
 from meerschaum.api.dash import dash_app
@@ -19,17 +20,19 @@ from meerschaum.api.dash.sessions import get_username_from_session
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html
 from meerschaum.api.dash.components import alert_from_success_tuple
 from meerschaum.api.dash.jobs import (
+    build_job_card,
     build_manage_job_buttons_div_children,
     build_status_children,
     build_process_timestamps_children,
 )
-from meerschaum.jobs import Job
+from meerschaum.api.routes._jobs import _get_job
 from meerschaum.api.dash.sessions import is_session_authenticated
 dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 from dash.exceptions import PreventUpdate
 from dash.dependencies import Input, Output, State, ALL, MATCH
 import dash_bootstrap_components as dbc
+from dash import no_update
 
 
 @dash_app.callback(
@@ -53,6 +56,13 @@ def download_job_logs(n_clicks):
 
     component_dict = json.loads(ctx[0]['prop_id'].split('.' + 'n_clicks')[0])
     job_name = component_dict['index']
+    try:
+        job = _get_job(job_name)
+    except Exception:
+        job = None
+    if job is None or not job.exists():
+        raise PreventUpdate
+
     now = datetime.now(timezone.utc)
     filename = job_name + '_' + str(int(now.timestamp())) + '.log'
     return {
@@ -69,7 +79,7 @@ def download_job_logs(n_clicks):
     Input({'type': 'manage-job-button', 'action': ALL, 'index': MATCH}, 'n_clicks'),
     State('session-store', 'data'),
     State({'type': 'job-label-p', 'index': MATCH}, 'children'),
-    prevent_initial_call = True,
+    prevent_initial_call=True,
 )
 def manage_job_button_click(
     n_clicks: Optional[int] = None,
@@ -102,10 +112,10 @@ def manage_job_button_click(
     job_name = component_dict['index']
     manage_job_action = component_dict['action']
     try:
-        job = Job(job_name, job_label.replace('\n', ' ') if job_label else None)
-    except Exception as e:
+        job = _get_job(job_name, job_label.replace('\n', ' ') if job_label else None)
+    except Exception:
         job = None
-    if job is None:
+    if job is None or not job.exists():
         raise PreventUpdate
 
     manage_functions = {
@@ -191,7 +201,7 @@ dash_app.clientside_callback(
     Output({'type': 'process-timestamps-div', 'index': ALL}, 'children'),
     Input('refresh-jobs-interval', 'n_intervals'),
     State('session-store', 'data'),
-    prevent_initial_call = True,
+    prevent_initial_call=True,
 )
 def refresh_jobs_on_interval(
     n_intervals: Optional[int] = None,
@@ -209,7 +219,7 @@ def refresh_jobs_on_interval(
     ]
 
     ### NOTE: The job may have been deleted, but the card may still exist.
-    jobs = [Job(name) for name in job_names]
+    jobs = [_get_job(name) for name in job_names]
 
     return (
         [
@@ -229,3 +239,39 @@ def refresh_jobs_on_interval(
             for job in jobs
         ],
     )
+
+
+@dash_app.callback(
+    Output('job-output-div', 'children'),
+    Input('job-location', 'pathname'),
+    State('session-store', 'data'),
+)
+def render_job_page_from_url(
+    pathname: str,
+    session_data: Optional[Dict[str, Any]],
+):
+    """
+    Load the `/job/{name}` page.
+    """
+    if not str(pathname).startswith('/dash/job'):
+        return no_update
+
+    session_id = (session_data or {}).get('session-id', None)
+    authenticated = is_session_authenticated(str(session_id))
+
+    job_name = pathname.replace('/dash/job', '').lstrip('/').rstrip('/')
+    if not job_name:
+        return no_update
+
+    job = _get_job(job_name)
+    if not job.exists():
+        return [
+            html.Br(),
+            html.H2("404: Job does not exist."),
+        ]
+
+    return [
+        html.Br(),
+        build_job_card(job, authenticated=authenticated, include_follow=False),
+        html.Br(),
+    ]