meerschaum 2.4.12__py3-none-any.whl → 2.5.0__py3-none-any.whl

meerschaum/_internal/entry.py

@@ -43,6 +43,7 @@ if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
     if _systemd_stdin_path:
         sys.stdin = _StdinFile(_systemd_stdin_path)
 
+
 def entry(
     sysargs: Optional[List[str]] = None,
     _patch_args: Optional[Dict[str, Any]] = None,

meerschaum/actions/bootstrap.py

@@ -19,7 +19,7 @@ def bootstrap(
 ) -> SuccessTuple:
     """
     Launch an interactive wizard to bootstrap pipes or connectors.
-    
+
     Example:
         `bootstrap pipes`
 
@@ -462,7 +462,6 @@ def _bootstrap_jobs(
     if not action:
         action = [prompt("What is the name of the job you'd like to create?")]
 
-    new_jobs = {}
     for name in action:
         clear_screen(debug=debug)
         job = mrsm.Job(name, executor_keys=executor_keys)
@@ -473,13 +472,14 @@ def _bootstrap_jobs(
             continue
 
         info(
-            "Press [Esc + Enter] to submit, [CTRL + C] to exit.\n"
+            f"Editing arguments for job '{name}'.\n"
+            "    Press [Esc + Enter] to submit, [CTRL + C] to exit.\n\n"
             "    Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
             "    https://meerschaum.io/reference/actions/#chaining-actions\n"
         )
         try:
             new_sysargs_str = prompt(
-                f"Arguments for job '{name}':",
+                "",
                 multiline=True,
                 icon=False,
                 completer=ShellCompleter(),
@@ -491,6 +491,7 @@ def _bootstrap_jobs(
         new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
         chained_sysargs = split_chained_sysargs(new_sysargs)
 
+        clear_screen(debug=debug)
         if len(chained_sysargs) > 1:
             print_options(
                 [
@@ -508,6 +509,7 @@ def _bootstrap_jobs(
         if pipeline_args:
             print('\n' + make_header("Pipeline Arguments:"))
             print(shlex.join(pipeline_args))
+            print()
 
         if not yes_no(
             (
@@ -524,15 +526,11 @@ def _bootstrap_jobs(
         if not start_success:
             return start_success, start_msg
 
-        new_jobs[name] = new_job
-
-    if not new_jobs:
-        return False, "No new jobs were created."
-
     msg = (
         "Successfully bootstrapped job"
-        + ('s' if len(new_jobs) != 1 else '')
-        + items_str(list(new_jobs.keys()))
+        + ('s' if len(action) != 1 else '')
+        + ' '
+        + items_str(action)
         + '.'
     )
     return True, msg

meerschaum/actions/delete.py

@@ -498,7 +498,7 @@ def _complete_delete_jobs(
     action: Optional[List[str]] = None,
     executor_keys: Optional[str] = None,
     line: str = '',
-    _get_job_method: Optional[str, List[str]] = None,
+    _get_job_method: Union[str, List[str], None] = None,
     **kw
 ) -> List[str]:
     from meerschaum._internal.shell.Shell import shell_attrs

meerschaum/actions/edit.py

@@ -38,10 +38,17 @@ def _complete_edit(
     """
     Override the default Meerschaum `complete_` function.
     """
+    from meerschaum.actions.delete import _complete_delete_jobs
+
+    if action is None:
+        action = []
+
     options = {
         'config': _complete_edit_config,
         'plugin': _complete_edit_plugins,
         'plugins': _complete_edit_plugins,
+        'job': _complete_delete_jobs,
+        'jobs': _complete_delete_jobs,
     }
 
     if action is None:
@@ -133,7 +140,7 @@ def _edit_pipes(
 
     pipes = get_pipes(debug=debug, as_list=True, **kw)
     if pipes:
-        print_options(pipes, header=f'Pipes to be edited:')
+        print_options(pipes, header='Pipes to be edited:')
     else:
         return False, "No pipes to edit."
 
@@ -144,10 +151,10 @@ def _edit_pipes(
                     f"Press [Enter] to begin editing the above {len(pipes)} pipe"
                     + ("s" if len(pipes) != 1 else "")
                     + " or [CTRL-C] to cancel:",
-                    icon = False,
+                    icon=False,
                 )
         except KeyboardInterrupt:
-            return False, f"No pipes changed."
+            return False, "No pipes changed."
 
     interactive = (not bool(params))
     success, msg = True, ""
@@ -393,41 +400,51 @@ def _edit_jobs(
     )
     from meerschaum.utils.formatting import make_header, print_options
     from meerschaum.utils.warnings import info
-    from meerschaum.actions import actions
+    from meerschaum._internal.shell.ShellCompleter import ShellCompleter
+    from meerschaum.utils.misc import items_str
+    from meerschaum.utils.formatting._shell import clear_screen
+
     jobs = get_filtered_jobs(executor_keys, action, debug=debug)
     if not jobs:
         return False, "No jobs to edit."
 
-    info(
-        "Press [Esc + Enter] to submit, [CTRL + C] to exit.\n"
-        "    Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
-        "    https://meerschaum.io/reference/actions/#chaining-actions\n"
-    )
-
+    num_edited = 0
     for name, job in jobs.items():
         sysargs_str = shlex.join(job.sysargs)
+        clear_screen(debug=debug)
+        info(
+            f"Editing arguments for job '{name}'.\n"
+            "    Press [Esc + Enter] to submit, [CTRL + C] to exit.\n\n"
+            "    Tip: join actions with `+`, manage pipeline with `:`.\n"
+            "    https://meerschaum.io/reference/actions/#chaining-actions\n"
+        )
 
         try:
             new_sysargs_str = prompt(
-                f"Arguments for job '{name}':",
-                default_editable=sysargs_str.lstrip().rstrip(),
+                "",
+                default_editable=sysargs_str.lstrip().rstrip().replace(' + ', '\n+ '),
                 multiline=True,
                 icon=False,
+                completer=ShellCompleter(),
             )
         except KeyboardInterrupt:
             return True, "Nothing was changed."
 
+        if new_sysargs_str.strip() == sysargs_str.strip():
+            continue
+
         new_sysargs = shlex.split(new_sysargs_str)
         new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
         chained_sysargs = split_chained_sysargs(new_sysargs)
 
+        clear_screen(debug=debug)
         if len(chained_sysargs) > 1:
             print_options(
                 [
                     shlex.join(step_sysargs)
                     for step_sysargs in chained_sysargs
                 ],
-                header=f"Steps in Job '{name}':",
+                header=f"\nSteps in Job '{name}':",
                 number_options=True,
                 **kwargs
             )
@@ -438,6 +455,7 @@ def _edit_jobs(
         if pipeline_args:
             print('\n' + make_header("Pipeline Arguments:"))
             print(shlex.join(pipeline_args))
+            print()
 
         if not yes_no(
             (
@@ -457,8 +475,16 @@ def _edit_jobs(
         start_success, start_msg = new_job.start()
         if not start_success:
             return start_success, start_msg
+        num_edited += 1
 
-    return True, "Success"
+    msg = (
+        "Successfully edit job"
+        + ('s' if len(jobs) != 1 else '')
+        + ' '
+        + items_str(list(jobs.keys()))
+        + '.'
+        ) if num_edited > 0 else "Nothing was edited."
+    return True, msg
 
 
 ### NOTE: This must be the final statement of the module.

meerschaum/actions/sh.py

@@ -9,14 +9,15 @@ NOTE: This action may be a huge security vulnerability
 
 from meerschaum.utils.typing import SuccessTuple, List, Any, Optional
 
+
 def sh(
-        action: Optional[List[str]] = None,
-        sub_args: Optional[List[str]] = None,
-        filtered_sysargs: Optional[List[str]] = None,
-        use_bash: bool = True,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    sub_args: Optional[List[str]] = None,
+    filtered_sysargs: Optional[List[str]] = None,
+    use_bash: bool = True,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Execute system commands.
     """
@@ -50,7 +51,7 @@ def sh(
         if len(action) != 0:
             try:
                 command_list += ["-c", shlex.join(cmd_list)]
-            except Exception as e:
+            except Exception:
                 command_list += ["-c", ' '.join(cmd_list)]
     else:
         if len(action) == 0:
@@ -64,8 +65,8 @@ def sh(
     try:
         process = subprocess.Popen(
             command_list,
-            shell = False,
-            env = os.environ,
+            shell=False,
+            env=os.environ,
         )
         exit_code = process.wait()
     except FileNotFoundError:

meerschaum/actions/start.py

@@ -541,6 +541,7 @@ def _start_pipeline(
     action: Optional[List[str]] = None,
     loop: bool = False,
     min_seconds: Union[float, int, None] = 1.0,
+    timeout_seconds: Optional[int] = None,
     params: Optional[Dict[str, Any]] = None,
     **kwargs
 ) -> SuccessTuple:
@@ -557,10 +558,39 @@ def _start_pipeline(
     `show version + show arguments :: --loop`
 
     """
+    import json
     import time
+    import sys
     from meerschaum._internal.entry import entry
     from meerschaum.utils.warnings import info, warn
     from meerschaum.utils.misc import is_int
+    from meerschaum.utils.venv import venv_exec
+    from meerschaum.utils.process import poll_process
+    fence_begin, fence_end = '<MRSM_RESULT>', '</MRSM_RESULT>'
+
+    success, msg = False, "Did not run pipeline."
+    def write_line(line):
+        nonlocal success, msg
+        decoded = line.decode('utf-8')
+        begin_index, end_index = decoded.find(fence_begin), decoded.find(fence_end)
+
+        ### Found the beginning of the return value.
+        ### Don't write the parsed success tuple message.
+        if begin_index >= 0:
+            success, msg = tuple(json.loads(
+                decoded[begin_index + len(fence_begin):end_index]
+            ))
+            return
+
+        print(decoded)
+
+    def timeout_handler(*args, **kw):
+        nonlocal success, msg
+        success, msg = False, (
+            f"Failed to execute pipeline within {timeout_seconds} second"
+            + ('s' if timeout_seconds != 1 else '') + '.'
+        )
+        write_line((fence_begin + json.dumps((success, msg)) + fence_end).encode('utf-8'))
 
     do_n_times = (
         int(action[0].lstrip('x'))
@@ -578,12 +608,38 @@ def _start_pipeline(
     if min_seconds is None:
         min_seconds = 1.0
 
+    def do_entry() -> None:
+        nonlocal success, msg
+        if timeout_seconds is None:
+            success, msg = entry(sub_args_line, _patch_args=patch_args)
+            return
+
+        sub_args_line_escaped = sub_args_line.replace("'", "<QUOTE>")
+        patch_args_escaped_str = json.dumps(patch_args).replace("'", "<QUOTE>")
+        src = (
+            "import json\n"
+            "from meerschaum._internal.entry import entry\n\n"
+            f"sub_args_line = '{sub_args_line_escaped}'.replace(\"<QUOTE>\", \"'\")\n"
+            f"patch_args = json.loads('{patch_args_escaped_str}'.replace('<QUOTE>', \"'\"))\n"
+            "success, msg = entry(sub_args_line, _patch_args=patch_args)\n"
+            f"print('{fence_begin}' + json.dumps((success, msg)) + '{fence_end}')"
+        )
+        proc = venv_exec(src, venv=None, as_proc=True)
+        poll_process(
+            proc,
+            write_line,
+            timeout_seconds,
+            timeout_handler,
+        )
+
     ran_n_times = 0
-    success, msg = False, "Did not run pipeline."
     def run_loop():
         nonlocal ran_n_times, success, msg
         while True:
-            success, msg = entry(sub_args_line, _patch_args=patch_args)
+            try:
+                do_entry()
+            except Exception as e:
+                warn(e)
             ran_n_times += 1
 
             if not loop and do_n_times == 1:

meerschaum/actions/sync.py

@@ -129,7 +129,6 @@ def _pipes_lap(
             stack=False,
         )
 
-
     def _task_label(count: int):
         return f"[cyan]Syncing {count} pipe{'s' if count != 1 else ''}..."
 
@@ -137,7 +136,6 @@ def _pipes_lap(
         _progress.add_task(_task_label(len(pipes)), start=True, total=len(pipes))
     ) if _progress is not None else None
 
-
     def worker_fn():
         while not stop_event.is_set():
             try:
@@ -188,7 +186,7 @@ def _pipes_lap(
                     decoded[begin_index + len(fence_begin):end_index]
                 ))
                 return
-            sys.stdout.buffer.write(line)
+            print(decoded)
 
         def timeout_handler(p, *args, **kw):
             success, msg = False, (
@@ -227,7 +225,7 @@ def _pipes_lap(
             write_line,
             timeout_seconds,
             timeout_handler,
-            (p,)
+            (p,),
         )
         return _success_tuple
 
@@ -249,18 +247,18 @@ def _pipes_lap(
 
 
 def _sync_pipes(
-        loop: bool = False,
-        min_seconds: int = 1,
-        unblock: bool = False,
-        verify: bool = False,
-        deduplicate: bool = False,
-        bounded: Optional[bool] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        shell: bool = False,
-        nopretty: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    loop: bool = False,
+    min_seconds: int = 1,
+    unblock: bool = False,
+    verify: bool = False,
+    deduplicate: bool = False,
+    bounded: Optional[bool] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    shell: bool = False,
+    nopretty: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Fetch and sync new data for pipes.
 

meerschaum/actions/upgrade.py

@@ -9,16 +9,17 @@ Upgrade your current Meerschaum environment
 from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Union
 
+
 def upgrade(
     action: Optional[List[str]] = None,
     **kw: Any
 ) -> SuccessTuple:
     """
     Upgrade Meerschaum, plugins, or packages.
-    
+
     Command:
         `upgrade {option}`
-    
+
     Example:
         `upgrade meerschaum`
     """
@@ -77,7 +78,7 @@ def _upgrade_meerschaum(
         if force:
             answer = True
         else:
-            answer = yes_no(f"Take down the stack?", default='y', yes=yes, noask=noask)
+            answer = yes_no("Take down the stack?", default='n', yes=yes, noask=noask)
 
         if answer:
             if debug:
@@ -95,7 +96,7 @@ def _upgrade_meerschaum(
     if debug:
         dprint('Upgrade meerschaum with dependencies: \"' + f'{dependencies}' + '\"')
     if not pip_install(install_name, venv=None, debug=debug):
-        return False, f"Failed to upgrade Meerschaum via pip."
+        return False, "Failed to upgrade Meerschaum via pip."
 
     if debug:
         dprint("Pulling new Docker images...")
@@ -108,6 +109,7 @@ def _upgrade_meerschaum(
 class NoVenv:
     pass
 
+
 def _upgrade_packages(
     action: Optional[List[str]] = None,
     venv: Union[str, None, NoVenv] = NoVenv,
@@ -120,7 +122,7 @@ def _upgrade_packages(
     """
     Upgrade and install dependencies.
     If provided, upgrade only a dependency group, otherwise default to `full`.
-    
+
     Examples:
         upgrade packages
         upgrade packages full
@@ -159,7 +161,7 @@ def _upgrade_packages(
     to_install = [
         install_name
         for install_name in to_install
-        if install_name not in prereleases_to_install
+        if (install_name not in prereleases_to_install) or group == '_internal'
     ]
 
     success, msg = False, f"Nothing installed."

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.12"
+__version__ = "2.5.0"

meerschaum/connectors/sql/_create_engine.py

@@ -63,6 +63,8 @@ flavor_configs = {
             'fast_executemany': True,
             'isolation_level': 'AUTOCOMMIT',
             'use_setinputsizes': False,
+            'pool_pre_ping': True,
+            'ignore_no_transaction_on_rollback': True,
         },
         'omit_create_engine': {'method',},
         'to_sql': {
@@ -189,15 +191,18 @@ def create_engine(
     ### Install and patch required drivers.
     if self.flavor in install_flavor_drivers:
         attempt_import(*install_flavor_drivers[self.flavor], debug=debug, lazy=False, warn=False)
+        if self.flavor == 'mssql':
+            pyodbc = attempt_import('pyodbc', debug=debug, lazy=False, warn=False)
+            pyodbc.pooling = False
     if self.flavor in require_patching_flavors:
         from meerschaum.utils.packages import determine_version, _monkey_patch_get_distribution
         import pathlib
         for install_name, import_name in require_patching_flavors[self.flavor]:
             pkg = attempt_import(
                 import_name,
-                debug = debug,
-                lazy = False,
-                warn = False
+                debug=debug,
+                lazy=False,
+                warn=False
             )
             _monkey_patch_get_distribution(
                 install_name, determine_version(pathlib.Path(pkg.__file__), venv='mrsm')

meerschaum/connectors/sql/_fetch.py

@@ -31,7 +31,7 @@ def fetch(
     ----------
     pipe: mrsm.Pipe
         The pipe object which contains the `fetch` metadata.
-        
+
         - pipe.columns['datetime']: str
             - Name of the datetime column for the remote table.
         - pipe.parameters['fetch']: Dict[str, Any]
@@ -196,7 +196,7 @@ def get_pipe_metadef(
             dateadd_str(
                 flavor=self.flavor,
                 datepart='minute',
-                number=((-1 * btm) if apply_backtrack else 0), 
+                number=((-1 * btm) if apply_backtrack else 0),
                 begin=begin,
             )
             if begin
@@ -309,6 +309,8 @@ def _simple_fetch_query(
     """Build a fetch query from a pipe's definition."""
     from meerschaum.utils.sql import format_cte_subquery
     definition = get_pipe_query(pipe)
+    if definition is None:
+        raise ValueError(f"No SQL definition could be found for {pipe}.")
     return format_cte_subquery(definition, flavor, 'definition')
 
 

meerschaum/connectors/sql/_instance.py

@@ -88,9 +88,9 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
     from meerschaum.connectors.sql.tables import get_tables
     sqlalchemy = mrsm.attempt_import('sqlalchemy')
     temp_tables_table = get_tables(
-        mrsm_instance = self,
-        create = False,
-        debug = debug,
+        mrsm_instance=self,
+        create=False,
+        debug=debug,
     )['temp_tables']
     query = (
         sqlalchemy.select(temp_tables_table.c.table)

meerschaum/connectors/sql/_pipes.py

@@ -384,7 +384,7 @@ def get_create_index_queries(
 
     Returns
     -------
-    A dictionary of column names mapping to lists of queries.
+    A dictionary of index names mapping to lists of queries.
     """
     ### NOTE: Due to recent breaking changes in DuckDB, indices don't behave properly.
     if self.flavor == 'duckdb':
@@ -400,7 +400,8 @@ def get_create_index_queries(
     index_queries = {}
 
     upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in update_queries
-    indices = pipe.get_indices()
+    index_names = pipe.get_indices()
+    indices = pipe.indices
 
     _datetime = pipe.get_columns('datetime', error=False)
     _datetime_type = pipe.dtypes.get(_datetime, 'datetime64[ns]')
@@ -409,8 +410,8 @@ def get_create_index_queries(
         if _datetime is not None else None
     )
     _datetime_index_name = (
-        sql_item_name(indices['datetime'], self.flavor, None)
-        if indices.get('datetime', None)
+        sql_item_name(index_names['datetime'], self.flavor, None)
+        if index_names.get('datetime', None)
         else None
     )
     _id = pipe.get_columns('id', error=False)
@@ -421,8 +422,8 @@ def get_create_index_queries(
     )
 
     _id_index_name = (
-        sql_item_name(indices['id'], self.flavor, None)
-        if indices.get('id', None)
+        sql_item_name(index_names['id'], self.flavor, None)
+        if index_names.get('id', None)
         else None
     )
     _pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
@@ -491,18 +492,22 @@ def get_create_index_queries(
         if id_query is not None:
             index_queries[_id] = id_query if isinstance(id_query, list) else [id_query]
 
-
     ### Create indices for other labels in `pipe.columns`.
-    other_indices = {
+    other_index_names = {
         ix_key: ix_unquoted
-        for ix_key, ix_unquoted in pipe.get_indices().items()
+        for ix_key, ix_unquoted in index_names.items()
         if ix_key not in ('datetime', 'id')
     }
-    for ix_key, ix_unquoted in other_indices.items():
+    for ix_key, ix_unquoted in other_index_names.items():
         ix_name = sql_item_name(ix_unquoted, self.flavor, None)
-        col = pipe.columns[ix_key]
-        col_name = sql_item_name(col, self.flavor, None)
-        index_queries[col] = [f"CREATE INDEX {ix_name} ON {_pipe_name} ({col_name})"]
+        cols = indices[ix_key]
+        if not isinstance(cols, (list, tuple)):
+            cols = [cols]
+        cols_names = [sql_item_name(col, self.flavor, None) for col in cols if col]
+        if not cols_names:
+            continue
+        cols_names_str = ", ".join(cols_names)
+        index_queries[ix_key] = [f"CREATE INDEX {ix_name} ON {_pipe_name} ({cols_names_str})"]
 
     existing_cols_types = pipe.get_columns_types(debug=debug)
     indices_cols_str = ', '.join(