meerschaum 2.4.12__tar.gz → 2.4.13__tar.gz

{meerschaum-2.4.12/meerschaum.egg-info → meerschaum-2.4.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.4.12
+Version: 2.4.13
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/_internal/entry.py

@@ -43,6 +43,7 @@ if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
     if _systemd_stdin_path:
         sys.stdin = _StdinFile(_systemd_stdin_path)
 
+
 def entry(
     sysargs: Optional[List[str]] = None,
     _patch_args: Optional[Dict[str, Any]] = None,

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/actions/bootstrap.py

@@ -19,7 +19,7 @@ def bootstrap(
 ) -> SuccessTuple:
     """
     Launch an interactive wizard to bootstrap pipes or connectors.
-    
+
     Example:
         `bootstrap pipes`
 
@@ -462,7 +462,6 @@ def _bootstrap_jobs(
     if not action:
         action = [prompt("What is the name of the job you'd like to create?")]
 
-    new_jobs = {}
     for name in action:
         clear_screen(debug=debug)
         job = mrsm.Job(name, executor_keys=executor_keys)
@@ -473,13 +472,14 @@ def _bootstrap_jobs(
             continue
 
         info(
-            "Press [Esc + Enter] to submit, [CTRL + C] to exit.\n"
+            f"Editing arguments for job '{name}'.\n"
+            "    Press [Esc + Enter] to submit, [CTRL + C] to exit.\n\n"
             "    Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
             "    https://meerschaum.io/reference/actions/#chaining-actions\n"
         )
         try:
             new_sysargs_str = prompt(
-                f"Arguments for job '{name}':",
+                "",
                 multiline=True,
                 icon=False,
                 completer=ShellCompleter(),
@@ -491,6 +491,7 @@ def _bootstrap_jobs(
         new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
         chained_sysargs = split_chained_sysargs(new_sysargs)
 
+        clear_screen(debug=debug)
         if len(chained_sysargs) > 1:
             print_options(
                 [
@@ -508,6 +509,7 @@ def _bootstrap_jobs(
         if pipeline_args:
             print('\n' + make_header("Pipeline Arguments:"))
             print(shlex.join(pipeline_args))
+            print()
 
         if not yes_no(
             (
@@ -524,15 +526,11 @@ def _bootstrap_jobs(
         if not start_success:
             return start_success, start_msg
 
-        new_jobs[name] = new_job
-
-    if not new_jobs:
-        return False, "No new jobs were created."
-
     msg = (
         "Successfully bootstrapped job"
-        + ('s' if len(new_jobs) != 1 else '')
-        + items_str(list(new_jobs.keys()))
+        + ('s' if len(action) != 1 else '')
+        + ' '
+        + items_str(action)
         + '.'
     )
     return True, msg

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/actions/delete.py

@@ -498,7 +498,7 @@ def _complete_delete_jobs(
     action: Optional[List[str]] = None,
     executor_keys: Optional[str] = None,
     line: str = '',
-    _get_job_method: Optional[str, List[str]] = None,
+    _get_job_method: Union[str, List[str], None] = None,
     **kw
 ) -> List[str]:
     from meerschaum._internal.shell.Shell import shell_attrs

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/actions/edit.py

@@ -38,10 +38,17 @@ def _complete_edit(
     """
     Override the default Meerschaum `complete_` function.
     """
+    from meerschaum.actions.delete import _complete_delete_jobs
+
+    if action is None:
+        action = []
+
     options = {
         'config': _complete_edit_config,
         'plugin': _complete_edit_plugins,
         'plugins': _complete_edit_plugins,
+        'job': _complete_delete_jobs,
+        'jobs': _complete_delete_jobs,
     }
 
     if action is None:
@@ -133,7 +140,7 @@ def _edit_pipes(
 
     pipes = get_pipes(debug=debug, as_list=True, **kw)
     if pipes:
-        print_options(pipes, header=f'Pipes to be edited:')
+        print_options(pipes, header='Pipes to be edited:')
     else:
         return False, "No pipes to edit."
 
@@ -144,10 +151,10 @@ def _edit_pipes(
                     f"Press [Enter] to begin editing the above {len(pipes)} pipe"
                     + ("s" if len(pipes) != 1 else "")
                     + " or [CTRL-C] to cancel:",
-                    icon = False,
+                    icon=False,
                 )
         except KeyboardInterrupt:
-            return False, f"No pipes changed."
+            return False, "No pipes changed."
 
     interactive = (not bool(params))
     success, msg = True, ""
@@ -393,41 +400,51 @@ def _edit_jobs(
     )
     from meerschaum.utils.formatting import make_header, print_options
     from meerschaum.utils.warnings import info
-    from meerschaum.actions import actions
+    from meerschaum._internal.shell.ShellCompleter import ShellCompleter
+    from meerschaum.utils.misc import items_str
+    from meerschaum.utils.formatting._shell import clear_screen
+
     jobs = get_filtered_jobs(executor_keys, action, debug=debug)
     if not jobs:
         return False, "No jobs to edit."
 
-    info(
-        "Press [Esc + Enter] to submit, [CTRL + C] to exit.\n"
-        "    Tip: join multiple actions with `+`, add pipeline arguments with `:`.\n"
-        "    https://meerschaum.io/reference/actions/#chaining-actions\n"
-    )
-
+    num_edited = 0
     for name, job in jobs.items():
         sysargs_str = shlex.join(job.sysargs)
+        clear_screen(debug=debug)
+        info(
+            f"Editing arguments for job '{name}'.\n"
+            "    Press [Esc + Enter] to submit, [CTRL + C] to exit.\n\n"
+            "    Tip: join actions with `+`, manage pipeline with `:`.\n"
+            "    https://meerschaum.io/reference/actions/#chaining-actions\n"
+        )
 
         try:
             new_sysargs_str = prompt(
-                f"Arguments for job '{name}':",
-                default_editable=sysargs_str.lstrip().rstrip(),
+                "",
+                default_editable=sysargs_str.lstrip().rstrip().replace(' + ', '\n+ '),
                 multiline=True,
                 icon=False,
+                completer=ShellCompleter(),
             )
         except KeyboardInterrupt:
             return True, "Nothing was changed."
 
+        if new_sysargs_str.strip() == sysargs_str.strip():
+            continue
+
         new_sysargs = shlex.split(new_sysargs_str)
         new_sysargs, pipeline_args = split_pipeline_sysargs(new_sysargs)
         chained_sysargs = split_chained_sysargs(new_sysargs)
 
+        clear_screen(debug=debug)
         if len(chained_sysargs) > 1:
             print_options(
                 [
                     shlex.join(step_sysargs)
                     for step_sysargs in chained_sysargs
                 ],
-                header=f"Steps in Job '{name}':",
+                header=f"\nSteps in Job '{name}':",
                 number_options=True,
                 **kwargs
             )
@@ -438,6 +455,7 @@ def _edit_jobs(
         if pipeline_args:
             print('\n' + make_header("Pipeline Arguments:"))
             print(shlex.join(pipeline_args))
+            print()
 
         if not yes_no(
             (
@@ -457,8 +475,16 @@ def _edit_jobs(
         start_success, start_msg = new_job.start()
         if not start_success:
             return start_success, start_msg
+        num_edited += 1
 
-    return True, "Success"
+    msg = (
+        "Successfully edit job"
+        + ('s' if len(jobs) != 1 else '')
+        + ' '
+        + items_str(list(jobs.keys()))
+        + '.'
+        ) if num_edited > 0 else "Nothing was edited."
+    return True, msg
 
 
 ### NOTE: This must be the final statement of the module.

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/actions/sh.py

@@ -9,14 +9,15 @@ NOTE: This action may be a huge security vulnerability
 
 from meerschaum.utils.typing import SuccessTuple, List, Any, Optional
 
+
 def sh(
-        action: Optional[List[str]] = None,
-        sub_args: Optional[List[str]] = None,
-        filtered_sysargs: Optional[List[str]] = None,
-        use_bash: bool = True,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    sub_args: Optional[List[str]] = None,
+    filtered_sysargs: Optional[List[str]] = None,
+    use_bash: bool = True,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Execute system commands.
     """
@@ -50,7 +51,7 @@ def sh(
         if len(action) != 0:
             try:
                 command_list += ["-c", shlex.join(cmd_list)]
-            except Exception as e:
+            except Exception:
                 command_list += ["-c", ' '.join(cmd_list)]
     else:
         if len(action) == 0:
@@ -64,8 +65,8 @@ def sh(
     try:
         process = subprocess.Popen(
             command_list,
-            shell = False,
-            env = os.environ,
+            shell=False,
+            env=os.environ,
         )
         exit_code = process.wait()
     except FileNotFoundError:

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/actions/start.py

@@ -541,6 +541,7 @@ def _start_pipeline(
     action: Optional[List[str]] = None,
     loop: bool = False,
     min_seconds: Union[float, int, None] = 1.0,
+    timeout_seconds: Optional[int] = None,
     params: Optional[Dict[str, Any]] = None,
     **kwargs
 ) -> SuccessTuple:
@@ -557,10 +558,39 @@ def _start_pipeline(
     `show version + show arguments :: --loop`
 
     """
+    import json
     import time
+    import sys
     from meerschaum._internal.entry import entry
     from meerschaum.utils.warnings import info, warn
     from meerschaum.utils.misc import is_int
+    from meerschaum.utils.venv import venv_exec
+    from meerschaum.utils.process import poll_process
+    fence_begin, fence_end = '<MRSM_RESULT>', '</MRSM_RESULT>'
+
+    success, msg = False, "Did not run pipeline."
+    def write_line(line):
+        nonlocal success, msg
+        decoded = line.decode('utf-8')
+        begin_index, end_index = decoded.find(fence_begin), decoded.find(fence_end)
+
+        ### Found the beginning of the return value.
+        ### Don't write the parsed success tuple message.
+        if begin_index >= 0:
+            success, msg = tuple(json.loads(
+                decoded[begin_index + len(fence_begin):end_index]
+            ))
+            return
+
+        print(decoded)
+
+    def timeout_handler(*args, **kw):
+        nonlocal success, msg
+        success, msg = False, (
+            f"Failed to execute pipeline within {timeout_seconds} second"
+            + ('s' if timeout_seconds != 1 else '') + '.'
+        )
+        write_line((fence_begin + json.dumps((success, msg)) + fence_end).encode('utf-8'))
 
     do_n_times = (
         int(action[0].lstrip('x'))
@@ -578,12 +608,38 @@ def _start_pipeline(
     if min_seconds is None:
         min_seconds = 1.0
 
+    def do_entry() -> None:
+        nonlocal success, msg
+        if timeout_seconds is None:
+            success, msg = entry(sub_args_line, _patch_args=patch_args)
+            return
+
+        sub_args_line_escaped = sub_args_line.replace("'", "<QUOTE>")
+        patch_args_escaped_str = json.dumps(patch_args).replace("'", "<QUOTE>")
+        src = (
+            "import json\n"
+            "from meerschaum._internal.entry import entry\n\n"
+            f"sub_args_line = '{sub_args_line_escaped}'.replace(\"<QUOTE>\", \"'\")\n"
+            f"patch_args = json.loads('{patch_args_escaped_str}'.replace('<QUOTE>', \"'\"))\n"
+            "success, msg = entry(sub_args_line, _patch_args=patch_args)\n"
+            f"print('{fence_begin}' + json.dumps((success, msg)) + '{fence_end}')"
+        )
+        proc = venv_exec(src, venv=None, as_proc=True)
+        poll_process(
+            proc,
+            write_line,
+            timeout_seconds,
+            timeout_handler,
+        )
+
     ran_n_times = 0
-    success, msg = False, "Did not run pipeline."
     def run_loop():
         nonlocal ran_n_times, success, msg
         while True:
-            success, msg = entry(sub_args_line, _patch_args=patch_args)
+            try:
+                do_entry()
+            except Exception as e:
+                warn(e)
             ran_n_times += 1
 
             if not loop and do_n_times == 1:

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/actions/sync.py

@@ -129,7 +129,6 @@ def _pipes_lap(
             stack=False,
         )
 
-
     def _task_label(count: int):
         return f"[cyan]Syncing {count} pipe{'s' if count != 1 else ''}..."
 
@@ -137,7 +136,6 @@ def _pipes_lap(
         _progress.add_task(_task_label(len(pipes)), start=True, total=len(pipes))
     ) if _progress is not None else None
 
-
     def worker_fn():
         while not stop_event.is_set():
             try:
@@ -188,7 +186,7 @@ def _pipes_lap(
                     decoded[begin_index + len(fence_begin):end_index]
                 ))
                 return
-            sys.stdout.buffer.write(line)
+            print(decoded)
 
         def timeout_handler(p, *args, **kw):
             success, msg = False, (
@@ -227,7 +225,7 @@ def _pipes_lap(
             write_line,
             timeout_seconds,
             timeout_handler,
-            (p,)
+            (p,),
         )
         return _success_tuple
 
@@ -249,18 +247,18 @@ def _pipes_lap(
 
 
 def _sync_pipes(
-        loop: bool = False,
-        min_seconds: int = 1,
-        unblock: bool = False,
-        verify: bool = False,
-        deduplicate: bool = False,
-        bounded: Optional[bool] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        shell: bool = False,
-        nopretty: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    loop: bool = False,
+    min_seconds: int = 1,
+    unblock: bool = False,
+    verify: bool = False,
+    deduplicate: bool = False,
+    bounded: Optional[bool] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    shell: bool = False,
+    nopretty: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Fetch and sync new data for pipes.
 

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/actions/upgrade.py

@@ -9,16 +9,17 @@ Upgrade your current Meerschaum environment
 from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Union
 
+
 def upgrade(
     action: Optional[List[str]] = None,
     **kw: Any
 ) -> SuccessTuple:
     """
     Upgrade Meerschaum, plugins, or packages.
-    
+
     Command:
         `upgrade {option}`
-    
+
     Example:
         `upgrade meerschaum`
     """
@@ -77,7 +78,7 @@ def _upgrade_meerschaum(
         if force:
             answer = True
         else:
-            answer = yes_no(f"Take down the stack?", default='y', yes=yes, noask=noask)
+            answer = yes_no("Take down the stack?", default='n', yes=yes, noask=noask)
 
         if answer:
             if debug:
@@ -95,7 +96,7 @@ def _upgrade_meerschaum(
     if debug:
         dprint('Upgrade meerschaum with dependencies: \"' + f'{dependencies}' + '\"')
     if not pip_install(install_name, venv=None, debug=debug):
-        return False, f"Failed to upgrade Meerschaum via pip."
+        return False, "Failed to upgrade Meerschaum via pip."
 
     if debug:
         dprint("Pulling new Docker images...")

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.12"
+__version__ = "2.4.13"

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/connectors/sql/_fetch.py

@@ -309,6 +309,8 @@ def _simple_fetch_query(
     """Build a fetch query from a pipe's definition."""
     from meerschaum.utils.sql import format_cte_subquery
     definition = get_pipe_query(pipe)
+    if definition is None:
+        raise ValueError(f"No SQL definition could be found for {pipe}.")
     return format_cte_subquery(definition, flavor, 'definition')
 
 

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/plugins/__init__.py

@@ -316,7 +316,11 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
 
         ### NOTE: Allow plugins to be installed via `pip`.
         packaged_plugin_paths = []
-        discovered_packaged_plugins_eps = entry_points(group='meerschaum.plugins')
+        try:
+            discovered_packaged_plugins_eps = entry_points(group='meerschaum.plugins')
+        except TypeError:
+            discovered_packaged_plugins_eps = []
+
         for ep in discovered_packaged_plugins_eps:
             module_name = ep.name
             for package_file_path in ep.dist.files:
@@ -330,7 +334,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
         if is_symlink(PLUGINS_RESOURCES_PATH) or not PLUGINS_RESOURCES_PATH.exists():
             try:
                 PLUGINS_RESOURCES_PATH.unlink()
-            except Exception as e:
+            except Exception:
                 pass
 
         PLUGINS_RESOURCES_PATH.mkdir(exist_ok=True)

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/plugins/bootstrap.py

@@ -47,7 +47,6 @@ IMPORTS_LINES: Dict[str, str] = {
     ),
 }
 
-### TODO: Add feature for custom connectors.
 FEATURE_LINES: Dict[str, str] = {
     'header': (
         "#! /usr/bin/env python3\n"
@@ -94,7 +93,7 @@ FEATURE_LINES: Dict[str, str] = {
         "class {plugin_name_capitalized}Connector(Connector):\n"
         "    \"\"\"Implement '{plugin_name_lower}' connectors.\"\"\"\n\n"
         "    REQUIRED_ATTRIBUTES: list[str] = []\n"
-        "    \n"
+        "\n"
         "    def fetch(\n"
         "        self,\n"
         "        pipe: mrsm.Pipe,\n"
@@ -149,11 +148,12 @@ FEATURE_LINES: Dict[str, str] = {
     ),
 }
 
+
 def bootstrap_plugin(
-        plugin_name: str,
-        debug: bool = False,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    plugin_name: str,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Prompt the user for features and create a plugin file.
     """
@@ -177,9 +177,9 @@ def bootstrap_plugin(
     features: List[str] = choose(
         "Which of the following features would you like to add to your plugin?",
         list(FEATURE_CHOICES.items()),
-        default = 'fetch',
-        multiple = True,
-        as_indices = True,
+        default='fetch',
+        multiple=True,
+        as_indices=True,
         **kwargs
     )
 
@@ -256,7 +256,7 @@ def bootstrap_plugin(
         _ = prompt(
             f"Press [Enter] to edit plugin '{plugin_name}',"
             + " [CTRL+C] to skip.",
-            icon = False,
+            icon=False,
         )
     except (KeyboardInterrupt, Exception):
         return True, "Success"
@@ -267,7 +267,7 @@ def bootstrap_plugin(
 
 def _get_plugins_dir_path() -> pathlib.Path:
     from meerschaum.config.paths import PLUGINS_DIR_PATHS
-    
+
     if not PLUGINS_DIR_PATHS:
         raise EnvironmentError("No plugin dir path could be found.")
 
@@ -278,9 +278,9 @@ def _get_plugins_dir_path() -> pathlib.Path:
         choose(
             "In which directory do you want to write your plugin?",
             [path.as_posix() for path in PLUGINS_DIR_PATHS],
-            numeric = True,
-            multiple = False,
-            default = PLUGINS_DIR_PATHS[0].as_posix(),
+            numeric=True,
+            multiple=False,
+            default=PLUGINS_DIR_PATHS[0].as_posix(),
         )
     )
  
@@ -290,7 +290,7 @@ def _ask_to_uninstall(plugin: mrsm.Plugin, **kwargs: Any) -> SuccessTuple:
     warn(f"Plugin '{plugin}' is already installed!", stack=False)
     uninstall_plugin = yes_no(
         f"Do you want to first uninstall '{plugin}'?",
-        default = 'n',
+        default='n',
         **kwargs
     )
     if not uninstall_plugin:

{meerschaum-2.4.12 → meerschaum-2.4.13}/meerschaum/utils/process.py

@@ -178,22 +178,32 @@ def run_process(
 
     return ret
 
+
 def poll_process(
-        proc: subprocess.Popen,
-        line_callback: Callable[[bytes], Any],
-        timeout_seconds: Union[int, float, None] = None,
-        timeout_callback: Optional[Callable[[Any], Any]] = None,
-        timeout_callback_args: Optional[Tuple[Any]] = None,
-        timeout_callback_kwargs: Optional[Dict[str, Any]] = None,
-    ) -> int:
+    proc: subprocess.Popen,
+    line_callback: Callable[[bytes], Any],
+    timeout_seconds: Union[int, float, None] = None,
+    timeout_callback: Optional[Callable[[Any], Any]] = None,
+    timeout_callback_args: Optional[Tuple[Any]] = None,
+    timeout_callback_kwargs: Optional[Dict[str, Any]] = None,
+) -> int:
     """
     Poll a process and execute a callback function for each line printed to the process's `stdout`.
     """
     from meerschaum.utils.threading import Timer
+    from meerschaum.utils.warnings import warn
 
     def timeout_handler():
         nonlocal timeout_callback_args, timeout_callback_kwargs
-        proc.terminate()
+        try:
+            if platform.system() != 'Windows':
+                ### The process being killed may have children.
+                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+            else:
+                proc.send_signal(signal.CTRL_BREAK_EVENT)
+            proc.terminate()
+        except Exception as e:
+            warn(f"Failed to kill process:\n{e}")
         if timeout_callback_args is None:
             timeout_callback_args = []
         if timeout_callback_kwargs is None: