meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0__py3-none-any.whl

meerschaum/actions/bootstrap.py

@@ -234,7 +234,13 @@ def _bootstrap_connectors(
     Prompt the user for the details necessary to create a Connector.
     """
     from meerschaum.connectors.parse import is_valid_connector_keys
-    from meerschaum.connectors import connectors, get_connector, types, custom_types
+    from meerschaum.connectors import (
+        connectors,
+        get_connector,
+        types,
+        custom_types,
+        _load_builtin_custom_connectors,
+    )
     from meerschaum.utils.prompt import prompt, yes_no, choose
     from meerschaum.config import get_config
     from meerschaum.config._edit import write_config
@@ -246,6 +252,7 @@ def _bootstrap_connectors(
 
     abort_tuple = False, "No connectors bootstrapped."
     _clear = get_config('shell', 'clear_screen', patch=True)
+    _load_builtin_custom_connectors()
 
     if action is None:
         action = []
@@ -262,8 +269,8 @@ def _bootstrap_connectors(
                 + '    See https://meerschaum.io/reference/connectors '
                 + 'for documentation on connectors.\n'
             ),
-            sorted(list(connectors)),
-            default = 'sql'
+            sorted([k for k in connectors if k != 'plugin']),
+            default='sql',
         )
     except KeyboardInterrupt:
         return abort_tuple
@@ -288,21 +295,30 @@ def _bootstrap_connectors(
             warn(f"Connector '{_type}:{_label}' already exists.", stack=False)
             overwrite = yes_no(
                 f"Do you want to overwrite connector '{_type}:{_label}'?",
-                default = 'n',
-                yes = yes,
-                noask = noask,
+                default='n',
+                yes=yes,
+                noask=noask,
             )
             if not overwrite and not force:
-                return False, f"No changes made to connector configuration."
-                break
+                return False, "No changes made to connector configuration."
+            break
         elif _label == "":
-            warn(f"Please enter a label.", stack=False)
+            warn("Please enter a label.", stack=False)
         else:
             break
 
     cls = types.get(_type)
     cls_required_attrs = getattr(cls, 'REQUIRED_ATTRIBUTES', [])
-    type_attributes = connector_attributes.get(_type, {'required': cls_required_attrs})
+    cls_optional_attrs = getattr(cls, 'OPTIONAL_ATTRIBUTES', [])
+    cls_default_attrs = getattr(cls, 'DEFAULT_ATTRIBUTES', {})
+    type_attributes = connector_attributes.get(
+        _type,
+        {
+            'required': cls_required_attrs,
+            'optional': cls_optional_attrs,
+            'default': cls_default_attrs,
+        }
+    )
 
     new_attributes = {}
     if 'flavors' in type_attributes:
@@ -320,9 +336,11 @@ def _bootstrap_connectors(
             return abort_tuple
         new_attributes['flavor'] = flavor
         required = sorted(list(connector_attributes[_type]['flavors'][flavor]['requirements']))
+        required = sorted(list(connector_attributes[_type]['flavors'][flavor]['optional']))
         default = type_attributes['flavors'][flavor].get('defaults', {})
     else:
         required = sorted(list(type_attributes.get('required', {})))
+        optional = sorted(list(type_attributes.get('optional', {})))
         default = type_attributes.get('default', {})
     info(
         f"Please answer the following questions to configure the new connector '{_type}:{_label}'."
@@ -330,13 +348,22 @@ def _bootstrap_connectors(
     )
     for r in required:
         try:
-            val = prompt(f"Value for {r}:")
+            default_val = str(default.get(r)) if r in default else None
+            val = prompt(f"Value for {r}:", default=default_val)
         except KeyboardInterrupt:
             continue
         if is_int(val):
             val = int(val)
         new_attributes[r] = val
 
+    for o in optional:
+        try:
+            val = prompt(f"Value for {o} (optional; empty to omit):")
+        except KeyboardInterrupt:
+            continue
+        if val:
+            new_attributes[o] = val
+
     for k, v in default.items():
         ### skip already configured attributes, (e.g. flavor or from required)
         if k in new_attributes:

meerschaum/actions/copy.py

@@ -115,11 +115,11 @@ def _copy_pipes(
         ):
             _new_pipe.sync(
                 p.get_data(
-                    debug = debug,
-                    as_iterator = True,
+                    debug=debug,
+                    as_iterator=True,
                     **kw
                 ),
-                debug = debug,
+                debug=debug,
                 **kw
             )
 

meerschaum/actions/delete.py

@@ -521,7 +521,10 @@ def _complete_delete_jobs(
         )
     )
 
-    if parse_executor_keys(executor_keys, construct=False) is None:
+    if (
+        executor_keys != 'systemd'
+        and parse_executor_keys(executor_keys, construct=False) is None
+    ):
         return []
 
     jobs = get_jobs(executor_keys, include_hidden=False)

meerschaum/actions/register.py

@@ -118,7 +118,6 @@ def _register_pipes(
             )
 
     success, message = True, "Success"
-    failed_message = ""
     failed_pipes = []
     success_pipes = []
     for p in pipes:
@@ -127,7 +126,6 @@ def _register_pipes(
         ss, msg = p.register(debug=debug)
         if not ss:
             warn(f"{msg}", stack=False)
-            success = False
             failed_pipes.append(p)
         else:
             success_pipes.append(p)
@@ -356,7 +354,7 @@ def _register_users(
 
     msg = (
         f"Finished registering {succeeded + failed} users." + '\n' +
-        f"  ({succeeded} succeeded, {failed} failed)"
+        f"    ({succeeded} succeeded, {failed} failed)"
     )
     return succeeded > 0, msg
 

meerschaum/actions/stack.py

@@ -9,34 +9,32 @@ Functions for running the Docker Compose stack
 from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Union
 
+
 def stack(
-        action: Optional[List[str]] = None,
-        sysargs: Optional[List[str]] = None,
-        sub_args: Optional[List[str]] = None,
-        yes: bool = False,
-        noask: bool = False,
-        force: bool = False,
-        debug: bool = False,
-        _capture_output: bool = False,
-        **kw: Any
-    ) -> Union[SuccessTuple, 'subprocess.Popen']:
+    action: Optional[List[str]] = None,
+    sysargs: Optional[List[str]] = None,
+    sub_args: Optional[List[str]] = None,
+    yes: bool = False,
+    noask: bool = False,
+    force: bool = False,
+    debug: bool = False,
+    _capture_output: bool = False,
+    **kw: Any
+) -> Union[SuccessTuple, 'subprocess.Popen']:
     """
     Control the Meerschaum stack with Docker Compose.
     Usage: `stack {command}`
-    
+
     Command: action[0]: default 'up'
         Docker Compose command to run. E.g. 'config' will print Docker Compose configuration
     """
     import subprocess
-    import contextlib
-    import io
     import os
     import sys
     import pathlib
     import meerschaum.config.stack
     from meerschaum.config.stack import NECESSARY_FILES, write_stack
     from meerschaum.config._paths import STACK_COMPOSE_PATH
-    from meerschaum.utils.prompt import yes_no
     import meerschaum.config
     from meerschaum.config._patch import apply_patch_to_config
     from meerschaum.utils.packages import (
@@ -47,9 +45,9 @@ def stack(
     from meerschaum.config import get_config
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn
-    from meerschaum.utils.formatting import ANSI
     from meerschaum.utils.misc import is_docker_available
     from meerschaum.config._read_config import search_and_substitute_config
+    from meerschaum.utils.prompt import yes_no
 
     stack_env_dict = apply_patch_to_config(
         os.environ.copy(),
@@ -81,7 +79,7 @@ def stack(
             break
     if bootstrap:
         write_stack(debug=debug)
-    else: 
+    else:
         sync_files(['stack'])
 
     ### define project name when starting containers
@@ -91,7 +89,7 @@ def stack(
             'stack', 'project_name', patch=True, substitute=True,
         )
     ]
-    
+
     ### Debug list used to include --log-level DEBUG, but the flag is not supported on Windows (?)
     debug_list = []
 
@@ -102,7 +100,7 @@ def stack(
         print(
             "To start the Docker service, run `sudo systemctl start docker` or `sudo dockerd`.\n"
             + "On Windows or MacOS, make sure Docker Desktop is running.",
-            file = sys.stderr,
+            file=sys.stderr,
         )
         return False, "Failed to connect to the Docker engine."
 
@@ -115,7 +113,7 @@ def stack(
 
     if not has_builtin_compose:
         _compose_venv = 'mrsm'
-        compose = attempt_import('compose', lazy=False, venv=_compose_venv, debug=debug)
+        _ = attempt_import('compose', lazy=False, venv=_compose_venv, debug=debug)
 
         ### If docker-compose is installed globally, don't use the `mrsm` venv.
         if not venv_contains_package('compose', _compose_venv):
@@ -129,6 +127,13 @@ def stack(
             if not pip_install('pyyaml', venv=_compose_venv, debug=debug):
                 warn(f"Unable to install `pyyaml` into venv '{_compose_venv}'.")
 
+    if 'down' in sysargs and '-v' in sysargs:
+        if not yes_no(
+            "Are you sure you want to drop volumes?\n    This cannot be undone!",
+            default='n',
+        ):
+            return False, "Nothing was dropped."
+
     cmd_list = [
         _arg
         for _arg in (settings_list + sysargs[1:])

meerschaum/actions/start.py

@@ -89,6 +89,7 @@ def _start_jobs(
     name: Optional[str] = None,
     sysargs: Optional[List[str]] = None,
     executor_keys: Optional[str] = None,
+    rm: bool = False,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -210,7 +211,7 @@ def _start_jobs(
 
     def _run_new_job(name: Optional[str] = None):
         name = name or get_new_daemon_name()
-        job = Job(name, sysargs, executor_keys=executor_keys)
+        job = Job(name, sysargs, executor_keys=executor_keys, delete_after_completion=rm)
         return job.start(debug=debug), name
 
     def _run_existing_job(name: str):
@@ -303,24 +304,20 @@ def _start_jobs(
 
 
 def _start_gui(
-        action: Optional[List[str]] = None,
-        mrsm_instance: Optional[str] = None,
-        port: Optional[int] = None,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    mrsm_instance: Optional[str] = None,
+    port: Optional[int] = None,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Start the Meerschaum GUI application.
     """
-    from meerschaum.utils.daemon import Daemon
-    from meerschaum.utils.process import run_process
     from meerschaum.utils.venv import venv_exec
-    from meerschaum.utils.packages import (
-        run_python_package, attempt_import, venv_contains_package, pip_install
-    )
+    from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.networking import find_open_ports, is_port_in_use
+    from meerschaum.utils.networking import find_open_ports
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum._internal.term.tools import is_webterm_running
     import platform
@@ -394,13 +391,14 @@ def _start_gui(
 
 
 def _start_webterm(
-        port: Optional[int] = None,
-        host: Optional[str] = None,
-        force: bool = False,
-        nopretty: bool = False,
-        sysargs: Optional[List[str]] = None,
-        **kw
-    ) -> SuccessTuple:
+    port: Optional[int] = None,
+    host: Optional[str] = None,
+    mrsm_instance: Optional[str] = None,
+    force: bool = False,
+    nopretty: bool = False,
+    sysargs: Optional[List[str]] = None,
+    **kw
+) -> SuccessTuple:
     """
     Start the Meerschaum Web Terminal.
     
@@ -408,15 +406,17 @@ def _start_webterm(
         - `-p`, `--port`
             The port to which the webterm binds.
             Defaults to 8765, and `--force` will choose the next available port.
-    
+
         - `--host`
             The host interface to which the webterm binds.
             Defaults to '127.0.0.1'.
+
+        - `-i`, '--instance'
+            The default instance to use for the Webterm shell.
     """
-    from meerschaum._internal.term import get_webterm_app_and_manager, tornado, tornado_ioloop
+    from meerschaum._internal.term import get_webterm_app_and_manager, tornado_ioloop
     from meerschaum._internal.term.tools import is_webterm_running
-    from meerschaum.utils.networking import find_open_ports, is_port_in_use
-    from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.networking import find_open_ports
     from meerschaum.utils.warnings import info
 
     if host is None:
@@ -425,7 +425,7 @@ def _start_webterm(
         port = 8765
     if sysargs is None:
         sysargs = ['start', 'webterm']
-    tornado_app, term_manager = get_webterm_app_and_manager()
+    tornado_app, term_manager = get_webterm_app_and_manager(instance_keys=mrsm_instance)
 
     if is_webterm_running(host, port):
         if force:
@@ -454,12 +454,12 @@ def _start_webterm(
 
 
 def _start_connectors(
-        action: Optional[List[str]] = None,
-        connector_keys: Optional[List[str]] = None,
-        min_seconds: int = 3,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    connector_keys: Optional[List[str]] = None,
+    min_seconds: int = 3,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Start polling connectors to verify a connection can be made.
     """
@@ -479,7 +479,7 @@ def _start_connectors(
     for keys in unique_keys:
         try:
             conn = parse_instance_keys(keys)
-        except Exception as e:
+        except Exception:
             warn(f"Invalid connector keys: '{keys}'. Skipping...", stack=False)
             continue
         valid_conns.append(conn)
@@ -487,20 +487,19 @@ def _start_connectors(
     if not valid_conns:
         return False, "No valid connector keys were provided."
 
-
     connected = {}
     def connect(conn):
         success = retry_connect(
             conn,
-            retry_wait = min_seconds,
-            enforce_chaining = False,
-            enforce_login = False,
-            print_on_connect = True,
-            debug = debug,
+            retry_wait=min_seconds,
+            enforce_chaining=False,
+            enforce_login=False,
+            print_on_connect=True,
+            debug=debug,
         )
         connected[conn] = success
         return success
-    
+
     pool = get_pool()
     try:
         pool.map(connect, valid_conns)
@@ -517,12 +516,12 @@ def _start_connectors(
 
     success = len(fails) == 0
     msg = (
-        f"Successfully started connector" + ('s' if len(successes) != 1 else '')
+        "Successfully started connector" + ('s' if len(successes) != 1 else '')
         + ' ' + items_str(successes) + '.'
     ) if success else f"Failed to start {len(fails)} connectors."
     if len(fails) > 0:
         msg += (
-            f"\n    Failed to start connector" + ('s' if len(fails) != 1 else '')
+            "\n    Failed to start connector" + ('s' if len(fails) != 1 else '')
             + ' ' + items_str(fails) + '.'
         )
 
@@ -568,6 +567,7 @@ def _start_pipeline(
         else 1
     )
 
+    params = params or {}
     sub_args_line = params.get('sub_args_line', None)
     patch_args = params.get('patch_args', None)
 

meerschaum/actions/sync.py

@@ -9,10 +9,12 @@ NOTE: `sync` required a SQL connection and is not intended for client use
 """
 
 from __future__ import annotations
-from datetime import timedelta, datetime, timezone
+from datetime import timedelta
+
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Tuple, Union
 
+
 def sync(
     action: Optional[List[str]] = None,
     **kw: Any
@@ -38,47 +40,45 @@ def sync(
 
 
 def _pipes_lap(
-        workers: Optional[int] = None,
-        debug: Optional[bool] = None,
-        unblock: bool = False,
-        force: bool = False,
-        min_seconds: int = 1,
-        verify: bool = False,
-        deduplicate: bool = False,
-        bounded: Optional[bool] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        mrsm_instance: Optional[str] = None,
-        timeout_seconds: Optional[int] = None,
-        nopretty: bool = False,
-        _progress: Optional['rich.progress.Progress'] = None,
-        **kw: Any
-    ) -> Tuple[List[mrsm.Pipe], List[mrsm.Pipe]]:
+    workers: Optional[int] = None,
+    debug: Optional[bool] = None,
+    unblock: bool = False,
+    force: bool = False,
+    min_seconds: int = 1,
+    verify: bool = False,
+    deduplicate: bool = False,
+    bounded: Optional[bool] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    mrsm_instance: Optional[str] = None,
+    timeout_seconds: Optional[int] = None,
+    nopretty: bool = False,
+    _progress: Optional['rich.progress.Progress'] = None,
+    **kw: Any
+) -> Tuple[List[mrsm.Pipe], List[mrsm.Pipe]]:
     """
     Do a lap of syncing pipes.
     """
+    import queue
+    import multiprocessing
+    import time
+    import copy
+    import json
+    import sys
+
     from meerschaum import get_pipes
     from meerschaum.utils.debug import dprint, _checkpoint
-    from meerschaum.utils.packages import attempt_import, import_rich
-    from meerschaum.utils.formatting import print_tuple, ANSI, UNICODE, get_console
+    from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.formatting import print_tuple
     from meerschaum.utils.warnings import warn
-    from meerschaum.utils.threading import Lock, RLock, Thread, Event
-    from meerschaum.utils.misc import print_options, get_cols_lines
-    from meerschaum.utils.pool import get_pool_executor, get_pool
+    from meerschaum.utils.threading import Lock, Thread, Event
     from meerschaum.connectors.parse import parse_instance_keys
-    from meerschaum import Plugin
-    import queue
-    import multiprocessing
-    import contextlib
-    import time, os, copy
     from meerschaum.utils.packages import venv_exec
     from meerschaum.utils.process import poll_process
-    import json
-    import sys
     dill = attempt_import('dill')
 
     rich_table, rich_text, rich_box = attempt_import(
         'rich.table', 'rich.text', 'rich.box',
-        lazy = False,
+        lazy=False,
     )
     all_kw = copy.deepcopy(kw)
     all_kw.update({
@@ -126,7 +126,7 @@ def _pipes_lap(
             + "may lead to concurrency issues.\n    You can change the pool size with "
             + "`edit config system` under the keys connectors:sql:create_engine:pool_size,\n    "
             + "and a size of 0 will not limit the number of connections.",
-            stack = False,
+            stack=False,
         )
 
 
@@ -155,8 +155,8 @@ def _pipes_lap(
                 ) + msg + '\n'
                 print_tuple(
                     (success, msg),
-                    calm = True,
-                    _progress = _progress,
+                    calm=True,
+                    _progress=_progress,
                 )
             _checkpoint(_progress=_progress, _task=_task)
             if _progress is not None:
@@ -274,25 +274,26 @@ def _sync_pipes(
         - `--debug`
             - Print verbose messages.
     """
-    from meerschaum.utils.debug import dprint
+    import time
+    import os
+    import contextlib
+
     from meerschaum.utils.warnings import warn, info
-    from meerschaum.utils.formatting import UNICODE
-    from meerschaum.utils.formatting._shell import progress, live
-    from meerschaum.utils.formatting._shell import clear_screen, flush_with_newlines
+    from meerschaum.utils.formatting._shell import progress
+    from meerschaum.utils.formatting._shell import clear_screen
     from meerschaum.utils.formatting import print_pipes_results
-    import contextlib
-    import time
-    import sys
-    import json
-    import asyncio
+    from meerschaum.config.static import STATIC_CONFIG
+
+    noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
+    is_noninterative = noninteractive_val in ('1', 'true', 'True', 'yes')
+
     run = True
     msg = ""
     interrupt_warning_msg = "Syncing was interrupted due to a keyboard interrupt."
     cooldown = 2 * (min_seconds + 1)
-    underline = '\u2015' if UNICODE else '-'
     success_pipes, failure_pipes = [], []
     while run:
-        _progress = progress() if shell else None
+        _progress = progress() if shell and not is_noninterative else None
         cm = _progress if _progress is not None else contextlib.nullcontext()
 
         lap_begin = time.perf_counter()
@@ -301,15 +302,15 @@ def _sync_pipes(
             results_dict = {}
             with cm:
                 results_dict = _pipes_lap(
-                    min_seconds = min_seconds,
-                    _progress = _progress,
-                    verify = verify,
-                    deduplicate = deduplicate,
-                    bounded = bounded,
-                    chunk_interval = chunk_interval,
-                    unblock = unblock,
-                    debug = debug,
-                    nopretty = nopretty,
+                    min_seconds=min_seconds,
+                    _progress=_progress,
+                    verify=verify,
+                    deduplicate=deduplicate,
+                    bounded=bounded,
+                    chunk_interval=chunk_interval,
+                    unblock=unblock,
+                    debug=debug,
+                    nopretty=nopretty,
                     **kw
                 )
                 success_pipes = [