meerschaum 2.7.4__tar.gz → 2.7.6__tar.gz

{meerschaum-2.7.4/meerschaum.egg-info → meerschaum-2.7.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.7.4
+Version: 2.7.6
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -127,7 +127,7 @@ Provides-Extra: sql
 Requires-Dist: numpy>=1.18.5; extra == "sql"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "sql"
 Requires-Dist: pyarrow>=16.1.0; extra == "sql"
-Requires-Dist: dask[complete]>=2024.5.1; extra == "sql"
+Requires-Dist: dask[complete]>=2024.12.1; extra == "sql"
 Requires-Dist: partd>=1.4.2; extra == "sql"
 Requires-Dist: pytz; extra == "sql"
 Requires-Dist: joblib>=0.17.0; extra == "sql"
@@ -187,7 +187,7 @@ Requires-Dist: valkey>=6.0.0; extra == "api"
 Requires-Dist: numpy>=1.18.5; extra == "api"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "api"
 Requires-Dist: pyarrow>=16.1.0; extra == "api"
-Requires-Dist: dask[complete]>=2024.5.1; extra == "api"
+Requires-Dist: dask[complete]>=2024.12.1; extra == "api"
 Requires-Dist: partd>=1.4.2; extra == "api"
 Requires-Dist: pytz; extra == "api"
 Requires-Dist: joblib>=0.17.0; extra == "api"
@@ -292,7 +292,7 @@ Requires-Dist: pycparser>=2.21.0; extra == "full"
 Requires-Dist: numpy>=1.18.5; extra == "full"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "full"
 Requires-Dist: pyarrow>=16.1.0; extra == "full"
-Requires-Dist: dask[complete]>=2024.5.1; extra == "full"
+Requires-Dist: dask[complete]>=2024.12.1; extra == "full"
 Requires-Dist: partd>=1.4.2; extra == "full"
 Requires-Dist: pytz; extra == "full"
 Requires-Dist: joblib>=0.17.0; extra == "full"

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/_internal/shell/Shell.py

@@ -742,8 +742,7 @@ class Shell(cmd.Cmd):
         """
         from meerschaum import get_connector
         from meerschaum.connectors.parse import parse_instance_keys
-        from meerschaum.utils.warnings import warn, info
-        from meerschaum.utils.misc import remove_ansi
+        from meerschaum.utils.warnings import info
 
         if action is None:
             action = []
@@ -829,7 +828,7 @@ class Shell(cmd.Cmd):
         """
         from meerschaum import get_connector
         from meerschaum.connectors.parse import parse_repo_keys
-        from meerschaum.utils.warnings import warn, info
+        from meerschaum.utils.warnings import info
 
         if action is None:
             action = []
@@ -878,7 +877,6 @@ class Shell(cmd.Cmd):
         
         Note that executors are API instances.
         """
-        from meerschaum import get_connector
         from meerschaum.connectors.parse import parse_executor_keys
         from meerschaum.utils.warnings import warn, info
         from meerschaum.jobs import get_executor_keys_from_context
@@ -894,7 +892,7 @@ class Shell(cmd.Cmd):
             executor_keys = get_executor_keys_from_context()
 
         if executor_keys == 'systemd' and get_executor_keys_from_context() != 'systemd':
-            warn(f"Cannot execute via `systemd`, falling back to `local`...", stack=False)
+            warn("Cannot execute via `systemd`, falling back to `local`...", stack=False)
             executor_keys = 'local'
         
         conn = parse_executor_keys(executor_keys, debug=debug)
@@ -935,7 +933,7 @@ class Shell(cmd.Cmd):
         if args['action'][0] not in shell_attrs['_actions']:
             try:
                 print(textwrap.dedent(getattr(self, f"do_{args['action'][0]}").__doc__))
-            except Exception as e:
+            except Exception:
                 print(f"No help on '{args['action'][0]}'.")
             return ""
         parse_help(args)

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/_internal/shell/ShellCompleter.py

@@ -7,13 +7,15 @@ Implement the prompt_toolkit Completer base class.
 """
 
 from __future__ import annotations
-from prompt_toolkit.completion import Completer, Completion
-from meerschaum.utils.typing import Optional
-from meerschaum.actions import get_shell, get_completer, get_main_action_name, get_action
+from meerschaum.actions import get_shell, get_main_action_name, get_action
 from meerschaum._internal.arguments import parse_line
 
 from meerschaum.utils.packages import attempt_import, ensure_readline
-prompt_toolkit = attempt_import('prompt_toolkit', lazy=False, install=True)
+
+prompt_toolkit_completion = attempt_import('prompt_toolkit.completion', lazy=False, install=True)
+Completer = prompt_toolkit_completion.Completer
+Completion = prompt_toolkit_completion.Completion
+
 
 class ShellCompleter(Completer):
     """
@@ -30,7 +32,6 @@ class ShellCompleter(Completer):
         yielded = []
         ensure_readline()
         parts = document.text.split('-')
-        ends_with_space = parts[0].endswith(' ')
         last_action_line = parts[0].split('+')[-1]
         part_0_subbed_spaces = last_action_line.replace(' ', '_')
         parsed_text = (part_0_subbed_spaces + '-'.join(parts[1:]))

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/actions/clear.py

@@ -130,12 +130,15 @@ def _ask_with_rowcounts(
                     )
 
 
-    pipes_rowcounts = {p: p.get_rowcount(begin=begin, end=end, debug=debug) for p in pipes} 
+    pipes_rowcounts = {
+        pipe: pipe.get_rowcount(begin=begin, end=end, debug=debug)
+        for pipe in pipes
+    }
     print_options(
-        [str(p) + f'\n{rc}\n' for p, rc in pipes_rowcounts.items()],
+        [str(pipe) + f'\n{rowcount:,}\n' for pipe, rowcount in pipes_rowcounts.items()],
         header='Number of Rows to be Deleted'
     )
-    total_num_rows = sum([rc for p, rc in pipes_rowcounts.items()])
+    total_num_rows = sum([rowcount for rowcount in pipes_rowcounts.values()])
     question = (
         f"Are you sure you want to delete {total_num_rows:,} rows across {len(pipes)} pipe"
         + ('s' if len(pipes) != 1 else '')

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/actions/copy.py

@@ -7,7 +7,7 @@ Functions for copying elements.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional, Tuple, List
+from meerschaum.utils.typing import Any, SuccessTuple, Optional, List
 
 def copy(
     action: Optional[List[str]] = None,
@@ -69,44 +69,53 @@ def _copy_pipes(
     Copy pipes' attributes and make new pipes.
     """
     from meerschaum import get_pipes, Pipe
-    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum.utils.prompt import prompt, yes_no, get_connectors_completer
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.formatting import print_tuple
     from meerschaum.utils.formatting._shell import clear_screen
     pipes = get_pipes(as_list=True, **kw)
     successes = 0
-    for p in pipes:
-        ck = prompt(f"Connector keys for copy of {p}:", default=p.connector_keys)
-        mk = prompt(f"Metric key for copy of {p}:", default=p.metric_key)
-        lk = prompt(f"Location key for copy of {p} ('None' to omit):", default=str(p.location_key))
+    for pipe in pipes:
+        ck = prompt(
+            f"Connector keys for copy of {pipe}:",
+            default=pipe.connector_keys,
+            completer=get_connectors_completer(),
+        )
+        mk = prompt(f"Metric key for copy of {pipe}:", default=pipe.metric_key)
+        lk = prompt(
+            f"Location key for copy of {pipe} ('None' to omit):",
+            default=str(pipe.location_key),
+        )
         if lk in ('', 'None', '[None]'):
             lk = None
-        _new_pipe = Pipe(
-            ck, mk, lk,
-            parameters=p.parameters.copy(),
-        )
+
         instance_keys = prompt(
-            f"Meerschaum instance to store the new {_new_pipe}:",
-            default=p.instance_keys
+            f"Meerschaum instance for copy of {pipe}:",
+            default=pipe.instance_keys
         )
-        _new_pipe.instance_keys = instance_keys
-        if _new_pipe.get_id(debug=debug) is not None:
-            warn(f"New {_new_pipe} already exists. Skipping...", stack=False)
+        new_pipe = Pipe(
+            ck, mk, lk,
+            instance=instance_keys,
+            parameters=pipe.parameters.copy(),
+        )
+
+        if new_pipe.get_id(debug=debug) is not None:
+            warn(f"{new_pipe} already exists. Skipping...", stack=False)
             continue
-        _register_success_tuple = _new_pipe.register(debug=debug)
+        _register_success_tuple = new_pipe.register(debug=debug)
         if not _register_success_tuple[0]:
-            warn(f"Failed to register new {_new_pipe}.", stack=False)
+            warn(f"Failed to register new {new_pipe}.", stack=False)
             continue
 
         clear_screen(debug=debug)
         successes += 1
         print_tuple(
-            (True, f"Successfully copied attributes of {p} " + f" into {_new_pipe}.")
+            (True, f"Successfully copied attributes of {pipe} " + f" into {new_pipe}.")
         )
         if (
             force or yes_no(
                 (
-                    f"Do you want to copy data from {p} into {_new_pipe}?\n\n"
+                    f"Do you want to copy data from {pipe} into {new_pipe}?\n\n"
                     + "If you specified `--begin`, `--end` or `--params`, data will be filtered."
                 ),
                     noask=noask,
@@ -114,8 +123,8 @@ def _copy_pipes(
                     default='n',
                 )
         ):
-            _new_pipe.sync(
-                p.get_data(
+            new_pipe.sync(
+                pipe.get_data(
                     debug=debug,
                     as_iterator=True,
                     **kw
@@ -142,17 +151,14 @@ def _copy_connectors(
 ) -> SuccessTuple:
     """
     Create a new connector from an existing one.
-
     """
-    import os, pathlib
     from meerschaum.utils.prompt import yes_no, prompt
     from meerschaum.connectors.parse import parse_connector_keys
     from meerschaum.config import _config, get_config
-    from meerschaum.config._edit import write_config
-    from meerschaum.utils.warnings import info, warn
+    from meerschaum.utils.warnings import info
     from meerschaum.utils.formatting import pprint
     from meerschaum.actions import get_action
-    cf = _config()
+    _ = _config()
     if action is None:
         action = []
     if connector_keys is None:
@@ -170,7 +176,7 @@ def _copy_connectors(
 
     try:
         conn = parse_connector_keys(ck)
-    except Exception as e:
+    except Exception:
         return False, f"Unable to parse connector '{ck}'."
 
     if len(_keys) == 2:

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/actions/sql.py

@@ -21,7 +21,7 @@ def sql(
     nopretty: bool = False,
     debug: bool = False,
     **kw: Any
-):
+) -> SuccessTuple:
     """Execute a SQL query or launch an interactive CLI. All positional arguments are optional.
 
     Usage:
@@ -125,10 +125,9 @@ def sql(
     if result is False:
         return (False, f"Failed to execute query:\n\n{query}")
     
-    from meerschaum.utils.packages import attempt_import, import_pandas
+    from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.formatting import print_tuple, pprint
-    sqlalchemy_engine_result = attempt_import('sqlalchemy.engine.result')
-    pd = import_pandas()
+    _ = attempt_import('sqlalchemy.engine.result')
     if 'sqlalchemy' in str(type(result)):
         if not nopretty:
             print_tuple((True, f"Successfully executed query:\n\n{query}"))
@@ -145,3 +144,14 @@ def sql(
             )
 
     return (True, "Success")
+
+
+def _complete_sql(
+    action: Optional[List[str]] = None, **kw: Any
+) -> List[str]:
+    from meerschaum.utils.misc import get_connector_labels
+    _text = action[0] if action else ""
+    return [
+        label.split('sql:', maxsplit=1)[-1]
+        for label in get_connector_labels('sql', search_term=_text, ignore_exact_match=True)
+    ]

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/actions/sync.py

@@ -275,12 +275,14 @@ def _sync_pipes(
     import time
     import os
     import contextlib
+    from datetime import timedelta
 
     from meerschaum.utils.warnings import warn, info
     from meerschaum.utils.formatting._shell import progress
     from meerschaum.utils.formatting._shell import clear_screen
     from meerschaum.utils.formatting import print_pipes_results
     from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum.utils.misc import interval_str
 
     noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
     noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
@@ -321,7 +323,7 @@ def _sync_pipes(
                     for pipe, (_success, _msg) in results_dict.items()
                     if not _success
                 ]
-        except Exception as e:
+        except Exception:
             import traceback
             traceback.print_exc()
             warn(
@@ -351,9 +353,9 @@ def _sync_pipes(
         success_msg = (
             "Successfully spawned threads for pipes:"
             if unblock
-            else f"Successfully synced pipes:"
+            else "Successfully synced pipes:"
         )
-        fail_msg = f"Failed to sync pipes:"
+        fail_msg = "Failed to sync pipes:"
         if results_dict:
             print_pipes_results(
                 results_dict,
@@ -362,8 +364,10 @@ def _sync_pipes(
                 nopretty = nopretty,
             )
 
+        lap_duration_text = interval_str(timedelta(seconds=(lap_end - lap_begin)))
+
         msg = (
-            f"It took {round(lap_end - lap_begin, 2)} seconds to sync " +
+            f"It took {lap_duration_text} to sync " +
             f"{len(success_pipes) + len(failure_pipes)} pipe" +
                 ("s" if (len(success_pipes) + len(failure_pipes)) != 1 else "") + "\n" +
             f"    ({len(success_pipes)} succeeded, {len(failure_pipes)} failed)."
@@ -385,26 +389,26 @@ def _sync_pipes(
 
 
 def _wrap_pipe(
-        pipe,
-        unblock: bool = False,
-        force: bool = False,
-        debug: bool = False,
-        min_seconds: int = 1,
-        workers = None,
-        verify: bool = False,
-        deduplicate: bool = False,
-        bounded: Optional[bool] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        **kw
-    ):
+    pipe,
+    unblock: bool = False,
+    force: bool = False,
+    debug: bool = False,
+    min_seconds: int = 1,
+    workers = None,
+    verify: bool = False,
+    deduplicate: bool = False,
+    bounded: Optional[bool] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    **kw
+):
     """
     Wrapper function for handling exceptions.
     """
     import time
     import traceback
-    from datetime import datetime, timedelta, timezone
+    from datetime import datetime, timezone
     import meerschaum as mrsm
-    from meerschaum.utils.typing import is_success_tuple, SuccessTuple
+    from meerschaum.utils.typing import is_success_tuple
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.venv import Venv
     from meerschaum.plugins import _pre_sync_hooks, _post_sync_hooks

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/api/dash/pipes.py

@@ -12,7 +12,6 @@ import shlex
 from textwrap import dedent
 from urllib.parse import urlencode
 
-from dash.dependencies import Input, Output, State
 from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
 from meerschaum.utils.misc import string_to_dict
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
@@ -76,7 +75,7 @@ def keys_from_state(
         try:
             #  params = string_to_dict(state['params-textarea.value'])
             params = string_to_dict(state['search-parameters-editor.value'])
-        except Exception as e:
+        except Exception:
             params = None
     else:
         params = None
@@ -506,7 +505,7 @@ def accordion_items_from_pipe(
 
         stats_rows = []
         if rowcount is not None:
-            stats_rows.append(html.Tr([html.Td("Row-count"), html.Td(f"{rowcount}")]))
+            stats_rows.append(html.Tr([html.Td("Row Count"), html.Td(f"{rowcount:,}")]))
         if interval is not None:
             stats_rows.append(
                 html.Tr([html.Td("Timespan"), html.Td(humanfriendly.format_timespan(interval))])

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/config/_default.py

@@ -68,8 +68,19 @@ default_system_config = {
             'pandas': 'pandas',
         },
         'sql': {
+            'bulk_insert': {
+                'postgresql': True,
+                'citus': True,
+                'timescaledb': True,
+                'mssql': True,
+            },
             'instance': {
                 'stale_temporary_tables_minutes': 1440,
+                'temporary_target': {
+                    'prefix': '_',
+                    'transaction_id_length': 4,
+                    'separator': '_',
+                },
             },
             'chunksize': 100_000,
             'poolclass': 'sqlalchemy.pool.QueuePool',

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.7.4"
+__version__ = "2.7.6"

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/connectors/api/_misc.py

@@ -20,12 +20,13 @@ def get_mrsm_version(self, **kw) -> Optional[str]:
             use_token=False,
             **kw
         ).json()
-    except Exception as e:
+    except Exception:
         return None
     if isinstance(j, dict) and 'detail' in j:
         return None
     return j
 
+
 def get_chaining_status(self, **kw) -> Optional[bool]:
     """
     Fetch the chaining status of the API instance.
@@ -39,7 +40,7 @@ def get_chaining_status(self, **kw) -> Optional[bool]:
         )
         if not response:
             return None
-    except Exception as e:
+    except Exception:
         return None
 
     return response.json()

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/connectors/api/_pipes.py

@@ -9,8 +9,7 @@ Register or fetch Pipes from the API
 from __future__ import annotations
 import time
 import json
-from io import StringIO
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
@@ -178,11 +177,11 @@ def sync_pipe(
     """Sync a DataFrame into a Pipe."""
     from decimal import Decimal
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.misc import json_serialize_datetime, items_str
+    from meerschaum.utils.misc import json_serialize_datetime, items_str, interval_str
     from meerschaum.config import get_config
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.dataframe import get_numeric_cols, to_json, get_bytes_cols
-    begin = time.time()
+    from meerschaum.utils.dataframe import get_numeric_cols, to_json
+    begin = time.perf_counter()
     more_itertools = attempt_import('more_itertools')
     if df is None:
         msg = f"DataFrame is `None`. Cannot sync {pipe}."
@@ -304,9 +303,10 @@ def sync_pipe(
         num_success_chunks += 1
 
     success_tuple = True, (
-        f"It took {round(time.time() - begin, 2)} seconds to sync {rowcount} row"
+        f"It took {interval_str(timedelta(seconds=(time.perf_counter() - begin)))} "
+        + "to sync {rowcount:,} row"
         + ('s' if rowcount != 1 else '')
-        + f" across {num_success_chunks} chunk" + ('s' if num_success_chunks != 1 else '') +
+        + f" across {num_success_chunks:,} chunk" + ('s' if num_success_chunks != 1 else '') +
         f" to {pipe}."
     )
     return success_tuple
@@ -549,10 +549,9 @@ def create_metadata(
     if debug:
         dprint("Create metadata response: {response.text}")
     try:
-        metadata_response = json.loads(response.text)
+        _ = json.loads(response.text)
     except Exception as e:
         warn(f"Failed to create metadata on {self}:\n{e}")
-        metadata_response = False
     return False
 
 

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/connectors/sql/_SQLConnector.py

@@ -69,6 +69,7 @@ class SQLConnector(Connector):
         get_pipe_schema,
         create_pipe_table_from_df,
         get_pipe_columns_indices,
+        get_temporary_target,
     )
     from ._plugins import (
         register_plugin,

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/connectors/sql/_cli.py

@@ -26,46 +26,52 @@ flavor_clis = {
     'duckdb'      : 'gadwall',
 }
 cli_deps = {
-    'pgcli': ['pgspecial', 'pendulum'],
+    'pgcli': ['pgspecial', 'pendulum', 'cli_helpers'],
     'mycli': ['cryptography'],
+    'mssql': ['cli_helpers'],
 }
 
 
 def cli(
-        self,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    debug: bool = False,
+) -> SuccessTuple:
     """
     Launch a subprocess for an interactive CLI.
     """
+    from meerschaum.utils.warnings import dprint
     from meerschaum.utils.venv import venv_exec
     env = copy.deepcopy(dict(os.environ))
-    env[f'MRSM_SQL_{self.label.upper()}'] = json.dumps(self.meta)
+    env_key = f"MRSM_SQL_{self.label.upper()}"
+    env_val = json.dumps(self.meta)
+    env[env_key] = env_val
     cli_code = (
         "import sys\n"
         "import meerschaum as mrsm\n"
+        "import os\n"
         f"conn = mrsm.get_connector('sql:{self.label}')\n"
         "success, msg = conn._cli_exit()\n"
         "mrsm.pprint((success, msg))\n"
         "if not success:\n"
         "    raise Exception(msg)"
     )
+    if debug:
+        dprint(cli_code)
     try:
-        _ = venv_exec(cli_code, venv=None, debug=debug, capture_output=False)
+        _ = venv_exec(cli_code, venv=None, env=env, debug=debug, capture_output=False)
     except Exception as e:
         return False, f"[{self}] Failed to start CLI:\n{e}"
     return True, "Success"
 
 
 def _cli_exit(
-        self,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    debug: bool = False
+) -> SuccessTuple:
     """Launch an interactive CLI for the SQLConnector's flavor."""
-    from meerschaum.utils.packages import venv_exec, attempt_import
+    import  os
+    from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.warnings import error
-    import sys, subprocess, os
 
     if self.flavor not in flavor_clis:
         return False, f"No CLI available for flavor '{self.flavor}'."

{meerschaum-2.7.4 → meerschaum-2.7.6}/meerschaum/connectors/sql/_create_engine.py

@@ -61,6 +61,7 @@ flavor_configs = {
         'engine': 'mssql+pyodbc',
         'create_engine': {
             'fast_executemany': True,
+            'use_insertmanyvalues': False,
             'isolation_level': 'AUTOCOMMIT',
             'use_setinputsizes': False,
             'pool_pre_ping': True,