meerschaum 2.7.3__tar.gz → 2.7.5__tar.gz

{meerschaum-2.7.3/meerschaum.egg-info → meerschaum-2.7.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.7.3
+Version: 2.7.5
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/actions/sql.py

@@ -21,7 +21,7 @@ def sql(
     nopretty: bool = False,
     debug: bool = False,
     **kw: Any
-):
+) -> SuccessTuple:
     """Execute a SQL query or launch an interactive CLI. All positional arguments are optional.
 
     Usage:
@@ -125,10 +125,9 @@ def sql(
     if result is False:
         return (False, f"Failed to execute query:\n\n{query}")
     
-    from meerschaum.utils.packages import attempt_import, import_pandas
+    from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.formatting import print_tuple, pprint
-    sqlalchemy_engine_result = attempt_import('sqlalchemy.engine.result')
-    pd = import_pandas()
+    _ = attempt_import('sqlalchemy.engine.result')
     if 'sqlalchemy' in str(type(result)):
         if not nopretty:
             print_tuple((True, f"Successfully executed query:\n\n{query}"))
@@ -145,3 +144,14 @@ def sql(
             )
 
     return (True, "Success")
+
+
+def _complete_sql(
+    action: Optional[List[str]] = None, **kw: Any
+) -> List[str]:
+    from meerschaum.utils.misc import get_connector_labels
+    _text = action[0] if action else ""
+    return [
+        label.split('sql:', maxsplit=1)[-1]
+        for label in get_connector_labels('sql', search_term=_text, ignore_exact_match=True)
+    ]

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/actions/start.py

@@ -183,7 +183,7 @@ def _start_jobs(
         ### Cannot find dameon_id
         else:
             msg = (
-                f"Unknown job" + ('s' if len(action) != 1 else '') + ' '
+                "Unknown job" + ('s' if len(action) != 1 else '') + ' '
                 + items_str(action, and_str='or') + '.'
             )
             return False, msg
@@ -216,7 +216,7 @@ def _start_jobs(
         return job.start(debug=debug), name
 
     def _run_existing_job(name: str):
-        job = Job(name, executor_keys=executor_keys)
+        job = jobs.get(name, Job(name, executor_keys=executor_keys))
         return job.start(debug=debug), name
 
     if not names:

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/config/_default.py

@@ -68,6 +68,12 @@ default_system_config = {
             'pandas': 'pandas',
         },
         'sql': {
+            'bulk_insert': {
+                'postgresql': True,
+                'citus': True,
+                'timescaledb': True,
+                'mssql': True,
+            },
             'instance': {
                 'stale_temporary_tables_minutes': 1440,
             },

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.7.3"
+__version__ = "2.7.5"

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/connectors/sql/_cli.py

@@ -26,46 +26,52 @@ flavor_clis = {
     'duckdb'      : 'gadwall',
 }
 cli_deps = {
-    'pgcli': ['pgspecial', 'pendulum'],
+    'pgcli': ['pgspecial', 'pendulum', 'cli_helpers'],
     'mycli': ['cryptography'],
+    'mssql': ['cli_helpers'],
 }
 
 
 def cli(
-        self,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    debug: bool = False,
+) -> SuccessTuple:
     """
     Launch a subprocess for an interactive CLI.
     """
+    from meerschaum.utils.warnings import dprint
     from meerschaum.utils.venv import venv_exec
     env = copy.deepcopy(dict(os.environ))
-    env[f'MRSM_SQL_{self.label.upper()}'] = json.dumps(self.meta)
+    env_key = f"MRSM_SQL_{self.label.upper()}"
+    env_val = json.dumps(self.meta)
+    env[env_key] = env_val
     cli_code = (
         "import sys\n"
         "import meerschaum as mrsm\n"
+        "import os\n"
         f"conn = mrsm.get_connector('sql:{self.label}')\n"
         "success, msg = conn._cli_exit()\n"
         "mrsm.pprint((success, msg))\n"
         "if not success:\n"
         "    raise Exception(msg)"
     )
+    if debug:
+        dprint(cli_code)
     try:
-        _ = venv_exec(cli_code, venv=None, debug=debug, capture_output=False)
+        _ = venv_exec(cli_code, venv=None, env=env, debug=debug, capture_output=False)
     except Exception as e:
         return False, f"[{self}] Failed to start CLI:\n{e}"
     return True, "Success"
 
 
 def _cli_exit(
-        self,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    debug: bool = False
+) -> SuccessTuple:
     """Launch an interactive CLI for the SQLConnector's flavor."""
-    from meerschaum.utils.packages import venv_exec, attempt_import
+    import  os
+    from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.warnings import error
-    import sys, subprocess, os
 
     if self.flavor not in flavor_clis:
         return False, f"No CLI available for flavor '{self.flavor}'."

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/connectors/sql/_create_engine.py

@@ -61,6 +61,7 @@ flavor_configs = {
         'engine': 'mssql+pyodbc',
         'create_engine': {
             'fast_executemany': True,
+            'use_insertmanyvalues': False,
             'isolation_level': 'AUTOCOMMIT',
             'use_setinputsizes': False,
             'pool_pre_ping': True,

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/connectors/sql/_pipes.py

@@ -3064,7 +3064,12 @@ def get_alter_columns_queries(
         return []
     if pipe.static:
         return
-    from meerschaum.utils.sql import sql_item_name, DROP_IF_EXISTS_FLAVORS, get_table_cols_types
+    from meerschaum.utils.sql import (
+        sql_item_name,
+        get_table_cols_types,
+        DROP_IF_EXISTS_FLAVORS,
+        SINGLE_ALTER_TABLE_FLAVORS,
+    )
     from meerschaum.utils.dataframe import get_numeric_cols
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import (
@@ -3308,14 +3313,19 @@ def get_alter_columns_queries(
             else 'TYPE '
         )
         column_str = 'COLUMN' if self.flavor != 'oracle' else ''
-        query += (
+        query_suffix = (
             f"\n{alter_col_prefix} {column_str} "
             + sql_item_name(col, self.flavor, None)
             + " " + type_prefix + typ + ","
         )
+        if self.flavor not in SINGLE_ALTER_TABLE_FLAVORS:
+            query += query_suffix
+        else:
+            queries.append(query + query_suffix[:-1])
+
+    if self.flavor not in SINGLE_ALTER_TABLE_FLAVORS:
+        queries.append(query[:-1])
 
-    query = query[:-1]
-    queries.append(query)
     if self.flavor != 'duckdb':
         return queries
 

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/connectors/sql/_sql.py

@@ -6,8 +6,10 @@ This module contains SQLConnector functions for executing SQL queries.
 """
 
 from __future__ import annotations
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import (
-    Union, Mapping, List, Dict, SuccessTuple, Optional, Any, Iterable, Callable,
+    Union, List, Dict, SuccessTuple, Optional, Any, Iterable, Callable,
     Tuple, Hashable,
 )
 
@@ -15,7 +17,7 @@ from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn
 
 ### database flavors that can use bulk insert
-_bulk_flavors = {'postgresql', 'timescaledb', 'citus'}
+_bulk_flavors = {'postgresql', 'timescaledb', 'citus', 'mssql'}
 ### flavors that do not support chunks
 _disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000}
@@ -779,6 +781,7 @@ def to_sql(
     from meerschaum.utils.warnings import error, warn
     import warnings
     import functools
+
     if name is None:
         error(f"Name must not be `None` to insert data into {self}.")
 
@@ -805,6 +808,7 @@ def to_sql(
         quantize_decimal,
         coerce_timezone,
         encode_bytes_for_bytea,
+        serialize_bytes,
     )
     from meerschaum.utils.dtypes.sql import (
         NUMERIC_PRECISION_FLAVORS,
@@ -821,24 +825,36 @@ def to_sql(
     bytes_cols = get_bytes_cols(df)
     numeric_cols = get_numeric_cols(df)
 
-    stats = {'target': name,}
+    enable_bulk_insert = mrsm.get_config(
+        'system', 'connectors', 'sql', 'bulk_insert'
+    ).get(self.flavor, False)
+    stats = {'target': name}
     ### resort to defaults if None
     copied = False
-    use_psql_copy = False
+    use_bulk_insert = False
     if method == "":
-        if self.flavor in _bulk_flavors:
-            method = functools.partial(psql_insert_copy, schema=self.schema)
-            use_psql_copy = True
+        if enable_bulk_insert:
+            method = (
+                functools.partial(mssql_insert_json, debug=debug)
+                if self.flavor == 'mssql'
+                else functools.partial(psql_insert_copy, debug=debug)
+            )
+            use_bulk_insert = True
         else:
             ### Should resolve to 'multi' or `None`.
             method = flavor_configs.get(self.flavor, {}).get('to_sql', {}).get('method', 'multi')
 
-    if bytes_cols and (use_psql_copy or self.flavor == 'oracle'):
+    if bytes_cols and (use_bulk_insert or self.flavor == 'oracle'):
         if safe_copy and not copied:
             df = df.copy()
             copied = True
+        bytes_serializer = (
+            functools.partial(encode_bytes_for_bytea, with_prefix=(self.flavor != 'oracle'))
+            if self.flavor != 'mssql'
+            else serialize_bytes
+        )
         for col in bytes_cols:
-            df[col] = df[col].apply(encode_bytes_for_bytea, with_prefix=(self.flavor != 'oracle'))
+            df[col] = df[col].apply(bytes_serializer)
 
     if self.flavor in NUMERIC_AS_TEXT_FLAVORS:
         if safe_copy and not copied:
@@ -988,7 +1004,7 @@ def to_sql(
     stats['duration'] = end - start
 
     if debug:
-        print(f" done.", flush=True)
+        print(" done.", flush=True)
         dprint(msg)
 
     stats['success'] = success
@@ -1005,7 +1021,7 @@ def psql_insert_copy(
     conn: Union[sqlalchemy.engine.Engine, sqlalchemy.engine.Connection],
     keys: List[str],
     data_iter: Iterable[Any],
-    schema: Optional[str] = None,
+    debug: bool = False,
 ) -> None:
     """
     Execute SQL statement inserting data for PostgreSQL.
@@ -1022,18 +1038,15 @@ def psql_insert_copy(
     data_iter: Iterable[Any]
         Iterable that iterates the values to be inserted
 
-    schema: Optional[str], default None
-        Optionally specify the schema of the table to be inserted into.
-
     Returns
     -------
     None
     """
     import csv
-    from io import StringIO
     import json
 
     from meerschaum.utils.sql import sql_item_name
+    from meerschaum.utils.warnings import dprint
 
     ### NOTE: PostgreSQL doesn't support NUL chars in text, so they're removed from strings.
     data_iter = (
@@ -1057,6 +1070,8 @@ def psql_insert_copy(
     table_name = sql_item_name(table.name, 'postgresql', table.schema)
     columns = ', '.join(f'"{k}"' for k in keys)
     sql = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV NULL '\\N'"
+    if debug:
+        dprint(sql)
 
     dbapi_conn = conn.connection
     with dbapi_conn.cursor() as cur:
@@ -1065,6 +1080,76 @@ def psql_insert_copy(
             writer.writerows(data_iter)
 
 
+def mssql_insert_json(
+    table: pandas.io.sql.SQLTable,
+    conn: Union[sqlalchemy.engine.Engine, sqlalchemy.engine.Connection],
+    keys: List[str],
+    data_iter: Iterable[Any],
+    cols_types: Optional[Dict[str, str]] = None,
+    debug: bool = False,
+):
+    """
+    Execute SQL statement inserting data via OPENJSON.
+
+    Adapted from this snippet:
+    https://gist.github.com/gordthompson/1fb0f1c3f5edbf6192e596de8350f205
+
+    Parameters
+    ----------
+    table: pandas.io.sql.SQLTable
+    
+    conn: Union[sqlalchemy.engine.Engine, sqlalchemy.engine.Connection]
+    
+    keys: List[str]
+        Column names
+    
+    data_iter: Iterable[Any]
+        Iterable that iterates the values to be inserted
+
+    cols_types: Optional[Dict[str, str]], default None
+        If provided, use these as the columns and types for the table.
+
+    Returns
+    -------
+    None
+    """
+    import json
+    from meerschaum.utils.sql import sql_item_name
+    from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type, get_db_type_from_pd_type
+    from meerschaum.utils.warnings import dprint
+    table_name = sql_item_name(table.name, 'mssql', table.schema)
+    if not cols_types:
+        pd_types = {
+            str(column.name): get_pd_type_from_db_type(str(column.type))
+            for column in table.table.columns
+        }
+        cols_types = {
+            col: get_db_type_from_pd_type(typ, 'mssql')
+            for col, typ in pd_types.items()
+        }
+    columns = ",\n    ".join([f"[{k}]" for k in keys])
+    json_data = [dict(zip(keys, row)) for row in data_iter]
+    with_clause = ",\n    ".join(
+        [
+            f"[{col_name}] {col_type} '$.\"{col_name}\"'"
+            for col_name, col_type in cols_types.items()
+        ]
+    )
+    placeholder = "?" if conn.dialect.paramstyle == "qmark" else "%s"
+    sql = (
+        f"INSERT INTO {table_name} (\n    {columns}\n)\n"
+        f"SELECT\n    {columns}\n"
+        f"FROM OPENJSON({placeholder})\n"
+        "WITH (\n"
+        f"    {with_clause}\n"
+        ");"
+    )
+    if debug:
+        dprint(sql)
+
+    conn.exec_driver_sql(sql, (json.dumps(json_data, default=str),))
+
+
 def format_sql_query_for_dask(query: str) -> 'sqlalchemy.sql.selectable.Select':
     """
     Given a `SELECT` query, return a `sqlalchemy` query for Dask to use.

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/jobs/__init__.py

@@ -6,8 +6,6 @@
 Higher-level utilities for managing `meerschaum.utils.daemon.Daemon`.
 """
 
-import pathlib
-
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Optional, List, SuccessTuple
 

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/utils/daemon/Daemon.py

@@ -13,7 +13,6 @@ import pathlib
 import json
 import shutil
 import signal
-import sys
 import time
 import traceback
 from functools import partial
@@ -262,7 +261,10 @@ class Daemon:
         -------
         Nothing — this will exit the parent process.
         """
-        import platform, sys, os, traceback
+        import platform
+        import sys
+        import os
+        import traceback
         from meerschaum.utils.warnings import warn
         from meerschaum.config import get_config
         daemon = attempt_import('daemon')
@@ -301,6 +303,7 @@ class Daemon:
             os.environ['LINES'], os.environ['COLUMNS'] = str(int(lines)), str(int(columns))
             with self._daemon_context:
                 sys.stdin = self.stdin_file
+                _ = os.environ.pop(STATIC_CONFIG['environment']['systemd_stdin_path'], None)
                 os.environ[STATIC_CONFIG['environment']['daemon_id']] = self.daemon_id
                 os.environ['PYTHONUNBUFFERED'] = '1'
 
@@ -1085,7 +1088,8 @@ class Daemon:
 
     def write_pickle(self) -> SuccessTuple:
         """Write the pickle file for the daemon."""
-        import pickle, traceback
+        import pickle
+        import traceback
         try:
             self.path.mkdir(parents=True, exist_ok=True)
             with open(self.pickle_path, 'wb+') as pickle_file:

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/utils/daemon/StdinFile.py

@@ -104,7 +104,10 @@ class StdinFile(io.TextIOBase):
         if self._file_handler is not None:
             self.sel.unregister(self._file_handler)
             self._file_handler.close()
-            os.close(self._fd)
+            try:
+                os.close(self._fd)
+            except OSError:
+                pass
             self._file_handler = None
             self._fd = None
 

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/utils/packages/__init__.py

@@ -831,7 +831,6 @@ def pip_install(
 
     """
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
-    from meerschaum.config import get_config
     from meerschaum.config.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
     if args is None:
@@ -966,6 +965,10 @@ def pip_install(
 
         if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall):
             if venv is not None:
+                vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
+                if not vtp.exists():
+                    if not init_venv(venv, force=True):
+                        vtp.mkdir(parents=True, exist_ok=True)
                 _args += ['--target', venv_target_path(venv, debug=debug)]
         elif (
             '--target' not in _args

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/utils/venv/_Venv.py

@@ -62,8 +62,13 @@ class Venv:
         If a `meerschaum.plugins.Plugin` was provided, its dependent virtual environments
         will also be activated.
         """
-        from meerschaum.utils.venv import active_venvs
+        from meerschaum.utils.venv import active_venvs, init_venv
         self._kwargs['previously_active_venvs'] = copy.deepcopy(active_venvs)
+        try:
+            return self._activate(debug=(debug or self._debug), **self._kwargs)
+        except OSError as e:
+            if not init_venv(self._venv, force=True):
+                raise e
         return self._activate(debug=(debug or self._debug), **self._kwargs)
 
 

{meerschaum-2.7.3 → meerschaum-2.7.5}/meerschaum/utils/venv/__init__.py

@@ -176,23 +176,26 @@ def deactivate_venv(
     if sys.path is None:
         return False
 
-    target = venv_target_path(venv, allow_nonexistent=force, debug=debug).as_posix()
+    target = venv_target_path(venv, allow_nonexistent=True, debug=debug).as_posix()
     with LOCKS['sys.path']:
         if target in sys.path:
-            sys.path.remove(target)
+            try:
+                sys.path.remove(target)
+            except Exception:
+                pass
             try:
                 active_venvs_order.remove(venv)
-            except Exception as e:
+            except Exception:
                 pass
 
     return True
 
 
 def is_venv_active(
-        venv: str = 'mrsm',
-        color : bool = True,
-        debug: bool = False
-    ) -> bool:
+    venv: str = 'mrsm',
+    color : bool = True,
+    debug: bool = False
+) -> bool:
     """
     Check if a virtual environment is active.
 
@@ -395,9 +398,13 @@ def init_venv(
 
     def update_lock(active: bool):
         try:
-            if active:
+            if not active:
+                if debug:
+                    print(f"Releasing lock: '{lock_path}'")
                 lock_path.unlink()
             else:
+                if debug:
+                    print(f"Acquiring lock: '{lock_path}'")
                 lock_path.touch()
         except Exception:
             pass
@@ -406,9 +413,12 @@ def init_venv(
         max_lock_seconds = 1.0
         step_sleep_seconds = 0.1
         init_venv_check_start = time.perf_counter()
-        while (time.perf_counter() - init_venv_check_start < max_lock_seconds):
+        while ((time.perf_counter() - init_venv_check_start) < max_lock_seconds):
             if not lock_path.exists():
-                continue
+                break
+
+            if debug:
+                print(f"Lock exists for '{venv}', sleeping...")
             time.sleep(step_sleep_seconds)
         update_lock(False)
 
@@ -441,10 +451,9 @@ def init_venv(
     rename_vtp = vtp.exists() and not temp_vtp.exists()
 
     if rename_vtp:
-        try:
-            vtp.rename(temp_vtp)
-        except FileExistsError:
-            pass
+        if debug:
+            print(f"Moving '{vtp}' to '{temp_vtp}'...")
+        shutil.move(vtp, temp_vtp)
 
     wait_for_lock()
     update_lock(True)
@@ -491,7 +500,9 @@ def init_venv(
                 + "Please install `virtualenv` via pip then restart Meerschaum."
             )
             if rename_vtp and temp_vtp.exists():
-                temp_vtp.rename(vtp)
+                if debug:
+                    print(f"Moving '{temp_vtp}' back to '{vtp}'...")
+                shutil.move(temp_vtp, vtp)
             update_lock(False)
             return False
 
@@ -530,7 +541,7 @@ def init_venv(
             import traceback
             traceback.print_exc()
             if rename_vtp and temp_vtp.exists():
-                temp_vtp.rename(vtp)
+                shutil.move(temp_vtp, vtp)
             update_lock(False)
             return False
     if verify:
@@ -538,7 +549,9 @@ def init_venv(
         verified_venvs.add(venv)
 
     if rename_vtp and temp_vtp.exists():
-        temp_vtp.rename(vtp)
+        if debug:
+            print(f"Cleanup: move '{temp_vtp}' back to '{vtp}'.")
+        shutil.move(temp_vtp, vtp)
 
     update_lock(False)
     return True
@@ -653,10 +666,10 @@ def venv_exists(venv: Union[str, None], debug: bool = False) -> bool:
 
 
 def venv_target_path(
-        venv: Union[str, None],
-        allow_nonexistent: bool = False,
-        debug: bool = False,
-    ) -> 'pathlib.Path':
+    venv: Union[str, None],
+    allow_nonexistent: bool = False,
+    debug: bool = False,
+) -> 'pathlib.Path':
     """
     Return a virtual environment's site-package path.
 
@@ -673,7 +686,11 @@ def venv_target_path(
     The `pathlib.Path` object for the virtual environment's path.
 
     """
-    import os, sys, platform, pathlib, site
+    import os
+    import sys
+    import platform
+    import pathlib
+    import site
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
     from meerschaum.config.static import STATIC_CONFIG
 

{meerschaum-2.7.3 → meerschaum-2.7.5/meerschaum.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.7.3
+Version: 2.7.5
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.7.3 → meerschaum-2.7.5}/tests/test_jobs.py

@@ -6,7 +6,6 @@ Test Meerschaum jobs.
 """
 
 import time
-import pytest
 
 import meerschaum as mrsm
 
@@ -16,18 +15,16 @@ def test_create_job():
     Test creating and running a new job.
     """
     sysargs = ['show', 'version', ':', '--loop', '--min-seconds', '0.1']
-    job = mrsm.Job(
-        'test', sysargs, executor_keys='local'
-    )
+    job = mrsm.Job('test', sysargs, executor_keys='local')
     job.delete()
     success, msg = job.start()
     assert success, msg
 
-    duration = 4.0
-    time.sleep(duration)
+    time.sleep(4.0)
 
     success, msg = job.stop()
     assert success, msg
+    time.sleep(1.0)
 
     success, msg = job.result
     assert success, msg