meerschaum 2.7.4__py3-none-any.whl → 2.7.6__py3-none-any.whl

meerschaum/connectors/sql/_sql.py

@@ -6,8 +6,10 @@ This module contains SQLConnector functions for executing SQL queries.
 """
 
 from __future__ import annotations
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import (
-    Union, Mapping, List, Dict, SuccessTuple, Optional, Any, Iterable, Callable,
+    Union, List, Dict, SuccessTuple, Optional, Any, Iterable, Callable,
     Tuple, Hashable,
 )
 
@@ -15,7 +17,7 @@ from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn
 
 ### database flavors that can use bulk insert
-_bulk_flavors = {'postgresql', 'timescaledb', 'citus'}
+_bulk_flavors = {'postgresql', 'timescaledb', 'citus', 'mssql'}
 ### flavors that do not support chunks
 _disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000}
@@ -776,9 +778,11 @@ def to_sql(
     import time
     import json
     from decimal import Decimal
+    from datetime import timedelta
     from meerschaum.utils.warnings import error, warn
     import warnings
     import functools
+
     if name is None:
         error(f"Name must not be `None` to insert data into {self}.")
 
@@ -805,6 +809,7 @@ def to_sql(
         quantize_decimal,
         coerce_timezone,
         encode_bytes_for_bytea,
+        serialize_bytes,
     )
     from meerschaum.utils.dtypes.sql import (
         NUMERIC_PRECISION_FLAVORS,
@@ -812,6 +817,7 @@ def to_sql(
         PD_TO_SQLALCHEMY_DTYPES_FLAVORS,
         get_db_type_from_pd_type,
     )
+    from meerschaum.utils.misc import interval_str
     from meerschaum.connectors.sql._create_engine import flavor_configs
     from meerschaum.utils.packages import attempt_import, import_pandas
     sqlalchemy = attempt_import('sqlalchemy', debug=debug)
@@ -821,24 +827,36 @@ def to_sql(
     bytes_cols = get_bytes_cols(df)
     numeric_cols = get_numeric_cols(df)
 
-    stats = {'target': name,}
+    enable_bulk_insert = mrsm.get_config(
+        'system', 'connectors', 'sql', 'bulk_insert'
+    ).get(self.flavor, False)
+    stats = {'target': name}
     ### resort to defaults if None
     copied = False
-    use_psql_copy = False
+    use_bulk_insert = False
     if method == "":
-        if self.flavor in _bulk_flavors:
-            method = functools.partial(psql_insert_copy, schema=self.schema)
-            use_psql_copy = True
+        if enable_bulk_insert:
+            method = (
+                functools.partial(mssql_insert_json, debug=debug)
+                if self.flavor == 'mssql'
+                else functools.partial(psql_insert_copy, debug=debug)
+            )
+            use_bulk_insert = True
         else:
             ### Should resolve to 'multi' or `None`.
             method = flavor_configs.get(self.flavor, {}).get('to_sql', {}).get('method', 'multi')
 
-    if bytes_cols and (use_psql_copy or self.flavor == 'oracle'):
+    if bytes_cols and (use_bulk_insert or self.flavor == 'oracle'):
         if safe_copy and not copied:
             df = df.copy()
             copied = True
+        bytes_serializer = (
+            functools.partial(encode_bytes_for_bytea, with_prefix=(self.flavor != 'oracle'))
+            if self.flavor != 'mssql'
+            else serialize_bytes
+        )
         for col in bytes_cols:
-            df[col] = df[col].apply(encode_bytes_for_bytea, with_prefix=(self.flavor != 'oracle'))
+            df[col] = df[col].apply(bytes_serializer)
 
     if self.flavor in NUMERIC_AS_TEXT_FLAVORS:
         if safe_copy and not copied:
@@ -982,13 +1000,19 @@ def to_sql(
 
     end = time.perf_counter()
     if success:
-        msg = f"It took {round(end - start, 2)} seconds to sync {len(df)} rows to {name}."
+        num_rows = len(df)
+        msg = (
+            f"It took {interval_str(timedelta(seconds=(end - start)))} "
+            + f"to sync {num_rows:,} row"
+            + ('s' if num_rows != 1 else '')
+            + f" to {name}."
+        )
     stats['start'] = start
     stats['end'] = end
     stats['duration'] = end - start
 
     if debug:
-        print(f" done.", flush=True)
+        print(" done.", flush=True)
         dprint(msg)
 
     stats['success'] = success
@@ -1005,7 +1029,7 @@ def psql_insert_copy(
     conn: Union[sqlalchemy.engine.Engine, sqlalchemy.engine.Connection],
     keys: List[str],
     data_iter: Iterable[Any],
-    schema: Optional[str] = None,
+    debug: bool = False,
 ) -> None:
     """
     Execute SQL statement inserting data for PostgreSQL.
@@ -1022,18 +1046,15 @@ def psql_insert_copy(
     data_iter: Iterable[Any]
         Iterable that iterates the values to be inserted
 
-    schema: Optional[str], default None
-        Optionally specify the schema of the table to be inserted into.
-
     Returns
     -------
     None
     """
     import csv
-    from io import StringIO
     import json
 
     from meerschaum.utils.sql import sql_item_name
+    from meerschaum.utils.warnings import dprint
 
     ### NOTE: PostgreSQL doesn't support NUL chars in text, so they're removed from strings.
     data_iter = (
@@ -1057,6 +1078,8 @@ def psql_insert_copy(
     table_name = sql_item_name(table.name, 'postgresql', table.schema)
     columns = ', '.join(f'"{k}"' for k in keys)
     sql = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV NULL '\\N'"
+    if debug:
+        dprint(sql)
 
     dbapi_conn = conn.connection
     with dbapi_conn.cursor() as cur:
@@ -1065,6 +1088,76 @@ def psql_insert_copy(
             writer.writerows(data_iter)
 
 
+def mssql_insert_json(
+    table: pandas.io.sql.SQLTable,
+    conn: Union[sqlalchemy.engine.Engine, sqlalchemy.engine.Connection],
+    keys: List[str],
+    data_iter: Iterable[Any],
+    cols_types: Optional[Dict[str, str]] = None,
+    debug: bool = False,
+):
+    """
+    Execute SQL statement inserting data via OPENJSON.
+
+    Adapted from this snippet:
+    https://gist.github.com/gordthompson/1fb0f1c3f5edbf6192e596de8350f205
+
+    Parameters
+    ----------
+    table: pandas.io.sql.SQLTable
+    
+    conn: Union[sqlalchemy.engine.Engine, sqlalchemy.engine.Connection]
+    
+    keys: List[str]
+        Column names
+    
+    data_iter: Iterable[Any]
+        Iterable that iterates the values to be inserted
+
+    cols_types: Optional[Dict[str, str]], default None
+        If provided, use these as the columns and types for the table.
+
+    Returns
+    -------
+    None
+    """
+    import json
+    from meerschaum.utils.sql import sql_item_name
+    from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type, get_db_type_from_pd_type
+    from meerschaum.utils.warnings import dprint
+    table_name = sql_item_name(table.name, 'mssql', table.schema)
+    if not cols_types:
+        pd_types = {
+            str(column.name): get_pd_type_from_db_type(str(column.type))
+            for column in table.table.columns
+        }
+        cols_types = {
+            col: get_db_type_from_pd_type(typ, 'mssql')
+            for col, typ in pd_types.items()
+        }
+    columns = ",\n    ".join([f"[{k}]" for k in keys])
+    json_data = [dict(zip(keys, row)) for row in data_iter]
+    with_clause = ",\n    ".join(
+        [
+            f"[{col_name}] {col_type} '$.\"{col_name}\"'"
+            for col_name, col_type in cols_types.items()
+        ]
+    )
+    placeholder = "?" if conn.dialect.paramstyle == "qmark" else "%s"
+    sql = (
+        f"INSERT INTO {table_name} (\n    {columns}\n)\n"
+        f"SELECT\n    {columns}\n"
+        f"FROM OPENJSON({placeholder})\n"
+        "WITH (\n"
+        f"    {with_clause}\n"
+        ");"
+    )
+    if debug:
+        dprint(sql)
+
+    conn.exec_driver_sql(sql, (json.dumps(json_data, default=str),))
+
+
 def format_sql_query_for_dask(query: str) -> 'sqlalchemy.sql.selectable.Select':
     """
     Given a `SELECT` query, return a `sqlalchemy` query for Dask to use.

meerschaum/jobs/_Job.py

@@ -313,6 +313,7 @@ class Job:
         if not cleanup_success:
             return cleanup_success, cleanup_msg
 
+        _ = self.daemon._properties.pop('result', None)
         return cleanup_success, f"Deleted {self}."
 
     def is_running(self) -> bool:

meerschaum/plugins/__init__.py

@@ -126,8 +126,8 @@ def pre_sync_hook(
 
 
 def post_sync_hook(
-        function: Callable[[Any], Any],
-    ) -> Callable[[Any], Any]:
+    function: Callable[[Any], Any],
+) -> Callable[[Any], Any]:
     """
     Register a function as a sync hook to be executed upon completion of a sync.
     
@@ -143,10 +143,14 @@ def post_sync_hook(
     Examples
     --------
     >>> from meerschaum.plugins import post_sync_hook
+    >>> from meerschaum.utils.misc import interval_str
+    >>> from datetime import timedelta
     >>>
     >>> @post_sync_hook
     ... def log_sync(pipe, success_tuple, duration=None, **kwargs):
-    ...     print(f"It took {round(duration, 2)} seconds to sync {pipe}.")
+    ...     duration_delta = timedelta(seconds=duration)
+    ...     duration_text = interval_str(duration_delta)
+    ...     print(f"It took {duration_text} to sync {pipe}.")
     >>>
     """
     with _locks['_post_sync_hooks']:

meerschaum/utils/daemon/Daemon.py

@@ -261,7 +261,10 @@ class Daemon:
         -------
         Nothing — this will exit the parent process.
         """
-        import platform, sys, os, traceback
+        import platform
+        import sys
+        import os
+        import traceback
         from meerschaum.utils.warnings import warn
         from meerschaum.config import get_config
         daemon = attempt_import('daemon')
@@ -1014,7 +1017,8 @@ class Daemon:
 
     def read_pickle(self) -> Daemon:
         """Read a Daemon's pickle file and return the `Daemon`."""
-        import pickle, traceback
+        import pickle
+        import traceback
         if not self.pickle_path.exists():
             error(f"Pickle file does not exist for daemon '{self.daemon_id}'.")
 
@@ -1050,6 +1054,9 @@ class Daemon:
         if self._properties is None:
             self._properties = {}
 
+        if self._properties.get('result', None) is None:
+            _ = self._properties.pop('result', None)
+
         if _file_properties is not None:
             self._properties = apply_patch_to_config(
                 _file_properties,
@@ -1085,7 +1092,8 @@ class Daemon:
 
     def write_pickle(self) -> SuccessTuple:
         """Write the pickle file for the daemon."""
-        import pickle, traceback
+        import pickle
+        import traceback
         try:
             self.path.mkdir(parents=True, exist_ok=True)
             with open(self.pickle_path, 'wb+') as pickle_file:

meerschaum/utils/daemon/__init__.py

@@ -37,7 +37,7 @@ def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     filtered_sysargs = [arg for arg in sysargs if arg not in ('-d', '--daemon')]
     try:
         label = shlex.join(filtered_sysargs) if sysargs else None
-    except Exception as e:
+    except Exception:
         label = ' '.join(filtered_sysargs) if sysargs else None
 
     name = _args.get('name', None)
@@ -45,7 +45,7 @@ def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     if name:
         try:
             daemon = Daemon(daemon_id=name)
-        except Exception as e:
+        except Exception:
             daemon = None
 
     if daemon is not None:

meerschaum/utils/misc.py

@@ -90,20 +90,21 @@ def add_method_to_class(
 
 
 def generate_password(length: int = 12) -> str:
-    """Generate a secure password of given length.
+    """
+    Generate a secure password of given length.
 
     Parameters
     ----------
-    length : int, default 12
+    length: int, default 12
         The length of the password.
 
     Returns
     -------
     A random password string.
-
     """
-    import secrets, string
-    return ''.join((secrets.choice(string.ascii_letters) for i in range(length)))
+    import secrets
+    import string
+    return ''.join((secrets.choice(string.ascii_letters + string.digits) for i in range(length)))
 
 def is_int(s : str) -> bool:
     """
@@ -121,7 +122,7 @@ def is_int(s : str) -> bool:
     """
     try:
         float(s)
-    except Exception as e:
+    except Exception:
         return False
     
     return float(s).is_integer()

meerschaum/utils/packages/__init__.py

@@ -337,14 +337,14 @@ def get_install_no_version(install_name: str) -> str:
 
 import_versions = {}
 def determine_version(
-        path: pathlib.Path,
-        import_name: Optional[str] = None,
-        venv: Optional[str] = 'mrsm',
-        search_for_metadata: bool = True,
-        split: bool = True,
-        warn: bool = False,
-        debug: bool = False,
-    ) -> Union[str, None]:
+    path: pathlib.Path,
+    import_name: Optional[str] = None,
+    venv: Optional[str] = 'mrsm',
+    search_for_metadata: bool = True,
+    split: bool = True,
+    warn: bool = False,
+    debug: bool = False,
+) -> Union[str, None]:
     """
     Determine a module's `__version__` string from its filepath.
     
@@ -381,11 +381,8 @@ def determine_version(
     with _locks['import_versions']:
         if venv not in import_versions:
             import_versions[venv] = {}
-    import importlib.metadata
-    import re, os
+    import os
     old_cwd = os.getcwd()
-    if debug:
-        from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn as warn_function
     if import_name is None:
         import_name = path.parent.stem if path.stem == '__init__' else path.stem
@@ -395,7 +392,10 @@ def determine_version(
     _version = None
     module_parent_dir = (
         path.parent.parent if path.stem == '__init__' else path.parent
-    ) if path is not None else venv_target_path(venv, debug=debug)
+    ) if path is not None else venv_target_path(venv, allow_nonexistent=True, debug=debug)
+
+    if not module_parent_dir.exists():
+        return None
 
     installed_dir_name = _import_to_dir_name(import_name)
     clean_installed_dir_name = installed_dir_name.lower().replace('-', '_')
@@ -403,7 +403,11 @@ def determine_version(
     ### First, check if a dist-info directory exists.
     _found_versions = []
     if search_for_metadata:
-        for filename in os.listdir(module_parent_dir):
+        try:
+            filenames = os.listdir(module_parent_dir)
+        except FileNotFoundError:
+            filenames = []
+        for filename in filenames:
             if not filename.endswith('.dist-info'):
                 continue
             filename_lower = filename.lower()
@@ -430,7 +434,7 @@ def determine_version(
         try:
             os.chdir(module_parent_dir)
             _version = importlib_metadata.metadata(import_name)['Version']
-        except Exception as e:
+        except Exception:
             _version = None
         finally:
             os.chdir(old_cwd)
@@ -698,7 +702,7 @@ def need_update(
             (not semver.Version.parse(version).match(required_version))
             if required_version else False
         )
-    except AttributeError as e:
+    except AttributeError:
         pip_install(_import_to_install_name('semver'), venv='mrsm', debug=debug)
         semver = manually_import_module('semver', venv='mrsm', debug=debug)
         return (
@@ -724,10 +728,10 @@ def need_update(
 
 
 def get_pip(
-        venv: Optional[str] = 'mrsm',
-        color: bool = True,
-        debug: bool = False,
-    ) -> bool:
+    venv: Optional[str] = 'mrsm',
+    color: bool = True,
+    debug: bool = False,
+) -> bool:
     """
     Download and run the get-pip.py script.
 
@@ -747,7 +751,8 @@ def get_pip(
     A bool indicating success.
 
     """
-    import sys, subprocess
+    import sys
+    import subprocess
     from meerschaum.utils.misc import wget
     from meerschaum.config._paths import CACHE_RESOURCES_PATH
     from meerschaum.config.static import STATIC_CONFIG
@@ -755,7 +760,7 @@ def get_pip(
     dest = CACHE_RESOURCES_PATH / 'get-pip.py'
     try:
         wget(url, dest, color=False, debug=debug)
-    except Exception as e:
+    except Exception:
         print(f"Failed to fetch pip from '{url}'. Please install pip and restart Meerschaum.") 
         sys.exit(1)
     if venv is not None:
@@ -776,6 +781,7 @@ def pip_install(
     _uninstall: bool = False,
     _from_completely_uninstall: bool = False,
     _install_uv_pip: bool = True,
+    _use_uv_pip: bool = True,
     color: bool = True,
     silent: bool = False,
     debug: bool = False,
@@ -831,15 +837,11 @@ def pip_install(
 
     """
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
-    from meerschaum.config import get_config
     from meerschaum.config.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
     if args is None:
         args = ['--upgrade'] if not _uninstall else []
-    if color:
-        ANSI, UNICODE = True, True
-    else:
-        ANSI, UNICODE = False, False
+    ANSI = True if color else False
     if check_wheel:
         have_wheel = venv_contains_package('wheel', venv=venv, debug=debug)
 
@@ -878,7 +880,8 @@ def pip_install(
             )
 
     use_uv_pip = (
-        venv_contains_package('uv', venv=None, debug=debug)
+        _use_uv_pip
+        and venv_contains_package('uv', venv=None, debug=debug)
         and uv_bin is not None
         and venv is not None
         and is_uv_enabled()
@@ -966,6 +969,10 @@ def pip_install(
 
         if '--target' not in _args and '-t' not in _args and not (not use_uv_pip and _uninstall):
             if venv is not None:
+                vtp = venv_target_path(venv, allow_nonexistent=True, debug=debug)
+                if not vtp.exists():
+                    if not init_venv(venv, force=True):
+                        vtp.mkdir(parents=True, exist_ok=True)
                 _args += ['--target', venv_target_path(venv, debug=debug)]
         elif (
             '--target' not in _args

meerschaum/utils/packages/_packages.py

@@ -136,7 +136,7 @@ packages['sql'] = {
     'numpy'                          : 'numpy>=1.18.5',
     'pandas'                         : 'pandas[parquet]>=2.0.1',
     'pyarrow'                        : 'pyarrow>=16.1.0',
-    'dask'                           : 'dask[complete]>=2024.5.1',
+    'dask'                           : 'dask[complete]>=2024.12.1',
     'partd'                          : 'partd>=1.4.2',
     'pytz'                           : 'pytz',
     'joblib'                         : 'joblib>=0.17.0',