meerschaum 2.1.6__py3-none-any.whl → 2.2.0__py3-none-any.whl

meerschaum/utils/dtypes/__init__.py

@@ -6,8 +6,10 @@
 Utility functions for working with data types.
 """
 
+import traceback
 from decimal import Decimal, Context, InvalidOperation
 from meerschaum.utils.typing import Dict, Union, Any
+from meerschaum.utils.warnings import warn
 
 MRSM_PD_DTYPES: Dict[str, str] = {
     'json': 'object',
@@ -37,9 +39,7 @@ def to_pandas_dtype(dtype: str) -> str:
         from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
         return get_pd_type_from_db_type(dtype)
 
-    import traceback
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.warnings import warn
     pandas = attempt_import('pandas', lazy=False)
 
     try:
@@ -88,8 +88,12 @@ def are_dtypes_equal(
                 return False
         return True
 
-    if ldtype == rdtype:
-        return True
+    try:
+        if ldtype == rdtype:
+            return True
+    except Exception as e:
+        warn(f"Exception when comparing dtypes, returning False:\n{traceback.format_exc()}")
+        return False
 
     ### Sometimes pandas dtype objects are passed.
     ldtype = str(ldtype)
@@ -177,7 +181,7 @@ def attempt_cast_to_numeric(value: Any) -> Any:
         return value
 
 
-def value_is_null(value: Any) -> Any:
+def value_is_null(value: Any) -> bool:
     """
     Determine if a value is a null-like string.
     """

meerschaum/utils/formatting/_pipes.py

@@ -9,7 +9,7 @@ Formatting functions for printing pipes
 from __future__ import annotations
 import json
 import meerschaum as mrsm
-from meerschaum.utils.typing import PipesDict, Dict, Union, Optional, SuccessTuple, Any
+from meerschaum.utils.typing import PipesDict, Dict, Union, Optional, SuccessTuple, Any, List
 from meerschaum.config import get_config
 
 def pprint_pipes(pipes: PipesDict) -> None:
@@ -481,22 +481,54 @@ def print_pipes_results(
         )
 
 
-def extract_stats_from_message(message: str) -> Dict[str, int]:
+def extract_stats_from_message(
+        message: str,
+        stat_keys: Optional[List[str]] = None,
+    ) -> Dict[str, int]:
     """
-    Given a sync message, return the insert, update stats from within.
+    Given a sync message, return the insert, update, upsert stats from within.
+
+    Parameters
+    ----------
+    message: str
+        The message to parse for statistics.
+
+    stat_keys: Optional[List[str]], default None
+        If provided, search for these words (case insensitive) in the message.
+        Defaults to `['inserted', 'updated', 'upserted']`.
+
+    Returns
+    -------
+    A dictionary mapping the stat keys to the total number of rows affected.
     """
-    stats = {
-        'inserted': 0,
-        'updated': 0,
-        'upserted': 0,
+    stat_keys = stat_keys or ['inserted', 'updated', 'upserted']
+    lines_stats = [extract_stats_from_line(line, stat_keys) for line in message.split('\n')]
+    message_stats = {
+        stat_key: sum(stats.get(stat_key, 0) for stats in lines_stats)
+        for stat_key in stat_keys
     }
+    return message_stats
 
-    for search_key in list(stats.keys()):
-        if search_key not in message.lower():
+
+def extract_stats_from_line(
+        line: str,
+        stat_keys: List[str],
+    ) -> Dict[str, int]:
+    """
+    Return the insert, update, upsert stats from a single line.
+    """
+    stats = {key: 0 for key in stat_keys}
+
+    for stat_key in stat_keys:
+        search_key = stat_key.lower()
+        if search_key not in line.lower():
             continue
 
         ### stat_text starts with the digits we want.
-        stat_text = message.lower().split(search_key + ' ')[1]
+        try:
+            stat_text = line.lower().split(search_key + ' ')[1]
+        except IndexError:
+            continue
 
         ### find the first non-digit value.
         end_of_num_ix = -1
@@ -504,6 +536,8 @@ def extract_stats_from_message(message: str) -> Dict[str, int]:
             if not char.isdigit():
                 end_of_num_ix = i
                 break
+            if i == len(stat_text) - 1:
+                end_of_num_ix = i + 1
         if end_of_num_ix == -1:
             continue
 

meerschaum/utils/misc.py

@@ -1234,7 +1234,7 @@ def truncate_string_sections(item: str, delimeter: str = '_', max_len: int = 128
 
 
 def separate_negation_values(
-        vals: List[str],
+        vals: Union[List[str], Tuple[str]],
         negation_prefix: Optional[str] = None,
     ) -> Tuple[List[str], List[str]]:
     """
@@ -1243,7 +1243,7 @@ def separate_negation_values(
 
     Parameters
     ----------
-    vals: List[str]
+    vals: Union[List[str], Tuple[str]]
         A list of strings to parse.
 
     negation_prefix: Optional[str], default None
@@ -1263,6 +1263,38 @@ def separate_negation_values(
     return _in_vals, _ex_vals
 
 
+def get_in_ex_params(params: Optional[Dict[str, Any]]) -> Dict[str, Tuple[List[Any], List[Any]]]:
+    """
+    Translate a params dictionary into lists of include- and exclude-values.
+
+    Parameters
+    ----------
+    params: Optional[Dict[str, Any]]
+        A params query dictionary.
+
+    Returns
+    -------
+    A dictionary mapping keys to a tuple of lists for include and exclude values.
+
+    Examples
+    --------
+    >>> get_in_ex_params({'a': ['b', 'c', '_d', 'e', '_f']})
+    {'a': (['b', 'c', 'e'], ['d', 'f'])}
+    """
+    if not params:
+        return {}
+    return {
+        col: separate_negation_values(
+            (
+                val
+                if isinstance(val, (list, tuple))
+                else [val]
+            )
+        )
+        for col, val in params.items()
+    }
+
+
 def flatten_list(list_: List[Any]) -> List[Any]:
     """
     Recursively flatten a list.

meerschaum/utils/packages/__init__.py

@@ -8,7 +8,7 @@ Functions for managing packages and virtual environments reside here.
 
 from __future__ import annotations
 
-import importlib.util, os, pathlib
+import importlib.util, os, pathlib, re
 from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
 from meerschaum.utils.threading import Lock, RLock
 from meerschaum.utils.packages._packages import packages, all_packages, get_install_names
@@ -35,6 +35,7 @@ _locks = {
 }
 _checked_for_updates = set()
 _is_installed_first_check: Dict[str, bool] = {}
+_MRSM_PACKAGE_ARCHIVES_PREFIX: str = "https://meerschaum.io/files/archives/"
 
 def get_module_path(
         import_name: str,
@@ -350,6 +351,7 @@ def determine_version(
     with _locks['import_versions']:
         if venv not in import_versions:
             import_versions[venv] = {}
+    import importlib.metadata
     import re, os
     old_cwd = os.getcwd()
     if debug:
@@ -639,6 +641,15 @@ def need_update(
 
     ### We might be depending on a prerelease.
     ### Sanity check that the required version is not greater than the installed version. 
+    required_version = (
+        required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '')
+        .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '')
+        .replace('-py3-none-any.whl', '')
+    )
+
+    if 'a' in required_version:
+        required_version = required_version.replace('a', '-dev').replace('+mrsm', '')
+        version = version.replace('a', '-dev').replace('+mrsm', '')
     try:
         return (
             (not semver.Version.parse(version).match(required_version))
@@ -818,8 +829,11 @@ def pip_install(
                     check_wheel = False, debug = debug,
                 ):
                     warn(
-                        f"Failed to install `setuptools` and `wheel` for virtual environment '{venv}'.",
-                        color=False,
+                        (
+                            "Failed to install `setuptools` and `wheel` for virtual "
+                            + f"environment '{venv}'."
+                        ),
+                        color = False,
                     )
 
         if requirements_file_path is not None:
@@ -882,13 +896,16 @@ def pip_install(
                         f"Failed to clean up package '{_install_no_version}'.",
                     )
 
-        success = run_python_package(
+        rc = run_python_package(
             'pip',
             _args + _packages,
             venv = venv,
             env = _get_pip_os_env(),
             debug = debug,
-        ) == 0
+        )
+        if debug:
+            print(f"{rc=}")
+        success = rc == 0
 
     msg = (
         "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 
@@ -1379,13 +1396,13 @@ def get_modules_from_package(
     Returns
     -------
     Either list of modules or tuple of lists.
-
     """
     from os.path import dirname, join, isfile, isdir, basename
     import glob
 
     pattern = '*' if recursive else '*.py'
-    module_names = glob.glob(join(dirname(package.__file__), pattern), recursive=recursive)
+    package_path = dirname(package.__file__ or package.__path__[0])
+    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
     _all = [
         basename(f)[:-3] if isfile(f) else basename(f)
         for f in module_names
@@ -1410,7 +1427,7 @@ def get_modules_from_package(
             modules.append(m)
         except Exception as e:
             if debug:
-                dprint(e)
+                dprint(str(e))
         finally:
             if modules_venvs:
                 deactivate_venv(module_name.split('.')[-1], debug=debug)

meerschaum/utils/packages/_packages.py

@@ -30,7 +30,7 @@ packages: Dict[str, Dict[str, str]] = {
         'more_termcolor'             : 'more-termcolor>=1.1.3',
         'humanfriendly'              : 'humanfriendly>=10.0.0',
     },
-    '_required': {
+    'core': {
         'wheel'                      : 'wheel>=0.34.2',
         'setuptools'                 : 'setuptools>=63.3.0',
         'yaml'                       : 'PyYAML>=5.3.1',
@@ -49,21 +49,21 @@ packages: Dict[str, Dict[str, str]] = {
         'daemon'                     : 'python-daemon>=0.2.3',
         'fasteners'                  : 'fasteners>=0.18.0',
         'psutil'                     : 'psutil>=5.8.0',
-        'watchgod'                   : 'watchgod>=0.7.0',
+        'watchfiles'                 : 'watchfiles>=0.21.0',
         'dill'                       : 'dill>=0.3.3',
         'virtualenv'                 : 'virtualenv>=20.1.0',
-        'rocketry'                   : 'rocketry>=2.5.1',
+        'apscheduler'                : 'APScheduler>=4.0.0a5',
     },
     'drivers': {
         'cryptography'               : 'cryptography>=38.0.1',
-        'psycopg2'                   : 'psycopg2-binary>=2.8.6',
+        'psycopg'                    : 'psycopg[binary]>=3.1.18',
         'pymysql'                    : 'PyMySQL>=0.9.0',
         'aiomysql'                   : 'aiomysql>=0.0.21',
         'sqlalchemy_cockroachdb'     : 'sqlalchemy-cockroachdb>=2.0.0',
-        'duckdb'                     : 'duckdb>=0.9.0',
+        'duckdb'                     : 'duckdb<0.10.3',
         'duckdb_engine'              : 'duckdb-engine>=0.9.2',
     },
-    '_drivers': {
+    'drivers-extras': {
         'pyodbc'                     : 'pyodbc>=4.0.30',
         'cx_Oracle'                  : 'cx_Oracle>=8.3.0',
     },
@@ -75,11 +75,11 @@ packages: Dict[str, Dict[str, str]] = {
         'gadwall'                    : 'gadwall>=0.2.0',
     },
     'stack': {
-        'compose'                    : 'docker-compose>=1.27.4',
+        'compose'                    : 'docker-compose>=1.29.2',
     },
     'build': {
-        'cx_Freeze'                  : 'cx_Freeze>=6.5.1',
-        'PyInstaller'                : 'pyinstaller>=5.0.0-dev0',
+        'cx_Freeze'                  : 'cx_Freeze>=7.0.0',
+        'PyInstaller'                : 'pyinstaller>6.6.0',
     },
     'dev-tools': {
         'twine'                      : 'twine>=3.2.0',
@@ -89,6 +89,7 @@ packages: Dict[str, Dict[str, str]] = {
         'pytest'                     : 'pytest>=6.2.2',
         'pytest_xdist'               : 'pytest-xdist>=3.2.1',
         'heartrate'                  : 'heartrate>=0.2.1',
+        'build'                      : 'build>=1.2.1',
     },
     'setup': {
     },
@@ -119,8 +120,8 @@ packages: Dict[str, Dict[str, str]] = {
 packages['sql'] = {
     'numpy'                          : 'numpy>=1.18.5',
     'pandas'                         : 'pandas[parquet]>=2.0.1',
-    'pyarrow'                        : 'pyarrow>=7.0.0',
-    'dask'                           : 'dask>=2023.5.0',
+    'pyarrow'                        : 'pyarrow>=16.1.0',
+    'dask'                           : 'dask[dataframe]>=2024.5.1',
     'pytz'                           : 'pytz',
     'joblib'                         : 'joblib>=0.17.0',
     'sqlalchemy'                     : 'SQLAlchemy>=2.0.5',
@@ -129,7 +130,7 @@ packages['sql'] = {
     'asyncpg'                        : 'asyncpg>=0.21.0',
 }
 packages['sql'].update(packages['drivers'])
-packages['sql'].update(packages['_required'])
+packages['sql'].update(packages['core'])
 packages['dash'] = {
     'flask_compress'                 : 'Flask-Compress>=1.10.1',
     'dash'                           : 'dash>=2.6.2',
@@ -141,17 +142,14 @@ packages['dash'] = {
     'tornado'                        : 'tornado>=6.1.0',
 }
 packages['api'] = {
-    'uvicorn'                        : 'uvicorn[standard]>=0.22.0',
-    'gunicorn'                       : 'gunicorn>=20.1.0',
+    'uvicorn'                        : 'uvicorn[standard]>=0.29.0',
+    'gunicorn'                       : 'gunicorn>=22.0.0',
     'dotenv'                         : 'python-dotenv>=0.20.0',
     'websockets'                     : 'websockets>=11.0.3',
-    'fastapi'                        : 'fastapi>=0.100.0',
-    'passlib'                        : 'passlib>=1.7.4',
+    'fastapi'                        : 'fastapi>=0.111.0',
     'fastapi_login'                  : 'fastapi-login>=1.7.2',
-    'multipart'                      : 'python-multipart>=0.0.5',
-    'pydantic'                       : 'pydantic<2.0.0',
+    'multipart'                      : 'python-multipart>=0.0.9',
     'httpx'                          : 'httpx>=0.24.1',
-    'websockets'                     : 'websockets>=11.0.3',
 }
 packages['api'].update(packages['sql'])
 packages['api'].update(packages['formatting'])
@@ -171,7 +169,7 @@ def get_install_names():
             install_names[get_install_no_version(_install_name)] = _import_name
     return install_names
 
-skip_groups = {'docs', 'build', 'cli', 'dev-tools', 'portable', 'extras', 'stack', '_drivers'}
+skip_groups = {'docs', 'build', 'cli', 'dev-tools', 'portable', 'extras', 'stack', 'drivers-extras'}
 full = []
 _full = {}
 for group, import_names in packages.items():

meerschaum/utils/process.py

@@ -11,6 +11,7 @@ See `meerschaum.utils.pool` for multiprocessing and
 from __future__ import annotations
 import os, signal, subprocess, sys, platform
 from meerschaum.utils.typing import Union, Optional, Any, Callable, Dict, Tuple
+from meerschaum.config.static import STATIC_CONFIG
 
 def run_process(
         *args,
@@ -68,9 +69,18 @@ def run_process(
     if platform.system() == 'Windows':
         foreground = False
 
-    if line_callback is not None:
+    def print_line(line):
+        sys.stdout.write(line.decode('utf-8'))
+        sys.stdout.flush()
+
+    if capture_output or line_callback is not None:
+        kw['stdout'] = subprocess.PIPE
+        kw['stderr'] = subprocess.STDOUT
+    elif os.environ.get(STATIC_CONFIG['environment']['daemon_id']):
         kw['stdout'] = subprocess.PIPE
         kw['stderr'] = subprocess.STDOUT
+        if line_callback is None:
+            line_callback = print_line
 
     if 'env' not in kw:
         kw['env'] = os.environ
@@ -112,15 +122,6 @@ def run_process(
         kw['preexec_fn'] = new_pgid
 
     try:
-        # fork the child
-        #  stdout, stderr = (
-            #  (sys.stdout, sys.stderr) if not capture_output
-            #  else (subprocess.PIPE, subprocess.PIPE)
-        #  )
-        if capture_output:
-            kw['stdout'] = subprocess.PIPE
-            kw['stderr'] = subprocess.PIPE
-
         child = subprocess.Popen(*args, **kw)
 
         # we can't set the process group id from the parent since the child
@@ -197,6 +198,8 @@ def poll_process(
     while proc.poll() is None:
         line = proc.stdout.readline()
         line_callback(line)
+
     if timeout_seconds is not None:
         watchdog_thread.cancel()
+
     return proc.poll()