meerschaum 2.7.7__py3-none-any.whl → 2.7.9__py3-none-any.whl

meerschaum/utils/dataframe.py

@@ -85,6 +85,7 @@ def filter_unseen_df(
     safe_copy: bool = True,
     dtypes: Optional[Dict[str, Any]] = None,
     include_unchanged_columns: bool = False,
+    coerce_mixed_numerics: bool = True,
     debug: bool = False,
 ) -> 'pd.DataFrame':
     """
@@ -108,6 +109,10 @@ def filter_unseen_df(
     include_unchanged_columns: bool, default False
         If `True`, include columns which haven't changed on rows which have changed.
 
+    coerce_mixed_numerics: bool, default True
+        If `True`, cast mixed integer and float columns between the old and new dataframes into
+        numeric values (`decimal.Decimal`).
+
     debug: bool, default False
         Verbosity toggle.
 
@@ -138,7 +143,6 @@ def filter_unseen_df(
     import json
     import functools
     import traceback
-    from decimal import Decimal
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.packages import import_pandas, attempt_import
     from meerschaum.utils.dtypes import (
@@ -148,7 +152,9 @@ def filter_unseen_df(
         attempt_cast_to_uuid,
         attempt_cast_to_bytes,
         coerce_timezone,
+        serialize_decimal,
     )
+    from meerschaum.utils.dtypes.sql import get_numeric_precision_scale
     pd = import_pandas(debug=debug)
     is_dask = 'dask' in new_df.__module__
     if is_dask:
@@ -211,6 +217,12 @@ def filter_unseen_df(
         if col not in dtypes:
             dtypes[col] = typ
 
+    numeric_cols_precisions_scales = {
+        col: get_numeric_precision_scale(None, typ)
+        for col, typ in dtypes.items()
+        if col and typ and typ.startswith('numeric')
+    }
+
     dt_dtypes = {
         col: typ
         for col, typ in dtypes.items()
@@ -259,6 +271,8 @@ def filter_unseen_df(
             old_is_numeric = col in old_numeric_cols
 
             if (
+                coerce_mixed_numerics
+                and
                 (new_is_float or new_is_int or new_is_numeric)
                 and
                 (old_is_float or old_is_int or old_is_numeric)
@@ -300,13 +314,9 @@ def filter_unseen_df(
     new_numeric_cols = get_numeric_cols(new_df)
     numeric_cols = set(new_numeric_cols + old_numeric_cols)
     for numeric_col in old_numeric_cols:
-        old_df[numeric_col] = old_df[numeric_col].apply(
-            lambda x: f'{x:f}' if isinstance(x, Decimal) else x
-        )
+        old_df[numeric_col] = old_df[numeric_col].apply(serialize_decimal)
     for numeric_col in new_numeric_cols:
-        new_df[numeric_col] = new_df[numeric_col].apply(
-            lambda x: f'{x:f}' if isinstance(x, Decimal) else x
-        )
+        new_df[numeric_col] = new_df[numeric_col].apply(serialize_decimal)
 
     old_dt_cols = [
         col
@@ -361,7 +371,14 @@ def filter_unseen_df(
         if numeric_col not in delta_df.columns:
             continue
         try:
-            delta_df[numeric_col] = delta_df[numeric_col].apply(attempt_cast_to_numeric)
+            delta_df[numeric_col] = delta_df[numeric_col].apply(
+                functools.partial(
+                    attempt_cast_to_numeric,
+                    quantize=True,
+                    precision=numeric_cols_precisions_scales.get(numeric_col, (None, None)[0]),
+                    scale=numeric_cols_precisions_scales.get(numeric_col, (None, None)[1]),
+                )
+            )
         except Exception:
             warn(f"Unable to parse numeric column '{numeric_col}':\n{traceback.format_exc()}")
 
@@ -882,6 +899,7 @@ def enforce_dtypes(
     The Pandas DataFrame with the types enforced.
     """
     import json
+    import functools
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.formatting import pprint
     from meerschaum.utils.dtypes import (
@@ -893,6 +911,7 @@ def enforce_dtypes(
         attempt_cast_to_bytes,
         coerce_timezone as _coerce_timezone,
     )
+    from meerschaum.utils.dtypes.sql import get_numeric_precision_scale
     pandas = mrsm.attempt_import('pandas')
     is_dask = 'dask' in df.__module__
     if safe_copy:
@@ -914,7 +933,7 @@ def enforce_dtypes(
     numeric_cols = [
         col
         for col, typ in dtypes.items()
-        if typ == 'numeric'
+        if typ.startswith('numeric')
     ]
     uuid_cols = [
         col
@@ -961,9 +980,17 @@ def enforce_dtypes(
         if debug:
             dprint(f"Checking for numerics: {numeric_cols}")
         for col in numeric_cols:
+            precision, scale = get_numeric_precision_scale(None, dtypes.get(col, ''))
             if col in df.columns:
                 try:
-                    df[col] = df[col].apply(attempt_cast_to_numeric)
+                    df[col] = df[col].apply(
+                        functools.partial(
+                            attempt_cast_to_numeric,
+                            quantize=True,
+                            precision=precision,
+                            scale=scale,
+                        )
+                    )
                 except Exception as e:
                     if debug:
                         dprint(f"Unable to parse column '{col}' as NUMERIC:\n{e}")
@@ -1040,7 +1067,7 @@ def enforce_dtypes(
         previous_typ = common_dtypes[col]
         mixed_numeric_types = (is_dtype_numeric(typ) and is_dtype_numeric(previous_typ))
         explicitly_float = are_dtypes_equal(dtypes.get(col, 'object'), 'float')
-        explicitly_numeric = dtypes.get(col, 'numeric') == 'numeric'
+        explicitly_numeric = dtypes.get(col, 'numeric').startswith('numeric')
         cast_to_numeric = (
             explicitly_numeric
             or col in df_numeric_cols
@@ -1574,16 +1601,19 @@ def to_json(
     A JSON string.
     """
     from meerschaum.utils.packages import import_pandas
-    from meerschaum.utils.dtypes import serialize_bytes
+    from meerschaum.utils.dtypes import serialize_bytes, serialize_decimal
     pd = import_pandas()
     uuid_cols = get_uuid_cols(df)
     bytes_cols = get_bytes_cols(df)
+    numeric_cols = get_numeric_cols(df)
     if safe_copy and bool(uuid_cols or bytes_cols):
         df = df.copy()
     for col in uuid_cols:
         df[col] = df[col].astype(str)
     for col in bytes_cols:
         df[col] = df[col].apply(serialize_bytes)
+    for col in numeric_cols:
+        df[col] = df[col].apply(serialize_decimal)
     return df.infer_objects(copy=False).fillna(pd.NA).to_json(
         date_format=date_format,
         date_unit=date_unit,

meerschaum/utils/dtypes/__init__.py

@@ -8,15 +8,16 @@ Utility functions for working with data types.
 
 import traceback
 import uuid
-from datetime import timezone
-from decimal import Decimal, Context, InvalidOperation
+from datetime import timezone, datetime
+from decimal import Decimal, Context, InvalidOperation, ROUND_HALF_UP
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Dict, Union, Any
+from meerschaum.utils.typing import Dict, Union, Any, Optional
 from meerschaum.utils.warnings import warn
 
 MRSM_ALIAS_DTYPES: Dict[str, str] = {
     'decimal': 'numeric',
+    'Decimal': 'numeric',
     'number': 'numeric',
     'jsonl': 'json',
     'JSON': 'json',
@@ -56,6 +57,9 @@ def to_pandas_dtype(dtype: str) -> str:
     if alias_dtype is not None:
         return MRSM_PD_DTYPES[alias_dtype]
 
+    if dtype.startswith('numeric'):
+        return MRSM_PD_DTYPES['numeric']
+
     ### NOTE: Kind of a hack, but if the first word of the given dtype is in all caps,
     ### treat it as a SQL db type.
     if dtype.split(' ')[0].isupper():
@@ -118,8 +122,14 @@ def are_dtypes_equal(
         return False
 
     ### Sometimes pandas dtype objects are passed.
-    ldtype = str(ldtype)
-    rdtype = str(rdtype)
+    ldtype = str(ldtype).split('[', maxsplit=1)[0]
+    rdtype = str(rdtype).split('[', maxsplit=1)[0]
+
+    if ldtype in MRSM_ALIAS_DTYPES:
+        ldtype = MRSM_ALIAS_DTYPES[ldtype]
+
+    if rdtype in MRSM_ALIAS_DTYPES:
+        rdtype = MRSM_ALIAS_DTYPES[rdtype]
 
     json_dtypes = ('json', 'object')
     if ldtype in json_dtypes and rdtype in json_dtypes:
@@ -137,10 +147,7 @@ def are_dtypes_equal(
     if ldtype in bytes_dtypes and rdtype in bytes_dtypes:
         return True
 
-    ldtype_clean = ldtype.split('[', maxsplit=1)[0]
-    rdtype_clean = rdtype.split('[', maxsplit=1)[0]
-
-    if ldtype_clean.lower() == rdtype_clean.lower():
+    if ldtype.lower() == rdtype.lower():
         return True
 
     datetime_dtypes = ('datetime', 'timestamp')
@@ -153,19 +160,19 @@ def are_dtypes_equal(
         return True
 
     string_dtypes = ('str', 'string', 'object')
-    if ldtype_clean in string_dtypes and rdtype_clean in string_dtypes:
+    if ldtype in string_dtypes and rdtype in string_dtypes:
         return True
 
     int_dtypes = ('int', 'int64', 'int32', 'int16', 'int8')
-    if ldtype_clean.lower() in int_dtypes and rdtype_clean.lower() in int_dtypes:
+    if ldtype.lower() in int_dtypes and rdtype.lower() in int_dtypes:
         return True
 
     float_dtypes = ('float', 'float64', 'float32', 'float16', 'float128', 'double')
-    if ldtype_clean.lower() in float_dtypes and rdtype_clean.lower() in float_dtypes:
+    if ldtype.lower() in float_dtypes and rdtype.lower() in float_dtypes:
         return True
 
     bool_dtypes = ('bool', 'boolean')
-    if ldtype_clean in bool_dtypes and rdtype_clean in bool_dtypes:
+    if ldtype in bool_dtypes and rdtype in bool_dtypes:
         return True
 
     return False
@@ -195,18 +202,45 @@ def is_dtype_numeric(dtype: str) -> bool:
     return False
 
 
-def attempt_cast_to_numeric(value: Any) -> Any:
+def attempt_cast_to_numeric(
+    value: Any,
+    quantize: bool = False,
+    precision: Optional[int] = None,
+    scale: Optional[int] = None,
+)-> Any:
     """
     Given a value, attempt to coerce it into a numeric (Decimal).
+
+    Parameters
+    ----------
+    value: Any
+        The value to be cast to a Decimal.
+
+    quantize: bool, default False
+        If `True`, quantize the decimal to the specified precision and scale.
+
+    precision: Optional[int], default None
+        If `quantize` is `True`, use this precision.
+
+    scale: Optional[int], default None
+        If `quantize` is `True`, use this scale.
+
+    Returns
+    -------
+    A `Decimal` if possible, or `value`.
     """
     if isinstance(value, Decimal):
+        if quantize and precision and scale:
+            return quantize_decimal(value, precision, scale)
         return value
     try:
-        return (
-            Decimal(str(value))
-            if not value_is_null(value)
-            else Decimal('NaN')
-        )
+        if value_is_null(value):
+            return Decimal('NaN')
+
+        dec = Decimal(str(value))
+        if not quantize or not precision or not scale:
+            return dec
+        return quantize_decimal(dec, precision, scale)
     except Exception:
         return value
 
@@ -257,7 +291,7 @@ def none_if_null(value: Any) -> Any:
     return (None if value_is_null(value) else value)
 
 
-def quantize_decimal(x: Decimal, scale: int, precision: int) -> Decimal:
+def quantize_decimal(x: Decimal, precision: int, scale: int) -> Decimal:
     """
     Quantize a given `Decimal` to a known scale and precision.
 
@@ -266,22 +300,64 @@ def quantize_decimal(x: Decimal, scale: int, precision: int) -> Decimal:
     x: Decimal
         The `Decimal` to be quantized.
 
-    scale: int
+    precision: int
         The total number of significant digits.
 
-    precision: int
+    scale: int
         The number of significant digits after the decimal point.
 
     Returns
     -------
     A `Decimal` quantized to the specified scale and precision.
     """
-    precision_decimal = Decimal((('1' * scale) + '.' + ('1' * precision)))
+    precision_decimal = Decimal(('1' * (precision - scale)) + '.' + ('1' * scale))
     try:
-        return x.quantize(precision_decimal, context=Context(prec=scale))
+        return x.quantize(precision_decimal, context=Context(prec=precision), rounding=ROUND_HALF_UP)
     except InvalidOperation:
+        pass
+
+    raise ValueError(f"Cannot quantize value '{x}' to {precision=}, {scale=}.")
+
+
+def serialize_decimal(
+    x: Any,
+    quantize: bool = False,
+    precision: Optional[int] = None,
+    scale: Optional[int] = None,
+) -> Any:
+    """
+    Return a quantized string of an input decimal.
+
+    Parameters
+    ----------
+    x: Any
+        The potential decimal to be serialized.
+
+    quantize: bool, default False
+        If `True`, quantize the incoming Decimal to the specified scale and precision
+        before serialization.
+
+    precision: Optional[int], default None
+        The precision of the decimal to be quantized.
+
+    scale: Optional[int], default None
+        The scale of the decimal to be quantized.
+
+    Returns
+    -------
+    A string of the input decimal or the input if not a Decimal.
+    """
+    if not isinstance(x, Decimal):
         return x
 
+    if value_is_null(x):
+        return None
+
+    if quantize and scale and precision:
+        x = quantize_decimal(x, precision, scale)
+
+    return f"{x:f}"
+
 
 def coerce_timezone(
     dt: Any,
@@ -434,3 +510,56 @@ def encode_bytes_for_bytea(data: bytes, with_prefix: bool = True) -> str | None:
     if not isinstance(data, bytes) and value_is_null(data):
         return data
     return ('\\x' if with_prefix else '') + binascii.hexlify(data).decode('utf-8')
+
+
+def serialize_datetime(dt: datetime) -> Union[str, None]:
+    """
+    Serialize a datetime object into JSON (ISO format string).
+
+    Examples
+    --------
+    >>> import json
+    >>> from datetime import datetime
+    >>> json.dumps({'a': datetime(2022, 1, 1)}, default=json_serialize_datetime)
+    '{"a": "2022-01-01T00:00:00Z"}'
+
+    """
+    if not isinstance(dt, datetime):
+        return None
+    tz_suffix = 'Z' if dt.tzinfo is None else ''
+    return dt.isoformat() + tz_suffix
+
+
+def json_serialize_value(x: Any, default_to_str: bool = True) -> str:
+    """
+    Serialize the given value to a JSON value. Accounts for datetimes, bytes, decimals, etc.
+
+    Parameters
+    ----------
+    x: Any
+        The value to serialize.
+
+    default_to_str: bool, default True
+        If `True`, return a string of `x` if x is not a designated type.
+        Otherwise return x.
+
+    Returns
+    -------
+    A serialized version of x, or x.
+    """
+    if isinstance(x, (mrsm.Pipe, mrsm.connectors.Connector)):
+        return x.meta
+
+    if hasattr(x, 'tzinfo'):
+        return serialize_datetime(x)
+
+    if isinstance(x, bytes):
+        return serialize_bytes(x)
+
+    if isinstance(x, Decimal):
+        return serialize_decimal(x)
+
+    if value_is_null(x):
+        return None
+
+    return str(x) if default_to_str else x

meerschaum/utils/dtypes/sql.py

@@ -7,7 +7,7 @@ Utility functions for working with SQL data types.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Dict, Union, Tuple
+from meerschaum.utils.typing import Dict, Union, Tuple, Optional
 
 NUMERIC_PRECISION_FLAVORS: Dict[str, Tuple[int, int]] = {
     'mariadb': (38, 20),
@@ -170,7 +170,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'mariadb': 'DATETIME',
         'mysql': 'DATETIME',
         'mssql': 'DATETIME2',
-        'oracle': 'TIMESTAMP',
+        'oracle': 'TIMESTAMP(9)',
         'sqlite': 'DATETIME',
         'duckdb': 'TIMESTAMP',
         'citus': 'TIMESTAMP',
@@ -183,7 +183,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'mariadb': 'DATETIME',
         'mysql': 'DATETIME',
         'mssql': 'DATETIMEOFFSET',
-        'oracle': 'TIMESTAMP',
+        'oracle': 'TIMESTAMP(9)',
         'sqlite': 'TIMESTAMP',
         'duckdb': 'TIMESTAMPTZ',
         'citus': 'TIMESTAMPTZ',
@@ -196,7 +196,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'mariadb': 'DATETIME',
         'mysql': 'DATETIME',
         'mssql': 'DATETIMEOFFSET',
-        'oracle': 'TIMESTAMP',
+        'oracle': 'TIMESTAMP(9)',
         'sqlite': 'TIMESTAMP',
         'duckdb': 'TIMESTAMPTZ',
         'citus': 'TIMESTAMPTZ',
@@ -470,7 +470,7 @@ AUTO_INCREMENT_COLUMN_FLAVORS: Dict[str, str] = {
 }
 
 
-def get_pd_type_from_db_type(db_type: str, allow_custom_dtypes: bool = False) -> str:
+def get_pd_type_from_db_type(db_type: str, allow_custom_dtypes: bool = True) -> str:
     """
     Parse a database type to a pandas data type.
 
@@ -486,12 +486,17 @@ def get_pd_type_from_db_type(db_type: str, allow_custom_dtypes: bool = False) ->
     -------
     The equivalent datatype for a pandas DataFrame.
     """
+    from meerschaum.utils.dtypes import are_dtypes_equal
     def parse_custom(_pd_type: str, _db_type: str) -> str:
         if 'json' in _db_type.lower():
             return 'json'
+        if are_dtypes_equal(_pd_type, 'numeric') and _pd_type != 'object':
+            precision, scale = get_numeric_precision_scale(None, dtype=_db_type.upper())
+            if precision and scale:
+                return f"numeric[{precision},{scale}]"
         return _pd_type
 
-    pd_type = DB_TO_PD_DTYPES.get(db_type.upper(), None)
+    pd_type = DB_TO_PD_DTYPES.get(db_type.upper().split('(', maxsplit=1)[0].strip(), None)
     if pd_type is not None:
         return (
             parse_custom(pd_type, db_type)
@@ -544,17 +549,24 @@ def get_db_type_from_pd_type(
         else PD_TO_SQLALCHEMY_DTYPES_FLAVORS
     )
 
+    precision, scale = None, None
+    og_pd_type = pd_type
     if pd_type in MRSM_ALIAS_DTYPES:
         pd_type = MRSM_ALIAS_DTYPES[pd_type]
 
     ### Check whether we are able to match this type (e.g. pyarrow support).
     found_db_type = False
-    if pd_type not in types_registry:
+    if pd_type not in types_registry and not pd_type.startswith('numeric['):
         for mapped_pd_type in types_registry:
             if are_dtypes_equal(mapped_pd_type, pd_type):
                 pd_type = mapped_pd_type
                 found_db_type = True
                 break
+    elif pd_type.startswith('numeric['):
+        og_pd_type = pd_type
+        pd_type = 'numeric'
+        precision, scale = get_numeric_precision_scale(flavor, og_pd_type)
+        found_db_type = True
     else:
         found_db_type = True
 
@@ -587,6 +599,9 @@ def get_db_type_from_pd_type(
         warn(f"Unknown flavor '{flavor}'. Falling back to '{default_flavor_type}' (default).")
     db_type = flavor_types.get(flavor, default_flavor_type)
     if not as_sqlalchemy:
+        if precision is not None and scale is not None:
+            db_type_bare = db_type.split('(', maxsplit=1)[0]
+            return f"{db_type_bare}({precision},{scale})"
         return db_type
 
     if db_type.startswith('sqlalchemy.dialects'):
@@ -603,9 +618,8 @@ def get_db_type_from_pd_type(
         return cls(*cls_args, **cls_kwargs)
 
     if 'numeric' in db_type.lower():
-        if flavor not in NUMERIC_PRECISION_FLAVORS:
+        if precision is None or scale is None:
             return sqlalchemy_types.Numeric
-        precision, scale = NUMERIC_PRECISION_FLAVORS[flavor]
         return sqlalchemy_types.Numeric(precision, scale)
 
     cls_args, cls_kwargs = None, None
@@ -619,3 +633,38 @@ def get_db_type_from_pd_type(
     if cls_args is None:
         return cls
     return cls(*cls_args, **cls_kwargs)
+
+
+def get_numeric_precision_scale(
+    flavor: str,
+    dtype: Optional[str] = None,
+) -> Union[Tuple[int, int], Tuple[None, None]]:
+    """
+    Return the precision and scale to use for a numeric column for a given database flavor.
+
+    Parameters
+    ----------
+    flavor: str
+        The database flavor for which to return the precision and scale.
+    
+    dtype: Optional[str], default None
+        If provided, return the precision and scale provided in the dtype (if applicable).
+        If all caps, treat this as a DB type.
+
+    Returns
+    -------
+    A tuple of ints or a tuple of Nones.
+    """
+    if not dtype:
+        return None, None
+
+    lbracket = '[' if '[' in dtype else '('
+    rbracket = ']' if lbracket == '[' else ')'
+    if lbracket in dtype and dtype.count(',') == 1 and dtype.endswith(rbracket):
+        try:
+            parts = dtype.split(lbracket, maxsplit=1)[-1].rstrip(rbracket).split(',', maxsplit=1)
+            return int(parts[0].strip()), int(parts[1].strip())
+        except Exception:
+            pass
+
+    return NUMERIC_PRECISION_FLAVORS.get(flavor, (None, None))

meerschaum/utils/formatting/__init__.py

@@ -217,8 +217,8 @@ def print_tuple(
     tup: mrsm.SuccessTuple,
     skip_common: bool = True,
     common_only: bool = False,
-    upper_padding: int = 0,
-    lower_padding: int = 0,
+    upper_padding: int = 1,
+    lower_padding: int = 1,
     left_padding: int = 1,
     calm: bool = False,
     _progress: Optional['rich.progress.Progress'] = None,

meerschaum/utils/formatting/_pprint.py

@@ -7,21 +7,22 @@ Pretty printing wrapper
 """
 
 def pprint(
-        *args,
-        detect_password: bool = True,
-        nopretty: bool = False,
-        **kw
-    ) -> None:
+    *args,
+    detect_password: bool = True,
+    nopretty: bool = False,
+    **kw
+) -> None:
     """Pretty print an object according to the configured ANSI and UNICODE settings.
     If detect_password is True (default), search and replace passwords with '*' characters.
     Does not mutate objects.
     """
+    import copy
+    import json
     from meerschaum.utils.packages import attempt_import, import_rich
-    from meerschaum.utils.formatting import ANSI, UNICODE, get_console, print_tuple
+    from meerschaum.utils.formatting import ANSI, get_console, print_tuple
     from meerschaum.utils.warnings import error
     from meerschaum.utils.misc import replace_password, dict_from_od, filter_keywords
     from collections import OrderedDict
-    import copy, json
 
     if (
         len(args) == 1
@@ -34,7 +35,7 @@ def pprint(
         and
         isinstance(args[0][1], str)
     ):
-        return print_tuple(args[0])
+        return print_tuple(args[0], **filter_keywords(print_tuple, **kw))
 
     modify = True
     rich_pprint = None
@@ -52,7 +53,7 @@ def pprint(
         pprintpp = attempt_import('pprintpp', warn=False)
         try:
             _pprint = pprintpp.pprint
-        except Exception as e:
+        except Exception :
             import pprint as _pprint_module
             _pprint = _pprint_module.pprint
 
@@ -62,7 +63,7 @@ def pprint(
 
     try:
         args_copy = copy.deepcopy(args)
-    except Exception as e:
+    except Exception:
         args_copy = args
         modify = False
     _args = []
@@ -85,12 +86,12 @@ def pprint(
                 try:
                     c = json.dumps(c)
                     is_json = True
-                except Exception as e:
+                except Exception:
                     is_json = False
                 if not is_json:
                     try:
                         c = str(c)
-                    except Exception as e:
+                    except Exception:
                         pass
             _args.append(c)