meerschaum 2.7.6__py3-none-any.whl → 2.7.8__py3-none-any.whl

meerschaum/utils/dtypes/__init__.py

@@ -8,15 +8,16 @@ Utility functions for working with data types.
 
 import traceback
 import uuid
-from datetime import timezone
-from decimal import Decimal, Context, InvalidOperation
+from datetime import timezone, datetime
+from decimal import Decimal, Context, InvalidOperation, ROUND_HALF_UP
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Dict, Union, Any
+from meerschaum.utils.typing import Dict, Union, Any, Optional
 from meerschaum.utils.warnings import warn
 
 MRSM_ALIAS_DTYPES: Dict[str, str] = {
     'decimal': 'numeric',
+    'Decimal': 'numeric',
     'number': 'numeric',
     'jsonl': 'json',
     'JSON': 'json',
@@ -56,6 +57,9 @@ def to_pandas_dtype(dtype: str) -> str:
     if alias_dtype is not None:
         return MRSM_PD_DTYPES[alias_dtype]
 
+    if dtype.startswith('numeric'):
+        return MRSM_PD_DTYPES['numeric']
+
     ### NOTE: Kind of a hack, but if the first word of the given dtype is in all caps,
     ### treat it as a SQL db type.
     if dtype.split(' ')[0].isupper():
@@ -118,8 +122,14 @@ def are_dtypes_equal(
         return False
 
     ### Sometimes pandas dtype objects are passed.
-    ldtype = str(ldtype)
-    rdtype = str(rdtype)
+    ldtype = str(ldtype).split('[', maxsplit=1)[0]
+    rdtype = str(rdtype).split('[', maxsplit=1)[0]
+
+    if ldtype in MRSM_ALIAS_DTYPES:
+        ldtype = MRSM_ALIAS_DTYPES[ldtype]
+
+    if rdtype in MRSM_ALIAS_DTYPES:
+        rdtype = MRSM_ALIAS_DTYPES[rdtype]
 
     json_dtypes = ('json', 'object')
     if ldtype in json_dtypes and rdtype in json_dtypes:
@@ -137,10 +147,7 @@ def are_dtypes_equal(
     if ldtype in bytes_dtypes and rdtype in bytes_dtypes:
         return True
 
-    ldtype_clean = ldtype.split('[', maxsplit=1)[0]
-    rdtype_clean = rdtype.split('[', maxsplit=1)[0]
-
-    if ldtype_clean.lower() == rdtype_clean.lower():
+    if ldtype.lower() == rdtype.lower():
         return True
 
     datetime_dtypes = ('datetime', 'timestamp')
@@ -153,19 +160,19 @@ def are_dtypes_equal(
         return True
 
     string_dtypes = ('str', 'string', 'object')
-    if ldtype_clean in string_dtypes and rdtype_clean in string_dtypes:
+    if ldtype in string_dtypes and rdtype in string_dtypes:
         return True
 
     int_dtypes = ('int', 'int64', 'int32', 'int16', 'int8')
-    if ldtype_clean.lower() in int_dtypes and rdtype_clean.lower() in int_dtypes:
+    if ldtype.lower() in int_dtypes and rdtype.lower() in int_dtypes:
         return True
 
     float_dtypes = ('float', 'float64', 'float32', 'float16', 'float128', 'double')
-    if ldtype_clean.lower() in float_dtypes and rdtype_clean.lower() in float_dtypes:
+    if ldtype.lower() in float_dtypes and rdtype.lower() in float_dtypes:
         return True
 
     bool_dtypes = ('bool', 'boolean')
-    if ldtype_clean in bool_dtypes and rdtype_clean in bool_dtypes:
+    if ldtype in bool_dtypes and rdtype in bool_dtypes:
         return True
 
     return False
@@ -195,18 +202,45 @@ def is_dtype_numeric(dtype: str) -> bool:
     return False
 
 
-def attempt_cast_to_numeric(value: Any) -> Any:
+def attempt_cast_to_numeric(
+    value: Any,
+    quantize: bool = False,
+    precision: Optional[int] = None,
+    scale: Optional[int] = None,
+)-> Any:
     """
     Given a value, attempt to coerce it into a numeric (Decimal).
+
+    Parameters
+    ----------
+    value: Any
+        The value to be cast to a Decimal.
+
+    quantize: bool, default False
+        If `True`, quantize the decimal to the specified precision and scale.
+
+    precision: Optional[int], default None
+        If `quantize` is `True`, use this precision.
+
+    scale: Optional[int], default None
+        If `quantize` is `True`, use this scale.
+
+    Returns
+    -------
+    A `Decimal` if possible, or `value`.
     """
     if isinstance(value, Decimal):
+        if quantize and precision and scale:
+            return quantize_decimal(value, precision, scale)
         return value
     try:
-        return (
-            Decimal(str(value))
-            if not value_is_null(value)
-            else Decimal('NaN')
-        )
+        if value_is_null(value):
+            return Decimal('NaN')
+
+        dec = Decimal(str(value))
+        if not quantize or not precision or not scale:
+            return dec
+        return quantize_decimal(dec, precision, scale)
     except Exception:
         return value
 
@@ -257,7 +291,7 @@ def none_if_null(value: Any) -> Any:
     return (None if value_is_null(value) else value)
 
 
-def quantize_decimal(x: Decimal, scale: int, precision: int) -> Decimal:
+def quantize_decimal(x: Decimal, precision: int, scale: int) -> Decimal:
     """
     Quantize a given `Decimal` to a known scale and precision.
 
@@ -266,22 +300,61 @@ def quantize_decimal(x: Decimal, scale: int, precision: int) -> Decimal:
     x: Decimal
         The `Decimal` to be quantized.
 
-    scale: int
+    precision: int
         The total number of significant digits.
 
-    precision: int
+    scale: int
         The number of significant digits after the decimal point.
 
     Returns
     -------
     A `Decimal` quantized to the specified scale and precision.
     """
-    precision_decimal = Decimal((('1' * scale) + '.' + ('1' * precision)))
+    precision_decimal = Decimal(('1' * (precision - scale)) + '.' + ('1' * scale))
     try:
-        return x.quantize(precision_decimal, context=Context(prec=scale))
+        return x.quantize(precision_decimal, context=Context(prec=precision), rounding=ROUND_HALF_UP)
     except InvalidOperation:
+        pass
+
+    raise ValueError(f"Cannot quantize value '{x}' to {precision=}, {scale=}.")
+
+
+def serialize_decimal(
+    x: Any,
+    quantize: bool = False,
+    precision: Optional[int] = None,
+    scale: Optional[int] = None,
+) -> Any:
+    """
+    Return a quantized string of an input decimal.
+
+    Parameters
+    ----------
+    x: Any
+        The potential decimal to be serialized.
+
+    quantize: bool, default False
+        If `True`, quantize the incoming Decimal to the specified scale and precision
+        before serialization.
+
+    precision: Optional[int], default None
+        The precision of the decimal to be quantized.
+
+    scale: Optional[int], default None
+        The scale of the decimal to be quantized.
+
+    Returns
+    -------
+    A string of the input decimal or the input if not a Decimal.
+    """
+    if not isinstance(x, Decimal):
         return x
 
+    if quantize and scale and precision:
+        x = quantize_decimal(x, precision, scale)
+
+    return f"{x:f}"
+
 
 def coerce_timezone(
     dt: Any,
@@ -434,3 +507,50 @@ def encode_bytes_for_bytea(data: bytes, with_prefix: bool = True) -> str | None:
     if not isinstance(data, bytes) and value_is_null(data):
         return data
     return ('\\x' if with_prefix else '') + binascii.hexlify(data).decode('utf-8')
+
+
+def serialize_datetime(dt: datetime) -> Union[str, None]:
+    """
+    Serialize a datetime object into JSON (ISO format string).
+
+    Examples
+    --------
+    >>> import json
+    >>> from datetime import datetime
+    >>> json.dumps({'a': datetime(2022, 1, 1)}, default=json_serialize_datetime)
+    '{"a": "2022-01-01T00:00:00Z"}'
+
+    """
+    if not isinstance(dt, datetime):
+        return None
+    tz_suffix = 'Z' if dt.tzinfo is None else ''
+    return dt.isoformat() + tz_suffix
+
+
+def json_serialize_value(x: Any, default_to_str: bool = True) -> str:
+    """
+    Serialize the given value to a JSON value. Accounts for datetimes, bytes, decimals, etc.
+
+    Parameters
+    ----------
+    x: Any
+        The value to serialize.
+
+    default_to_str: bool, default True
+        If `True`, return a string of `x` if x is not a designated type.
+        Otherwise return x.
+
+    Returns
+    -------
+    A serialized version of x, or x.
+    """
+    if hasattr(x, 'tzinfo'):
+        return serialize_datetime(x)
+
+    if isinstance(x, bytes):
+        return serialize_bytes(x)
+
+    if isinstance(x, Decimal):
+        return serialize_decimal(x)
+
+    return str(x) if default_to_str else x

meerschaum/utils/dtypes/sql.py

@@ -7,7 +7,7 @@ Utility functions for working with SQL data types.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Dict, Union, Tuple
+from meerschaum.utils.typing import Dict, Union, Tuple, Optional
 
 NUMERIC_PRECISION_FLAVORS: Dict[str, Tuple[int, int]] = {
     'mariadb': (38, 20),
@@ -170,7 +170,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'mariadb': 'DATETIME',
         'mysql': 'DATETIME',
         'mssql': 'DATETIME2',
-        'oracle': 'TIMESTAMP',
+        'oracle': 'TIMESTAMP(9)',
         'sqlite': 'DATETIME',
         'duckdb': 'TIMESTAMP',
         'citus': 'TIMESTAMP',
@@ -183,7 +183,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'mariadb': 'DATETIME',
         'mysql': 'DATETIME',
         'mssql': 'DATETIMEOFFSET',
-        'oracle': 'TIMESTAMP',
+        'oracle': 'TIMESTAMP(9)',
         'sqlite': 'TIMESTAMP',
         'duckdb': 'TIMESTAMPTZ',
         'citus': 'TIMESTAMPTZ',
@@ -196,7 +196,7 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
         'mariadb': 'DATETIME',
         'mysql': 'DATETIME',
         'mssql': 'DATETIMEOFFSET',
-        'oracle': 'TIMESTAMP',
+        'oracle': 'TIMESTAMP(9)',
         'sqlite': 'TIMESTAMP',
         'duckdb': 'TIMESTAMPTZ',
         'citus': 'TIMESTAMPTZ',
@@ -536,7 +536,7 @@ def get_db_type_from_pd_type(
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.dtypes import are_dtypes_equal, MRSM_ALIAS_DTYPES
     from meerschaum.utils.misc import parse_arguments_str
-    sqlalchemy_types = attempt_import('sqlalchemy.types')
+    sqlalchemy_types = attempt_import('sqlalchemy.types', lazy=False)
 
     types_registry = (
         PD_TO_DB_DTYPES_FLAVORS
@@ -544,22 +544,29 @@ def get_db_type_from_pd_type(
         else PD_TO_SQLALCHEMY_DTYPES_FLAVORS
     )
 
+    precision, scale = None, None
+    og_pd_type = pd_type
     if pd_type in MRSM_ALIAS_DTYPES:
         pd_type = MRSM_ALIAS_DTYPES[pd_type]
 
     ### Check whether we are able to match this type (e.g. pyarrow support).
     found_db_type = False
-    if pd_type not in types_registry:
+    if pd_type not in types_registry and not pd_type.startswith('numeric['):
         for mapped_pd_type in types_registry:
             if are_dtypes_equal(mapped_pd_type, pd_type):
                 pd_type = mapped_pd_type
                 found_db_type = True
                 break
+    elif pd_type.startswith('numeric['):
+        og_pd_type = pd_type
+        pd_type = 'numeric'
+        precision, scale = get_numeric_precision_scale(flavor, og_pd_type)
+        found_db_type = True
     else:
         found_db_type = True
 
     if not found_db_type:
-        warn(f"Unknown Pandas data type '{pd_type}'. Falling back to 'TEXT'.")
+        warn(f"Unknown Pandas data type '{pd_type}'. Falling back to 'TEXT'.", stacklevel=3)
         return (
             'TEXT'
             if not as_sqlalchemy
@@ -587,6 +594,9 @@ def get_db_type_from_pd_type(
         warn(f"Unknown flavor '{flavor}'. Falling back to '{default_flavor_type}' (default).")
     db_type = flavor_types.get(flavor, default_flavor_type)
     if not as_sqlalchemy:
+        if precision is not None and scale is not None:
+            db_type_bare = db_type.split('(', maxsplit=1)[0]
+            return f"{db_type_bare}({precision},{scale})"
         return db_type
 
     if db_type.startswith('sqlalchemy.dialects'):
@@ -603,9 +613,8 @@ def get_db_type_from_pd_type(
         return cls(*cls_args, **cls_kwargs)
 
     if 'numeric' in db_type.lower():
-        if flavor not in NUMERIC_PRECISION_FLAVORS:
+        if precision is None or scale is None:
             return sqlalchemy_types.Numeric
-        precision, scale = NUMERIC_PRECISION_FLAVORS[flavor]
         return sqlalchemy_types.Numeric(precision, scale)
 
     cls_args, cls_kwargs = None, None
@@ -619,3 +628,37 @@ def get_db_type_from_pd_type(
     if cls_args is None:
         return cls
     return cls(*cls_args, **cls_kwargs)
+
+
+def get_numeric_precision_scale(
+    flavor: str,
+    dtype: Optional[str] = None,
+) -> Union[Tuple[int, int], Tuple[None, None]]:
+    """
+    Return the precision and scale to use for a numeric column for a given database flavor.
+
+    Parameters
+    ----------
+    flavor: str
+        The database flavor for which to return the precision and scale.
+    
+    dtype: Optional[str], default None
+        If provided, return the precision and scale provided in the dtype (if applicable).
+
+    Returns
+    -------
+    A tuple of ints or a tuple of Nones.
+    """
+    from meerschaum.utils.dtypes import are_dtypes_equal
+    if dtype and are_dtypes_equal(dtype, 'numeric'):
+        if '[' in dtype and ',' in dtype:
+            try:
+                parts = dtype.split('[', maxsplit=1)[-1].rstrip(']').split(',', maxsplit=1)
+                return int(parts[0].strip()), int(parts[1].strip())
+            except Exception:
+                pass
+
+    if flavor not in NUMERIC_PRECISION_FLAVORS:
+        return None, None
+
+    return NUMERIC_PRECISION_FLAVORS[flavor]

meerschaum/utils/formatting/__init__.py

@@ -217,8 +217,8 @@ def print_tuple(
     tup: mrsm.SuccessTuple,
     skip_common: bool = True,
     common_only: bool = False,
-    upper_padding: int = 0,
-    lower_padding: int = 0,
+    upper_padding: int = 1,
+    lower_padding: int = 1,
     left_padding: int = 1,
     calm: bool = False,
     _progress: Optional['rich.progress.Progress'] = None,

meerschaum/utils/formatting/_pprint.py

@@ -7,21 +7,22 @@ Pretty printing wrapper
 """
 
 def pprint(
-        *args,
-        detect_password: bool = True,
-        nopretty: bool = False,
-        **kw
-    ) -> None:
+    *args,
+    detect_password: bool = True,
+    nopretty: bool = False,
+    **kw
+) -> None:
     """Pretty print an object according to the configured ANSI and UNICODE settings.
     If detect_password is True (default), search and replace passwords with '*' characters.
     Does not mutate objects.
     """
+    import copy
+    import json
     from meerschaum.utils.packages import attempt_import, import_rich
-    from meerschaum.utils.formatting import ANSI, UNICODE, get_console, print_tuple
+    from meerschaum.utils.formatting import ANSI, get_console, print_tuple
     from meerschaum.utils.warnings import error
     from meerschaum.utils.misc import replace_password, dict_from_od, filter_keywords
     from collections import OrderedDict
-    import copy, json
 
     if (
         len(args) == 1
@@ -52,7 +53,7 @@ def pprint(
         pprintpp = attempt_import('pprintpp', warn=False)
         try:
             _pprint = pprintpp.pprint
-        except Exception as e:
+        except Exception :
             import pprint as _pprint_module
             _pprint = _pprint_module.pprint
 
@@ -62,7 +63,7 @@ def pprint(
 
     try:
         args_copy = copy.deepcopy(args)
-    except Exception as e:
+    except Exception:
         args_copy = args
         modify = False
     _args = []
@@ -85,12 +86,12 @@ def pprint(
                 try:
                     c = json.dumps(c)
                     is_json = True
-                except Exception as e:
+                except Exception:
                     is_json = False
                 if not is_json:
                     try:
                         c = str(c)
-                    except Exception as e:
+                    except Exception:
                         pass
             _args.append(c)
 

meerschaum/utils/misc.py

@@ -957,24 +957,6 @@ def get_connector_labels(
     return sorted(possibilities)
 
 
-def json_serialize_datetime(dt: datetime) -> Union[str, None]:
-    """
-    Serialize a datetime object into JSON (ISO format string).
-
-    Examples
-    --------
-    >>> import json
-    >>> from datetime import datetime
-    >>> json.dumps({'a': datetime(2022, 1, 1)}, default=json_serialize_datetime)
-    '{"a": "2022-01-01T00:00:00Z"}'
-
-    """
-    if not isinstance(dt, datetime):
-        return None
-    tz_suffix = 'Z' if dt.tzinfo is None else ''
-    return dt.isoformat() + tz_suffix
-
-
 def wget(
     url: str,
     dest: Optional[Union[str, 'pathlib.Path']] = None,
@@ -1705,6 +1687,22 @@ def _get_subaction_names(*args, **kwargs) -> Any:
     return real_function(*args, **kwargs)
 
 
+def json_serialize_datetime(dt: datetime) -> Union[str, None]:
+    """
+    Serialize a datetime object into JSON (ISO format string).
+
+    Examples
+    --------
+    >>> import json
+    >>> from datetime import datetime
+    >>> json.dumps({'a': datetime(2022, 1, 1)}, default=json_serialize_datetime)
+    '{"a": "2022-01-01T00:00:00Z"}'
+
+    """
+    from meerschaum.utils.dtypes import serialize_datetime
+    return serialize_datetime(dt)
+
+
 _current_module = sys.modules[__name__]
 __all__ = tuple(
     name

meerschaum/utils/prompt.py

@@ -585,7 +585,7 @@ def get_connectors_completer(*types: str):
 
     class ConnectorCompleter(Completer):
         def get_completions(self, document, complete_event):
-            for label in get_connector_labels(*types):
+            for label in get_connector_labels(*types, search_term=document.text):
                 yield Completion(label, start_position=(-1 * len(document.text)))
 
     return ConnectorCompleter()