meerschaum 3.0.0rc1__py3-none-any.whl → 3.0.0rc3__py3-none-any.whl

meerschaum/utils/debug.py

@@ -10,16 +10,16 @@ from __future__ import annotations
 from meerschaum.utils.typing import Union, Optional, List
 
 def dprint(
-        msg: str,
-        leader: bool = True,
-        package: bool = True,
-        color: Optional[Union[str, List[str]]] = None,
-        attrs: Optional[List[str]] = None,
-        nopretty: bool = False,
-        _progress: Optional['rich.progress.Progress'] = None,
-        _task: Optional[int] = None,
-        **kw
-    ) -> None:
+    msg: str,
+    leader: bool = True,
+    package: bool = True,
+    color: Optional[Union[str, List[str]]] = None,
+    attrs: Optional[List[str]] = None,
+    nopretty: bool = False,
+    _progress: Optional['rich.progress.Progress'] = None,
+    _task: Optional[int] = None,
+    **kw
+) -> None:
     """Print a debug message."""
     if attrs is None:
         attrs = []
@@ -86,11 +86,11 @@ def dprint(
 
 
 def _checkpoint(
-        _progress: Optional['rich.progress.Progress'] = None,
-        _task: Optional[int] = None,
-        _total: Optional[int] = None,
-        **kw
-    ) -> None:
+    _progress: Optional['rich.progress.Progress'] = None,
+    _task: Optional[int] = None,
+    _total: Optional[int] = None,
+    **kw
+) -> None:
     """If the `_progress` and `_task` objects are provided, increment the task by one step.
     If `_total` is provided, update the total instead.
     """

meerschaum/utils/dtypes/__init__.py

@@ -9,12 +9,14 @@ Utility functions for working with data types.
 import traceback
 import json
 import uuid
-from datetime import timezone, datetime
+import time
+from datetime import timezone, datetime, date, timedelta
 from decimal import Decimal, Context, InvalidOperation, ROUND_HALF_UP
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Union, Any, Optional, Tuple
 from meerschaum.utils.warnings import warn
+from meerschaum._internal.static import STATIC_CONFIG as _STATIC_CONFIG
 
 MRSM_ALIAS_DTYPES: Dict[str, str] = {
     'decimal': 'numeric',
@@ -30,6 +32,8 @@ MRSM_ALIAS_DTYPES: Dict[str, str] = {
     'UUID': 'uuid',
     'geom': 'geometry',
     'geog': 'geography',
+    'boolean': 'bool',
+    'day': 'date',
 }
 MRSM_PD_DTYPES: Dict[Union[str, None], str] = {
     'json': 'object',
@@ -37,18 +41,52 @@ MRSM_PD_DTYPES: Dict[Union[str, None], str] = {
     'geometry': 'object',
     'geography': 'object',
     'uuid': 'object',
-    'datetime': 'datetime64[ns, UTC]',
+    'date': 'date32[day][pyarrow]',
+    'datetime': 'datetime64[us, UTC]',
     'bool': 'bool[pyarrow]',
-    'int': 'Int64',
-    'int8': 'Int8',
-    'int16': 'Int16',
-    'int32': 'Int32',
-    'int64': 'Int64',
-    'str': 'string[python]',
-    'bytes': 'object',
+    'int': 'int64[pyarrow]',
+    'int8': 'int8[pyarrow]',
+    'int16': 'int16[pyarrow]',
+    'int32': 'int32[pyarrow]',
+    'int64': 'int64[pyarrow]',
+    'str': 'string',
+    'bytes': 'binary[pyarrow]',
     None: 'object',
 }
 
+MRSM_PRECISION_UNITS_SCALARS: Dict[str, Union[int, float]] = {
+    'nanosecond': 1_000_000_000,
+    'microsecond': 1_000_000,
+    'millisecond': 1000,
+    'second': 1,
+    'minute': (1 / 60),
+    'hour': (1 / 3600),
+    'day': (1 / 86400),
+}
+
+MRSM_PRECISION_UNITS_ALIASES: Dict[str, str] = {
+    'ns': 'nanosecond',
+    'us': 'microsecond',
+    'ms': 'millisecond',
+    's': 'second',
+    'sec': 'second',
+    'm': 'minute',
+    'min': 'minute',
+    'h': 'hour',
+    'hr': 'hour',
+    'd': 'day',
+    'D': 'day',
+}
+MRSM_PRECISION_UNITS_ABBREVIATIONS: Dict[str, str] = {
+    'nanosecond': 'ns',
+    'microsecond': 'us',
+    'millisecond': 'ms',
+    'second': 's',
+    'minute': 'min',
+    'hour': 'hr',
+    'day': 'D',
+}
+
 
 def to_pandas_dtype(dtype: str) -> str:
     """
@@ -147,7 +185,7 @@ def are_dtypes_equal(
     if ldtype in json_dtypes and rdtype in json_dtypes:
         return True
 
-    numeric_dtypes = ('numeric', 'object')
+    numeric_dtypes = ('numeric', 'decimal', 'object')
     if ldtype in numeric_dtypes and rdtype in numeric_dtypes:
         return True
 
@@ -155,7 +193,7 @@ def are_dtypes_equal(
     if ldtype in uuid_dtypes and rdtype in uuid_dtypes:
         return True
 
-    bytes_dtypes = ('bytes', 'object')
+    bytes_dtypes = ('bytes', 'object', 'binary')
     if ldtype in bytes_dtypes and rdtype in bytes_dtypes:
         return True
 
@@ -179,7 +217,10 @@ def are_dtypes_equal(
     if ldtype in string_dtypes and rdtype in string_dtypes:
         return True
 
-    int_dtypes = ('int', 'int64', 'int32', 'int16', 'int8')
+    int_dtypes = (
+        'int', 'int64', 'int32', 'int16', 'int8',
+        'uint', 'uint64', 'uint32', 'uint16', 'uint8',
+    )
     if ldtype.lower() in int_dtypes and rdtype.lower() in int_dtypes:
         return True
 
@@ -191,6 +232,13 @@ def are_dtypes_equal(
     if ldtype in bool_dtypes and rdtype in bool_dtypes:
         return True
 
+    date_dtypes = (
+        'date', 'date32', 'date32[pyarrow]', 'date32[day][pyarrow]',
+        'date64', 'date64[pyarrow]', 'date64[ms][pyarrow]',
+    )
+    if ldtype in date_dtypes and rdtype in date_dtypes:
+        return True
+
     return False
 
 
@@ -461,7 +509,6 @@ def coerce_timezone(
             return dt
 
     dt_is_series = hasattr(dt, 'dtype') and hasattr(dt, '__module__')
-
     if dt_is_series:
         pandas = mrsm.attempt_import('pandas', lazy=False)
 
@@ -497,15 +544,40 @@ def coerce_timezone(
     return utc_dt
 
 
-def to_datetime(dt_val: Any, as_pydatetime: bool = False, coerce_utc: bool = True) -> Any:
+def to_datetime(
+    dt_val: Any,
+    as_pydatetime: bool = False,
+    coerce_utc: bool = True,
+    precision_unit: Optional[str] = None,
+) -> Any:
     """
     Wrap `pd.to_datetime()` and add support for out-of-bounds values.
+
+    Parameters
+    ----------
+    dt_val: Any
+        The value to coerce to Pandas Timestamps.
+
+    as_pydatetime: bool, default False
+        If `True`, return a Python datetime object.
+
+    coerce_utc: bool, default True
+        If `True`, ensure the value has UTC tzinfo.
+
+    precision_unit: Optional[str], default None
+        If provided, enforce the provided precision unit.
     """
     pandas, dateutil_parser = mrsm.attempt_import('pandas', 'dateutil.parser', lazy=False)
     is_dask = 'dask' in getattr(dt_val, '__module__', '')
     dd = mrsm.attempt_import('dask.dataframe') if is_dask else None
     dt_is_series = hasattr(dt_val, 'dtype') and hasattr(dt_val, '__module__')
     pd = pandas if dd is None else dd
+    enforce_precision = precision_unit is not None
+    precision_unit = precision_unit or 'microsecond'
+    true_precision_unit = MRSM_PRECISION_UNITS_ALIASES.get(precision_unit, precision_unit)
+    precision_abbreviation = MRSM_PRECISION_UNITS_ABBREVIATIONS.get(true_precision_unit, None)
+    if not precision_abbreviation:
+        raise ValueError(f"Invalid precision '{precision_unit}'.")
 
     def parse(x: Any) -> Any:
         try:
@@ -513,6 +585,25 @@ def to_datetime(dt_val: Any, as_pydatetime: bool = False, coerce_utc: bool = Tru
         except Exception:
             return x
 
+    def check_dtype(dtype_to_check: str, with_utc: bool = True) -> bool:
+        dtype_check_against = (
+            f"datetime64[{precision_abbreviation}, UTC]"
+            if with_utc
+            else f"datetime64[{precision_abbreviation}]"
+        )
+        return (
+            dtype_to_check == dtype_check_against
+            if enforce_precision
+            else (
+                dtype_to_check.startswith('datetime64[')
+                and (
+                    ('utc' in dtype_to_check.lower())
+                    if with_utc
+                    else ('utc' not in dtype_to_check.lower())
+                )
+            )
+        )
+
     if isinstance(dt_val, pd.Timestamp):
         dt_val_to_return = dt_val if not as_pydatetime else dt_val.to_pydatetime()
         return (
@@ -525,7 +616,11 @@ def to_datetime(dt_val: Any, as_pydatetime: bool = False, coerce_utc: bool = Tru
         changed_tz = False
         original_tz = None
         dtype = str(getattr(dt_val, 'dtype', 'object'))
-        if are_dtypes_equal(dtype, 'datetime') and 'utc' not in dtype.lower():
+        if (
+            are_dtypes_equal(dtype, 'datetime')
+            and 'utc' not in dtype.lower()
+            and hasattr(dt_val, 'dt')
+        ):
             original_tz = dt_val.dt.tz
             dt_val = dt_val.dt.tz_localize(timezone.utc)
             changed_tz = True
@@ -533,12 +628,14 @@ def to_datetime(dt_val: Any, as_pydatetime: bool = False, coerce_utc: bool = Tru
         try:
             new_dt_series = (
                 dt_val
-                if dtype == 'datetime64[ns, UTC]'
-                else dt_val.astype("datetime64[ns, UTC]")
+                if check_dtype(dtype, with_utc=True)
+                else dt_val.astype(f"datetime64[{precision_abbreviation}, UTC]")
             )
         except pd.errors.OutOfBoundsDatetime:
             try:
-                new_dt_series = dt_val.astype("datetime64[ms, UTC]")
+                next_precision = get_next_precision_unit(true_precision_unit)
+                next_precision_abbrevation = MRSM_PRECISION_UNITS_ABBREVIATIONS[next_precision]
+                new_dt_series = dt_val.astype(f"datetime64[{next_precision_abbrevation}, UTC]")
             except Exception:
                 new_dt_series = None
         except ValueError:
@@ -547,8 +644,8 @@ def to_datetime(dt_val: Any, as_pydatetime: bool = False, coerce_utc: bool = Tru
             try:
                 new_dt_series = (
                     new_dt_series
-                    if str(getattr(new_dt_series, 'dtype', None)) == 'datetime64[ns]'
-                    else dt_val.astype("datetime64[ns]")
+                    if check_dtype(str(getattr(new_dt_series, 'dtype', None)), with_utc=False)
+                    else dt_val.astype(f"datetime64[{precision_abbreviation}]")
                 )
             except Exception:
                 new_dt_series = None
@@ -565,6 +662,8 @@ def to_datetime(dt_val: Any, as_pydatetime: bool = False, coerce_utc: bool = Tru
 
     try:
         new_dt_val = pd.to_datetime(dt_val, utc=True, format='ISO8601')
+        if new_dt_val.unit != precision_abbreviation:
+            new_dt_val = new_dt_val.as_unit(precision_abbreviation)
         if as_pydatetime:
             return new_dt_val.to_pydatetime()
         return new_dt_val
@@ -716,12 +815,20 @@ def serialize_datetime(dt: datetime) -> Union[str, None]:
     '{"a": "2022-01-01T00:00:00Z"}'
 
     """
-    if not isinstance(dt, datetime):
+    if not hasattr(dt, 'isoformat'):
         return None
-    tz_suffix = 'Z' if dt.tzinfo is None else ''
+
+    tz_suffix = 'Z' if getattr(dt, 'tzinfo', None) is None else ''
     return dt.isoformat() + tz_suffix
 
 
+def serialize_date(d: date) -> Union[str, None]:
+    """
+    Serialize a date object into its ISO representation.
+    """
+    return d.isoformat() if hasattr(d, 'isoformat') else None
+
+
 def json_serialize_value(x: Any, default_to_str: bool = True) -> Union[str, None]:
     """
     Serialize the given value to a JSON value. Accounts for datetimes, bytes, decimals, etc.
@@ -745,6 +852,9 @@ def json_serialize_value(x: Any, default_to_str: bool = True) -> Union[str, None
     if hasattr(x, 'tzinfo'):
         return serialize_datetime(x)
 
+    if hasattr(x, 'isoformat'):
+        return serialize_date(x)
+
     if isinstance(x, bytes):
         return serialize_bytes(x)
 
@@ -846,3 +956,258 @@ def get_geometry_type_srid(
             break
 
     return geometry_type, srid
+
+
+def get_current_timestamp(
+    precision_unit: str = _STATIC_CONFIG['dtypes']['datetime']['default_precision_unit'],
+    precision_interval: int = 1,
+    round_to: str = 'down',
+    as_pandas: bool = False,
+    as_int: bool = False,
+    _now: Union[datetime, int, None] = None,
+) -> 'Union[datetime, pd.Timestamp, int]':
+    """
+    Return the current UTC timestamp to nanosecond precision.
+
+    Parameters
+    ----------
+    precision_unit: str, default 'us'
+        The precision of the timestamp to be returned.
+        Valid values are the following:
+            - `ns` / `nanosecond`
+            - `us` / `microsecond`
+            - `ms` / `millisecond`
+            - `s` / `sec` / `second`
+            - `m` / `min` / `minute`
+            - `h` / `hr` / `hour`
+            - `d` / `day`
+
+    precision_interval: int, default 1
+        Round the timestamp to the `precision_interval` units.
+        For example, `precision='minute'` and `precision_interval=15` will round to 15-minute intervals.
+        Note: `precision_interval` must be 1 when `precision='nanosecond'`.
+
+    round_to: str, default 'down'
+        The direction to which to round the timestamp.
+        Available options are `down`, `up`, and `closest`.
+
+    as_pandas: bool, default False
+        If `True`, return a Pandas Timestamp.
+        This is always true if `unit` is `nanosecond`.
+
+    as_int: bool, default False
+        If `True`, return the timestamp to an integer.
+        Overrides `as_pandas`.
+
+    Returns
+    -------
+    A Pandas Timestamp, datetime object, or integer with precision to the provided unit.
+
+    Examples
+    --------
+    >>> get_current_timestamp('ns')
+    Timestamp('2025-07-17 17:59:16.423644369+0000', tz='UTC')
+    >>> get_current_timestamp('ms')
+    Timestamp('2025-07-17 17:59:16.424000+0000', tz='UTC')
+    """
+    true_precision_unit = MRSM_PRECISION_UNITS_ALIASES.get(precision_unit, precision_unit)
+    if true_precision_unit not in MRSM_PRECISION_UNITS_SCALARS:
+        from meerschaum.utils.misc import items_str
+        raise ValueError(
+            f"Unknown precision unit '{precision_unit}'. "
+            "Accepted values are "
+            f"{items_str(list(MRSM_PRECISION_UNITS_SCALARS) + list(MRSM_PRECISION_UNITS_ALIASES))}."
+        )
+
+    if not as_int:
+        as_pandas = as_pandas or true_precision_unit == 'nanosecond'
+    pd = mrsm.attempt_import('pandas', lazy=False) if as_pandas else None
+
+    if true_precision_unit == 'nanosecond':
+        if precision_interval != 1:
+            warn("`precision_interval` must be 1 for nanosecond precision.")
+        now_ts = time.time_ns() if not isinstance(_now, int) else _now
+        if as_int:
+            return now_ts
+        return pd.to_datetime(now_ts, unit='ns', utc=True)
+
+    now = datetime.now(timezone.utc) if not isinstance(_now, datetime) else _now
+    delta = timedelta(**{true_precision_unit + 's': precision_interval})
+    rounded_now = round_time(now, delta, to=round_to)
+
+    if as_int:
+        return int(rounded_now.timestamp() * MRSM_PRECISION_UNITS_SCALARS[true_precision_unit])
+
+    ts_val = (
+        pd.to_datetime(rounded_now, utc=True)
+        if as_pandas
+        else rounded_now
+    )
+
+    if not as_pandas:
+        return ts_val
+
+    as_unit_precisions = ('microsecond', 'millisecond', 'second')
+    if true_precision_unit not in as_unit_precisions:
+        return ts_val
+
+    return ts_val.as_unit(MRSM_PRECISION_UNITS_ABBREVIATIONS[true_precision_unit])
+
+
+def dtype_is_special(type_: str) -> bool:
+    """
+    Return whether a dtype should be treated as a special Meerschaum dtype.
+    This is not the same as a Meerschaum alias.
+    """
+    true_type = MRSM_ALIAS_DTYPES.get(type_, type_)
+    if true_type in (
+        'uuid',
+        'json',
+        'bytes',
+        'numeric',
+        'datetime',
+        'geometry',
+        'geography',
+        'date',
+    ):
+        return True
+
+    if are_dtypes_equal(true_type, 'datetime'):
+        return True
+
+    if are_dtypes_equal(true_type, 'date'):
+        return True
+
+    if true_type.startswith('numeric'):
+        return True
+
+    if true_type.startswith('geometry'):
+        return True
+
+    if true_type.startswith('geography'):
+        return True
+
+    return False
+
+
+def get_next_precision_unit(precision_unit: str, decrease: bool = True) -> str:
+    """
+    Get the next precision string in order of value.
+
+    Parameters
+    ----------
+    precision_unit: str
+        The precision string (`'nanosecond'`, `'ms'`, etc.).
+
+    decrease: bool, defaul True
+        If `True` return the precision unit which is lower (e.g. `nanosecond` -> `millisecond`).
+        If `False`, return the precision unit which is higher.
+
+    Returns
+    -------
+    A `precision` string which is lower or higher than the given precision unit.
+
+    Examples
+    --------
+    >>> get_next_precision_unit('nanosecond')
+    'microsecond'
+    >>> get_next_precision_unit('ms')
+    'second'
+    >>> get_next_precision_unit('hour', decrease=False)
+    'minute'
+    """
+    true_precision_unit = MRSM_PRECISION_UNITS_ALIASES.get(precision_unit, precision_unit)
+    precision_scalar = MRSM_PRECISION_UNITS_SCALARS.get(true_precision_unit, None)
+    if not precision_scalar:
+        raise ValueError(f"Invalid precision unit '{precision_unit}'.")
+
+    precisions = sorted(
+        list(MRSM_PRECISION_UNITS_SCALARS),
+        key=lambda p: MRSM_PRECISION_UNITS_SCALARS[p]
+    )
+
+    precision_index = precisions.index(true_precision_unit)
+    new_precision_index = precision_index + (-1 if decrease else 1)
+    if new_precision_index < 0 or new_precision_index >= len(precisions):
+        raise ValueError(f"No precision {'below' if decrease else 'above'} '{precision_unit}'.")
+
+    return precisions[new_precision_index]
+
+
+def round_time(
+    dt: Optional[datetime] = None,
+    date_delta: Optional[timedelta] = None,
+    to: 'str' = 'down'
+) -> datetime:
+    """
+    Round a datetime object to a multiple of a timedelta.
+
+    Parameters
+    ----------
+    dt: Optional[datetime], default None
+        If `None`, grab the current UTC datetime.
+
+    date_delta: Optional[timedelta], default None
+        If `None`, use a delta of 1 minute.
+
+    to: 'str', default 'down'
+        Available options are `'up'`, `'down'`, and `'closest'`.
+
+    Returns
+    -------
+    A rounded `datetime` object.
+
+    Examples
+    --------
+    >>> round_time(datetime(2022, 1, 1, 12, 15, 57, 200))
+    datetime.datetime(2022, 1, 1, 12, 15)
+    >>> round_time(datetime(2022, 1, 1, 12, 15, 57, 200), to='up')
+    datetime.datetime(2022, 1, 1, 12, 16)
+    >>> round_time(datetime(2022, 1, 1, 12, 15, 57, 200), timedelta(hours=1))
+    datetime.datetime(2022, 1, 1, 12, 0)
+    >>> round_time(
+    ...   datetime(2022, 1, 1, 12, 15, 57, 200),
+    ...   timedelta(hours=1),
+    ...   to = 'closest'
+    ... )
+    datetime.datetime(2022, 1, 1, 12, 0)
+    >>> round_time(
+    ...   datetime(2022, 1, 1, 12, 45, 57, 200),
+    ...   datetime.timedelta(hours=1),
+    ...   to = 'closest'
+    ... )
+    datetime.datetime(2022, 1, 1, 13, 0)
+
+    """
+    from decimal import Decimal, ROUND_HALF_UP, ROUND_DOWN, ROUND_UP
+    if date_delta is None:
+        date_delta = timedelta(minutes=1)
+
+    if dt is None:
+        dt = datetime.now(timezone.utc).replace(tzinfo=None)
+
+    def get_total_microseconds(td: timedelta) -> int:
+        return (td.days * 86400 + td.seconds) * 1_000_000 + td.microseconds
+
+    round_to_microseconds = get_total_microseconds(date_delta)
+    if round_to_microseconds == 0:
+        return dt
+
+    dt_delta_from_min = dt.replace(tzinfo=None) - datetime.min
+    dt_total_microseconds = get_total_microseconds(dt_delta_from_min)
+
+    dt_dec = Decimal(dt_total_microseconds)
+    round_to_dec = Decimal(round_to_microseconds)
+
+    div = dt_dec / round_to_dec
+    if to == 'down':
+        num_intervals = div.to_integral_value(rounding=ROUND_DOWN)
+    elif to == 'up':
+        num_intervals = div.to_integral_value(rounding=ROUND_UP)
+    else:
+        num_intervals = div.to_integral_value(rounding=ROUND_HALF_UP)
+
+    rounded_dt_total_microseconds = num_intervals * round_to_dec
+    adjustment_microseconds = int(rounded_dt_total_microseconds) - dt_total_microseconds
+
+    return dt + timedelta(microseconds=adjustment_microseconds)