meerschaum 2.6.16__py3-none-any.whl → 2.7.0__py3-none-any.whl

meerschaum/utils/daemon/Daemon.py

@@ -432,7 +432,7 @@ class Daemon:
             + "allow_dirty_run=True)"
         )
         env = dict(os.environ)
-        env['MRSM_NOASK'] = 'true'
+        env[STATIC_CONFIG['environment']['noninteractive']] = 'true'
         _launch_success_bool = venv_exec(_launch_daemon_code, debug=debug, venv=None, env=env)
         msg = (
             "Success"
@@ -465,18 +465,25 @@ class Daemon:
             self._write_stop_file('kill')
             return True, "Process has already stopped."
 
+        psutil = attempt_import('psutil')
         process = self.process
         try:
             process.terminate()
             process.kill()
             process.wait(timeout=timeout)
         except Exception as e:
-            return False, f"Failed to kill job {self} with exception: {e}"
+            return False, f"Failed to kill job {self} ({process}) with exception: {e}"
+
+        try:
+            if process.status():
+                return False, "Failed to stop daemon '{self}' ({process})."
+        except psutil.NoSuchProcess:
+            pass
 
         if self.pid_path.exists():
             try:
                 self.pid_path.unlink()
-            except Exception as e:
+            except Exception:
                 pass
 
         self._write_stop_file('kill')
@@ -534,7 +541,7 @@ class Daemon:
         if not timeout:
             try:
                 success = self.process.status() == 'stopped'
-            except psutil.NoSuchProcess as e:
+            except psutil.NoSuchProcess:
                 success = True
             msg = "Success" if success else f"Failed to suspend daemon '{self.daemon_id}'."
             if success:
@@ -677,11 +684,11 @@ class Daemon:
         raise SystemExit(0)
 
     def _send_signal(
-            self,
-            signal_to_send,
-            timeout: Union[float, int, None] = None,
-            check_timeout_interval: Union[float, int, None] = None,
-        ) -> SuccessTuple:
+        self,
+        signal_to_send,
+        timeout: Union[float, int, None] = None,
+        check_timeout_interval: Union[float, int, None] = None,
+    ) -> SuccessTuple:
         """Send a signal to the daemon process.
 
         Parameters
@@ -709,7 +716,7 @@ class Daemon:
                 )
             
             os.kill(pid, signal_to_send)
-        except Exception as e:
+        except Exception:
             return False, f"Failed to send signal {signal_to_send}:\n{traceback.format_exc()}"
 
         timeout = self.get_timeout_seconds(timeout)
@@ -727,7 +734,7 @@ class Daemon:
             time.sleep(check_timeout_interval)
 
         return False, (
-            f"Failed to stop daemon '{self.daemon_id}' within {timeout} second"
+            f"Failed to stop daemon '{self.daemon_id}' (PID: {pid}) within {timeout} second"
             + ('s' if timeout != 1 else '') + '.'
         )
 
@@ -745,7 +752,7 @@ class Daemon:
         if _already_exists and not allow_dirty_run:
             error(
                 f"Daemon '{self.daemon_id}' already exists. " +
-                f"To allow this daemon to run, do one of the following:\n"
+                "To allow this daemon to run, do one of the following:\n"
                 + "  - Execute `daemon.cleanup()`.\n"
                 + f"  - Delete the directory '{self.path}'.\n"
                 + "  - Pass `allow_dirty_run=True` to `daemon.run()`.\n",
@@ -764,7 +771,7 @@ class Daemon:
         if '_process' not in self.__dict__ or self.__dict__['_process'].pid != int(pid):
             try:
                 self._process = psutil.Process(int(pid))
-            except Exception as e:
+            except Exception:
                 if self.pid_path.exists():
                     self.pid_path.unlink()
                 return None
@@ -788,7 +795,7 @@ class Daemon:
             if self.pid_path.exists():
                 try:
                     self.pid_path.unlink()
-                except Exception as e:
+                except Exception:
                     pass
             return 'stopped'
 
@@ -1000,7 +1007,7 @@ class Daemon:
         try:
             with open(self.properties_path, 'r', encoding='utf-8') as file:
                 properties = json.load(file)
-        except Exception as e:
+        except Exception:
             properties = {}
         
         return properties

meerschaum/utils/dataframe.py

@@ -139,7 +139,6 @@ def filter_unseen_df(
     import functools
     import traceback
     from decimal import Decimal
-    from uuid import UUID
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.packages import import_pandas, attempt_import
     from meerschaum.utils.dtypes import (
@@ -147,6 +146,7 @@ def filter_unseen_df(
         are_dtypes_equal,
         attempt_cast_to_numeric,
         attempt_cast_to_uuid,
+        attempt_cast_to_bytes,
         coerce_timezone,
     )
     pd = import_pandas(debug=debug)
@@ -333,6 +333,11 @@ def filter_unseen_df(
     old_uuid_cols = get_uuid_cols(old_df)
     new_uuid_cols = get_uuid_cols(new_df)
     uuid_cols = set(new_uuid_cols + old_uuid_cols)
+
+    old_bytes_cols = get_bytes_cols(old_df)
+    new_bytes_cols = get_bytes_cols(new_df)
+    bytes_cols = set(new_bytes_cols + old_bytes_cols)
+
     joined_df = merge(
         new_df.infer_objects(copy=False).fillna(NA),
         old_df.infer_objects(copy=False).fillna(NA),
@@ -368,6 +373,14 @@ def filter_unseen_df(
         except Exception:
             warn(f"Unable to parse numeric column '{uuid_col}':\n{traceback.format_exc()}")
 
+    for bytes_col in bytes_cols:
+        if bytes_col not in delta_df.columns:
+            continue
+        try:
+            delta_df[bytes_col] = delta_df[bytes_col].apply(attempt_cast_to_bytes)
+        except Exception:
+            warn(f"Unable to parse bytes column '{bytes_col}':\n{traceback.format_exc()}")
+
     return delta_df
 
 
@@ -429,6 +442,7 @@ def parse_df_datetimes(
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.misc import items_str
+    from meerschaum.utils.dtypes import to_datetime
     import traceback
     pd = import_pandas()
     pandas = attempt_import('pandas')
@@ -480,7 +494,7 @@ def parse_df_datetimes(
     ### skip parsing if DataFrame is empty
     if len(pdf) == 0:
         if debug:
-            dprint(f"df is empty. Returning original DataFrame without casting datetime columns...")
+            dprint("df is empty. Returning original DataFrame without casting datetime columns...")
         return df
 
     ignore_cols = set(
@@ -494,8 +508,8 @@ def parse_df_datetimes(
 
     if len(cols_to_inspect) == 0:
         if debug:
-            dprint(f"All columns are ignored, skipping datetime detection...")
-        return df.fillna(pandas.NA)
+            dprint("All columns are ignored, skipping datetime detection...")
+        return df.infer_objects(copy=False).fillna(pandas.NA)
 
     ### apply regex to columns to determine which are ISO datetimes
     iso_dt_regex = r'\d{4}-\d{2}-\d{2}.\d{2}\:\d{2}\:\d+'
@@ -508,21 +522,17 @@ def parse_df_datetimes(
     if not datetime_cols:
         if debug:
             dprint("No columns detected as datetimes, returning...")
-        return df.fillna(pandas.NA)
+        return df.infer_objects(copy=False).fillna(pandas.NA)
 
     if debug:
         dprint("Converting columns to datetimes: " + str(datetime_cols))
 
     try:
         if not using_dask:
-            df[datetime_cols] = df[datetime_cols].apply(
-                pd.to_datetime,
-                utc=True,
-                format='ISO8601',
-            )
+            df[datetime_cols] = df[datetime_cols].apply(to_datetime)
         else:
             df[datetime_cols] = df[datetime_cols].apply(
-                pd.to_datetime,
+                to_datetime,
                 utc=True,
                 axis=1,
                 meta={
@@ -665,7 +675,7 @@ def get_uuid_cols(df: 'pd.DataFrame') -> List[str]:
 
     Returns
     -------
-    A list of columns to treat as numerics.
+    A list of columns to treat as UUIDs.
     """
     if df is None:
         return []
@@ -692,6 +702,135 @@ def get_uuid_cols(df: 'pd.DataFrame') -> List[str]:
     ]
 
 
+def get_datetime_cols(
+    df: 'pd.DataFrame',
+    timezone_aware: bool = True,
+    timezone_naive: bool = True,
+) -> List[str]:
+    """
+    Get the columns which contain `datetime` or `Timestamp` objects from a Pandas DataFrame.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        The DataFrame which may contain `datetime` or `Timestamp` objects.
+
+    timezone_aware: bool, default True
+        If `True`, include timezone-aware datetime columns.
+
+    timezone_naive: bool, default True
+        If `True`, include timezone-naive datetime columns.
+
+    Returns
+    -------
+    A list of columns to treat as datetimes.
+    """
+    if not timezone_aware and not timezone_naive:
+        raise ValueError("`timezone_aware` and `timezone_naive` cannot both be `False`.")
+
+    if df is None:
+        return []
+
+    from datetime import datetime
+    from meerschaum.utils.dtypes import are_dtypes_equal
+    is_dask = 'dask' in df.__module__
+    if is_dask:
+        df = get_first_valid_dask_partition(df)
+
+    known_dt_cols = [
+        col
+        for col, typ in df.dtypes.items()
+        if are_dtypes_equal('datetime', str(typ))
+    ]
+
+    if len(df) == 0:
+        return known_dt_cols
+
+    cols_indices = {
+        col: df[col].first_valid_index()
+        for col in df.columns
+        if col not in known_dt_cols
+    }
+    pydt_cols = [
+        col
+        for col, ix in cols_indices.items()
+        if (
+            ix is not None
+            and
+            isinstance(df.loc[ix][col], datetime)
+        )
+    ]
+    dt_cols_set = set(known_dt_cols + pydt_cols)
+    all_dt_cols = [
+        col
+        for col in df.columns
+        if col in dt_cols_set
+    ]
+    if timezone_aware and timezone_naive:
+        return all_dt_cols
+
+    known_timezone_aware_dt_cols = [
+        col
+        for col in known_dt_cols
+        if getattr(df[col], 'tz', None) is not None
+    ]
+    timezone_aware_pydt_cols = [
+        col
+        for col in pydt_cols
+        if df.loc[cols_indices[col]][col].tzinfo is not None
+    ]
+    timezone_aware_dt_cols_set = set(known_timezone_aware_dt_cols + timezone_aware_pydt_cols)
+    if timezone_aware:
+        return [
+            col
+            for col in all_dt_cols
+            if col in timezone_aware_pydt_cols
+        ]
+
+    return [
+        col
+        for col in all_dt_cols
+        if col not in timezone_aware_dt_cols_set
+    ]
+
+
+def get_bytes_cols(df: 'pd.DataFrame') -> List[str]:
+    """
+    Get the columns which contain bytes strings from a Pandas DataFrame.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        The DataFrame which may contain bytes strings.
+
+    Returns
+    -------
+    A list of columns to treat as bytes.
+    """
+    if df is None:
+        return []
+    is_dask = 'dask' in df.__module__
+    if is_dask:
+        df = get_first_valid_dask_partition(df)
+
+    if len(df) == 0:
+        return []
+
+    cols_indices = {
+        col: df[col].first_valid_index()
+        for col in df.columns
+    }
+    return [
+        col
+        for col, ix in cols_indices.items()
+        if (
+            ix is not None
+            and
+            isinstance(df.loc[ix][col], bytes)
+        )
+    ]
+
+
 def enforce_dtypes(
     df: 'pd.DataFrame',
     dtypes: Dict[str, str],
@@ -743,6 +882,7 @@ def enforce_dtypes(
         is_dtype_numeric,
         attempt_cast_to_numeric,
         attempt_cast_to_uuid,
+        attempt_cast_to_bytes,
         coerce_timezone as _coerce_timezone,
     )
     pandas = mrsm.attempt_import('pandas')
@@ -773,6 +913,11 @@ def enforce_dtypes(
         for col, typ in dtypes.items()
         if typ == 'uuid'
     ]
+    bytes_cols = [
+        col
+        for col, typ in dtypes.items()
+        if typ == 'bytes'
+    ]
     datetime_cols = [
         col
         for col, typ in dtypes.items()
@@ -826,6 +971,17 @@ def enforce_dtypes(
                     if debug:
                         dprint(f"Unable to parse column '{col}' as UUID:\n{e}")
 
+    if bytes_cols:
+        if debug:
+            dprint(f"Checking for bytes: {bytes_cols}")
+        for col in bytes_cols:
+            if col in df.columns:
+                try:
+                    df[col] = df[col].apply(attempt_cast_to_bytes)
+                except Exception as e:
+                    if debug:
+                        dprint(f"Unable to parse column '{col}' as bytes:\n{e}")
+
     if datetime_cols and coerce_timezone:
         if debug:
             dprint(f"Checking for datetime conversion: {datetime_cols}")
@@ -931,6 +1087,8 @@ def get_datetime_bound_from_df(
     -------
     The minimum or maximum datetime value in the dataframe, or `None`.
     """
+    from meerschaum.utils.dtypes import to_datetime, value_is_null
+
     if df is None:
         return None
     if not datetime_column:
@@ -982,9 +1140,9 @@ def get_datetime_bound_from_df(
             dt_val = dt_val.compute()
 
         return (
-            pandas.to_datetime(dt_val).to_pydatetime()
+            to_datetime(dt_val, as_pydatetime=True)
             if are_dtypes_equal(str(type(dt_val)), 'datetime')
-            else (dt_val if dt_val is not pandas.NA else None)
+            else (dt_val if not value_is_null(dt_val) else None)
         )
 
     return None
@@ -1127,7 +1285,7 @@ def get_first_valid_dask_partition(ddf: 'dask.dataframe.DataFrame') -> Union['pd
     for partition in ddf.partitions:
         try:
             pdf = partition.compute()
-        except Exception as e:
+        except Exception:
             continue
         if len(pdf) > 0:
             return pdf
@@ -1408,12 +1566,16 @@ def to_json(
     A JSON string.
     """
     from meerschaum.utils.packages import import_pandas
+    from meerschaum.utils.dtypes import serialize_bytes
     pd = import_pandas()
     uuid_cols = get_uuid_cols(df)
-    if uuid_cols and safe_copy:
+    bytes_cols = get_bytes_cols(df)
+    if safe_copy and bool(uuid_cols or bytes_cols):
         df = df.copy()
     for col in uuid_cols:
         df[col] = df[col].astype(str)
+    for col in bytes_cols:
+        df[col] = df[col].apply(serialize_bytes)
     return df.infer_objects(copy=False).fillna(pd.NA).to_json(
         date_format=date_format,
         date_unit=date_unit,

meerschaum/utils/dtypes/__init__.py

@@ -15,7 +15,19 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Union, Any
 from meerschaum.utils.warnings import warn
 
-MRSM_PD_DTYPES: Dict[str, str] = {
+MRSM_ALIAS_DTYPES: Dict[str, str] = {
+    'decimal': 'numeric',
+    'number': 'numeric',
+    'jsonl': 'json',
+    'JSON': 'json',
+    'binary': 'bytes',
+    'blob': 'bytes',
+    'varbinary': 'bytes',
+    'bytea': 'bytes',
+    'guid': 'uuid',
+    'UUID': 'uuid',
+}
+MRSM_PD_DTYPES: Dict[Union[str, None], str] = {
     'json': 'object',
     'numeric': 'object',
     'uuid': 'object',
@@ -27,6 +39,8 @@ MRSM_PD_DTYPES: Dict[str, str] = {
     'int32': 'Int32',
     'int64': 'Int64',
     'str': 'string[python]',
+    'bytes': 'object',
+    None: 'object',
 }
 
 
@@ -38,6 +52,10 @@ def to_pandas_dtype(dtype: str) -> str:
     if known_dtype is not None:
         return known_dtype
 
+    alias_dtype = MRSM_ALIAS_DTYPES.get(dtype, None)
+    if alias_dtype is not None:
+        return MRSM_PD_DTYPES[alias_dtype]
+
     ### NOTE: Kind of a hack, but if the first word of the given dtype is in all caps,
     ### treat it as a SQL db type.
     if dtype.split(' ')[0].isupper():
@@ -95,7 +113,7 @@ def are_dtypes_equal(
     try:
         if ldtype == rdtype:
             return True
-    except Exception as e:
+    except Exception:
         warn(f"Exception when comparing dtypes, returning False:\n{traceback.format_exc()}")
         return False
 
@@ -115,6 +133,10 @@ def are_dtypes_equal(
     if ldtype in uuid_dtypes and rdtype in uuid_dtypes:
         return True
 
+    bytes_dtypes = ('bytes', 'object')
+    if ldtype in bytes_dtypes and rdtype in bytes_dtypes:
+        return True
+
     ldtype_clean = ldtype.split('[', maxsplit=1)[0]
     rdtype_clean = rdtype.split('[', maxsplit=1)[0]
 
@@ -185,7 +207,7 @@ def attempt_cast_to_numeric(value: Any) -> Any:
             if not value_is_null(value)
             else Decimal('NaN')
         )
-    except Exception as e:
+    except Exception:
         return value
 
 
@@ -201,7 +223,23 @@ def attempt_cast_to_uuid(value: Any) -> Any:
             if not value_is_null(value)
             else None
         )
-    except Exception as e:
+    except Exception:
+        return value
+
+
+def attempt_cast_to_bytes(value: Any) -> Any:
+    """
+    Given a value, attempt to coerce it into a bytestring.
+    """
+    if isinstance(value, bytes):
+        return value
+    try:
+        return (
+            deserialize_bytes_string(str(value))
+            if not value_is_null(value)
+            else None
+        )
+    except Exception:
         return value
 
 
@@ -251,7 +289,7 @@ def coerce_timezone(
 ) -> Any:
     """
     Given a `datetime`, pandas `Timestamp` or `Series` of `Timestamp`,
-    return a naive datetime in terms of UTC.
+    return a UTC timestamp (strip timezone if `strip_utc` is `True`.
     """
     if dt is None:
         return None
@@ -266,9 +304,7 @@ def coerce_timezone(
     dt_is_series = hasattr(dt, 'dtype') and hasattr(dt, '__module__')
 
     if dt_is_series:
-        is_dask = 'dask' in dt.__module__
         pandas = mrsm.attempt_import('pandas', lazy=False)
-        dd = mrsm.attempt_import('dask.dataframe') if is_dask else None
 
         if (
             pandas.api.types.is_datetime64_any_dtype(dt) and (
@@ -279,14 +315,13 @@ def coerce_timezone(
         ):
             return dt
 
-        dt_series = (
-            pandas.to_datetime(dt, utc=True, format='ISO8601')
-            if dd is None
-            else dd.to_datetime(dt, utc=True, format='ISO8601')
-        )
+        dt_series = to_datetime(dt, coerce_utc=False)
         if strip_utc:
-            if dt_series.dt.tz is not None:
-                dt_series = dt_series.dt.tz_localize(None)
+            try:
+                if dt_series.dt.tz is not None:
+                    dt_series = dt_series.dt.tz_localize(None)
+            except Exception:
+                pass
 
         return dt_series
 
@@ -299,3 +334,103 @@ def coerce_timezone(
     if strip_utc:
         return utc_dt.replace(tzinfo=None)
     return utc_dt
+
+
+def to_datetime(dt_val: Any, as_pydatetime: bool = False, coerce_utc: bool = True) -> Any:
+    """
+    Wrap `pd.to_datetime()` and add support for out-of-bounds values.
+    """
+    pandas, dateutil_parser = mrsm.attempt_import('pandas', 'dateutil.parser', lazy=False)
+    is_dask = 'dask' in getattr(dt_val, '__module__', '')
+    dd = mrsm.attempt_import('dask.dataframe') if is_dask else None
+    dt_is_series = hasattr(dt_val, 'dtype') and hasattr(dt_val, '__module__')
+    pd = pandas if dd is None else dd
+
+    try:
+        new_dt_val = pd.to_datetime(dt_val, utc=True, format='ISO8601')
+        if as_pydatetime:
+            return new_dt_val.to_pydatetime()
+        return new_dt_val
+    except (pd.errors.OutOfBoundsDatetime, ValueError):
+        pass
+
+    def parse(x: Any) -> Any:
+        try:
+            return dateutil_parser.parse(x)
+        except Exception:
+            return x
+
+    if dt_is_series:
+        new_series = dt_val.apply(parse)
+        if coerce_utc:
+            return coerce_timezone(new_series)
+        return new_series
+
+    new_dt_val = parse(dt_val)
+    if not coerce_utc:
+        return new_dt_val
+    return coerce_timezone(new_dt_val)
+
+
+def serialize_bytes(data: bytes) -> str:
+    """
+    Return the given bytes as a base64-encoded string.
+    """
+    import base64
+    if not isinstance(data, bytes) and value_is_null(data):
+        return data
+    return base64.b64encode(data).decode('utf-8')
+
+
+def deserialize_bytes_string(data: str | None, force_hex: bool = False) -> bytes | None:
+    """
+    Given a serialized ASCII string of bytes data, return the original bytes.
+    The input data may either be base64- or hex-encoded.
+
+    Parameters
+    ----------
+    data: str | None
+        The string to be deserialized into bytes.
+        May be base64- or hex-encoded (prefixed with `'\\x'`).
+
+    force_hex: bool = False
+        If `True`, treat the input string as hex-encoded.
+        If `data` does not begin with the prefix `'\\x'`, set `force_hex` to `True`.
+        This will still strip the leading `'\\x'` prefix if present.
+
+    Returns
+    -------
+    The original bytes used to produce the encoded string `data`.
+    """
+    if not isinstance(data, str) and value_is_null(data):
+        return data
+
+    import binascii
+    import base64
+
+    is_hex = force_hex or data.startswith('\\x')
+
+    if is_hex:
+        if data.startswith('\\x'):
+            data = data[2:]
+        return binascii.unhexlify(data)
+
+    return base64.b64decode(data)
+
+
+def deserialize_base64(data: str) -> bytes:
+    """
+    Return the original bytestring from the given base64-encoded string.
+    """
+    import base64
+    return base64.b64decode(data)
+
+
+def encode_bytes_for_bytea(data: bytes, with_prefix: bool = True) -> str | None:
+    """
+    Return the given bytes as a hex string for PostgreSQL's `BYTEA` type.
+    """
+    import binascii
+    if not isinstance(data, bytes) and value_is_null(data):
+        return data
+    return ('\\x' if with_prefix else '') + binascii.hexlify(data).decode('utf-8')