meerschaum 3.0.0rc1__py3-none-any.whl → 3.0.0rc2__py3-none-any.whl

meerschaum/connectors/valkey/_pipes.py

@@ -321,14 +321,40 @@ def drop_pipe(
     -------
     A `SuccessTuple` indicating success.
     """
-    for chunk_begin, chunk_end in pipe.get_chunk_bounds(debug=debug):
-        clear_chunk_success, clear_chunk_msg = pipe.clear(
-            begin=chunk_begin,
-            end=chunk_end,
-            debug=debug,
+    if not pipe.exists(debug=debug):
+        return True, f"{pipe} does not exist, so it was not dropped."
+
+    table_name = self.quote_table(pipe.target)
+    dt_col = pipe.columns.get('datetime', None)
+
+    try:
+        members = (
+            self.client.zrange(table_name, 0, -1)
+            if dt_col
+            else self.client.smembers(table_name)
         )
-        if not clear_chunk_success:
-            return clear_chunk_success, clear_chunk_msg
+        
+        keys_to_delete = []
+        for member_bytes in members:
+            member_str = member_bytes.decode('utf-8')
+            member_doc = json.loads(member_str)
+            ix_str = member_doc.get('ix')
+            if not ix_str:
+                continue
+            
+            ix_doc = string_to_dict(ix_str.replace(COLON, ':'))
+            doc_key = self.get_document_key(ix_doc, list(ix_doc.keys()), table_name)
+            keys_to_delete.append(doc_key)
+
+        if keys_to_delete:
+            batch_size = 1000
+            for i in range(0, len(keys_to_delete), batch_size):
+                batch = keys_to_delete[i:i+batch_size]
+                self.client.delete(*batch)
+
+    except Exception as e:
+        return False, f"Failed to delete documents for {pipe}:\n{e}"
+
     try:
         self.drop_table(pipe.target, debug=debug)
     except Exception as e:
@@ -558,11 +584,7 @@ def sync_pipe(
 
     valkey_dtypes = pipe.parameters.get('valkey', {}).get('dtypes', {})
     new_dtypes = {
-        str(key): (
-            str(val)
-            if not are_dtypes_equal(str(val), 'datetime')
-            else 'datetime64[ns, UTC]'
-        )
+        str(key): str(val)
         for key, val in df.dtypes.items()
         if str(key) not in valkey_dtypes
     }
@@ -571,6 +593,8 @@ def sync_pipe(
             try:
                 df[col] = df[col].astype(typ)
             except Exception:
+                import traceback
+                traceback.print_exc()
                 valkey_dtypes[col] = 'string'
                 new_dtypes[col] = 'string'
                 df[col] = df[col].astype('string')
@@ -780,7 +804,7 @@ def get_sync_time(
     """
     from meerschaum.utils.dtypes import are_dtypes_equal
     dt_col = pipe.columns.get('datetime', None)
-    dt_typ = pipe.dtypes.get(dt_col, 'datetime64[ns, UTC]')
+    dt_typ = pipe.dtypes.get(dt_col, 'datetime')
     if not dt_col:
         return None
 
@@ -788,14 +812,18 @@ def get_sync_time(
     table_name = self.quote_table(pipe.target)
     try:
         vals = (
-            self.client.zrevrange(table_name, 0, 0)
+            self.client.zrevrange(table_name, 0, 0, withscores=True)
             if newest
-            else self.client.zrange(table_name, 0, 0)
+            else self.client.zrange(table_name, 0, 0, withscores=True)
         )
         if not vals:
             return None
-        val = vals[0]
+        val = vals[0][0]
+        if isinstance(val, bytes):
+            val = val.decode('utf-8')
     except Exception:
+        import traceback
+        traceback.print_exc()
         return None
 
     doc = json.loads(val)

meerschaum/core/Pipe/__init__.py

@@ -117,11 +117,13 @@ class Pipe:
         tzinfo,
         enforce,
         null_indices,
+        mixed_numerics,
         get_columns,
         get_columns_types,
         get_columns_indices,
         get_indices,
         get_parameters,
+        get_dtypes,
         update_parameters,
         tags,
         get_id,
@@ -133,6 +135,9 @@ class Pipe:
         target,
         _target_legacy,
         guess_datetime,
+        precision,
+        get_precision,
+        _invalidate_cache,
     )
     from ._show import show
     from ._edit import edit, edit_definition, update
@@ -143,11 +148,7 @@ class Pipe:
         filter_existing,
         _get_chunk_label,
         get_num_workers,
-        _persist_new_json_columns,
-        _persist_new_numeric_columns,
-        _persist_new_uuid_columns,
-        _persist_new_bytes_columns,
-        _persist_new_geometry_columns,
+        _persist_new_special_columns,
     )
     from ._verify import (
         verify,
@@ -179,9 +180,11 @@ class Pipe:
         upsert: Optional[bool] = None,
         autoincrement: Optional[bool] = None,
         autotime: Optional[bool] = None,
+        precision: Union[str, Dict[str, Union[str, int]], None] = None,
         static: Optional[bool] = None,
         enforce: Optional[bool] = None,
         null_indices: Optional[bool] = None,
+        mixed_numerics: Optional[bool] = None,
         mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
         cache: bool = False,
         debug: bool = False,
@@ -240,6 +243,13 @@ class Pipe:
         autotime: Optional[bool], default None
             If `True`, set `autotime` in the parameters.
 
+        precision: Union[str, Dict[str, Union[str, int]], None], default None
+            If provided, set `precision` in the parameters.
+            This may be either a string (the precision unit) or a dictionary of in the form
+            `{'unit': <unit>, 'interval': <interval>}`.
+            Default is determined by the `datetime` column dtype
+            (e.g. `datetime64[us]` is `microsecond` precision).
+
         static: Optional[bool], default None
             If `True`, set `static` in the parameters.
 
@@ -251,6 +261,11 @@ class Pipe:
             Set to `False` if there will be no null values in the index columns.
             Defaults to `True`.
 
+        mixed_numerics: bool, default None
+            If `True`, integer columns will be converted to `numeric` when floats are synced.
+            Set to `False` to disable this behavior.
+            Defaults to `True`.
+
         temporary: bool, default False
             If `True`, prevent instance tables (pipes, users, plugins) from being created.
 
@@ -353,6 +368,11 @@ class Pipe:
         if isinstance(autotime, bool):
             self._attributes['parameters']['autotime'] = autotime
 
+        if isinstance(precision, dict):
+            self._attributes['parameters']['precision'] = precision
+        elif isinstance(precision, str):
+            self._attributes['parameters']['precision'] = {'unit': precision}
+
         if isinstance(static, bool):
             self._attributes['parameters']['static'] = static
 
@@ -362,6 +382,9 @@ class Pipe:
         if isinstance(null_indices, bool):
             self._attributes['parameters']['null_indices'] = null_indices
 
+        if isinstance(mixed_numerics, bool):
+            self._attributes['parameters']['mixed_numerics'] = mixed_numerics
+
         ### NOTE: The parameters dictionary is {} by default.
         ###       A Pipe may be registered without parameters, then edited,
         ###       or a Pipe may be registered with parameters set in-memory first.

meerschaum/core/Pipe/_attributes.py

@@ -13,7 +13,7 @@ from datetime import timezone
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import Tuple, Dict, Any, Union, Optional, List
-from meerschaum.utils.warnings import warn
+from meerschaum.utils.warnings import warn, dprint
 
 
 @property
@@ -52,18 +52,34 @@ def attributes(self) -> Dict[str, Any]:
 def get_parameters(
     self,
     apply_symlinks: bool = True,
+    refresh: bool = False,
+    debug: bool = False,
     _visited: 'Optional[set[mrsm.Pipe]]' = None,
 ) -> Dict[str, Any]:
     """
     Return the `parameters` dictionary of the pipe.
+
+    Parameters
+    ----------
+    apply_symlinks: bool, default True
+        If `True`, resolve references to parameters from other pipes.
+
+    refresh: bool, default False
+        If `True`, pull the latest attributes for the pipe.
+
+    Returns
+    -------
+    The pipe's parameters dictionary.
     """
     from meerschaum.config._patch import apply_patch_to_config
-    from meerschaum.utils.warnings import warn
     from meerschaum.config._read_config import search_and_substitute_config
 
     if _visited is None:
         _visited = {self}
 
+    if refresh:
+        self._invalidate_cache(hard=True)
+
     raw_parameters = self.attributes.get('parameters', {})
     ref_keys = raw_parameters.get('reference')
     if not apply_symlinks:
@@ -71,13 +87,15 @@ def get_parameters(
 
     if ref_keys:
         try:
+            if debug:
+                dprint(f"Building reference pipe from keys: {ref_keys}")
             ref_pipe = mrsm.Pipe(**ref_keys)
             if ref_pipe in _visited:
                 warn(f"Circular reference detected in {self}: chain involves {ref_pipe}.")
                 return search_and_substitute_config(raw_parameters)
 
             _visited.add(ref_pipe)
-            base_params = ref_pipe.get_parameters(_visited=_visited)
+            base_params = ref_pipe.get_parameters(_visited=_visited, debug=debug)
         except Exception as e:
             warn(f"Failed to resolve reference pipe for {self}: {e}")
             base_params = {}
@@ -113,10 +131,7 @@ def parameters(self) -> Optional[Dict[str, Any]]:
     """
     Return the parameters dictionary of the pipe.
     """
-    if (_parameters := self.__dict__.get('_parameters', None)) is not None:
-        return _parameters
-    self._parameters = self.get_parameters()
-    return self._parameters
+    return self.get_parameters()
 
 
 @parameters.setter
@@ -165,14 +180,12 @@ def indices(self) -> Union[Dict[str, Union[str, List[str]]], None]:
         if 'indexes' in self.parameters
         else 'indices'
     )
-    if indices_key not in self.parameters:
-        self.parameters[indices_key] = {}
-    _indices = self.parameters[indices_key]
+
+    _indices = self.parameters.get(indices_key, {})
     _columns = self.columns
     dt_col = _columns.get('datetime', None)
     if not isinstance(_indices, dict):
         _indices = {}
-        self.parameters[indices_key] = _indices
     unique_cols = list(set((
         [dt_col]
         if dt_col
@@ -246,17 +259,62 @@ def tags(self, _tags: List[str]) -> None:
 
 
 @property
-def dtypes(self) -> Union[Dict[str, Any], None]:
+def dtypes(self) -> Dict[str, Any]:
     """
     If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
     """
+    return self.get_dtypes(refresh=False)
+
+
+@dtypes.setter
+def dtypes(self, _dtypes: Dict[str, Any]) -> None:
+    """
+    Override the dtypes dictionary of the in-memory pipe.
+    Call `meerschaum.Pipe.edit()` to persist changes.
+    """
+    self.update_parameters({'dtypes': _dtypes}, persist=False)
+    _ = self.__dict__.pop('_remote_dtypes', None)
+    _ = self.__dict__.pop('_remote_dtypes_timestamp', None)
+
+
+def get_dtypes(
+    self,
+    infer: bool = True,
+    refresh: bool = False,
+    debug: bool = False,
+) -> Dict[str, Any]:
+    """
+    If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
+
+
+    Parameters
+    ----------
+    infer: bool, default True
+        If `True`, include the implicit existing dtypes.
+        Else only return the explicitly configured dtypes (e.g. `Pipe.parameters['dtypes']`).
+
+    refresh: bool, default False
+        If `True`, invalidate any cache and return the latest known dtypes.
+
+    Returns
+    -------
+    A dictionary mapping column names to dtypes.
+    """
+    import time
     from meerschaum.config._patch import apply_patch_to_config
     from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
-    configured_dtypes = self.parameters.get('dtypes', {})
-    remote_dtypes = self.infer_dtypes(persist=False)
-    patched_dtypes = apply_patch_to_config(remote_dtypes, configured_dtypes)
-    dt_col = self.columns.get('datetime', None)
-    primary_col = self.columns.get('primary', None)
+    from meerschaum._internal.static import STATIC_CONFIG
+    parameters = self.get_parameters(refresh=refresh, debug=debug)
+    configured_dtypes = parameters.get('dtypes', {})
+    if debug:
+        dprint(f"Configured dtypes for {self}:")
+        mrsm.pprint(configured_dtypes)
+
+    remote_dtypes = self.infer_dtypes(persist=False, refresh=refresh, debug=debug)
+    patched_dtypes = apply_patch_to_config((remote_dtypes or {}), (configured_dtypes or {}))
+
+    dt_col = parameters.get('columns', {}).get('datetime', None)
+    primary_col = parameters.get('columns', {}).get('primary', None)
     _dtypes = {
         col: MRSM_ALIAS_DTYPES.get(typ, typ)
         for col, typ in patched_dtypes.items()
@@ -264,18 +322,10 @@ def dtypes(self) -> Union[Dict[str, Any], None]:
     }
     if dt_col and dt_col not in configured_dtypes:
         _dtypes[dt_col] = 'datetime'
-    if primary_col and self.autoincrement and primary_col not in _dtypes:
+    if primary_col and parameters.get('autoincrement', False) and primary_col not in _dtypes:
         _dtypes[primary_col] = 'int'
-    return _dtypes
-
 
-@dtypes.setter
-def dtypes(self, _dtypes: Dict[str, Any]) -> None:
-    """
-    Override the dtypes dictionary of the in-memory pipe.
-    Call `meerschaum.Pipe.edit()` to persist changes.
-    """
-    self.update_parameters({'dtypes': _dtypes}, persist=False)
+    return _dtypes
 
 
 @property
@@ -347,18 +397,22 @@ def tzinfo(self) -> Union[None, timezone]:
     """
     Return `timezone.utc` if the pipe is timezone-aware.
     """
-    dt_col = self.columns.get('datetime', None)
-    if not dt_col:
-        return None
+    if '_tzinfo' in self.__dict__:
+        return self.__dict__['_tzinfo']
 
-    dt_typ = str(self.dtypes.get(dt_col, 'datetime64[ns, UTC]'))
-    if 'utc' in dt_typ.lower() or dt_typ == 'datetime':
-        return timezone.utc
+    _tzinfo = None
+    dt_col = self.columns.get('datetime', None)
+    dt_typ = str(self.dtypes.get(dt_col, 'datetime')) if dt_col else None
+    if self.autotime:
+        ts_col = mrsm.get_config('pipes', 'autotime', 'column_name_if_datetime_missing')
+        ts_typ = self.dtypes.get(ts_col, 'datetime')
+        dt_typ = ts_typ
 
-    if dt_typ == 'datetime64[ns]':
-        return None
+    if dt_typ and 'utc' in dt_typ.lower() or dt_typ == 'datetime':
+        _tzinfo = timezone.utc
 
-    return None
+    self._tzinfo = _tzinfo
+    return _tzinfo
 
 
 @property
@@ -393,6 +447,22 @@ def null_indices(self, _null_indices: bool) -> None:
     self.update_parameters({'null_indices': _null_indices}, persist=False)
 
 
+@property
+def mixed_numerics(self) -> bool:
+    """
+    Return the `mixed_numerics` parameter for the pipe.
+    """
+    return self.parameters.get('mixed_numerics', True)
+
+
+@mixed_numerics.setter
+def mixed_numerics(self, _mixed_numerics: bool) -> None:
+    """
+    Set the `mixed_numerics` parameter for the pipe.
+    """
+    self.update_parameters({'mixed_numerics': _mixed_numerics}, persist=False)
+
+
 def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
     """
     Check if the requested columns are defined.
@@ -471,15 +541,17 @@ def get_columns_types(
     import time
     from meerschaum.connectors import get_connector_plugin
     from meerschaum._internal.static import STATIC_CONFIG
-    from meerschaum.utils.warnings import dprint
 
     now = time.perf_counter()
-    cache_seconds = STATIC_CONFIG['pipes']['static_schema_cache_seconds']
-    if not self.static:
-        refresh = True
+    cache_seconds = (
+        mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
+        if self.static
+        else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
+    )
     if refresh:
         _ = self.__dict__.pop('_columns_types_timestamp', None)
         _ = self.__dict__.pop('_columns_types', None)
+
     _columns_types = self.__dict__.get('_columns_types', None)
     if _columns_types:
         columns_types_timestamp = self.__dict__.get('_columns_types_timestamp', None)
@@ -516,13 +588,12 @@ def get_columns_indices(
     import time
     from meerschaum.connectors import get_connector_plugin
     from meerschaum._internal.static import STATIC_CONFIG
-    from meerschaum.utils.warnings import dprint
 
     now = time.perf_counter()
     cache_seconds = (
-        STATIC_CONFIG['pipes']['static_schema_cache_seconds']
+        mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
         if self.static
-        else STATIC_CONFIG['pipes']['exists_timeout_seconds']
+        else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
     )
     if refresh:
         _ = self.__dict__.pop('_columns_indices_timestamp', None)
@@ -595,7 +666,6 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
     -------
     Either a string or `None`.
     """
-    from meerschaum.utils.debug import dprint
     if debug:
         dprint('Attempting to determine the value column...')
     try:
@@ -657,6 +727,7 @@ def parents(self) -> List[mrsm.Pipe]:
     """
     if 'parents' not in self.parameters:
         return []
+
     from meerschaum.utils.warnings import warn
     _parents_keys = self.parameters['parents']
     if not isinstance(_parents_keys, list):
@@ -695,6 +766,7 @@ def children(self) -> List[mrsm.Pipe]:
     """
     if 'children' not in self.parameters:
         return []
+
     from meerschaum.utils.warnings import warn
     _children_keys = self.parameters['children']
     if not isinstance(_children_keys, list):
@@ -836,11 +908,12 @@ def update_parameters(
         If `True`, call `Pipe.edit()` to persist the new parameters.
     """
     from meerschaum.config import apply_patch_to_config
-    if '_parameters' in self.__dict__:
-        del self.__dict__['_parameters']
     if 'parameters' not in self._attributes:
         self._attributes['parameters'] = {}
 
+    if '_parameters' not in self.__dict__:
+        self._parameters = {}
+
     self._attributes['parameters'] = apply_patch_to_config(
         self._attributes['parameters'],
         parameters_patch,
@@ -853,3 +926,154 @@ def update_parameters(
         return True, "Success"
 
     return self.edit(debug=debug)
+
+
+def get_precision(self, debug: bool = False) -> Dict[str, Union[str, int]]:
+    """
+    Return the timestamp precision unit and interval for the `datetime` axis.
+    """
+    from meerschaum.utils.dtypes import (
+        MRSM_PRECISION_UNITS_SCALARS,
+        MRSM_PRECISION_UNITS_ALIASES,
+        MRSM_PD_DTYPES,
+        are_dtypes_equal,
+    )
+    from meerschaum._internal.static import STATIC_CONFIG
+
+    if self.__dict__.get('_precision', None):
+        if debug:
+            dprint(f"Returning cached precision: {self._precision}")
+        return self._precision
+
+    parameters = self.parameters
+    _precision = parameters.get('precision', {})
+    if isinstance(_precision, str):
+        _precision = {'unit': _precision}
+    default_precision_unit = STATIC_CONFIG['dtypes']['datetime']['default_precision_unit']
+
+    if not _precision:
+
+        dt_col = parameters.get('columns', {}).get('datetime', None)
+        if not dt_col and self.autotime:
+            dt_col = mrsm.get_config('pipes', 'autotime', 'column_name_if_datetime_missing')
+        if not dt_col:
+            if debug:
+                dprint(f"No datetime axis, returning default precision '{default_precision_unit}'.")
+            return {'unit': default_precision_unit}
+
+        dt_typ = self.dtypes.get(dt_col, 'datetime')
+        if are_dtypes_equal(dt_typ, 'datetime'):
+            if dt_typ == 'datetime':
+                dt_typ = MRSM_PD_DTYPES['datetime']
+                if debug:
+                    dprint(f"Datetime type is `datetime`, assuming {dt_typ} precision.")
+
+            _precision = {
+                'unit': (
+                    dt_typ
+                    .split('[', maxsplit=1)[-1]
+                    .split(',', maxsplit=1)[0]
+                    .split(' ', maxsplit=1)[0]
+                ).rstrip(']')
+            }
+
+            if debug:
+                dprint(f"Extracted precision '{_precision['unit']}' from type '{dt_typ}'.")
+
+        elif are_dtypes_equal(dt_typ, 'int'):
+            _precision = {
+                'unit': (
+                    'second'
+                    if '32' in dt_typ
+                    else default_precision_unit
+                )
+            }
+        elif are_dtypes_equal(dt_typ, 'date'):
+            if debug:
+                dprint("Datetime axis is 'date', falling back to 'day' precision.")
+            _precision = {'unit': 'day'}
+
+    precision_unit = _precision.get('unit', default_precision_unit)
+    precision_interval = _precision.get('interval', None)
+    true_precision_unit = MRSM_PRECISION_UNITS_ALIASES.get(precision_unit, precision_unit)
+    if true_precision_unit is None:
+        if debug:
+            dprint(f"No precision could be determined, falling back to '{default_precision_unit}'.")
+        true_precision_unit = default_precision_unit
+
+    if true_precision_unit not in MRSM_PRECISION_UNITS_SCALARS:
+        from meerschaum.utils.misc import items_str
+        raise ValueError(
+            f"Invalid precision unit '{true_precision_unit}'.\n"
+            "Accepted values are "
+            f"{items_str(list(MRSM_PRECISION_UNITS_SCALARS) + list(MRSM_PRECISION_UNITS_ALIASES))}."
+        )
+
+    self._precision = {'unit': true_precision_unit}
+    if precision_interval:
+        self._precision['interval'] = precision_interval
+    return self._precision
+
+
+@property
+def precision(self) -> Dict[str, Union[str, int]]:
+    """
+    Return the configured or detected precision.
+    """
+    return self.get_precision()
+
+
+@precision.setter
+def precision(self, _precision: Union[str, Dict[str, Union[str, int]]]) -> None:
+    """
+    Update the `precision` parameter.
+    """
+    existing_precision = self._attributes.get('parameters', {}).get('precision', None)
+    if isinstance(existing_precision, str):
+        existing_precision = {'unit': existing_precision}
+
+    true_precision = (
+        _precision
+        if isinstance(_precision, dict)
+        else {
+            'unit': _precision,
+            **(
+                {
+                    'interval': existing_precision['interval'],
+                } if existing_precision else {}
+            )
+        }
+    )
+
+    self.update_parameters({'precision': true_precision}, persist=False)
+    _ = self.__dict__.pop('_precision', None)
+
+
+def _invalidate_cache(
+    self,
+    hard: bool = False,
+    debug: bool = False,
+) -> None:
+    """
+    Invalidate temporary metadata cache.
+
+    Parameters
+    ----------
+    hard: bool, default False
+        If `True`, clear all temporary cache.
+        Otherwise only clear soft cache.
+    """
+    if debug:
+        dprint(f"Invalidating {'some' if not hard else 'all'} cache for {self}.")
+
+    self._exists = None
+    self._sync_ts = None
+
+    if not hard:
+        return
+
+    _ = self.__dict__.pop('_parameters', None)
+    _ = self.__dict__.pop('_precision', None)
+    self._columns_types_timestamp = None
+    self._columns_types = None
+    self._attributes_sync_time = None