meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

meerschaum/core/Pipe/__init__.py

@@ -137,7 +137,28 @@ class Pipe:
         guess_datetime,
         precision,
         get_precision,
+    )
+    from ._cache import (
+        _get_cache_connector,
+        _cache_value,
+        _get_cached_value,
         _invalidate_cache,
+        _get_cache_dir_path,
+        _write_cache_key,
+        _write_cache_file,
+        _write_cache_conn_key,
+        _read_cache_key,
+        _read_cache_file,
+        _read_cache_conn_key,
+        _load_cache_keys,
+        _load_cache_files,
+        _load_cache_conn_keys,
+        _get_cache_keys,
+        _get_cache_file_keys,
+        _get_cache_conn_keys,
+        _clear_cache_key,
+        _clear_cache_file,
+        _clear_cache_conn_key,
     )
     from ._show import show
     from ._edit import edit, edit_definition, update
@@ -176,7 +197,6 @@ class Pipe:
         target: Optional[str] = None,
         dtypes: Optional[Dict[str, str]] = None,
         instance: Optional[Union[str, InstanceConnector]] = None,
-        temporary: bool = False,
         upsert: Optional[bool] = None,
         autoincrement: Optional[bool] = None,
         autotime: Optional[bool] = None,
@@ -185,14 +205,16 @@ class Pipe:
         enforce: Optional[bool] = None,
         null_indices: Optional[bool] = None,
         mixed_numerics: Optional[bool] = None,
+        temporary: bool = False,
+        cache: Optional[bool] = None,
+        cache_connector_keys: Optional[str] = None,
         mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
-        cache: bool = False,
-        debug: bool = False,
         connector_keys: Optional[str] = None,
         metric_key: Optional[str] = None,
         location_key: Optional[str] = None,
         instance_keys: Optional[str] = None,
         indexes: Union[Dict[str, str], List[str], None] = None,
+        debug: bool = False,
     ):
         """
         Parameters
@@ -269,9 +291,13 @@ class Pipe:
         temporary: bool, default False
             If `True`, prevent instance tables (pipes, users, plugins) from being created.
 
-        cache: bool, default False
-            If `True`, cache fetched data into a local database file.
-            Defaults to `False`.
+        cache: Optional[bool], default None
+            If `True`, cache the pipe's metadata to disk (in addition to in-memory caching).
+            If `cache` is not explicitly `True`, it is set to `False` if `temporary` is `True`.
+            Defaults to `True` (from `None`).
+
+        cache_connector_keys: Optional[str], default None
+            If provided, use the keys to a Valkey connector (e.g. `valkey:main`).
         """
         from meerschaum.utils.warnings import error, warn
         if (not connector and not connector_keys) or (not metric and not metric_key):
@@ -304,8 +330,15 @@ class Pipe:
         self.metric_key = metric
         self.location_key = location
         self.temporary = temporary
+        self.cache = cache if cache is not None else (not temporary)
+        self.cache_connector_keys = (
+            str(cache_connector_keys)
+            if cache_connector_keys is not None
+            else None
+        )
+        self.debug = debug
 
-        self._attributes = {
+        self._attributes: Dict[str, Any] = {
             'connector_keys': self.connector_keys,
             'metric_key': self.metric_key,
             'location_key': self.location_key,
@@ -375,6 +408,7 @@ class Pipe:
 
         if isinstance(static, bool):
             self._attributes['parameters']['static'] = static
+            self._static = static
 
         if isinstance(enforce, bool):
             self._attributes['parameters']['enforce'] = enforce
@@ -395,10 +429,12 @@ class Pipe:
         if not isinstance(_mrsm_instance, str):
             self._instance_connector = _mrsm_instance
             self.instance_keys = str(_mrsm_instance)
-        else: ### NOTE: must be SQL or API Connector for this work
+        else:
             self.instance_keys = _mrsm_instance
 
-        self._cache = cache and get_config('system', 'experimental', 'cache')
+        if self.instance_keys == 'sql:memory':
+            self.cache = False
+
 
     @property
     def meta(self):
@@ -456,68 +492,6 @@ class Pipe:
                 return None
         return self._connector
 
-    @property
-    def cache_connector(self) -> Union['Connector', None]:
-        """
-        If the pipe was created with `cache=True`, return the connector to the pipe's
-        SQLite database for caching.
-        """
-        if not self._cache:
-            return None
-
-        if '_cache_connector' not in self.__dict__:
-            from meerschaum.connectors import get_connector
-            from meerschaum.config._paths import DUCKDB_RESOURCES_PATH, SQLITE_RESOURCES_PATH
-            _resources_path = SQLITE_RESOURCES_PATH
-            self._cache_connector = get_connector(
-                'sql', '_cache_' + str(self),
-                flavor='sqlite',
-                database=str(_resources_path / ('_cache_' + str(self) + '.db')),
-            )
-
-        return self._cache_connector
-
-    @property
-    def cache_pipe(self) -> Union[mrsm.Pipe, None]:
-        """
-        If the pipe was created with `cache=True`, return another `meerschaum.Pipe` used to
-        manage the local data.
-        """
-        if self.cache_connector is None:
-            return None
-        if '_cache_pipe' not in self.__dict__:
-            from meerschaum.config._patch import apply_patch_to_config
-            from meerschaum.utils.sql import sql_item_name
-            _parameters = copy.deepcopy(self.parameters)
-            _fetch_patch = {
-                'fetch': ({
-                    'definition': (
-                        "SELECT * FROM "
-                        + sql_item_name(
-                            str(self.target),
-                            self.instance_connector.flavor,
-                            self.instance_connector.get_pipe_schema(self),
-                        )
-                    ),
-                }) if self.instance_connector.type == 'sql' else ({
-                    'connector_keys': self.connector_keys,
-                    'metric_key': self.metric_key,
-                    'location_key': self.location_key,
-                })
-            }
-            _parameters = apply_patch_to_config(_parameters, _fetch_patch)
-            self._cache_pipe = Pipe(
-                self.instance_keys,
-                (self.connector_keys + '_' + self.metric_key + '_cache'),
-                self.location_key,
-                mrsm_instance=self.cache_connector,
-                parameters=_parameters,
-                cache=False,
-                temporary=True,
-            )
-
-        return self._cache_pipe
-
     def __str__(self, ansi: bool=False):
         return pipe_repr(self, ansi=ansi)
 

meerschaum/core/Pipe/_attributes.py

@@ -22,30 +22,34 @@ def attributes(self) -> Dict[str, Any]:
     Return a dictionary of a pipe's keys and parameters.
     These values are reflected directly from the pipes table of the instance.
     """
-    import time
     from meerschaum.config import get_config
     from meerschaum.config._patch import apply_patch_to_config
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
+    from meerschaum.utils.dtypes import get_current_timestamp
 
     timeout_seconds = get_config('pipes', 'attributes', 'local_cache_timeout_seconds')
 
-    if '_attributes' not in self.__dict__:
-        self._attributes = {}
-
-    now = time.perf_counter()
-    last_refresh = self.__dict__.get('_attributes_sync_time', None)
+    now = get_current_timestamp('ms', as_int=True) / 1000
+    _attributes_sync_time = self._get_cached_value('_attributes_sync_time', debug=self.debug)
     timed_out = (
-        last_refresh is None
+        _attributes_sync_time is None
         or
-        (timeout_seconds is not None and (now - last_refresh) >= timeout_seconds)
+        (timeout_seconds is not None and (now - _attributes_sync_time) >= timeout_seconds)
     )
     if not self.temporary and timed_out:
-        self._attributes_sync_time = now
-        local_attributes = self.__dict__.get('_attributes', {})
+        self._cache_value('_attributes_sync_time', now, memory_only=True, debug=self.debug)
+        local_attributes = self._get_cached_value('attributes', debug=self.debug) or {}
         with Venv(get_connector_plugin(self.instance_connector)):
             instance_attributes = self.instance_connector.get_pipe_attributes(self)
-        self._attributes = apply_patch_to_config(instance_attributes, local_attributes)
+
+        self._cache_value(
+            'attributes',
+            apply_patch_to_config(instance_attributes, local_attributes),
+            memory_only=True,
+            debug=self.debug,
+        )
+
     return self._attributes
 
 
@@ -78,7 +82,7 @@ def get_parameters(
         _visited = {self}
 
     if refresh:
-        self._invalidate_cache(hard=True)
+        _ = self._invalidate_cache(hard=True)
 
     raw_parameters = self.attributes.get('parameters', {})
     ref_keys = raw_parameters.get('reference')
@@ -131,18 +135,16 @@ def parameters(self) -> Optional[Dict[str, Any]]:
     """
     Return the parameters dictionary of the pipe.
     """
-    return self.get_parameters()
+    return self.get_parameters(debug=self.debug)
 
 
 @parameters.setter
-def parameters(self, parameters: Dict[str, Any]) -> None:
+def parameters(self, _parameters: Dict[str, Any]) -> None:
     """
     Set the parameters dictionary of the in-memory pipe.
     Call `meerschaum.Pipe.edit()` to persist changes.
     """
-    self._attributes['parameters'] = parameters
-    if '_parameters' in self.__dict__:
-        del self.__dict__['_parameters']
+    self._attributes['parameters'] = _parameters
 
 
 @property
@@ -175,13 +177,14 @@ def indices(self) -> Union[Dict[str, Union[str, List[str]]], None]:
     """
     Return the `indices` dictionary defined in `meerschaum.Pipe.parameters`.
     """
+    _parameters = self.get_parameters(debug=self.debug)
     indices_key = (
         'indexes'
-        if 'indexes' in self.parameters
+        if 'indexes' in _parameters
         else 'indices'
     )
 
-    _indices = self.parameters.get(indices_key, {})
+    _indices = _parameters.get(indices_key, {})
     _columns = self.columns
     dt_col = _columns.get('datetime', None)
     if not isinstance(_indices, dict):
@@ -263,7 +266,7 @@ def dtypes(self) -> Dict[str, Any]:
     """
     If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
     """
-    return self.get_dtypes(refresh=False)
+    return self.get_dtypes(refresh=False, debug=self.debug)
 
 
 @dtypes.setter
@@ -273,8 +276,8 @@ def dtypes(self, _dtypes: Dict[str, Any]) -> None:
     Call `meerschaum.Pipe.edit()` to persist changes.
     """
     self.update_parameters({'dtypes': _dtypes}, persist=False)
-    _ = self.__dict__.pop('_remote_dtypes', None)
-    _ = self.__dict__.pop('_remote_dtypes_timestamp', None)
+    self._clear_cache_key('_remote_dtypes', debug=self.debug)
+    self._clear_cache_key('_remote_dtypes_timestamp', debug=self.debug)
 
 
 def get_dtypes(
@@ -299,10 +302,8 @@ def get_dtypes(
     -------
     A dictionary mapping column names to dtypes.
     """
-    import time
     from meerschaum.config._patch import apply_patch_to_config
     from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
-    from meerschaum._internal.static import STATIC_CONFIG
     parameters = self.get_parameters(refresh=refresh, debug=debug)
     configured_dtypes = parameters.get('dtypes', {})
     if debug:
@@ -361,6 +362,7 @@ def static(self, _static: bool) -> None:
     Set the `static` parameter for the pipe.
     """
     self.update_parameters({'static': _static}, persist=False)
+    self._static = _static
 
 
 @property
@@ -400,8 +402,9 @@ def tzinfo(self) -> Union[None, timezone]:
     """
     Return `timezone.utc` if the pipe is timezone-aware.
     """
-    if '_tzinfo' in self.__dict__:
-        return self.__dict__['_tzinfo']
+    _tzinfo = self._get_cached_value('tzinfo', debug=self.debug)
+    if _tzinfo is not None:
+        return _tzinfo if _tzinfo != 'None' else None
 
     _tzinfo = None
     dt_col = self.columns.get('datetime', None)
@@ -414,7 +417,7 @@ def tzinfo(self) -> Union[None, timezone]:
     if dt_typ and 'utc' in dt_typ.lower() or dt_typ == 'datetime':
         _tzinfo = timezone.utc
 
-    self._tzinfo = _tzinfo
+    self._cache_value('tzinfo', (_tzinfo if _tzinfo is not None else 'None'), debug=self.debug)
     return _tzinfo
 
 
@@ -541,23 +544,22 @@ def get_columns_types(
     }
     >>>
     """
-    import time
     from meerschaum.connectors import get_connector_plugin
-    from meerschaum._internal.static import STATIC_CONFIG
+    from meerschaum.utils.dtypes import get_current_timestamp
 
-    now = time.perf_counter()
+    now = get_current_timestamp('ms', as_int=True) / 1000
     cache_seconds = (
         mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
         if self.static
         else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
     )
     if refresh:
-        _ = self.__dict__.pop('_columns_types_timestamp', None)
-        _ = self.__dict__.pop('_columns_types', None)
+        self._clear_cache_key('_columns_types_timestamp', debug=debug)
+        self._clear_cache_key('_columns_types', debug=debug)
 
-    _columns_types = self.__dict__.get('_columns_types', None)
+    _columns_types = self._get_cached_value('_columns_types', debug=debug)
     if _columns_types:
-        columns_types_timestamp = self.__dict__.get('_columns_types_timestamp', None)
+        columns_types_timestamp = self._get_cached_value('_columns_types_timestamp', debug=debug)
         if columns_types_timestamp is not None:
             delta = now - columns_types_timestamp
             if delta < cache_seconds:
@@ -575,8 +577,8 @@ def get_columns_types(
             else None
         )
 
-    self.__dict__['_columns_types'] = _columns_types
-    self.__dict__['_columns_types_timestamp'] = now
+    self._cache_value('_columns_types', _columns_types, debug=debug)
+    self._cache_value('_columns_types_timestamp', now, debug=debug)
     return _columns_types or {}
 
 
@@ -588,22 +590,23 @@ def get_columns_indices(
     """
     Return a dictionary mapping columns to index information.
     """
-    import time
     from meerschaum.connectors import get_connector_plugin
-    from meerschaum._internal.static import STATIC_CONFIG
+    from meerschaum.utils.dtypes import get_current_timestamp
 
-    now = time.perf_counter()
+    now = get_current_timestamp('ms', as_int=True) / 1000
     cache_seconds = (
         mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
         if self.static
         else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
     )
     if refresh:
-        _ = self.__dict__.pop('_columns_indices_timestamp', None)
-        _ = self.__dict__.pop('_columns_indices', None)
-    _columns_indices = self.__dict__.get('_columns_indices', None)
+        self._clear_cache_key('_columns_indices_timestamp', debug=debug)
+        self._clear_cache_key('_columns_indices', debug=debug)
+
+    _columns_indices = self._get_cached_value('_columns_indices', debug=debug)
+
     if _columns_indices:
-        columns_indices_timestamp = self.__dict__.get('_columns_indices_timestamp', None)
+        columns_indices_timestamp = self._get_cached_value('_columns_indices_timestamp', debug=debug)
         if columns_indices_timestamp is not None:
             delta = now - columns_indices_timestamp
             if delta < cache_seconds:
@@ -621,18 +624,19 @@ def get_columns_indices(
             else None
         )
 
-    self.__dict__['_columns_indices'] = _columns_indices
-    self.__dict__['_columns_indices_timestamp'] = now
+    self._cache_value('_columns_indices', _columns_indices, debug=debug)
+    self._cache_value('_columns_indices_timestamp', now, debug=debug)
     return {k: v for k, v in _columns_indices.items() if k and v} or {}
 
 
-def get_id(self, **kw: Any) -> Union[int, None]:
+def get_id(self, **kw: Any) -> Union[int, str, None]:
     """
     Fetch a pipe's ID from its instance connector.
-    If the pipe does not exist, return `None`.
+    If the pipe is not registered, return `None`.
     """
     if self.temporary:
         return None
+
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
 
@@ -648,9 +652,12 @@ def id(self) -> Union[int, str, uuid.UUID, None]:
     """
     Fetch and cache a pipe's ID.
     """
-    if not ('_id' in self.__dict__ and self._id):
-        self._id = self.get_id()
-    return self._id
+    _id = self._get_cached_value('_id', debug=self.debug)
+    if not _id:
+        _id = self.get_id(debug=self.debug)
+        if _id is not None:
+            self._cache_value('_id', _id, debug=self.debug)
+    return _id
 
 
 def get_val_column(self, debug: bool = False) -> Union[str, None]:
@@ -914,9 +921,6 @@ def update_parameters(
     if 'parameters' not in self._attributes:
         self._attributes['parameters'] = {}
 
-    if '_parameters' not in self.__dict__:
-        self._parameters = {}
-
     self._attributes['parameters'] = apply_patch_to_config(
         self._attributes['parameters'],
         parameters_patch,
@@ -943,10 +947,11 @@ def get_precision(self, debug: bool = False) -> Dict[str, Union[str, int]]:
     )
     from meerschaum._internal.static import STATIC_CONFIG
 
-    if self.__dict__.get('_precision', None):
+    _precision = self._get_cached_value('precision', debug=debug)
+    if _precision:
         if debug:
-            dprint(f"Returning cached precision: {self._precision}")
-        return self._precision
+            dprint(f"Returning cached precision: {_precision}")
+        return _precision
 
     parameters = self.parameters
     _precision = parameters.get('precision', {})
@@ -1012,9 +1017,10 @@ def get_precision(self, debug: bool = False) -> Dict[str, Union[str, int]]:
             f"{items_str(list(MRSM_PRECISION_UNITS_SCALARS) + list(MRSM_PRECISION_UNITS_ALIASES))}."
         )
 
-    self._precision = {'unit': true_precision_unit}
+    _precision = {'unit': true_precision_unit}
     if precision_interval:
-        self._precision['interval'] = precision_interval
+        _precision['interval'] = precision_interval
+    self._cache_value('precision', _precision, debug=debug)
     return self._precision
 
 
@@ -1023,7 +1029,7 @@ def precision(self) -> Dict[str, Union[str, int]]:
     """
     Return the configured or detected precision.
     """
-    return self.get_precision()
+    return self.get_precision(debug=self.debug)
 
 
 @precision.setter
@@ -1049,34 +1055,4 @@ def precision(self, _precision: Union[str, Dict[str, Union[str, int]]]) -> None:
     )
 
     self.update_parameters({'precision': true_precision}, persist=False)
-    _ = self.__dict__.pop('_precision', None)
-
-
-def _invalidate_cache(
-    self,
-    hard: bool = False,
-    debug: bool = False,
-) -> None:
-    """
-    Invalidate temporary metadata cache.
-
-    Parameters
-    ----------
-    hard: bool, default False
-        If `True`, clear all temporary cache.
-        Otherwise only clear soft cache.
-    """
-    if debug:
-        dprint(f"Invalidating {'some' if not hard else 'all'} cache for {self}.")
-
-    self._exists = None
-    self._sync_ts = None
-
-    if not hard:
-        return
-
-    _ = self.__dict__.pop('_parameters', None)
-    _ = self.__dict__.pop('_precision', None)
-    self._columns_types_timestamp = None
-    self._columns_types = None
-    self._attributes_sync_time = None
+    self._clear_cache_key('precision', debug=self.debug)