meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

meerschaum/core/Pipe/_delete.py

@@ -9,11 +9,11 @@ Delete a Pipe's contents and registration
 from meerschaum.utils.typing import SuccessTuple
 
 def delete(
-        self,
-        drop: bool = True,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    self,
+    drop: bool = True,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Call the Pipe's instance connector's `delete_pipe()` method.
 
@@ -30,29 +30,22 @@ def delete(
     A `SuccessTuple` of success (`bool`), message (`str`).
 
     """
-    import os, pathlib
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
 
     if self.temporary:
+        if self.cache:
+            invalidate_success, invalidate_msg = self._invalidate_cache(hard=True, debug=debug)
+            if not invalidate_success:
+                return invalidate_success, invalidate_msg
+
         return (
             False,
             "Cannot delete pipes created with `temporary=True` (read-only). "
             + "You may want to call `pipe.drop()` instead."
         )
 
-    if self.cache_pipe is not None:
-        _drop_cache_tuple = self.cache_pipe.drop(debug=debug, **kw)
-        if not _drop_cache_tuple[0]:
-            warn(_drop_cache_tuple[1])
-        if getattr(self.cache_connector, 'flavor', None) == 'sqlite':
-            _cache_db_path = pathlib.Path(self.cache_connector.database)
-            try:
-                os.remove(_cache_db_path)
-            except Exception as e:
-                warn(f"Could not delete cache file '{_cache_db_path}' for {self}:\n{e}")
-
     if drop:
         drop_success, drop_msg = self.drop(debug=debug)
         if not drop_success:
@@ -65,8 +58,6 @@ def delete(
         return False, f"Received an unexpected result from '{self.instance_connector}': {result}"
 
     if result[0]:
-        to_delete = ['_id']
-        for member in to_delete:
-            if member in self.__dict__:
-                del self.__dict__[member]
+        self._invalidate_cache(hard=True, debug=debug)
+
     return result

meerschaum/core/Pipe/_drop.py

@@ -28,15 +28,10 @@ def drop(
     A `SuccessTuple` of success, message.
 
     """
-    self._exists = False
-    from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
 
-    if self.cache_pipe is not None:
-        _drop_cache_tuple = self.cache_pipe.drop(debug=debug, **kw)
-        if not _drop_cache_tuple[0]:
-            warn(_drop_cache_tuple[1])
+    self._clear_cache_key('_exists', debug=debug)
 
     with Venv(get_connector_plugin(self.instance_connector)):
         if hasattr(self.instance_connector, 'drop_pipe'):
@@ -50,9 +45,8 @@ def drop(
                 )
             )
 
-
-    _ = self.__dict__.pop('_exists', None)
-    _ = self.__dict__.pop('_exists_timestamp', None)
+    self._clear_cache_key('_exists', debug=debug)
+    self._clear_cache_key('_exists_timestamp', debug=debug)
 
     return result
 
@@ -79,19 +73,13 @@ def drop_indices(
     A `SuccessTuple` of success, message.
 
     """
-    from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
 
-    _ = self.__dict__.pop('_columns_indices', None)
-    _ = self.__dict__.pop('_columns_indices_timestamp', None)
-    _ = self.__dict__.pop('_columns_types_timestamp', None)
-    _ = self.__dict__.pop('_columns_types', None)
-
-    if self.cache_pipe is not None:
-        _drop_cache_tuple = self.cache_pipe.drop_indices(columns=columns, debug=debug, **kw)
-        if not _drop_cache_tuple[0]:
-            warn(_drop_cache_tuple[1])
+    self._clear_cache_key('_columns_indices', debug=debug)
+    self._clear_cache_key('_columns_indices_timestamp', debug=debug)
+    self._clear_cache_key('_columns_types', debug=debug)
+    self._clear_cache_key('_columns_types_timestamp', debug=debug)
 
     with Venv(get_connector_plugin(self.instance_connector)):
         if hasattr(self.instance_connector, 'drop_pipe_indices'):
@@ -110,9 +98,9 @@ def drop_indices(
                 )
             )
 
-    _ = self.__dict__.pop('_columns_indices', None)
-    _ = self.__dict__.pop('_columns_indices_timestamp', None)
-    _ = self.__dict__.pop('_columns_types_timestamp', None)
-    _ = self.__dict__.pop('_columns_types', None)
+    self._clear_cache_key('_columns_indices', debug=debug)
+    self._clear_cache_key('_columns_indices_timestamp', debug=debug)
+    self._clear_cache_key('_columns_types', debug=debug)
+    self._clear_cache_key('_columns_types_timestamp', debug=debug)
 
     return result

meerschaum/core/Pipe/_dtypes.py

@@ -98,7 +98,7 @@ def enforce_dtypes(
         if debug:
             dprint(
                 f"Could not find dtypes for {self}.\n"
-                + "    Skipping dtype enforcement..."
+                + "Skipping dtype enforcement..."
             )
         return df
 

meerschaum/core/Pipe/_index.py

@@ -29,19 +29,13 @@ def create_indices(
     A `SuccessTuple` of success, message.
 
     """
-    from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
 
-    _ = self.__dict__.pop('_columns_indices', None)
-    _ = self.__dict__.pop('_columns_indices_timestamp', None)
-    _ = self.__dict__.pop('_columns_types_timestamp', None)
-    _ = self.__dict__.pop('_columns_types', None)
-
-    if self.cache_pipe is not None:
-        cache_success, cache_msg = self.cache_pipe.index(columns=columns, debug=debug, **kw)
-        if not cache_success:
-            warn(cache_msg)
+    self._clear_cache_key('_columns_indices', debug=debug)
+    self._clear_cache_key('_columns_indices_timestamp', debug=debug)
+    self._clear_cache_key('_columns_types', debug=debug)
+    self._clear_cache_key('_columns_types_timestamp', debug=debug)
 
     with Venv(get_connector_plugin(self.instance_connector)):
         if hasattr(self.instance_connector, 'create_pipe_indices'):
@@ -60,9 +54,9 @@ def create_indices(
                 )
             )
 
-    _ = self.__dict__.pop('_columns_indices', None)
-    _ = self.__dict__.pop('_columns_indices_timestamp', None)
-    _ = self.__dict__.pop('_columns_types_timestamp', None)
-    _ = self.__dict__.pop('_columns_types', None)
+    self._clear_cache_key('_columns_indices', debug=debug)
+    self._clear_cache_key('_columns_indices_timestamp', debug=debug)
+    self._clear_cache_key('_columns_types', debug=debug)
+    self._clear_cache_key('_columns_types_timestamp', debug=debug)
 
     return result

meerschaum/core/Pipe/_sync.py

@@ -166,7 +166,7 @@ def sync(
     })
 
     self._invalidate_cache(debug=debug)
-    self._sync_ts = get_current_timestamp('ms')
+    self._cache_value('sync_ts', get_current_timestamp('ms'), debug=debug)
 
     def _sync(
         p: mrsm.Pipe,
@@ -477,15 +477,8 @@ def sync(
                         ("s" if retries != 1 else "") + "!"
                 )
 
-        ### CHECKPOINT: Finished syncing. Handle caching.
+        ### CHECKPOINT: Finished syncing.
         _checkpoint(**kw)
-        if p.cache_pipe is not None:
-            if debug:
-                dprint("Caching retrieved dataframe.", **kw)
-            _sync_cache_tuple = p.cache_pipe.sync(df, debug=debug, **kw)
-            if not _sync_cache_tuple[0]:
-                warn(f"Failed to sync local cache for {self}.")
-
         p._invalidate_cache(debug=debug)
         return return_tuple
 
@@ -620,16 +613,16 @@ def exists(
     A `bool` corresponding to whether a pipe's underlying table exists.
 
     """
-    import time
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.debug import dprint
-    now = time.perf_counter()
+    from meerschaum.utils.dtypes import get_current_timestamp
+    now = get_current_timestamp('ms', as_int=True) / 1000
     cache_seconds = mrsm.get_config('pipes', 'sync', 'exists_cache_seconds')
 
-    _exists = self.__dict__.get('_exists', None)
+    _exists = self._get_cached_value('_exists', debug=debug)
     if _exists:
-        exists_timestamp = self.__dict__.get('_exists_timestamp', None)
+        exists_timestamp = self._get_cached_value('_exists_timestamp', debug=debug)
         if exists_timestamp is not None:
             delta = now - exists_timestamp
             if delta < cache_seconds:
@@ -644,8 +637,8 @@ def exists(
             else False
         )
 
-    self.__dict__['_exists'] = _exists
-    self.__dict__['_exists_timestamp'] = now
+    self._cache_value('_exists', _exists, debug=debug)
+    self._cache_value('_exists_timestamp', now, debug=debug)
     return _exists
 
 
@@ -1058,7 +1051,7 @@ def _persist_new_special_columns(
     Check for new special columns and update the parameters accordingly.
     """
     from meerschaum.utils.dataframe import get_special_cols
-    from meerschaum.utils.dtypes import dtype_is_special
+    from meerschaum.utils.dtypes import is_dtype_special
     from meerschaum.utils.warnings import dprint
 
     special_cols = get_special_cols(df)
@@ -1066,18 +1059,19 @@ def _persist_new_special_columns(
     existing_special_cols = {
         col: typ
         for col, typ in dtypes.items()
-        if dtype_is_special(typ)
+        if is_dtype_special(typ)
     }
     new_special_cols = {
         col: typ
         for col, typ in special_cols.items()
         if col not in existing_special_cols
     }
+    self._cache_value('new_special_cols', new_special_cols, memory_only=True, debug=debug)
     if not new_special_cols:
         return True, "Success"
 
     if debug:
         dprint(f"New special columns:\n{new_special_cols}")
 
-    self._attributes_sync_time = None
+    self._clear_cache_key('_attributes_sync_time', debug=debug)
     return self.update_parameters({'dtypes': new_special_cols}, debug=debug)

meerschaum/core/Plugin/_Plugin.py

@@ -112,8 +112,10 @@ class Plugin:
         if '_module' not in self.__dict__ or self.__dict__.get('_module', None) is None:
             if self.__file__ is None:
                 return None
+
             from meerschaum.plugins import import_plugins
             self._module = import_plugins(str(self), warn=False)
+
         return self._module
 
 
@@ -363,6 +365,10 @@ class Plugin:
         ### Determine where to permanently store the new plugin.
         plugin_installation_dir_path = PLUGINS_DIR_PATHS[0]
         for path in PLUGINS_DIR_PATHS:
+            if not path.exists():
+                warn(f"Plugins path does not exist: {path}", stack=False)
+                continue
+
             files_in_plugins_dir = os.listdir(path)
             if (
                 self.name in files_in_plugins_dir
@@ -729,7 +735,7 @@ class Plugin:
             _d[len('plugin:'):] for _d in _deps
             if _d.startswith('plugin:') and len(_d) > len('plugin:')
         ]
-        default_repo_keys = get_config('meerschaum', 'default_repository')
+        default_repo_keys = get_config('meerschaum', 'repository')
         skipped_repo_keys = set()
 
         for _plugin_name in plugin_names:

meerschaum/core/Token/_Token.py

@@ -42,7 +42,7 @@ class Token:
         from meerschaum._internal.static import STATIC_CONFIG
         now = datetime.now(timezone.utc)
         default_expiration_days = mrsm.get_config(
-            'system', 'api', 'tokens', 'default_expiration_days',
+            'api', 'tokens', 'default_expiration_days',
         ) or 366
         default_expiration = round_time(
             now + timedelta(days=default_expiration_days),

meerschaum/core/User/_User.py

@@ -94,14 +94,13 @@ def verify_password(
     from meerschaum._internal.static import STATIC_CONFIG
     if password is None or password_hash is None:
         return False
-    hash_config = STATIC_CONFIG['users']['password_hash']
     try:
         digest, rounds_str, encoded_salt, encoded_checksum = password_hash.split('$')[1:]
         algorithm_name = digest.split('-')[-1]
         salt = ab64_decode(encoded_salt)
         checksum = ab64_decode(encoded_checksum)
         rounds = int(rounds_str)
-    except Exception as e:
+    except Exception:
         warn(f"Failed to extract context from password hash '{password_hash}'. Is it corrupted?")
         return False
 

meerschaum/jobs/_Executor.py

@@ -10,28 +10,72 @@ from __future__ import annotations
 from abc import abstractmethod
 
 from meerschaum.connectors import Connector
-from meerschaum.utils.typing import List, Dict, SuccessTuple, TYPE_CHECKING, Optional, Any
+from meerschaum.utils.typing import (
+    List, Dict, SuccessTuple, TYPE_CHECKING, Optional, Any, Union, Callable,
+)
 
 if TYPE_CHECKING:
     from meerschaum.jobs import Job
+    from datetime import datetime
 
 class Executor(Connector):
     """
     Define the methods for managing jobs.
     """
 
+    @abstractmethod
+    def get_job_names(self, debug: bool = False) -> List[str]:
+        """
+        Return a list of existing jobs, including hidden ones.
+        """
+
     @abstractmethod
     def get_job_exists(self, name: str, debug: bool = False) -> bool:
         """
         Return whether a job exists.
         """
-    
+
+    @abstractmethod
+    def get_jobs(self, debug: bool = False) -> Dict[str, Job]:
+        """
+        Return a dictionary of existing jobs.
+        """
+
+    @abstractmethod
+    def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
+        """
+        Return a job's metadata.
+        """
+
+    @abstractmethod
+    def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
+        """
+        Return the underlying daemon's properties.
+        """
     @abstractmethod
-    def get_jobs(self) -> Dict[str, Job]:
+    def get_job_status(self, name: str, debug: bool = False) -> str:
         """
-        Return a dictionary of names -> Jobs.
+        Return the job's status.
         """
 
+    @abstractmethod
+    def get_job_began(self, name: str, debug: bool = False) -> Union[str, None]:
+        """
+        Return when a job began running.
+        """
+
+    @abstractmethod
+    def get_job_ended(self, name: str, debug: bool = False) -> Union[str, None]:
+        """
+        Return when a job stopped running.
+        """
+
+    @abstractmethod
+    def get_job_paused(self, name: str, debug: bool = False) -> Union[str, None]:
+        """
+        Return a job's `paused` timestamp, if it exists.
+        """
+    
     @abstractmethod
     def create_job(
         self,
@@ -73,3 +117,43 @@ class Executor(Connector):
         """
         Return a job's log output.
         """
+
+    @abstractmethod
+    def get_job_stop_time(self, name: str, debug: bool = False) -> Union[datetime, None]:
+        """
+        Return the job's manual stop time.
+        """
+
+    @abstractmethod
+    async def monitor_logs_async(
+        self,
+        name: str,
+        callback_function: Callable[[Any], Any],
+        input_callback_function: Callable[[], str],
+        stop_callback_function: Callable[[SuccessTuple], str],
+        stop_on_exit: bool = False,
+        strip_timestamps: bool = False,
+        accept_input: bool = True,
+        debug: bool = False,
+    ):
+        """
+        Monitor a job's log files and await a callback with the changes.
+        """
+
+    @abstractmethod
+    def monitor_logs(self, *args, **kwargs):
+        """
+        Monitor a job's log files.
+        """
+
+    @abstractmethod
+    def get_job_is_blocking_on_stdin(self, name: str, debug: bool = False) -> bool:
+        """
+        Return whether a job is blocking on stdin.
+        """
+
+    @abstractmethod
+    def get_job_prompt_kwargs(self, name: str, debug: bool = False) -> Dict[str, Any]:
+        """
+        Return the kwargs to the blocking prompt.
+        """