meerschaum 2.1.6__py3-none-any.whl → 2.2.0__py3-none-any.whl

meerschaum/core/Pipe/_sync.py

@@ -12,6 +12,7 @@ import json
 import time
 import threading
 import multiprocessing
+import functools
 from datetime import datetime, timedelta
 
 from meerschaum.utils.typing import (
@@ -518,6 +519,8 @@ def exists(
 def filter_existing(
         self,
         df: 'pd.DataFrame',
+        safe_copy: bool = True,
+        date_bound_only: bool = False,
         chunksize: Optional[int] = -1,
         debug: bool = False,
         **kw
@@ -530,6 +533,14 @@ def filter_existing(
     df: 'pd.DataFrame'
         The dataframe to inspect and filter.
         
+    safe_copy: bool, default True
+        If `True`, create a copy before comparing and modifying the dataframes.
+        Setting to `False` may mutate the DataFrames.
+        See `meerschaum.utils.dataframe.filter_unseen_df`.
+
+    date_bound_only: bool, default False
+        If `True`, only use the datetime index to fetch the sample dataframe.
+
     chunksize: Optional[int], default -1
         The `chunksize` used when fetching existing data.
 
@@ -567,7 +578,8 @@ def filter_existing(
     else:
         merge = pd.merge
         NA = pd.NA
-
+    if df is None:
+        return df, df, df
     if (df.empty if not is_dask else len(df) == 0):
         return df, df, df
 
@@ -617,7 +629,7 @@ def filter_existing(
         traceback.print_exc()
         max_dt = None
 
-    if not ('datetime' in str(type(max_dt))) or str(min_dt) == 'NaT':
+    if ('datetime' not in str(type(max_dt))) or str(min_dt) == 'NaT':
         if 'int' not in str(type(max_dt)).lower():
             max_dt = None
 
@@ -645,7 +657,7 @@ def filter_existing(
         col: df[col].unique()
         for col in self.columns
         if col in df.columns and col != dt_col
-    }
+    } if not date_bound_only else {}
     filter_params_index_limit = get_config('pipes', 'sync', 'filter_params_index_limit')
     _ = kw.pop('params', None)
     params = {
@@ -655,7 +667,7 @@ def filter_existing(
         ]
         for col, unique_vals in unique_index_vals.items()
         if len(unique_vals) <= filter_params_index_limit
-    }
+    } if not date_bound_only else {}
 
     if debug:
         dprint(f"Looking at data between '{begin}' and '{end}':", **kw)
@@ -698,18 +710,23 @@ def filter_existing(
                 col: to_pandas_dtype(typ)
                 for col, typ in self_dtypes.items()
             },
+            safe_copy = safe_copy,
             debug = debug
         ),
         on_cols_dtypes,
     )
 
     ### Cast dicts or lists to strings so we can merge.
+    serializer = functools.partial(json.dumps, sort_keys=True, separators=(',', ':'), default=str)
+    def deserializer(x):
+        return json.loads(x) if isinstance(x, str) else x
+
     unhashable_delta_cols = get_unhashable_cols(delta_df)
     unhashable_backtrack_cols = get_unhashable_cols(backtrack_df)
     for col in unhashable_delta_cols:
-        delta_df[col] = delta_df[col].apply(json.dumps)
+        delta_df[col] = delta_df[col].apply(serializer)
     for col in unhashable_backtrack_cols:
-        backtrack_df[col] = backtrack_df[col].apply(json.dumps)
+        backtrack_df[col] = backtrack_df[col].apply(serializer)
     casted_cols = set(unhashable_delta_cols + unhashable_backtrack_cols)
 
     joined_df = merge(
@@ -722,13 +739,9 @@ def filter_existing(
     ) if on_cols else delta_df
     for col in casted_cols:
         if col in joined_df.columns:
-            joined_df[col] = joined_df[col].apply(
-                lambda x: (
-                    json.loads(x)
-                    if isinstance(x, str)
-                    else x
-                )
-            )
+            joined_df[col] = joined_df[col].apply(deserializer)
+        if col in delta_df.columns:
+            delta_df[col] = delta_df[col].apply(deserializer)
 
     ### Determine which rows are completely new.
     new_rows_mask = (joined_df['_merge'] == 'left_only') if on_cols else None

meerschaum/core/User/_User.py

@@ -7,22 +7,157 @@ User class definition
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Optional, Dict, Any
-
-pwd_context = None
-def get_pwd_context():
-    global pwd_context
-    if pwd_context is None:
-        from meerschaum.config.static import STATIC_CONFIG
-        from meerschaum.utils.packages import attempt_import
-        hash_config = STATIC_CONFIG['users']['password_hash']
-        passlib_context = attempt_import('passlib.context')
-        pwd_context = passlib_context.CryptContext(
-            schemes = hash_config['schemes'],
-            default = hash_config['default'],
-            pbkdf2_sha256__default_rounds = hash_config['pbkdf2_sha256__default_rounds']
+import os
+import hashlib
+import hmac
+from binascii import b2a_base64, a2b_base64, Error as _BinAsciiError
+from meerschaum.utils.typing import Optional, Dict, Any, Tuple
+from meerschaum.config.static import STATIC_CONFIG
+from meerschaum.utils.warnings import warn
+
+
+__all__ = ('hash_password', 'verify_password', 'User')
+
+def hash_password(
+        password: str,
+        salt: Optional[bytes] = None,
+        rounds: Optional[int] = None,
+    ) -> str:
+    """
+    Return an encoded hash string from the given password.
+
+    Parameters
+    ----------
+    password: str
+        The password to be hashed.
+
+    salt: Optional[str], default None
+        If provided, use these bytes for the salt in the hash.
+        Otherwise defaults to 16 random bytes.
+
+    rounds: Optional[int], default None
+        If provided, use this number of rounds to generate the hash.
+        Defaults to 3,000,000.
+        
+    Returns
+    -------
+    An encoded hash string to be stored in a database.
+    See the `passlib` documentation on the string format:
+    https://passlib.readthedocs.io/en/stable/lib/passlib.hash.pbkdf2_digest.html#format-algorithm
+    """
+    hash_config = STATIC_CONFIG['users']['password_hash']
+    if password is None:
+        password = ''
+    if salt is None:
+        salt = os.urandom(hash_config['salt_bytes'])
+    if rounds is None:
+        rounds = hash_config['pbkdf2_sha256__default_rounds']
+
+    pw_hash = hashlib.pbkdf2_hmac(
+        hash_config['algorithm_name'],
+        password.encode('utf-8'),
+        salt,
+        rounds,
+    ) 
+    return (
+        f"$pbkdf2-{hash_config['algorithm_name']}"
+        + f"${hash_config['pbkdf2_sha256__default_rounds']}"
+        + '$' + ab64_encode(salt).decode('utf-8')
+        + '$' + ab64_encode(pw_hash).decode('utf-8')
+    )
+
+
+def verify_password(
+        password: str,
+        password_hash: str,
+    ) -> bool:
+    """
+    Return `True` if the password matches the provided hash.
+
+    Parameters
+    ----------
+    password: str
+        The password to be checked.
+
+    password_hash: str
+        The encoded hash string as generated from `hash_password()`.
+
+    Returns
+    -------
+    A `bool` indicating whether `password` matches `password_hash`.
+    """
+    if password is None or password_hash is None:
+        return False
+    hash_config = STATIC_CONFIG['users']['password_hash']
+    try:
+        digest, rounds_str, encoded_salt, encoded_checksum = password_hash.split('$')[1:]
+        algorithm_name = digest.split('-')[-1]
+        salt = ab64_decode(encoded_salt)
+        checksum = ab64_decode(encoded_checksum)
+        rounds = int(rounds_str)
+    except Exception as e:
+        warn(f"Failed to extract context from password hash '{password_hash}'. Is it corrupted?")
+        return False
+
+    return hmac.compare_digest(
+        checksum,
+        hashlib.pbkdf2_hmac(
+            algorithm_name,
+            password.encode('utf-8'),
+            salt,
+            rounds,
         )
-    return pwd_context
+    )
+
+_BASE64_STRIP = b"=\n"
+_BASE64_PAD1 = b"="
+_BASE64_PAD2 = b"=="
+
+def ab64_encode(data):
+    return b64s_encode(data).replace(b"+", b".")
+
+def ab64_decode(data):
+    """
+    decode from shortened base64 format which omits padding & whitespace.
+    uses custom ``./`` altchars, but supports decoding normal ``+/`` altchars as well.
+    """
+    if isinstance(data, str):
+        # needs bytes for replace() call, but want to accept ascii-unicode ala a2b_base64()
+        try:
+            data = data.encode("ascii")
+        except UnicodeEncodeError:
+            raise ValueError("string argument should contain only ASCII characters")
+    return b64s_decode(data.replace(b".", b"+"))
+
+
+def b64s_encode(data):
+    return b2a_base64(data).rstrip(_BASE64_STRIP)
+
+def b64s_decode(data):
+    """
+    decode from shortened base64 format which omits padding & whitespace.
+    uses default ``+/`` altchars.
+    """
+    if isinstance(data, str):
+        # needs bytes for replace() call, but want to accept ascii-unicode ala a2b_base64()
+        try:
+            data = data.encode("ascii")
+        except UnicodeEncodeError as ue:
+            raise ValueError("string argument should contain only ASCII characters") from ue
+    off = len(data) & 3
+    if off == 0:
+        pass
+    elif off == 2:
+        data += _BASE64_PAD2
+    elif off == 3:
+        data += _BASE64_PAD1
+    else:  # off == 1
+        raise ValueError("Invalid base64 input")
+    try:
+        return a2b_base64(data)
+    except _BinAsciiError as err:
+        raise TypeError(err) from err
+
 
 class User:
     """
@@ -42,7 +177,6 @@ class User:
         if password is None:
             password = ''
         self.password = password
-        self.password_hash = get_pwd_context().hash(password)
         self.username = username
         self.email = email
         self.type = type
@@ -80,3 +214,11 @@ class User:
     @user_id.setter
     def user_id(self, user_id):
         self._user_id = user_id
+
+    @property
+    def password_hash(self):
+        _password_hash = self.__dict__.get('_password_hash', None)
+        if _password_hash is not None:
+            return _password_hash
+        self._password_hash = hash_password(self.password)
+        return self._password_hash

meerschaum/core/User/__init__.py

@@ -6,4 +6,4 @@
 Manager users' metadata via the User class
 """
 
-from meerschaum.core.User._User import User
+from meerschaum.core.User._User import User, hash_password, verify_password

meerschaum/plugins/_Plugin.py

@@ -209,7 +209,7 @@ class Plugin:
         def parse_gitignore() -> 'Set[str]':
             gitignore_path = pathlib.Path(path) / '.gitignore'
             if not gitignore_path.exists():
-                return set()
+                return set(default_patterns_to_ignore)
             with open(gitignore_path, 'r', encoding='utf-8') as f:
                 gitignore_text = f.read()
             return set(pathspec.PathSpec.from_lines(
@@ -252,6 +252,7 @@ class Plugin:
 
     def install(
             self,
+            skip_deps: bool = False,
             force: bool = False,
             debug: bool = False,
         ) -> SuccessTuple:
@@ -263,6 +264,9 @@ class Plugin:
 
         Parameters
         ----------
+        skip_deps: bool, default False
+            If `True`, do not install dependencies.
+
         force: bool, default False
             If `True`, continue with installation, even if required packages fail to install.
 
@@ -366,7 +370,11 @@ class Plugin:
                 plugin_installation_dir_path = path
                 break
 
-        success_msg = f"Successfully installed plugin '{self}'."
+        success_msg = (
+            f"Successfully installed plugin '{self}'"
+            + ("\n    (skipped dependencies)" if skip_deps else "")
+            + "."
+        )
         success, abort = None, None
 
         if is_same_version and not force:
@@ -423,7 +431,8 @@ class Plugin:
             return success, msg
 
         ### attempt to install dependencies
-        if not self.install_dependencies(force=force, debug=debug):
+        dependencies_installed = skip_deps or self.install_dependencies(force=force, debug=debug)
+        if not dependencies_installed:
             _ongoing_installations.remove(self.full_name)
             return False, f"Failed to install dependencies for plugin '{self}'."
 

meerschaum/plugins/__init__.py

@@ -247,6 +247,26 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
                 _warn(f"Unable to create lockfile {PLUGINS_INTERNAL_LOCK_PATH}:\n{e}")
 
     with _locks['internal_plugins']:
+
+        try:
+            from importlib.metadata import entry_points
+        except ImportError:
+            importlib_metadata = attempt_import('importlib_metadata', lazy=False)
+            entry_points = importlib_metadata.entry_points
+
+        ### NOTE: Allow plugins to be installed via `pip`.
+        packaged_plugin_paths = []
+        discovered_packaged_plugins_eps = entry_points(group='meerschaum.plugins')
+        for ep in discovered_packaged_plugins_eps:
+            module_name = ep.name
+            for package_file_path in ep.dist.files:
+                if package_file_path.suffix != '.py':
+                    continue
+                if str(package_file_path) == f'{module_name}.py':
+                    packaged_plugin_paths.append(package_file_path.locate())
+                elif str(package_file_path) == f'{module_name}/__init__.py':
+                    packaged_plugin_paths.append(package_file_path.locate().parent)
+
         if is_symlink(PLUGINS_RESOURCES_PATH) or not PLUGINS_RESOURCES_PATH.exists():
             try:
                 PLUGINS_RESOURCES_PATH.unlink()
@@ -255,7 +275,6 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
 
         PLUGINS_RESOURCES_PATH.mkdir(exist_ok=True)
 
-
         existing_symlinked_paths = [
             (PLUGINS_RESOURCES_PATH / item) 
             for item in os.listdir(PLUGINS_RESOURCES_PATH)
@@ -275,6 +294,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
                 for plugins_path in PLUGINS_DIR_PATHS
             ]
         ))
+        plugins_to_be_symlinked.extend(packaged_plugin_paths)
 
         ### Check for duplicates.
         seen_plugins = defaultdict(lambda: 0)
@@ -538,6 +558,8 @@ def get_plugins(*to_load, try_import: bool = True) -> Union[Tuple[Plugin], Plugi
     ]
     plugins = tuple(plugin for plugin in _plugins if plugin.is_installed(try_import=try_import))
     if len(to_load) == 1:
+        if len(plugins) == 0:
+            raise ValueError(f"Plugin '{to_load[0]}' is not installed.")
         return plugins[0]
     return plugins
 

meerschaum/utils/daemon/Daemon.py

@@ -15,9 +15,11 @@ import signal
 import sys
 import time
 import traceback
+from functools import partial
 from datetime import datetime, timezone
 from meerschaum.utils.typing import Optional, Dict, Any, SuccessTuple, Callable, List, Union
 from meerschaum.config import get_config
+from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.config._paths import DAEMON_RESOURCES_PATH, LOGS_RESOURCES_PATH
 from meerschaum.config._patch import apply_patch_to_config
 from meerschaum.utils.warnings import warn, error
@@ -36,7 +38,7 @@ class Daemon:
     def __new__(
         cls,
         *args,
-        daemon_id : Optional[str] = None,
+        daemon_id: Optional[str] = None,
         **kw
     ):
         """
@@ -129,7 +131,7 @@ class Daemon:
         keep_daemon_output: bool, default True
             If `False`, delete the daemon's output directory upon exiting.
             
-        allow_dirty_run :
+        allow_dirty_run, bool, default False:
             If `True`, run the daemon, even if the `daemon_id` directory exists.
             This option is dangerous because if the same `daemon_id` runs twice,
             the last to finish will overwrite the output of the first.
@@ -138,8 +140,13 @@ class Daemon:
         -------
         Nothing — this will exit the parent process.
         """
-        import platform, sys, os
+        import platform, sys, os, traceback
+        from meerschaum.config._paths import DAEMON_ERROR_LOG_PATH
+        from meerschaum.utils.warnings import warn
+        from meerschaum.config import get_config
         daemon = attempt_import('daemon')
+        lines = get_config('jobs', 'terminal', 'lines')
+        columns = get_config('jobs','terminal', 'columns')
 
         if platform.system() == 'Windows':
             return False, "Windows is no longer supported."
@@ -160,31 +167,45 @@ class Daemon:
         )
 
         log_refresh_seconds = get_config('jobs', 'logs', 'refresh_files_seconds')
-        self._log_refresh_timer = RepeatTimer(log_refresh_seconds, self.rotating_log.refresh_files)
-
-        with self._daemon_context:
-            try:
-                with open(self.pid_path, 'w+') as f:
-                    f.write(str(os.getpid()))
+        self._log_refresh_timer = RepeatTimer(
+            log_refresh_seconds,
+            partial(self.rotating_log.refresh_files, start_interception=True),
+        )
 
-                self._log_refresh_timer.start()
-                result = self.target(*self.target_args, **self.target_kw)
-                self.properties['result'] = result
-            except Exception as e:
-                warn(e, stacklevel=3)
-                result = e
-            finally:
-                self._log_refresh_timer.cancel()
-                self.rotating_log.close()
-                if self.pid is None and self.pid_path.exists():
-                    self.pid_path.unlink()
+        try:
+            os.environ['LINES'], os.environ['COLUMNS'] = str(int(lines)), str(int(columns))
+            with self._daemon_context:
+                os.environ[STATIC_CONFIG['environment']['daemon_id']] = self.daemon_id
+                self.rotating_log.refresh_files(start_interception=True)
+                try:
+                    with open(self.pid_path, 'w+', encoding='utf-8') as f:
+                        f.write(str(os.getpid()))
 
-            if keep_daemon_output:
-                self._capture_process_timestamp('ended')
-            else:
-                self.cleanup()
+                    self._log_refresh_timer.start()
+                    result = self.target(*self.target_args, **self.target_kw)
+                    self.properties['result'] = result
+                except Exception as e:
+                    warn(e, stacklevel=3)
+                    result = e
+                finally:
+                    self._log_refresh_timer.cancel()
+                    self.rotating_log.close()
+                    if self.pid is None and self.pid_path.exists():
+                        self.pid_path.unlink()
+
+                if keep_daemon_output:
+                    self._capture_process_timestamp('ended')
+                else:
+                    self.cleanup()
+
+                return result
+        except Exception as e:
+            daemon_error = traceback.format_exc()
+            with open(DAEMON_ERROR_LOG_PATH, 'a+', encoding='utf-8') as f:
+                f.write(daemon_error)
 
-            return result
+        if daemon_error:
+            warn(f"Encountered an error while starting the daemon '{self}':\n{daemon_error}")
 
 
     def _capture_process_timestamp(
@@ -444,6 +465,9 @@ class Daemon:
         Handle `SIGINT` within the Daemon context.
         This method is injected into the `DaemonContext`.
         """
+        #  from meerschaum.utils.daemon.FileDescriptorInterceptor import STOP_READING_FD_EVENT
+        #  STOP_READING_FD_EVENT.set()
+        self.rotating_log.stop_log_fd_interception(unused_only=False)
         timer = self.__dict__.get('_log_refresh_timer', None)
         if timer is not None:
             timer.cancel()
@@ -453,9 +477,17 @@ class Daemon:
             daemon_context.close()
 
         _close_pools()
-
-        ### NOTE: SystemExit() does not work here.
-        sys.exit(0)
+        import threading
+        for thread in threading.enumerate():
+            if thread.name == 'MainThread':
+                continue
+            try:
+                if thread.is_alive():
+                    stack = traceback.format_stack(sys._current_frames()[thread.ident])
+                    thread.join()
+            except Exception as e:
+                warn(traceback.format_exc())
+        raise KeyboardInterrupt()
 
 
     def _handle_sigterm(self, signal_number: int, stack_frame: 'frame') -> None:
@@ -472,8 +504,7 @@ class Daemon:
             daemon_context.close()
 
         _close_pools()
-
-        raise SystemExit()
+        raise SystemExit(0)
 
  
     def _send_signal(
@@ -650,7 +681,12 @@ class Daemon:
         if '_rotating_log' in self.__dict__:
             return self._rotating_log
 
-        self._rotating_log = RotatingFile(self.log_path, redirect_streams=True)
+        self._rotating_log = RotatingFile(
+            self.log_path,
+            redirect_streams = True,
+            write_timestamps = get_config('jobs', 'logs', 'timestamps', 'enabled'),
+            timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format'),
+        )
         return self._rotating_log
 
 
@@ -663,6 +699,8 @@ class Daemon:
             self.rotating_log.file_path,
             num_files_to_keep = self.rotating_log.num_files_to_keep,
             max_file_size = self.rotating_log.max_file_size,
+            write_timestamps = get_config('jobs', 'logs', 'timestamps', 'enabled'),
+            timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format'),
         )
         return new_rotating_log.read()
 
@@ -714,7 +752,7 @@ class Daemon:
         if not self.pid_path.exists():
             return None
         try:
-            with open(self.pid_path, 'r') as f:
+            with open(self.pid_path, 'r', encoding='utf-8') as f:
                 text = f.read()
             pid = int(text.rstrip())
         except Exception as e:
@@ -815,7 +853,7 @@ class Daemon:
         if self.properties is not None:
             try:
                 self.path.mkdir(parents=True, exist_ok=True)
-                with open(self.properties_path, 'w+') as properties_file:
+                with open(self.properties_path, 'w+', encoding='utf-8') as properties_file:
                     json.dump(self.properties, properties_file)
                 success, msg = True, 'Success'
             except Exception as e:
@@ -865,21 +903,36 @@ class Daemon:
             error(_write_pickle_success_tuple[1])
 
 
-    def cleanup(self, keep_logs: bool = False) -> None:
-        """Remove a daemon's directory after execution.
+    def cleanup(self, keep_logs: bool = False) -> SuccessTuple:
+        """
+        Remove a daemon's directory after execution.
 
         Parameters
         ----------
         keep_logs: bool, default False
             If `True`, skip deleting the daemon's log files.
+
+        Returns
+        -------
+        A `SuccessTuple` indicating success.
         """
         if self.path.exists():
             try:
                 shutil.rmtree(self.path)
             except Exception as e:
-                warn(e)
+                msg = f"Failed to clean up '{self.daemon_id}':\n{e}"
+                warn(msg)
+                return False, msg
         if not keep_logs:
             self.rotating_log.delete()
+            try:
+                if self.log_offset_path.exists():
+                    self.log_offset_path.unlink()
+            except Exception as e:
+                msg = f"Failed to remove offset file for '{self.daemon_id}':\n{e}"
+                warn(msg)
+                return False, msg
+        return True, "Success"
 
 
     def get_timeout_seconds(self, timeout: Union[int, float, None] = None) -> Union[int, float]: