meerschaum 2.2.0.dev3__py3-none-any.whl → 2.2.0rc2__py3-none-any.whl

meerschaum/connectors/sql/_plugins.py

@@ -108,9 +108,7 @@ def get_plugin_version(
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
     sqlalchemy = attempt_import('sqlalchemy')
-
     query = sqlalchemy.select(plugins_tbl.c.version).where(plugins_tbl.c.plugin_name == plugin.name)
-
     return self.value(query, debug=debug)
 
 def get_plugin_user_id(

meerschaum/core/User/_User.py

@@ -7,22 +7,155 @@ User class definition
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Optional, Dict, Any
-
-pwd_context = None
-def get_pwd_context():
-    global pwd_context
-    if pwd_context is None:
-        from meerschaum.config.static import STATIC_CONFIG
-        from meerschaum.utils.packages import attempt_import
-        hash_config = STATIC_CONFIG['users']['password_hash']
-        passlib_context = attempt_import('passlib.context')
-        pwd_context = passlib_context.CryptContext(
-            schemes = hash_config['schemes'],
-            default = hash_config['default'],
-            pbkdf2_sha256__default_rounds = hash_config['pbkdf2_sha256__default_rounds']
+import os
+import hashlib
+import hmac
+from binascii import b2a_base64, a2b_base64, Error as _BinAsciiError
+from meerschaum.utils.typing import Optional, Dict, Any, Tuple
+from meerschaum.config.static import STATIC_CONFIG
+from meerschaum.utils.warnings import warn
+
+
+__all__ = ('hash_password', 'verify_password', 'User')
+
+def hash_password(
+        password: str,
+        salt: Optional[bytes] = None,
+        rounds: Optional[int] = None,
+    ) -> str:
+    """
+    Return an encoded hash string from the given password.
+
+    Parameters
+    ----------
+    password: str
+        The password to be hashed.
+
+    salt: Optional[str], default None
+        If provided, use these bytes for the salt in the hash.
+        Otherwise defaults to 16 random bytes.
+
+    rounds: Optional[int], default None
+        If provided, use this number of rounds to generate the hash.
+        Defaults to 3,000,000.
+        
+    Returns
+    -------
+    An encoded hash string to be stored in a database.
+    See the `passlib` documentation on the string format:
+    https://passlib.readthedocs.io/en/stable/lib/passlib.hash.pbkdf2_digest.html#format-algorithm
+    """
+    hash_config = STATIC_CONFIG['users']['password_hash']
+    if password is None:
+        password = ''
+    if salt is None:
+        salt = os.urandom(hash_config['salt_bytes'])
+    if rounds is None:
+        rounds = hash_config['pbkdf2_sha256__default_rounds']
+
+    pw_hash = hashlib.pbkdf2_hmac(
+        hash_config['algorithm_name'],
+        password.encode('utf-8'),
+        salt,
+        rounds,
+    ) 
+    return (
+        f"$pbkdf2-{hash_config['algorithm_name']}"
+        + f"${hash_config['pbkdf2_sha256__default_rounds']}"
+        + '$' + ab64_encode(salt).decode('utf-8')
+        + '$' + ab64_encode(pw_hash).decode('utf-8')
+    )
+
+
+def verify_password(
+        password: str,
+        password_hash: str,
+    ) -> bool:
+    """
+    Return `True` if the password matches the provided hash.
+
+    Parameters
+    ----------
+    password: str
+        The password to be checked.
+
+    password_hash: str
+        The encoded hash string as generated from `hash_password()`.
+
+    Returns
+    -------
+    A `bool` indicating whether `password` matches `password_hash`.
+    """
+    hash_config = STATIC_CONFIG['users']['password_hash']
+    try:
+        digest, rounds_str, encoded_salt, encoded_checksum = password_hash.split('$')[1:]
+        algorithm_name = digest.split('-')[-1]
+        salt = ab64_decode(encoded_salt)
+        checksum = ab64_decode(encoded_checksum)
+        rounds = int(rounds_str)
+    except Exception as e:
+        warn(f"Failed to extract context from password hash '{password_hash}'. Is it corrupted?")
+        return False
+
+    return hmac.compare_digest(
+        checksum,
+        hashlib.pbkdf2_hmac(
+            algorithm_name,
+            password.encode('utf-8'),
+            salt,
+            rounds,
         )
-    return pwd_context
+    )
+
+_BASE64_STRIP = b"=\n"
+_BASE64_PAD1 = b"="
+_BASE64_PAD2 = b"=="
+
+def ab64_encode(data):
+    return b64s_encode(data).replace(b"+", b".")
+
+def ab64_decode(data):
+    """
+    decode from shortened base64 format which omits padding & whitespace.
+    uses custom ``./`` altchars, but supports decoding normal ``+/`` altchars as well.
+    """
+    if isinstance(data, str):
+        # needs bytes for replace() call, but want to accept ascii-unicode ala a2b_base64()
+        try:
+            data = data.encode("ascii")
+        except UnicodeEncodeError:
+            raise ValueError("string argument should contain only ASCII characters")
+    return b64s_decode(data.replace(b".", b"+"))
+
+
+def b64s_encode(data):
+    return b2a_base64(data).rstrip(_BASE64_STRIP)
+
+def b64s_decode(data):
+    """
+    decode from shortened base64 format which omits padding & whitespace.
+    uses default ``+/`` altchars.
+    """
+    if isinstance(data, str):
+        # needs bytes for replace() call, but want to accept ascii-unicode ala a2b_base64()
+        try:
+            data = data.encode("ascii")
+        except UnicodeEncodeError as ue:
+            raise ValueError("string argument should contain only ASCII characters") from ue
+    off = len(data) & 3
+    if off == 0:
+        pass
+    elif off == 2:
+        data += _BASE64_PAD2
+    elif off == 3:
+        data += _BASE64_PAD1
+    else:  # off == 1
+        raise ValueError("Invalid base64 input")
+    try:
+        return a2b_base64(data)
+    except _BinAsciiError as err:
+        raise TypeError(err) from err
+
 
 class User:
     """
@@ -42,7 +175,6 @@ class User:
         if password is None:
             password = ''
         self.password = password
-        self.password_hash = get_pwd_context().hash(password)
         self.username = username
         self.email = email
         self.type = type
@@ -80,3 +212,11 @@ class User:
     @user_id.setter
     def user_id(self, user_id):
         self._user_id = user_id
+
+    @property
+    def password_hash(self):
+        _password_hash = self.__dict__.get('_password_hash', None)
+        if _password_hash is not None:
+            return _password_hash
+        self._password_hash = hash_password(self.password)
+        return self._password_hash

meerschaum/core/User/__init__.py

@@ -6,4 +6,4 @@
 Manager users' metadata via the User class
 """
 
-from meerschaum.core.User._User import User
+from meerschaum.core.User._User import User, hash_password, verify_password

meerschaum/plugins/_Plugin.py

@@ -209,7 +209,7 @@ class Plugin:
         def parse_gitignore() -> 'Set[str]':
             gitignore_path = pathlib.Path(path) / '.gitignore'
             if not gitignore_path.exists():
-                return set()
+                return set(default_patterns_to_ignore)
             with open(gitignore_path, 'r', encoding='utf-8') as f:
                 gitignore_text = f.read()
             return set(pathspec.PathSpec.from_lines(

meerschaum/utils/daemon/Daemon.py

@@ -15,6 +15,7 @@ import signal
 import sys
 import time
 import traceback
+from functools import partial
 from datetime import datetime, timezone
 from meerschaum.utils.typing import Optional, Dict, Any, SuccessTuple, Callable, List, Union
 from meerschaum.config import get_config
@@ -36,7 +37,7 @@ class Daemon:
     def __new__(
         cls,
         *args,
-        daemon_id : Optional[str] = None,
+        daemon_id: Optional[str] = None,
         **kw
     ):
         """
@@ -129,7 +130,7 @@ class Daemon:
         keep_daemon_output: bool, default True
             If `False`, delete the daemon's output directory upon exiting.
             
-        allow_dirty_run :
+        allow_dirty_run, bool, default False:
             If `True`, run the daemon, even if the `daemon_id` directory exists.
             This option is dangerous because if the same `daemon_id` runs twice,
             the last to finish will overwrite the output of the first.
@@ -138,8 +139,13 @@ class Daemon:
         -------
         Nothing — this will exit the parent process.
         """
-        import platform, sys, os
+        import platform, sys, os, traceback
+        from meerschaum.config._paths import DAEMON_ERROR_LOG_PATH
+        from meerschaum.utils.warnings import warn
+        from meerschaum.config import get_config
         daemon = attempt_import('daemon')
+        lines = get_config('jobs', 'terminal', 'lines')
+        columns = get_config('jobs','terminal', 'columns')
 
         if platform.system() == 'Windows':
             return False, "Windows is no longer supported."
@@ -160,31 +166,43 @@ class Daemon:
         )
 
         log_refresh_seconds = get_config('jobs', 'logs', 'refresh_files_seconds')
-        self._log_refresh_timer = RepeatTimer(log_refresh_seconds, self.rotating_log.refresh_files)
-
-        with self._daemon_context:
-            try:
-                with open(self.pid_path, 'w+') as f:
-                    f.write(str(os.getpid()))
-
-                self._log_refresh_timer.start()
-                result = self.target(*self.target_args, **self.target_kw)
-                self.properties['result'] = result
-            except Exception as e:
-                warn(e, stacklevel=3)
-                result = e
-            finally:
-                self._log_refresh_timer.cancel()
-                self.rotating_log.close()
-                if self.pid is None and self.pid_path.exists():
-                    self.pid_path.unlink()
+        self._log_refresh_timer = RepeatTimer(
+            log_refresh_seconds,
+            partial(self.rotating_log.refresh_files, start_interception=True),
+        )
+        try:
+            os.environ['LINES'], os.environ['COLUMNS'] = str(int(lines)), str(int(columns))
+            with self._daemon_context:
+                self.rotating_log.refresh_files(start_interception=True)
+                try:
+                    with open(self.pid_path, 'w+', encoding='utf-8') as f:
+                        f.write(str(os.getpid()))
 
-            if keep_daemon_output:
-                self._capture_process_timestamp('ended')
-            else:
-                self.cleanup()
+                    self._log_refresh_timer.start()
+                    result = self.target(*self.target_args, **self.target_kw)
+                    self.properties['result'] = result
+                except Exception as e:
+                    warn(e, stacklevel=3)
+                    result = e
+                finally:
+                    self._log_refresh_timer.cancel()
+                    self.rotating_log.close()
+                    if self.pid is None and self.pid_path.exists():
+                        self.pid_path.unlink()
+
+                if keep_daemon_output:
+                    self._capture_process_timestamp('ended')
+                else:
+                    self.cleanup()
+
+                return result
+        except Exception as e:
+            daemon_error = traceback.format_exc()
+            with open(DAEMON_ERROR_LOG_PATH, 'a+', encoding='utf-8') as f:
+                f.write(daemon_error)
 
-            return result
+        if daemon_error:
+            warn(f"Encountered an error while starting the daemon '{self}':\n{daemon_error}")
 
 
     def _capture_process_timestamp(
@@ -453,9 +471,8 @@ class Daemon:
             daemon_context.close()
 
         _close_pools()
-
-        ### NOTE: SystemExit() does not work here.
-        sys.exit(0)
+        self.rotating_log.stop_log_fd_interception()
+        raise KeyboardInterrupt()
 
 
     def _handle_sigterm(self, signal_number: int, stack_frame: 'frame') -> None:
@@ -472,8 +489,7 @@ class Daemon:
             daemon_context.close()
 
         _close_pools()
-
-        raise SystemExit()
+        raise SystemExit(1)
 
  
     def _send_signal(
@@ -650,7 +666,12 @@ class Daemon:
         if '_rotating_log' in self.__dict__:
             return self._rotating_log
 
-        self._rotating_log = RotatingFile(self.log_path, redirect_streams=True)
+        self._rotating_log = RotatingFile(
+            self.log_path,
+            redirect_streams = True,
+            write_timestamps = True,
+            timestamp_format = get_config('jobs', 'logs', 'timestamp_format'),
+        )
         return self._rotating_log
 
 
@@ -663,6 +684,7 @@ class Daemon:
             self.rotating_log.file_path,
             num_files_to_keep = self.rotating_log.num_files_to_keep,
             max_file_size = self.rotating_log.max_file_size,
+            write_timestamps = True,
         )
         return new_rotating_log.read()
 
@@ -714,7 +736,7 @@ class Daemon:
         if not self.pid_path.exists():
             return None
         try:
-            with open(self.pid_path, 'r') as f:
+            with open(self.pid_path, 'r', encoding='utf-8') as f:
                 text = f.read()
             pid = int(text.rstrip())
         except Exception as e:
@@ -815,7 +837,7 @@ class Daemon:
         if self.properties is not None:
             try:
                 self.path.mkdir(parents=True, exist_ok=True)
-                with open(self.properties_path, 'w+') as properties_file:
+                with open(self.properties_path, 'w+', encoding='utf-8') as properties_file:
                     json.dump(self.properties, properties_file)
                 success, msg = True, 'Success'
             except Exception as e:
@@ -887,6 +909,13 @@ class Daemon:
                 return False, msg
         if not keep_logs:
             self.rotating_log.delete()
+            try:
+                if self.log_offset_path.exists():
+                    self.log_offset_path.unlink()
+            except Exception as e:
+                msg = f"Failed to remove offset file for '{self.daemon_id}':\n{e}"
+                warn(msg)
+                return False, msg
         return True, "Success"
 
 

meerschaum/utils/daemon/FileDescriptorInterceptor.py

@@ -0,0 +1,102 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Intercept OS-level file descriptors.
+"""
+
+import os
+import traceback
+from datetime import datetime
+from meerschaum.utils.typing import Callable
+from meerschaum.utils.warnings import warn
+
+FD_CLOSED: int = 9
+
+class FileDescriptorInterceptor:
+    """
+    A management class to intercept data written to a file descriptor.
+    """
+    def __init__(
+        self,
+        file_descriptor: int,
+        injection_hook: Callable[[], str],
+    ):
+        """
+        Parameters
+        ----------
+        file_descriptor: int
+            The OS file descriptor from which to read.
+
+        injection_hook: Callable[[], str]
+            A callable which returns a string to be injected into the written data.
+        """
+        self.injection_hook = injection_hook
+        self.original_file_descriptor = file_descriptor
+        self.new_file_descriptor = os.dup(file_descriptor)
+        self.read_pipe, self.write_pipe = os.pipe()
+        os.dup2(self.write_pipe, file_descriptor)
+
+    def start_interception(self):
+        """
+        Read from the file descriptor and write the modified data after injection.
+
+        NOTE: This is blocking and is meant to be run in a thread.
+        """
+        is_first_read = True
+        while True:
+            data = os.read(self.read_pipe, 1024)
+            if not data:
+                break
+
+            first_char_is_newline = data[0] == b'\n'
+            last_char_is_newline = data[-1] == b'\n'
+
+            injected_str = self.injection_hook()
+            injected_bytes = injected_str.encode('utf-8')
+
+            if is_first_read:
+                data = b'\n' + data
+                is_first_read = False
+
+            modified_data = (
+                (data[:-1].replace(b'\n', b'\n' + injected_bytes) + b'\n')
+                if last_char_is_newline
+                else data.replace(b'\n', b'\n' + injected_bytes)
+            )
+
+            os.write(self.new_file_descriptor, modified_data)
+
+    def stop_interception(self):
+        """
+        Restore the file descriptors and close the new pipes.
+        """
+        try:
+            os.dup2(self.new_file_descriptor, self.original_file_descriptor)
+            #  os.close(self.new_file_descriptor)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the duplicated file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+
+        try:
+            os.close(self.write_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the write-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+        try:
+            os.close(self.read_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the read-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )