meerschaum 2.2.0rc1__py3-none-any.whl → 2.2.0rc2__py3-none-any.whl

meerschaum/core/User/_User.py

@@ -7,22 +7,155 @@ User class definition
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Optional, Dict, Any
-
-pwd_context = None
-def get_pwd_context():
-    global pwd_context
-    if pwd_context is None:
-        from meerschaum.config.static import STATIC_CONFIG
-        from meerschaum.utils.packages import attempt_import
-        hash_config = STATIC_CONFIG['users']['password_hash']
-        passlib_context = attempt_import('passlib.context')
-        pwd_context = passlib_context.CryptContext(
-            schemes = hash_config['schemes'],
-            default = hash_config['default'],
-            pbkdf2_sha256__default_rounds = hash_config['pbkdf2_sha256__default_rounds']
+import os
+import hashlib
+import hmac
+from binascii import b2a_base64, a2b_base64, Error as _BinAsciiError
+from meerschaum.utils.typing import Optional, Dict, Any, Tuple
+from meerschaum.config.static import STATIC_CONFIG
+from meerschaum.utils.warnings import warn
+
+
+__all__ = ('hash_password', 'verify_password', 'User')
+
+def hash_password(
+        password: str,
+        salt: Optional[bytes] = None,
+        rounds: Optional[int] = None,
+    ) -> str:
+    """
+    Return an encoded hash string from the given password.
+
+    Parameters
+    ----------
+    password: str
+        The password to be hashed.
+
+    salt: Optional[str], default None
+        If provided, use these bytes for the salt in the hash.
+        Otherwise defaults to 16 random bytes.
+
+    rounds: Optional[int], default None
+        If provided, use this number of rounds to generate the hash.
+        Defaults to 3,000,000.
+        
+    Returns
+    -------
+    An encoded hash string to be stored in a database.
+    See the `passlib` documentation on the string format:
+    https://passlib.readthedocs.io/en/stable/lib/passlib.hash.pbkdf2_digest.html#format-algorithm
+    """
+    hash_config = STATIC_CONFIG['users']['password_hash']
+    if password is None:
+        password = ''
+    if salt is None:
+        salt = os.urandom(hash_config['salt_bytes'])
+    if rounds is None:
+        rounds = hash_config['pbkdf2_sha256__default_rounds']
+
+    pw_hash = hashlib.pbkdf2_hmac(
+        hash_config['algorithm_name'],
+        password.encode('utf-8'),
+        salt,
+        rounds,
+    ) 
+    return (
+        f"$pbkdf2-{hash_config['algorithm_name']}"
+        + f"${hash_config['pbkdf2_sha256__default_rounds']}"
+        + '$' + ab64_encode(salt).decode('utf-8')
+        + '$' + ab64_encode(pw_hash).decode('utf-8')
+    )
+
+
+def verify_password(
+        password: str,
+        password_hash: str,
+    ) -> bool:
+    """
+    Return `True` if the password matches the provided hash.
+
+    Parameters
+    ----------
+    password: str
+        The password to be checked.
+
+    password_hash: str
+        The encoded hash string as generated from `hash_password()`.
+
+    Returns
+    -------
+    A `bool` indicating whether `password` matches `password_hash`.
+    """
+    hash_config = STATIC_CONFIG['users']['password_hash']
+    try:
+        digest, rounds_str, encoded_salt, encoded_checksum = password_hash.split('$')[1:]
+        algorithm_name = digest.split('-')[-1]
+        salt = ab64_decode(encoded_salt)
+        checksum = ab64_decode(encoded_checksum)
+        rounds = int(rounds_str)
+    except Exception as e:
+        warn(f"Failed to extract context from password hash '{password_hash}'. Is it corrupted?")
+        return False
+
+    return hmac.compare_digest(
+        checksum,
+        hashlib.pbkdf2_hmac(
+            algorithm_name,
+            password.encode('utf-8'),
+            salt,
+            rounds,
         )
-    return pwd_context
+    )
+
+_BASE64_STRIP = b"=\n"
+_BASE64_PAD1 = b"="
+_BASE64_PAD2 = b"=="
+
+def ab64_encode(data):
+    return b64s_encode(data).replace(b"+", b".")
+
+def ab64_decode(data):
+    """
+    decode from shortened base64 format which omits padding & whitespace.
+    uses custom ``./`` altchars, but supports decoding normal ``+/`` altchars as well.
+    """
+    if isinstance(data, str):
+        # needs bytes for replace() call, but want to accept ascii-unicode ala a2b_base64()
+        try:
+            data = data.encode("ascii")
+        except UnicodeEncodeError:
+            raise ValueError("string argument should contain only ASCII characters")
+    return b64s_decode(data.replace(b".", b"+"))
+
+
+def b64s_encode(data):
+    return b2a_base64(data).rstrip(_BASE64_STRIP)
+
+def b64s_decode(data):
+    """
+    decode from shortened base64 format which omits padding & whitespace.
+    uses default ``+/`` altchars.
+    """
+    if isinstance(data, str):
+        # needs bytes for replace() call, but want to accept ascii-unicode ala a2b_base64()
+        try:
+            data = data.encode("ascii")
+        except UnicodeEncodeError as ue:
+            raise ValueError("string argument should contain only ASCII characters") from ue
+    off = len(data) & 3
+    if off == 0:
+        pass
+    elif off == 2:
+        data += _BASE64_PAD2
+    elif off == 3:
+        data += _BASE64_PAD1
+    else:  # off == 1
+        raise ValueError("Invalid base64 input")
+    try:
+        return a2b_base64(data)
+    except _BinAsciiError as err:
+        raise TypeError(err) from err
+
 
 class User:
     """
@@ -42,7 +175,6 @@ class User:
         if password is None:
             password = ''
         self.password = password
-        self.password_hash = get_pwd_context().hash(password)
         self.username = username
         self.email = email
         self.type = type
@@ -80,3 +212,11 @@ class User:
     @user_id.setter
     def user_id(self, user_id):
         self._user_id = user_id
+
+    @property
+    def password_hash(self):
+        _password_hash = self.__dict__.get('_password_hash', None)
+        if _password_hash is not None:
+            return _password_hash
+        self._password_hash = hash_password(self.password)
+        return self._password_hash

meerschaum/core/User/__init__.py

@@ -6,4 +6,4 @@
 Manager users' metadata via the User class
 """
 
-from meerschaum.core.User._User import User
+from meerschaum.core.User._User import User, hash_password, verify_password

meerschaum/plugins/_Plugin.py

@@ -209,7 +209,7 @@ class Plugin:
         def parse_gitignore() -> 'Set[str]':
             gitignore_path = pathlib.Path(path) / '.gitignore'
             if not gitignore_path.exists():
-                return set()
+                return set(default_patterns_to_ignore)
             with open(gitignore_path, 'r', encoding='utf-8') as f:
                 gitignore_text = f.read()
             return set(pathspec.PathSpec.from_lines(

meerschaum/utils/daemon/Daemon.py

@@ -15,6 +15,7 @@ import signal
 import sys
 import time
 import traceback
+from functools import partial
 from datetime import datetime, timezone
 from meerschaum.utils.typing import Optional, Dict, Any, SuccessTuple, Callable, List, Union
 from meerschaum.config import get_config
@@ -139,11 +140,12 @@ class Daemon:
         Nothing — this will exit the parent process.
         """
         import platform, sys, os, traceback
-        from meerschaum.config._paths import LOGS_RESOURCES_PATH
+        from meerschaum.config._paths import DAEMON_ERROR_LOG_PATH
         from meerschaum.utils.warnings import warn
-        daemons_error_log_path = LOGS_RESOURCES_PATH / 'daemons_error.log'
-
+        from meerschaum.config import get_config
         daemon = attempt_import('daemon')
+        lines = get_config('jobs', 'terminal', 'lines')
+        columns = get_config('jobs','terminal', 'columns')
 
         if platform.system() == 'Windows':
             return False, "Windows is no longer supported."
@@ -164,10 +166,14 @@ class Daemon:
         )
 
         log_refresh_seconds = get_config('jobs', 'logs', 'refresh_files_seconds')
-        self._log_refresh_timer = RepeatTimer(log_refresh_seconds, self.rotating_log.refresh_files)
-
+        self._log_refresh_timer = RepeatTimer(
+            log_refresh_seconds,
+            partial(self.rotating_log.refresh_files, start_interception=True),
+        )
         try:
+            os.environ['LINES'], os.environ['COLUMNS'] = str(int(lines)), str(int(columns))
             with self._daemon_context:
+                self.rotating_log.refresh_files(start_interception=True)
                 try:
                     with open(self.pid_path, 'w+', encoding='utf-8') as f:
                         f.write(str(os.getpid()))
@@ -192,11 +198,11 @@ class Daemon:
                 return result
         except Exception as e:
             daemon_error = traceback.format_exc()
-            with open(LOGS_RESOURCES_PATH, 'a+', encoding='utf-8') as f:
+            with open(DAEMON_ERROR_LOG_PATH, 'a+', encoding='utf-8') as f:
                 f.write(daemon_error)
 
         if daemon_error:
-            warn("Encountered an error while starting the daemon '{self}':\n{daemon_error}")
+            warn(f"Encountered an error while starting the daemon '{self}':\n{daemon_error}")
 
 
     def _capture_process_timestamp(
@@ -464,10 +470,9 @@ class Daemon:
         if daemon_context is not None:
             daemon_context.close()
 
-        self.rotating_log.stop_log_fd_interception()
-
         _close_pools()
-        raise SystemExit(0)
+        self.rotating_log.stop_log_fd_interception()
+        raise KeyboardInterrupt()
 
 
     def _handle_sigterm(self, signal_number: int, stack_frame: 'frame') -> None:
@@ -483,8 +488,6 @@ class Daemon:
         if daemon_context is not None:
             daemon_context.close()
 
-        self.rotating_log.stop_log_fd_interception()
-
         _close_pools()
         raise SystemExit(1)
 
@@ -667,6 +670,7 @@ class Daemon:
             self.log_path,
             redirect_streams = True,
             write_timestamps = True,
+            timestamp_format = get_config('jobs', 'logs', 'timestamp_format'),
         )
         return self._rotating_log
 
@@ -905,6 +909,13 @@ class Daemon:
                 return False, msg
         if not keep_logs:
             self.rotating_log.delete()
+            try:
+                if self.log_offset_path.exists():
+                    self.log_offset_path.unlink()
+            except Exception as e:
+                msg = f"Failed to remove offset file for '{self.daemon_id}':\n{e}"
+                warn(msg)
+                return False, msg
         return True, "Success"
 
 

meerschaum/utils/daemon/FileDescriptorInterceptor.py

@@ -7,8 +7,12 @@ Intercept OS-level file descriptors.
 """
 
 import os
+import traceback
 from datetime import datetime
 from meerschaum.utils.typing import Callable
+from meerschaum.utils.warnings import warn
+
+FD_CLOSED: int = 9
 
 class FileDescriptorInterceptor:
     """
@@ -40,12 +44,28 @@ class FileDescriptorInterceptor:
 
         NOTE: This is blocking and is meant to be run in a thread.
         """
+        is_first_read = True
         while True:
             data = os.read(self.read_pipe, 1024)
             if not data:
                 break
+
+            first_char_is_newline = data[0] == b'\n'
+            last_char_is_newline = data[-1] == b'\n'
+
             injected_str = self.injection_hook()
-            modified_data = data.replace(b'\n', f'\n{injected_str}'.encode('utf-8'))
+            injected_bytes = injected_str.encode('utf-8')
+
+            if is_first_read:
+                data = b'\n' + data
+                is_first_read = False
+
+            modified_data = (
+                (data[:-1].replace(b'\n', b'\n' + injected_bytes) + b'\n')
+                if last_char_is_newline
+                else data.replace(b'\n', b'\n' + injected_bytes)
+            )
+
             os.write(self.new_file_descriptor, modified_data)
 
     def stop_interception(self):
@@ -54,7 +74,29 @@ class FileDescriptorInterceptor:
         """
         try:
             os.dup2(self.new_file_descriptor, self.original_file_descriptor)
-            os.close(self.read_pipe)
+            #  os.close(self.new_file_descriptor)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the duplicated file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+
+        try:
             os.close(self.write_pipe)
-        except OSError:
-            pass
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the write-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+        try:
+            os.close(self.read_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the read-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )

meerschaum/utils/daemon/RotatingFile.py

@@ -38,7 +38,7 @@ class RotatingFile(io.IOBase):
             max_file_size: Optional[int] = None,
             redirect_streams: bool = False,
             write_timestamps: bool = False,
-            timestamps_format: str = '%Y-%m-%d %H:%M | ',
+            timestamp_format: str = '%Y-%m-%d %H:%M',
         ):
         """
         Create a file-like object which manages other files.
@@ -78,7 +78,7 @@ class RotatingFile(io.IOBase):
         self.max_file_size = max_file_size
         self.redirect_streams = redirect_streams
         self.write_timestamps = write_timestamps
-        self.timestamps_format = timestamps_format
+        self.timestamp_format = timestamp_format
         self.subfile_regex_pattern = re.compile(
             r'^'
             + self.file_path.name
@@ -98,34 +98,14 @@ class RotatingFile(io.IOBase):
         atexit.register(self.close)
 
 
-
     def fileno(self):
         """
         Return the file descriptor for the latest subfile.
         """
-        import inspect
-        stack = inspect.stack()
-        parent_level = stack[1]
-        parent_module = parent_level[0].f_globals.get('__file__')
-        #  if parent_module.endswith('daemon.py'):
-            #  self._monkey_patch_os_write()
-        self.refresh_files()
+        self.refresh_files(start_interception=False)
         return self._current_file_obj.fileno()
 
 
-    def _monkey_patch_os_write(self):
-        import os
-        import sys
-        import pathlib
-        path = pathlib.Path('/home/bmeares/test1.log')
-        original_write = os.write
-        def intercept(*args, **kwargs):
-            with open(path, 'w', encoding='utf-8') as f:
-                f.write(str(args))
-            original_write(*args, **kwargs)
-        os.write = intercept
-
-
     def get_latest_subfile_path(self) -> pathlib.Path:
         """
         Return the path for the latest subfile to which to write into.
@@ -252,7 +232,11 @@ class RotatingFile(io.IOBase):
         ]
 
 
-    def refresh_files(self, potential_new_len: int = 0) -> '_io.TextUIWrapper':
+    def refresh_files(
+            self,
+            potential_new_len: int = 0,
+            start_interception: bool = False,
+        ) -> '_io.TextUIWrapper':
         """
         Check the state of the subfiles.
         If the latest subfile is too large, create a new file and delete old ones.
@@ -260,6 +244,9 @@ class RotatingFile(io.IOBase):
         Parameters
         ----------
         potential_new_len: int, default 0
+
+        start_interception: bool, default False
+            If `True`, kick off the file interception threads.
         """
         self.flush()
 
@@ -278,10 +265,15 @@ class RotatingFile(io.IOBase):
         if is_first_run_with_logs or lost_latest_handle:
             self._current_file_obj = open(latest_subfile_path, 'a+', encoding='utf-8')
             if self.redirect_streams:
-                self.stop_log_fd_interception()
-                daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
-                daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
-                self.start_log_fd_interception()
+                try:
+                    daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
+                    daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
+                except OSError as e:
+                    warn(
+                        f"Encountered an issue when redirecting streams:\n{traceback.format_exc()}"
+                    )
+                if start_interception:
+                    self.start_log_fd_interception()
 
         create_new_file = (
             (latest_subfile_index == -1)
@@ -302,20 +294,17 @@ class RotatingFile(io.IOBase):
             if self._previous_file_obj is not None:
                 if self.redirect_streams:
                     self._redirected_subfile_objects[old_subfile_index] = self._previous_file_obj
-                    self.stop_log_fd_interception()
                     daemon.daemon.redirect_stream(self._previous_file_obj, self._current_file_obj)
                     daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
                     daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
-                    self.start_log_fd_interception()
                 self.close(unused_only=True)
 
             ### Sanity check in case writing somehow fails.
             if self._previous_file_obj is self._current_file_obj:
-                self._previous_file_obj is None
+                self._previous_file_obj = None
 
             self.delete(unused_only=True)
 
-
         return self._current_file_obj
 
 
@@ -328,6 +317,7 @@ class RotatingFile(io.IOBase):
         unused_only: bool, default False
             If `True`, only close file descriptors not currently in use.
         """
+        self.stop_log_fd_interception(unused_only=unused_only)
         subfile_indices = sorted(self.subfile_objects.keys())
         for subfile_index in subfile_indices:
             subfile_object = self.subfile_objects[subfile_index]
@@ -335,14 +325,14 @@ class RotatingFile(io.IOBase):
                 continue
             try:
                 if not subfile_object.closed:
-                    #  subfile_object.flush()
                     subfile_object.close()
-                _ = self.subfile_objects.pop(subfile_index, None)
-                if self.redirect_streams:
-                    _ = self._redirected_subfile_objects.pop(subfile_index, None)
             except Exception as e:
                 warn(f"Failed to close an open subfile:\n{traceback.format_exc()}")
 
+            _ = self.subfile_objects.pop(subfile_index, None)
+            if self.redirect_streams:
+                _ = self._redirected_subfile_objects.pop(subfile_index, None)
+
         if not unused_only:
             self._previous_file_obj = None
             self._current_file_obj = None
@@ -352,7 +342,7 @@ class RotatingFile(io.IOBase):
         """
         Return the current minute prefixm string.
         """
-        return datetime.now(timezone.utc).strftime(self.timestamps_format)
+        return datetime.now(timezone.utc).strftime(self.timestamp_format) + ' | '
 
 
     def write(self, data: str) -> None:
@@ -371,7 +361,10 @@ class RotatingFile(io.IOBase):
 
         prefix_str = self.get_timestamp_prefix_str() if self.write_timestamps else ""
         suffix_str = "\n" if self.write_timestamps else ""
-        self.refresh_files(potential_new_len=len(prefix_str + data + suffix_str))
+        self.refresh_files(
+            potential_new_len = len(prefix_str + data + suffix_str),
+            start_interception = True,
+        )
         try:
             if prefix_str:
                 self._current_file_obj.write(prefix_str)
@@ -582,16 +575,23 @@ class RotatingFile(io.IOBase):
                 try:
                     subfile_object.flush()
                 except Exception as e:
-                    warn(f"Failed to flush subfile:\n{traceback.format_exc()}")
+                    warn(f"Failed to flush subfile {subfile_index}:\n{traceback.format_exc()}")
         if self.redirect_streams:
-            sys.stdout.flush()
-            sys.stderr.flush()
+            try:
+                sys.stdout.flush()
+            except Exception as e:
+                warn(f"Failed to flush STDOUT:\n{traceback.format_exc()}")
+            try:
+                sys.stderr.flush()
+            except Exception as e:
+                warn(f"Failed to flush STDERR:\n{traceback.format_exc()}")
 
 
     def start_log_fd_interception(self):
         """
         Start the file descriptor monitoring threads.
         """
+        threads = self.__dict__.get('_interceptor_threads', [])
         self._stdout_interceptor = FileDescriptorInterceptor(
             sys.stdout.fileno(),
             self.get_timestamp_prefix_str,
@@ -600,29 +600,48 @@ class RotatingFile(io.IOBase):
             sys.stderr.fileno(),
             self.get_timestamp_prefix_str,
         )
-        self._stdout_interceptor_thread = Thread(target=self._stdout_interceptor.start_interception)
-        self._stderr_interceptor_thread = Thread(target=self._stderr_interceptor.start_interception)
+
+        self._stdout_interceptor_thread = Thread(
+            target = self._stdout_interceptor.start_interception,
+            daemon = True,
+        )
+        self._stderr_interceptor_thread = Thread(
+            target = self._stderr_interceptor.start_interception,
+            daemon = True,
+        )
         self._stdout_interceptor_thread.start()
         self._stderr_interceptor_thread.start()
+        self._intercepting = True
 
+        if '_interceptor_threads' not in self.__dict__:
+            self._interceptor_threads = []
+        if '_interceptors' not in self.__dict__:
+            self._interceptors = []
+        self._interceptor_threads.extend([
+            self._stdout_interceptor_thread,
+            self._stderr_interceptor_thread,
+        ])
+        self.stop_log_fd_interception(unused_only=True)
 
-    def stop_log_fd_interception(self):
+    def stop_log_fd_interception(self, unused_only: bool = False):
         """
         Stop the file descriptor monitoring threads.
         """
-        stdout_interceptor = self.__dict__.get('_stdout_interceptor', None)
-        stderr_interceptor = self.__dict__.get('_stderr_interceptor', None)
-        stdout_interceptor_thread = self.__dict__.get('_stdout_interceptor_thread', None)
-        stderr_interceptor_thread = self.__dict__.get('_stderr_interceptor_thread', None)
-        if stdout_interceptor is None:
-            return
-        stdout_interceptor.stop_interception()
-        stderr_interceptor.stop_interception()
-        try:
-            stdout_interceptor_thread.join()
-            stderr_interceptor_thread.join()
-        except Exception:
-            pass
+        interceptors = self.__dict__.get('_interceptors', [])
+        interceptor_threads = self.__dict__.get('_interceptor_threads', [])
+
+        end_ix = len(interceptors) if not unused_only else -2
+
+        for interceptor in interceptors[:end_ix]:
+            interceptor.stop_interception()
+        del interceptors[:end_ix]
+
+        for thread in interceptor_threads[:end_ix]:
+            try:
+                thread.join()
+            except Exception as e:
+                warn(f"Failed to join interceptor threads:\n{traceback.format_exc()}")
+        del interceptor_threads[:end_ix]
 
 
     def __repr__(self) -> str:

meerschaum/utils/daemon/__init__.py

@@ -12,6 +12,7 @@ from meerschaum.utils.typing import SuccessTuple, List, Optional, Callable, Any,
 from meerschaum.config._paths import DAEMON_RESOURCES_PATH
 from meerschaum.utils.daemon.Daemon import Daemon
 from meerschaum.utils.daemon.RotatingFile import RotatingFile
+from meerschaum.utils.daemon.FileDescriptorInterceptor import FileDescriptorInterceptor
 
 
 def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:

meerschaum/utils/threading.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 from meerschaum.utils.typing import Optional
 
 import threading
+import traceback
 Lock = threading.Lock
 RLock = threading.RLock
 Event = threading.Event

{meerschaum-2.2.0rc1.dist-info → meerschaum-2.2.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.2.0rc1
+Version: 2.2.0rc2
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares