meerschaum 2.1.7__py3-none-any.whl → 2.2.0__py3-none-any.whl

meerschaum/utils/daemon/RotatingFile.py

@@ -13,9 +13,13 @@ import pathlib
 import traceback
 import sys
 import atexit
+from datetime import datetime, timezone, timedelta
 from typing import List, Union, Optional, Tuple
 from meerschaum.config import get_config
 from meerschaum.utils.warnings import warn
+from meerschaum.utils.misc import round_time
+from meerschaum.utils.daemon.FileDescriptorInterceptor import FileDescriptorInterceptor
+from meerschaum.utils.threading import Thread
 import meerschaum as mrsm
 daemon = mrsm.attempt_import('daemon')
 
@@ -33,6 +37,8 @@ class RotatingFile(io.IOBase):
             num_files_to_keep: Optional[int] = None,
             max_file_size: Optional[int] = None,
             redirect_streams: bool = False,
+            write_timestamps: bool = False,
+            timestamp_format: str = '%Y-%m-%d %H:%M',
         ):
         """
         Create a file-like object which manages other files.
@@ -54,6 +60,9 @@ class RotatingFile(io.IOBase):
             
             NOTE: Only set this to `True` if you are entering into a daemon context.
             Doing so will redirect `sys.stdout` and `sys.stderr` into the log files.
+
+        write_timestamps: bool, default False
+            If `True`, prepend the current UTC timestamp to each line of the file.
         """
         self.file_path = pathlib.Path(file_path)
         if num_files_to_keep is None:
@@ -68,6 +77,8 @@ class RotatingFile(io.IOBase):
         self.num_files_to_keep = num_files_to_keep
         self.max_file_size = max_file_size
         self.redirect_streams = redirect_streams
+        self.write_timestamps = write_timestamps
+        self.timestamp_format = timestamp_format
         self.subfile_regex_pattern = re.compile(
             r'^'
             + self.file_path.name
@@ -91,7 +102,7 @@ class RotatingFile(io.IOBase):
         """
         Return the file descriptor for the latest subfile.
         """
-        self.refresh_files()
+        self.refresh_files(start_interception=False)
         return self._current_file_obj.fileno()
 
 
@@ -221,7 +232,11 @@ class RotatingFile(io.IOBase):
         ]
 
 
-    def refresh_files(self, potential_new_len: int = 0) -> '_io.TextUIWrapper':
+    def refresh_files(
+            self,
+            potential_new_len: int = 0,
+            start_interception: bool = False,
+        ) -> '_io.TextUIWrapper':
         """
         Check the state of the subfiles.
         If the latest subfile is too large, create a new file and delete old ones.
@@ -229,6 +244,9 @@ class RotatingFile(io.IOBase):
         Parameters
         ----------
         potential_new_len: int, default 0
+
+        start_interception: bool, default False
+            If `True`, kick off the file interception threads.
         """
         self.flush()
 
@@ -247,8 +265,15 @@ class RotatingFile(io.IOBase):
         if is_first_run_with_logs or lost_latest_handle:
             self._current_file_obj = open(latest_subfile_path, 'a+', encoding='utf-8')
             if self.redirect_streams:
-                daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
-                daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
+                try:
+                    daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
+                    daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
+                except OSError as e:
+                    warn(
+                        f"Encountered an issue when redirecting streams:\n{traceback.format_exc()}"
+                    )
+                if start_interception and self.write_timestamps:
+                    self.start_log_fd_interception()
 
         create_new_file = (
             (latest_subfile_index == -1)
@@ -276,9 +301,10 @@ class RotatingFile(io.IOBase):
 
             ### Sanity check in case writing somehow fails.
             if self._previous_file_obj is self._current_file_obj:
-                self._previous_file_obj is None
+                self._previous_file_obj = None
 
             self.delete(unused_only=True)
+
         return self._current_file_obj
 
 
@@ -291,6 +317,7 @@ class RotatingFile(io.IOBase):
         unused_only: bool, default False
             If `True`, only close file descriptors not currently in use.
         """
+        self.stop_log_fd_interception(unused_only=unused_only)
         subfile_indices = sorted(self.subfile_objects.keys())
         for subfile_index in subfile_indices:
             subfile_object = self.subfile_objects[subfile_index]
@@ -298,19 +325,26 @@ class RotatingFile(io.IOBase):
                 continue
             try:
                 if not subfile_object.closed:
-                    #  subfile_object.flush()
                     subfile_object.close()
-                _ = self.subfile_objects.pop(subfile_index, None)
-                if self.redirect_streams:
-                    _ = self._redirected_subfile_objects.pop(subfile_index, None)
             except Exception as e:
                 warn(f"Failed to close an open subfile:\n{traceback.format_exc()}")
 
+            _ = self.subfile_objects.pop(subfile_index, None)
+            if self.redirect_streams:
+                _ = self._redirected_subfile_objects.pop(subfile_index, None)
+
         if not unused_only:
             self._previous_file_obj = None
             self._current_file_obj = None
 
 
+    def get_timestamp_prefix_str(self) -> str:
+        """
+        Return the current minute prefixm string.
+        """
+        return datetime.now(timezone.utc).strftime(self.timestamp_format) + ' | '
+
+
     def write(self, data: str) -> None:
         """
         Write the given text into the latest subfile.
@@ -325,9 +359,18 @@ class RotatingFile(io.IOBase):
         if isinstance(data, bytes):
             data = data.decode('utf-8')
 
-        self.refresh_files(potential_new_len=len(data))
+        prefix_str = self.get_timestamp_prefix_str() if self.write_timestamps else ""
+        suffix_str = "\n" if self.write_timestamps else ""
+        self.refresh_files(
+            potential_new_len = len(prefix_str + data + suffix_str),
+            start_interception = self.write_timestamps,
+        )
         try:
+            if prefix_str:
+                self._current_file_obj.write(prefix_str)
             self._current_file_obj.write(data)
+            if suffix_str:
+                self._current_file_obj.write(suffix_str)
         except Exception as e:
             warn(f"Failed to write to subfile:\n{traceback.format_exc()}")
         self.flush()
@@ -471,7 +514,7 @@ class RotatingFile(io.IOBase):
                 subfile_object = self.subfile_objects[subfile_index]
                 for i in range(self.SEEK_BACK_ATTEMPTS):
                     try:
-                        subfile_object.seek(max(seek_ix - i), 0)
+                        subfile_object.seek(max((seek_ix - i), 0))
                         subfile_lines = subfile_object.readlines()
                     except UnicodeDecodeError:
                         continue
@@ -532,10 +575,83 @@ class RotatingFile(io.IOBase):
                 try:
                     subfile_object.flush()
                 except Exception as e:
-                    warn(f"Failed to flush subfile:\n{traceback.format_exc()}")
+                    warn(f"Failed to flush subfile {subfile_index}:\n{traceback.format_exc()}")
         if self.redirect_streams:
-            sys.stdout.flush()
-            sys.stderr.flush()
+            try:
+                sys.stdout.flush()
+            except Exception as e:
+                warn(f"Failed to flush STDOUT:\n{traceback.format_exc()}")
+            try:
+                sys.stderr.flush()
+            except Exception as e:
+                warn(f"Failed to flush STDERR:\n{traceback.format_exc()}")
+
+
+    def start_log_fd_interception(self):
+        """
+        Start the file descriptor monitoring threads.
+        """
+        if not self.write_timestamps:
+            return
+
+        threads = self.__dict__.get('_interceptor_threads', [])
+        self._stdout_interceptor = FileDescriptorInterceptor(
+            sys.stdout.fileno(),
+            self.get_timestamp_prefix_str,
+        )
+        self._stderr_interceptor = FileDescriptorInterceptor(
+            sys.stderr.fileno(),
+            self.get_timestamp_prefix_str,
+        )
+
+        self._stdout_interceptor_thread = Thread(
+            target = self._stdout_interceptor.start_interception,
+            daemon = True,
+        )
+        self._stderr_interceptor_thread = Thread(
+            target = self._stderr_interceptor.start_interception,
+            daemon = True,
+        )
+        self._stdout_interceptor_thread.start()
+        self._stderr_interceptor_thread.start()
+        self._intercepting = True
+
+        if '_interceptor_threads' not in self.__dict__:
+            self._interceptor_threads = []
+        if '_interceptors' not in self.__dict__:
+            self._interceptors = []
+        self._interceptor_threads.extend([
+            self._stdout_interceptor_thread,
+            self._stderr_interceptor_thread,
+        ])
+        self._interceptors.extend([
+            self._stdout_interceptor,
+            self._stderr_interceptor,
+        ])
+        self.stop_log_fd_interception(unused_only=True)
+
+
+    def stop_log_fd_interception(self, unused_only: bool = False):
+        """
+        Stop the file descriptor monitoring threads.
+        """
+        if not self.write_timestamps:
+            return
+        interceptors = self.__dict__.get('_interceptors', [])
+        interceptor_threads = self.__dict__.get('_interceptor_threads', [])
+
+        end_ix = len(interceptors) if not unused_only else -2
+
+        for interceptor in interceptors[:end_ix]:
+            interceptor.stop_interception()
+        del interceptors[:end_ix]
+
+        for thread in interceptor_threads[:end_ix]:
+            try:
+                thread.join()
+            except Exception as e:
+                warn(f"Failed to join interceptor threads:\n{traceback.format_exc()}")
+        del interceptor_threads[:end_ix]
 
 
     def __repr__(self) -> str:

meerschaum/utils/daemon/__init__.py

@@ -12,6 +12,7 @@ from meerschaum.utils.typing import SuccessTuple, List, Optional, Callable, Any,
 from meerschaum.config._paths import DAEMON_RESOURCES_PATH
 from meerschaum.utils.daemon.Daemon import Daemon
 from meerschaum.utils.daemon.RotatingFile import RotatingFile
+from meerschaum.utils.daemon.FileDescriptorInterceptor import FileDescriptorInterceptor
 
 
 def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
@@ -63,6 +64,8 @@ def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
 
         ### Only run if the kwargs equal or no actions are provided.
         if existing_kwargs == _args or not _args.get('action', []):
+            if daemon.status == 'running':
+                return True, f"Daemon '{daemon}' is already running."
             return daemon.run(
                 debug = debug,
                 allow_dirty_run = True,

meerschaum/utils/dtypes/__init__.py

@@ -6,8 +6,10 @@
 Utility functions for working with data types.
 """
 
+import traceback
 from decimal import Decimal, Context, InvalidOperation
 from meerschaum.utils.typing import Dict, Union, Any
+from meerschaum.utils.warnings import warn
 
 MRSM_PD_DTYPES: Dict[str, str] = {
     'json': 'object',
@@ -37,9 +39,7 @@ def to_pandas_dtype(dtype: str) -> str:
         from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
         return get_pd_type_from_db_type(dtype)
 
-    import traceback
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.warnings import warn
     pandas = attempt_import('pandas', lazy=False)
 
     try:
@@ -88,8 +88,12 @@ def are_dtypes_equal(
                 return False
         return True
 
-    if ldtype == rdtype:
-        return True
+    try:
+        if ldtype == rdtype:
+            return True
+    except Exception as e:
+        warn(f"Exception when comparing dtypes, returning False:\n{traceback.format_exc()}")
+        return False
 
     ### Sometimes pandas dtype objects are passed.
     ldtype = str(ldtype)
@@ -177,7 +181,7 @@ def attempt_cast_to_numeric(value: Any) -> Any:
         return value
 
 
-def value_is_null(value: Any) -> Any:
+def value_is_null(value: Any) -> bool:
     """
     Determine if a value is a null-like string.
     """

meerschaum/utils/packages/__init__.py

@@ -8,7 +8,7 @@ Functions for managing packages and virtual environments reside here.
 
 from __future__ import annotations
 
-import importlib.util, os, pathlib
+import importlib.util, os, pathlib, re
 from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
 from meerschaum.utils.threading import Lock, RLock
 from meerschaum.utils.packages._packages import packages, all_packages, get_install_names
@@ -35,6 +35,7 @@ _locks = {
 }
 _checked_for_updates = set()
 _is_installed_first_check: Dict[str, bool] = {}
+_MRSM_PACKAGE_ARCHIVES_PREFIX: str = "https://meerschaum.io/files/archives/"
 
 def get_module_path(
         import_name: str,
@@ -640,6 +641,15 @@ def need_update(
 
     ### We might be depending on a prerelease.
     ### Sanity check that the required version is not greater than the installed version. 
+    required_version = (
+        required_version.replace(_MRSM_PACKAGE_ARCHIVES_PREFIX, '')
+        .replace(' @ ', '').replace('wheels', '').replace('+mrsm', '').replace('/-', '')
+        .replace('-py3-none-any.whl', '')
+    )
+
+    if 'a' in required_version:
+        required_version = required_version.replace('a', '-dev').replace('+mrsm', '')
+        version = version.replace('a', '-dev').replace('+mrsm', '')
     try:
         return (
             (not semver.Version.parse(version).match(required_version))
@@ -819,8 +829,11 @@ def pip_install(
                     check_wheel = False, debug = debug,
                 ):
                     warn(
-                        f"Failed to install `setuptools` and `wheel` for virtual environment '{venv}'.",
-                        color=False,
+                        (
+                            "Failed to install `setuptools` and `wheel` for virtual "
+                            + f"environment '{venv}'."
+                        ),
+                        color = False,
                     )
 
         if requirements_file_path is not None:
@@ -883,13 +896,16 @@ def pip_install(
                         f"Failed to clean up package '{_install_no_version}'.",
                     )
 
-        success = run_python_package(
+        rc = run_python_package(
             'pip',
             _args + _packages,
             venv = venv,
             env = _get_pip_os_env(),
             debug = debug,
-        ) == 0
+        )
+        if debug:
+            print(f"{rc=}")
+        success = rc == 0
 
     msg = (
         "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 

meerschaum/utils/packages/_packages.py

@@ -30,7 +30,7 @@ packages: Dict[str, Dict[str, str]] = {
         'more_termcolor'             : 'more-termcolor>=1.1.3',
         'humanfriendly'              : 'humanfriendly>=10.0.0',
     },
-    '_required': {
+    'core': {
         'wheel'                      : 'wheel>=0.34.2',
         'setuptools'                 : 'setuptools>=63.3.0',
         'yaml'                       : 'PyYAML>=5.3.1',
@@ -49,21 +49,21 @@ packages: Dict[str, Dict[str, str]] = {
         'daemon'                     : 'python-daemon>=0.2.3',
         'fasteners'                  : 'fasteners>=0.18.0',
         'psutil'                     : 'psutil>=5.8.0',
-        'watchgod'                   : 'watchgod>=0.7.0',
+        'watchfiles'                 : 'watchfiles>=0.21.0',
         'dill'                       : 'dill>=0.3.3',
         'virtualenv'                 : 'virtualenv>=20.1.0',
-        'rocketry'                   : 'rocketry>=2.5.1',
+        'apscheduler'                : 'APScheduler>=4.0.0a5',
     },
     'drivers': {
         'cryptography'               : 'cryptography>=38.0.1',
-        'psycopg2'                   : 'psycopg2-binary>=2.8.6',
+        'psycopg'                    : 'psycopg[binary]>=3.1.18',
         'pymysql'                    : 'PyMySQL>=0.9.0',
         'aiomysql'                   : 'aiomysql>=0.0.21',
         'sqlalchemy_cockroachdb'     : 'sqlalchemy-cockroachdb>=2.0.0',
-        'duckdb'                     : 'duckdb>=0.9.0',
+        'duckdb'                     : 'duckdb<0.10.3',
         'duckdb_engine'              : 'duckdb-engine>=0.9.2',
     },
-    '_drivers': {
+    'drivers-extras': {
         'pyodbc'                     : 'pyodbc>=4.0.30',
         'cx_Oracle'                  : 'cx_Oracle>=8.3.0',
     },
@@ -75,11 +75,11 @@ packages: Dict[str, Dict[str, str]] = {
         'gadwall'                    : 'gadwall>=0.2.0',
     },
     'stack': {
-        'compose'                    : 'docker-compose>=1.27.4',
+        'compose'                    : 'docker-compose>=1.29.2',
     },
     'build': {
-        'cx_Freeze'                  : 'cx_Freeze>=6.5.1',
-        'PyInstaller'                : 'pyinstaller>=5.0.0-dev0',
+        'cx_Freeze'                  : 'cx_Freeze>=7.0.0',
+        'PyInstaller'                : 'pyinstaller>6.6.0',
     },
     'dev-tools': {
         'twine'                      : 'twine>=3.2.0',
@@ -89,6 +89,7 @@ packages: Dict[str, Dict[str, str]] = {
         'pytest'                     : 'pytest>=6.2.2',
         'pytest_xdist'               : 'pytest-xdist>=3.2.1',
         'heartrate'                  : 'heartrate>=0.2.1',
+        'build'                      : 'build>=1.2.1',
     },
     'setup': {
     },
@@ -119,8 +120,8 @@ packages: Dict[str, Dict[str, str]] = {
 packages['sql'] = {
     'numpy'                          : 'numpy>=1.18.5',
     'pandas'                         : 'pandas[parquet]>=2.0.1',
-    'pyarrow'                        : 'pyarrow>=7.0.0',
-    'dask'                           : 'dask>=2023.5.0',
+    'pyarrow'                        : 'pyarrow>=16.1.0',
+    'dask'                           : 'dask[dataframe]>=2024.5.1',
     'pytz'                           : 'pytz',
     'joblib'                         : 'joblib>=0.17.0',
     'sqlalchemy'                     : 'SQLAlchemy>=2.0.5',
@@ -129,7 +130,7 @@ packages['sql'] = {
     'asyncpg'                        : 'asyncpg>=0.21.0',
 }
 packages['sql'].update(packages['drivers'])
-packages['sql'].update(packages['_required'])
+packages['sql'].update(packages['core'])
 packages['dash'] = {
     'flask_compress'                 : 'Flask-Compress>=1.10.1',
     'dash'                           : 'dash>=2.6.2',
@@ -141,17 +142,14 @@ packages['dash'] = {
     'tornado'                        : 'tornado>=6.1.0',
 }
 packages['api'] = {
-    'uvicorn'                        : 'uvicorn[standard]>=0.22.0',
-    'gunicorn'                       : 'gunicorn>=20.1.0',
+    'uvicorn'                        : 'uvicorn[standard]>=0.29.0',
+    'gunicorn'                       : 'gunicorn>=22.0.0',
     'dotenv'                         : 'python-dotenv>=0.20.0',
     'websockets'                     : 'websockets>=11.0.3',
-    'fastapi'                        : 'fastapi>=0.100.0',
-    'passlib'                        : 'passlib>=1.7.4',
+    'fastapi'                        : 'fastapi>=0.111.0',
     'fastapi_login'                  : 'fastapi-login>=1.7.2',
-    'multipart'                      : 'python-multipart>=0.0.5',
-    'pydantic'                       : 'pydantic<2.0.0',
+    'multipart'                      : 'python-multipart>=0.0.9',
     'httpx'                          : 'httpx>=0.24.1',
-    'websockets'                     : 'websockets>=11.0.3',
 }
 packages['api'].update(packages['sql'])
 packages['api'].update(packages['formatting'])
@@ -171,7 +169,7 @@ def get_install_names():
             install_names[get_install_no_version(_install_name)] = _import_name
     return install_names
 
-skip_groups = {'docs', 'build', 'cli', 'dev-tools', 'portable', 'extras', 'stack', '_drivers'}
+skip_groups = {'docs', 'build', 'cli', 'dev-tools', 'portable', 'extras', 'stack', 'drivers-extras'}
 full = []
 _full = {}
 for group, import_names in packages.items():

meerschaum/utils/process.py

@@ -11,6 +11,7 @@ See `meerschaum.utils.pool` for multiprocessing and
 from __future__ import annotations
 import os, signal, subprocess, sys, platform
 from meerschaum.utils.typing import Union, Optional, Any, Callable, Dict, Tuple
+from meerschaum.config.static import STATIC_CONFIG
 
 def run_process(
         *args,
@@ -68,9 +69,18 @@ def run_process(
     if platform.system() == 'Windows':
         foreground = False
 
-    if line_callback is not None:
+    def print_line(line):
+        sys.stdout.write(line.decode('utf-8'))
+        sys.stdout.flush()
+
+    if capture_output or line_callback is not None:
+        kw['stdout'] = subprocess.PIPE
+        kw['stderr'] = subprocess.STDOUT
+    elif os.environ.get(STATIC_CONFIG['environment']['daemon_id']):
         kw['stdout'] = subprocess.PIPE
         kw['stderr'] = subprocess.STDOUT
+        if line_callback is None:
+            line_callback = print_line
 
     if 'env' not in kw:
         kw['env'] = os.environ
@@ -112,15 +122,6 @@ def run_process(
         kw['preexec_fn'] = new_pgid
 
     try:
-        # fork the child
-        #  stdout, stderr = (
-            #  (sys.stdout, sys.stderr) if not capture_output
-            #  else (subprocess.PIPE, subprocess.PIPE)
-        #  )
-        if capture_output:
-            kw['stdout'] = subprocess.PIPE
-            kw['stderr'] = subprocess.PIPE
-
         child = subprocess.Popen(*args, **kw)
 
         # we can't set the process group id from the parent since the child
@@ -197,6 +198,8 @@ def poll_process(
     while proc.poll() is None:
         line = proc.stdout.readline()
         line_callback(line)
+
     if timeout_seconds is not None:
         watchdog_thread.cancel()
+
     return proc.poll()