meerschaum 2.2.0rc1__py3-none-any.whl → 2.2.0rc3__py3-none-any.whl

meerschaum/utils/daemon/FileDescriptorInterceptor.py

@@ -7,8 +7,15 @@ Intercept OS-level file descriptors.
 """
 
 import os
+import select
+import traceback
+from threading import Event
 from datetime import datetime
 from meerschaum.utils.typing import Callable
+from meerschaum.utils.warnings import warn
+
+FD_CLOSED: int = 9
+STOP_READING_FD_EVENT: Event = Event()
 
 class FileDescriptorInterceptor:
     """
@@ -28,10 +35,12 @@ class FileDescriptorInterceptor:
         injection_hook: Callable[[], str]
             A callable which returns a string to be injected into the written data.
         """
+        self.stop_event = Event()
         self.injection_hook = injection_hook
         self.original_file_descriptor = file_descriptor
         self.new_file_descriptor = os.dup(file_descriptor)
         self.read_pipe, self.write_pipe = os.pipe()
+        self.signal_read_pipe, self.signal_write_pipe = os.pipe()
         os.dup2(self.write_pipe, file_descriptor)
 
     def start_interception(self):
@@ -40,21 +49,92 @@ class FileDescriptorInterceptor:
 
         NOTE: This is blocking and is meant to be run in a thread.
         """
-        while True:
-            data = os.read(self.read_pipe, 1024)
-            if not data:
-                break
+        os.set_blocking(self.read_pipe, False)
+        os.set_blocking(self.signal_read_pipe, False)
+        is_first_read = True
+        while not self.stop_event.is_set():
+            try:
+                rlist, _, _ = select.select([self.read_pipe, self.signal_read_pipe], [], [], 0.1)
+                if self.signal_read_pipe in rlist:
+                    break
+                if not rlist:
+                    continue
+                data = os.read(self.read_pipe, 1024)
+                if not data:
+                    break
+            except BlockingIOError:
+                continue
+            except OSError as e:
+                continue
+
+            first_char_is_newline = data[0] == b'\n'
+            last_char_is_newline = data[-1] == b'\n'
+
             injected_str = self.injection_hook()
-            modified_data = data.replace(b'\n', f'\n{injected_str}'.encode('utf-8'))
+            injected_bytes = injected_str.encode('utf-8')
+
+            if is_first_read:
+                data = b'\n' + data
+                is_first_read = False
+
+            modified_data = (
+                (data[:-1].replace(b'\n', b'\n' + injected_bytes) + b'\n')
+                if last_char_is_newline
+                else data.replace(b'\n', b'\n' + injected_bytes)
+            )
             os.write(self.new_file_descriptor, modified_data)
 
+
     def stop_interception(self):
         """
-        Restore the file descriptors and close the new pipes.
+        Close the new file descriptors.
         """
+        self.stop_event.set()
+        os.write(self.signal_write_pipe, b'\0')
+        try:
+            os.close(self.new_file_descriptor)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the duplicated file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+
         try:
-            os.dup2(self.new_file_descriptor, self.original_file_descriptor)
-            os.close(self.read_pipe)
             os.close(self.write_pipe)
-        except OSError:
-            pass
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the write-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+        try:
+            os.close(self.read_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the read-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+
+        try:
+            os.close(self.signal_read_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the signal-read-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+
+        try:
+            os.close(self.signal_write_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the signal-write-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )

meerschaum/utils/daemon/RotatingFile.py

@@ -38,7 +38,7 @@ class RotatingFile(io.IOBase):
             max_file_size: Optional[int] = None,
             redirect_streams: bool = False,
             write_timestamps: bool = False,
-            timestamps_format: str = '%Y-%m-%d %H:%M | ',
+            timestamp_format: str = '%Y-%m-%d %H:%M',
         ):
         """
         Create a file-like object which manages other files.
@@ -78,7 +78,7 @@ class RotatingFile(io.IOBase):
         self.max_file_size = max_file_size
         self.redirect_streams = redirect_streams
         self.write_timestamps = write_timestamps
-        self.timestamps_format = timestamps_format
+        self.timestamp_format = timestamp_format
         self.subfile_regex_pattern = re.compile(
             r'^'
             + self.file_path.name
@@ -98,34 +98,14 @@ class RotatingFile(io.IOBase):
         atexit.register(self.close)
 
 
-
     def fileno(self):
         """
         Return the file descriptor for the latest subfile.
         """
-        import inspect
-        stack = inspect.stack()
-        parent_level = stack[1]
-        parent_module = parent_level[0].f_globals.get('__file__')
-        #  if parent_module.endswith('daemon.py'):
-            #  self._monkey_patch_os_write()
-        self.refresh_files()
+        self.refresh_files(start_interception=False)
         return self._current_file_obj.fileno()
 
 
-    def _monkey_patch_os_write(self):
-        import os
-        import sys
-        import pathlib
-        path = pathlib.Path('/home/bmeares/test1.log')
-        original_write = os.write
-        def intercept(*args, **kwargs):
-            with open(path, 'w', encoding='utf-8') as f:
-                f.write(str(args))
-            original_write(*args, **kwargs)
-        os.write = intercept
-
-
     def get_latest_subfile_path(self) -> pathlib.Path:
         """
         Return the path for the latest subfile to which to write into.
@@ -252,7 +232,11 @@ class RotatingFile(io.IOBase):
         ]
 
 
-    def refresh_files(self, potential_new_len: int = 0) -> '_io.TextUIWrapper':
+    def refresh_files(
+            self,
+            potential_new_len: int = 0,
+            start_interception: bool = False,
+        ) -> '_io.TextUIWrapper':
         """
         Check the state of the subfiles.
         If the latest subfile is too large, create a new file and delete old ones.
@@ -260,6 +244,9 @@ class RotatingFile(io.IOBase):
         Parameters
         ----------
         potential_new_len: int, default 0
+
+        start_interception: bool, default False
+            If `True`, kick off the file interception threads.
         """
         self.flush()
 
@@ -278,10 +265,15 @@ class RotatingFile(io.IOBase):
         if is_first_run_with_logs or lost_latest_handle:
             self._current_file_obj = open(latest_subfile_path, 'a+', encoding='utf-8')
             if self.redirect_streams:
-                self.stop_log_fd_interception()
-                daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
-                daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
-                self.start_log_fd_interception()
+                try:
+                    daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
+                    daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
+                except OSError as e:
+                    warn(
+                        f"Encountered an issue when redirecting streams:\n{traceback.format_exc()}"
+                    )
+                if start_interception:
+                    self.start_log_fd_interception()
 
         create_new_file = (
             (latest_subfile_index == -1)
@@ -302,20 +294,17 @@ class RotatingFile(io.IOBase):
             if self._previous_file_obj is not None:
                 if self.redirect_streams:
                     self._redirected_subfile_objects[old_subfile_index] = self._previous_file_obj
-                    self.stop_log_fd_interception()
                     daemon.daemon.redirect_stream(self._previous_file_obj, self._current_file_obj)
                     daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
                     daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
-                    self.start_log_fd_interception()
                 self.close(unused_only=True)
 
             ### Sanity check in case writing somehow fails.
             if self._previous_file_obj is self._current_file_obj:
-                self._previous_file_obj is None
+                self._previous_file_obj = None
 
             self.delete(unused_only=True)
 
-
         return self._current_file_obj
 
 
@@ -328,6 +317,7 @@ class RotatingFile(io.IOBase):
         unused_only: bool, default False
             If `True`, only close file descriptors not currently in use.
         """
+        self.stop_log_fd_interception(unused_only=unused_only)
         subfile_indices = sorted(self.subfile_objects.keys())
         for subfile_index in subfile_indices:
             subfile_object = self.subfile_objects[subfile_index]
@@ -335,14 +325,14 @@ class RotatingFile(io.IOBase):
                 continue
             try:
                 if not subfile_object.closed:
-                    #  subfile_object.flush()
                     subfile_object.close()
-                _ = self.subfile_objects.pop(subfile_index, None)
-                if self.redirect_streams:
-                    _ = self._redirected_subfile_objects.pop(subfile_index, None)
             except Exception as e:
                 warn(f"Failed to close an open subfile:\n{traceback.format_exc()}")
 
+            _ = self.subfile_objects.pop(subfile_index, None)
+            if self.redirect_streams:
+                _ = self._redirected_subfile_objects.pop(subfile_index, None)
+
         if not unused_only:
             self._previous_file_obj = None
             self._current_file_obj = None
@@ -352,7 +342,7 @@ class RotatingFile(io.IOBase):
         """
         Return the current minute prefixm string.
         """
-        return datetime.now(timezone.utc).strftime(self.timestamps_format)
+        return datetime.now(timezone.utc).strftime(self.timestamp_format) + ' | '
 
 
     def write(self, data: str) -> None:
@@ -371,7 +361,10 @@ class RotatingFile(io.IOBase):
 
         prefix_str = self.get_timestamp_prefix_str() if self.write_timestamps else ""
         suffix_str = "\n" if self.write_timestamps else ""
-        self.refresh_files(potential_new_len=len(prefix_str + data + suffix_str))
+        self.refresh_files(
+            potential_new_len = len(prefix_str + data + suffix_str),
+            start_interception = True,
+        )
         try:
             if prefix_str:
                 self._current_file_obj.write(prefix_str)
@@ -582,16 +575,23 @@ class RotatingFile(io.IOBase):
                 try:
                     subfile_object.flush()
                 except Exception as e:
-                    warn(f"Failed to flush subfile:\n{traceback.format_exc()}")
+                    warn(f"Failed to flush subfile {subfile_index}:\n{traceback.format_exc()}")
         if self.redirect_streams:
-            sys.stdout.flush()
-            sys.stderr.flush()
+            try:
+                sys.stdout.flush()
+            except Exception as e:
+                warn(f"Failed to flush STDOUT:\n{traceback.format_exc()}")
+            try:
+                sys.stderr.flush()
+            except Exception as e:
+                warn(f"Failed to flush STDERR:\n{traceback.format_exc()}")
 
 
     def start_log_fd_interception(self):
         """
         Start the file descriptor monitoring threads.
         """
+        threads = self.__dict__.get('_interceptor_threads', [])
         self._stdout_interceptor = FileDescriptorInterceptor(
             sys.stdout.fileno(),
             self.get_timestamp_prefix_str,
@@ -600,29 +600,52 @@ class RotatingFile(io.IOBase):
             sys.stderr.fileno(),
             self.get_timestamp_prefix_str,
         )
-        self._stdout_interceptor_thread = Thread(target=self._stdout_interceptor.start_interception)
-        self._stderr_interceptor_thread = Thread(target=self._stderr_interceptor.start_interception)
+
+        self._stdout_interceptor_thread = Thread(
+            target = self._stdout_interceptor.start_interception,
+            daemon = True,
+        )
+        self._stderr_interceptor_thread = Thread(
+            target = self._stderr_interceptor.start_interception,
+            daemon = True,
+        )
         self._stdout_interceptor_thread.start()
         self._stderr_interceptor_thread.start()
-
-
-    def stop_log_fd_interception(self):
+        self._intercepting = True
+
+        if '_interceptor_threads' not in self.__dict__:
+            self._interceptor_threads = []
+        if '_interceptors' not in self.__dict__:
+            self._interceptors = []
+        self._interceptor_threads.extend([
+            self._stdout_interceptor_thread,
+            self._stderr_interceptor_thread,
+        ])
+        self._interceptors.extend([
+            self._stdout_interceptor,
+            self._stderr_interceptor,
+        ])
+        self.stop_log_fd_interception(unused_only=True)
+
+    def stop_log_fd_interception(self, unused_only: bool = False):
         """
         Stop the file descriptor monitoring threads.
         """
-        stdout_interceptor = self.__dict__.get('_stdout_interceptor', None)
-        stderr_interceptor = self.__dict__.get('_stderr_interceptor', None)
-        stdout_interceptor_thread = self.__dict__.get('_stdout_interceptor_thread', None)
-        stderr_interceptor_thread = self.__dict__.get('_stderr_interceptor_thread', None)
-        if stdout_interceptor is None:
-            return
-        stdout_interceptor.stop_interception()
-        stderr_interceptor.stop_interception()
-        try:
-            stdout_interceptor_thread.join()
-            stderr_interceptor_thread.join()
-        except Exception:
-            pass
+        interceptors = self.__dict__.get('_interceptors', [])
+        interceptor_threads = self.__dict__.get('_interceptor_threads', [])
+
+        end_ix = len(interceptors) if not unused_only else -2
+
+        for interceptor in interceptors[:end_ix]:
+            interceptor.stop_interception()
+        del interceptors[:end_ix]
+
+        for thread in interceptor_threads[:end_ix]:
+            try:
+                thread.join()
+            except Exception as e:
+                warn(f"Failed to join interceptor threads:\n{traceback.format_exc()}")
+        del interceptor_threads[:end_ix]
 
 
     def __repr__(self) -> str:

meerschaum/utils/daemon/__init__.py

@@ -12,6 +12,7 @@ from meerschaum.utils.typing import SuccessTuple, List, Optional, Callable, Any,
 from meerschaum.config._paths import DAEMON_RESOURCES_PATH
 from meerschaum.utils.daemon.Daemon import Daemon
 from meerschaum.utils.daemon.RotatingFile import RotatingFile
+from meerschaum.utils.daemon.FileDescriptorInterceptor import FileDescriptorInterceptor
 
 
 def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
@@ -63,6 +64,8 @@ def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
 
         ### Only run if the kwargs equal or no actions are provided.
         if existing_kwargs == _args or not _args.get('action', []):
+            if daemon.status == 'running':
+                return True, f"Daemon '{daemon}' is already running."
             return daemon.run(
                 debug = debug,
                 allow_dirty_run = True,

meerschaum/utils/packages/__init__.py

@@ -829,8 +829,11 @@ def pip_install(
                     check_wheel = False, debug = debug,
                 ):
                     warn(
-                        f"Failed to install `setuptools` and `wheel` for virtual environment '{venv}'.",
-                        color=False,
+                        (
+                            "Failed to install `setuptools` and `wheel` for virtual "
+                            + f"environment '{venv}'."
+                        ),
+                        color = False,
                     )
 
         if requirements_file_path is not None:
@@ -893,13 +896,16 @@ def pip_install(
                         f"Failed to clean up package '{_install_no_version}'.",
                     )
 
-        success = run_python_package(
+        rc = run_python_package(
             'pip',
             _args + _packages,
             venv = venv,
             env = _get_pip_os_env(),
             debug = debug,
-        ) == 0
+        )
+        if debug:
+            print(f"{rc=}")
+        success = rc == 0
 
     msg = (
         "Successfully " + ('un' if _uninstall else '') + "installed packages." if success 

meerschaum/utils/packages/_packages.py

@@ -60,7 +60,7 @@ packages: Dict[str, Dict[str, str]] = {
         'pymysql'                    : 'PyMySQL>=0.9.0',
         'aiomysql'                   : 'aiomysql>=0.0.21',
         'sqlalchemy_cockroachdb'     : 'sqlalchemy-cockroachdb>=2.0.0',
-        'duckdb'                     : 'duckdb>=0.9.0',
+        'duckdb'                     : 'duckdb<0.10.0',
         'duckdb_engine'              : 'duckdb-engine>=0.9.2',
     },
     '_drivers': {
@@ -120,8 +120,8 @@ packages: Dict[str, Dict[str, str]] = {
 packages['sql'] = {
     'numpy'                          : 'numpy>=1.18.5',
     'pandas'                         : 'pandas[parquet]>=2.0.1',
-    'pyarrow'                        : 'pyarrow>=7.0.0',
-    'dask'                           : 'dask>=2023.5.0',
+    'pyarrow'                        : 'pyarrow>=16.1.0',
+    'dask'                           : 'dask[dataframe]>=2024.5.1',
     'pytz'                           : 'pytz',
     'joblib'                         : 'joblib>=0.17.0',
     'sqlalchemy'                     : 'SQLAlchemy>=2.0.5',
@@ -142,14 +142,13 @@ packages['dash'] = {
     'tornado'                        : 'tornado>=6.1.0',
 }
 packages['api'] = {
-    'uvicorn'                        : 'uvicorn[standard]>=0.22.0',
-    'gunicorn'                       : 'gunicorn>=20.1.0',
+    'uvicorn'                        : 'uvicorn[standard]>=0.29.0',
+    'gunicorn'                       : 'gunicorn>=22.0.0',
     'dotenv'                         : 'python-dotenv>=0.20.0',
     'websockets'                     : 'websockets>=11.0.3',
-    'fastapi'                        : 'fastapi>=0.100.0',
-    'passlib'                        : 'passlib>=1.7.4',
+    'fastapi'                        : 'fastapi>=0.111.0',
     'fastapi_login'                  : 'fastapi-login>=1.7.2',
-    'multipart'                      : 'python-multipart>=0.0.5',
+    'multipart'                      : 'python-multipart>=0.0.9',
     'httpx'                          : 'httpx>=0.24.1',
     'websockets'                     : 'websockets>=11.0.3',
 }

meerschaum/utils/process.py

@@ -11,6 +11,7 @@ See `meerschaum.utils.pool` for multiprocessing and
 from __future__ import annotations
 import os, signal, subprocess, sys, platform
 from meerschaum.utils.typing import Union, Optional, Any, Callable, Dict, Tuple
+from meerschaum.config.static import STATIC_CONFIG
 
 def run_process(
         *args,
@@ -68,9 +69,18 @@ def run_process(
     if platform.system() == 'Windows':
         foreground = False
 
-    if line_callback is not None:
+    def print_line(line):
+        sys.stdout.write(line.decode('utf-8'))
+        sys.stdout.flush()
+
+    if capture_output or line_callback is not None:
+        kw['stdout'] = subprocess.PIPE
+        kw['stderr'] = subprocess.STDOUT
+    elif os.environ.get(STATIC_CONFIG['environment']['daemon_id']):
         kw['stdout'] = subprocess.PIPE
         kw['stderr'] = subprocess.STDOUT
+        if line_callback is None:
+            line_callback = print_line
 
     if 'env' not in kw:
         kw['env'] = os.environ
@@ -112,15 +122,6 @@ def run_process(
         kw['preexec_fn'] = new_pgid
 
     try:
-        # fork the child
-        #  stdout, stderr = (
-            #  (sys.stdout, sys.stderr) if not capture_output
-            #  else (subprocess.PIPE, subprocess.PIPE)
-        #  )
-        if capture_output:
-            kw['stdout'] = subprocess.PIPE
-            kw['stderr'] = subprocess.PIPE
-
         child = subprocess.Popen(*args, **kw)
 
         # we can't set the process group id from the parent since the child
@@ -197,6 +198,8 @@ def poll_process(
     while proc.poll() is None:
         line = proc.stdout.readline()
         line_callback(line)
+
     if timeout_seconds is not None:
         watchdog_thread.cancel()
+
     return proc.poll()

meerschaum/utils/schedule.py

@@ -278,7 +278,21 @@ def parse_start_time(schedule: str, now: Optional[datetime] = None) -> datetime:
     starting_str = ('now' if len(starting_parts) == 1 else starting_parts[-1]).strip()
     now = now or round_time(datetime.now(timezone.utc), timedelta(minutes=1))
     try:
-        starting_ts = now if starting_str == 'now' else dateutil_parser.parse(starting_str)
+        if starting_str == 'now':
+            starting_ts = now
+        elif 'tomorrow' in starting_str or 'today' in starting_str:
+            today = round_time(now, timedelta(days=1))
+            tomorrow = today + timedelta(days=1)
+            is_tomorrow = 'tomorrow' in starting_str
+            time_str = starting_str.replace('tomorrow', '').replace('today', '').strip()
+            time_ts = dateutil_parser.parse(time_str) if time_str else today
+            starting_ts = (
+                (tomorrow if is_tomorrow else today)
+                + timedelta(hours=time_ts.hour)
+                + timedelta(minutes=time_ts.minute)
+            )
+        else:
+            starting_ts = dateutil_parser.parse(starting_str)
         schedule_parse_error = None
     except Exception as e:
         warn(f"Unable to parse starting time from '{starting_str}'.", stack=False)

meerschaum/utils/threading.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 from meerschaum.utils.typing import Optional
 
 import threading
+import traceback
 Lock = threading.Lock
 RLock = threading.RLock
 Event = threading.Event