meerschaum 2.1.6__py3-none-any.whl → 2.2.0__py3-none-any.whl

meerschaum/utils/daemon/FileDescriptorInterceptor.py

@@ -0,0 +1,140 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Intercept OS-level file descriptors.
+"""
+
+import os
+import select
+import traceback
+from threading import Event
+from datetime import datetime
+from meerschaum.utils.typing import Callable
+from meerschaum.utils.warnings import warn
+
+FD_CLOSED: int = 9
+STOP_READING_FD_EVENT: Event = Event()
+
+class FileDescriptorInterceptor:
+    """
+    A management class to intercept data written to a file descriptor.
+    """
+    def __init__(
+        self,
+        file_descriptor: int,
+        injection_hook: Callable[[], str],
+    ):
+        """
+        Parameters
+        ----------
+        file_descriptor: int
+            The OS file descriptor from which to read.
+
+        injection_hook: Callable[[], str]
+            A callable which returns a string to be injected into the written data.
+        """
+        self.stop_event = Event()
+        self.injection_hook = injection_hook
+        self.original_file_descriptor = file_descriptor
+        self.new_file_descriptor = os.dup(file_descriptor)
+        self.read_pipe, self.write_pipe = os.pipe()
+        self.signal_read_pipe, self.signal_write_pipe = os.pipe()
+        os.dup2(self.write_pipe, file_descriptor)
+
+    def start_interception(self):
+        """
+        Read from the file descriptor and write the modified data after injection.
+
+        NOTE: This is blocking and is meant to be run in a thread.
+        """
+        os.set_blocking(self.read_pipe, False)
+        os.set_blocking(self.signal_read_pipe, False)
+        is_first_read = True
+        while not self.stop_event.is_set():
+            try:
+                rlist, _, _ = select.select([self.read_pipe, self.signal_read_pipe], [], [], 0.1)
+                if self.signal_read_pipe in rlist:
+                    break
+                if not rlist:
+                    continue
+                data = os.read(self.read_pipe, 1024)
+                if not data:
+                    break
+            except BlockingIOError:
+                continue
+            except OSError as e:
+                continue
+
+            first_char_is_newline = data[0] == b'\n'
+            last_char_is_newline = data[-1] == b'\n'
+
+            injected_str = self.injection_hook()
+            injected_bytes = injected_str.encode('utf-8')
+
+            if is_first_read:
+                data = b'\n' + data
+                is_first_read = False
+
+            modified_data = (
+                (data[:-1].replace(b'\n', b'\n' + injected_bytes) + b'\n')
+                if last_char_is_newline
+                else data.replace(b'\n', b'\n' + injected_bytes)
+            )
+            os.write(self.new_file_descriptor, modified_data)
+
+
+    def stop_interception(self):
+        """
+        Close the new file descriptors.
+        """
+        self.stop_event.set()
+        os.write(self.signal_write_pipe, b'\0')
+        try:
+            os.close(self.new_file_descriptor)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the duplicated file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+
+        try:
+            os.close(self.write_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the write-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+        try:
+            os.close(self.read_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the read-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+
+        try:
+            os.close(self.signal_read_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the signal-read-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )
+
+        try:
+            os.close(self.signal_write_pipe)
+        except OSError as e:
+            if e.errno != FD_CLOSED:
+                warn(
+                    f"Error while trying to close the signal-write-pipe "
+                    + "to the intercepted file descriptor:\n"
+                    + f"{traceback.format_exc()}"
+                )

meerschaum/utils/daemon/RotatingFile.py

@@ -13,9 +13,13 @@ import pathlib
 import traceback
 import sys
 import atexit
+from datetime import datetime, timezone, timedelta
 from typing import List, Union, Optional, Tuple
 from meerschaum.config import get_config
 from meerschaum.utils.warnings import warn
+from meerschaum.utils.misc import round_time
+from meerschaum.utils.daemon.FileDescriptorInterceptor import FileDescriptorInterceptor
+from meerschaum.utils.threading import Thread
 import meerschaum as mrsm
 daemon = mrsm.attempt_import('daemon')
 
@@ -33,6 +37,8 @@ class RotatingFile(io.IOBase):
             num_files_to_keep: Optional[int] = None,
             max_file_size: Optional[int] = None,
             redirect_streams: bool = False,
+            write_timestamps: bool = False,
+            timestamp_format: str = '%Y-%m-%d %H:%M',
         ):
         """
         Create a file-like object which manages other files.
@@ -54,6 +60,9 @@ class RotatingFile(io.IOBase):
             
             NOTE: Only set this to `True` if you are entering into a daemon context.
             Doing so will redirect `sys.stdout` and `sys.stderr` into the log files.
+
+        write_timestamps: bool, default False
+            If `True`, prepend the current UTC timestamp to each line of the file.
         """
         self.file_path = pathlib.Path(file_path)
         if num_files_to_keep is None:
@@ -68,6 +77,8 @@ class RotatingFile(io.IOBase):
         self.num_files_to_keep = num_files_to_keep
         self.max_file_size = max_file_size
         self.redirect_streams = redirect_streams
+        self.write_timestamps = write_timestamps
+        self.timestamp_format = timestamp_format
         self.subfile_regex_pattern = re.compile(
             r'^'
             + self.file_path.name
@@ -91,7 +102,7 @@ class RotatingFile(io.IOBase):
         """
         Return the file descriptor for the latest subfile.
         """
-        self.refresh_files()
+        self.refresh_files(start_interception=False)
         return self._current_file_obj.fileno()
 
 
@@ -221,7 +232,11 @@ class RotatingFile(io.IOBase):
         ]
 
 
-    def refresh_files(self, potential_new_len: int = 0) -> '_io.TextUIWrapper':
+    def refresh_files(
+            self,
+            potential_new_len: int = 0,
+            start_interception: bool = False,
+        ) -> '_io.TextUIWrapper':
         """
         Check the state of the subfiles.
         If the latest subfile is too large, create a new file and delete old ones.
@@ -229,6 +244,9 @@ class RotatingFile(io.IOBase):
         Parameters
         ----------
         potential_new_len: int, default 0
+
+        start_interception: bool, default False
+            If `True`, kick off the file interception threads.
         """
         self.flush()
 
@@ -247,8 +265,15 @@ class RotatingFile(io.IOBase):
         if is_first_run_with_logs or lost_latest_handle:
             self._current_file_obj = open(latest_subfile_path, 'a+', encoding='utf-8')
             if self.redirect_streams:
-                daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
-                daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
+                try:
+                    daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
+                    daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
+                except OSError as e:
+                    warn(
+                        f"Encountered an issue when redirecting streams:\n{traceback.format_exc()}"
+                    )
+                if start_interception and self.write_timestamps:
+                    self.start_log_fd_interception()
 
         create_new_file = (
             (latest_subfile_index == -1)
@@ -276,9 +301,10 @@ class RotatingFile(io.IOBase):
 
             ### Sanity check in case writing somehow fails.
             if self._previous_file_obj is self._current_file_obj:
-                self._previous_file_obj is None
+                self._previous_file_obj = None
 
             self.delete(unused_only=True)
+
         return self._current_file_obj
 
 
@@ -291,6 +317,7 @@ class RotatingFile(io.IOBase):
         unused_only: bool, default False
             If `True`, only close file descriptors not currently in use.
         """
+        self.stop_log_fd_interception(unused_only=unused_only)
         subfile_indices = sorted(self.subfile_objects.keys())
         for subfile_index in subfile_indices:
             subfile_object = self.subfile_objects[subfile_index]
@@ -298,19 +325,26 @@ class RotatingFile(io.IOBase):
                 continue
             try:
                 if not subfile_object.closed:
-                    #  subfile_object.flush()
                     subfile_object.close()
-                _ = self.subfile_objects.pop(subfile_index, None)
-                if self.redirect_streams:
-                    _ = self._redirected_subfile_objects.pop(subfile_index, None)
             except Exception as e:
                 warn(f"Failed to close an open subfile:\n{traceback.format_exc()}")
 
+            _ = self.subfile_objects.pop(subfile_index, None)
+            if self.redirect_streams:
+                _ = self._redirected_subfile_objects.pop(subfile_index, None)
+
         if not unused_only:
             self._previous_file_obj = None
             self._current_file_obj = None
 
 
+    def get_timestamp_prefix_str(self) -> str:
+        """
+        Return the current minute prefixm string.
+        """
+        return datetime.now(timezone.utc).strftime(self.timestamp_format) + ' | '
+
+
     def write(self, data: str) -> None:
         """
         Write the given text into the latest subfile.
@@ -325,9 +359,18 @@ class RotatingFile(io.IOBase):
         if isinstance(data, bytes):
             data = data.decode('utf-8')
 
-        self.refresh_files(potential_new_len=len(data))
+        prefix_str = self.get_timestamp_prefix_str() if self.write_timestamps else ""
+        suffix_str = "\n" if self.write_timestamps else ""
+        self.refresh_files(
+            potential_new_len = len(prefix_str + data + suffix_str),
+            start_interception = self.write_timestamps,
+        )
         try:
+            if prefix_str:
+                self._current_file_obj.write(prefix_str)
             self._current_file_obj.write(data)
+            if suffix_str:
+                self._current_file_obj.write(suffix_str)
         except Exception as e:
             warn(f"Failed to write to subfile:\n{traceback.format_exc()}")
         self.flush()
@@ -471,7 +514,7 @@ class RotatingFile(io.IOBase):
                 subfile_object = self.subfile_objects[subfile_index]
                 for i in range(self.SEEK_BACK_ATTEMPTS):
                     try:
-                        subfile_object.seek(max(seek_ix - i), 0)
+                        subfile_object.seek(max((seek_ix - i), 0))
                         subfile_lines = subfile_object.readlines()
                     except UnicodeDecodeError:
                         continue
@@ -532,10 +575,83 @@ class RotatingFile(io.IOBase):
                 try:
                     subfile_object.flush()
                 except Exception as e:
-                    warn(f"Failed to flush subfile:\n{traceback.format_exc()}")
+                    warn(f"Failed to flush subfile {subfile_index}:\n{traceback.format_exc()}")
         if self.redirect_streams:
-            sys.stdout.flush()
-            sys.stderr.flush()
+            try:
+                sys.stdout.flush()
+            except Exception as e:
+                warn(f"Failed to flush STDOUT:\n{traceback.format_exc()}")
+            try:
+                sys.stderr.flush()
+            except Exception as e:
+                warn(f"Failed to flush STDERR:\n{traceback.format_exc()}")
+
+
+    def start_log_fd_interception(self):
+        """
+        Start the file descriptor monitoring threads.
+        """
+        if not self.write_timestamps:
+            return
+
+        threads = self.__dict__.get('_interceptor_threads', [])
+        self._stdout_interceptor = FileDescriptorInterceptor(
+            sys.stdout.fileno(),
+            self.get_timestamp_prefix_str,
+        )
+        self._stderr_interceptor = FileDescriptorInterceptor(
+            sys.stderr.fileno(),
+            self.get_timestamp_prefix_str,
+        )
+
+        self._stdout_interceptor_thread = Thread(
+            target = self._stdout_interceptor.start_interception,
+            daemon = True,
+        )
+        self._stderr_interceptor_thread = Thread(
+            target = self._stderr_interceptor.start_interception,
+            daemon = True,
+        )
+        self._stdout_interceptor_thread.start()
+        self._stderr_interceptor_thread.start()
+        self._intercepting = True
+
+        if '_interceptor_threads' not in self.__dict__:
+            self._interceptor_threads = []
+        if '_interceptors' not in self.__dict__:
+            self._interceptors = []
+        self._interceptor_threads.extend([
+            self._stdout_interceptor_thread,
+            self._stderr_interceptor_thread,
+        ])
+        self._interceptors.extend([
+            self._stdout_interceptor,
+            self._stderr_interceptor,
+        ])
+        self.stop_log_fd_interception(unused_only=True)
+
+
+    def stop_log_fd_interception(self, unused_only: bool = False):
+        """
+        Stop the file descriptor monitoring threads.
+        """
+        if not self.write_timestamps:
+            return
+        interceptors = self.__dict__.get('_interceptors', [])
+        interceptor_threads = self.__dict__.get('_interceptor_threads', [])
+
+        end_ix = len(interceptors) if not unused_only else -2
+
+        for interceptor in interceptors[:end_ix]:
+            interceptor.stop_interception()
+        del interceptors[:end_ix]
+
+        for thread in interceptor_threads[:end_ix]:
+            try:
+                thread.join()
+            except Exception as e:
+                warn(f"Failed to join interceptor threads:\n{traceback.format_exc()}")
+        del interceptor_threads[:end_ix]
 
 
     def __repr__(self) -> str:

meerschaum/utils/daemon/__init__.py

@@ -12,6 +12,7 @@ from meerschaum.utils.typing import SuccessTuple, List, Optional, Callable, Any,
 from meerschaum.config._paths import DAEMON_RESOURCES_PATH
 from meerschaum.utils.daemon.Daemon import Daemon
 from meerschaum.utils.daemon.RotatingFile import RotatingFile
+from meerschaum.utils.daemon.FileDescriptorInterceptor import FileDescriptorInterceptor
 
 
 def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
@@ -63,6 +64,8 @@ def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
 
         ### Only run if the kwargs equal or no actions are provided.
         if existing_kwargs == _args or not _args.get('action', []):
+            if daemon.status == 'running':
+                return True, f"Daemon '{daemon}' is already running."
             return daemon.run(
                 debug = debug,
                 allow_dirty_run = True,

meerschaum/utils/dataframe.py

@@ -7,9 +7,10 @@ Utility functions for working with DataFrames.
 """
 
 from __future__ import annotations
+from datetime import datetime
 from meerschaum.utils.typing import (
     Optional, Dict, Any, List, Hashable, Generator,
-    Iterator, Iterable, Union,
+    Iterator, Iterable, Union, Tuple,
 )
 
 
@@ -71,6 +72,7 @@ def add_missing_cols_to_df(df: 'pd.DataFrame', dtypes: Dict[str, Any]) -> pd.Dat
 def filter_unseen_df(
         old_df: 'pd.DataFrame',
         new_df: 'pd.DataFrame',
+        safe_copy: bool = True,
         dtypes: Optional[Dict[str, Any]] = None,
         debug: bool = False,
     ) -> 'pd.DataFrame':
@@ -84,6 +86,10 @@ def filter_unseen_df(
         
     new_df: 'pd.DataFrame'
         The fetched (source) dataframe. Rows that are contained in `old_df` are removed.
+
+    safe_copy: bool, default True
+        If `True`, create a copy before comparing and modifying the dataframes.
+        Setting to `False` may mutate the DataFrames.
         
     dtypes: Optional[Dict[str, Any]], default None
         Optionally specify the datatypes of the dataframe.
@@ -111,6 +117,10 @@ def filter_unseen_df(
     if old_df is None:
         return new_df
 
+    if safe_copy:
+        old_df = old_df.copy()
+        new_df = new_df.copy()
+
     import json
     import functools
     import traceback
@@ -118,6 +128,7 @@ def filter_unseen_df(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.packages import import_pandas, attempt_import
     from meerschaum.utils.dtypes import to_pandas_dtype, are_dtypes_equal, attempt_cast_to_numeric
+    from meerschaum.utils.debug import dprint
     pd = import_pandas(debug=debug)
     is_dask = 'dask' in new_df.__module__
     if is_dask:
@@ -243,12 +254,7 @@ def filter_unseen_df(
         indicator = True,
     )
     changed_rows_mask = (joined_df['_merge'] == 'left_only')
-
-    delta_df = joined_df[
-        list(new_df_dtypes.keys())
-    ][
-        changed_rows_mask
-    ].reset_index(drop=True)
+    delta_df = joined_df[list(new_df_dtypes.keys())][changed_rows_mask].reset_index(drop=True)
 
     for json_col in json_cols:
         if json_col not in delta_df.columns:
@@ -535,6 +541,8 @@ def get_numeric_cols(df: 'pd.DataFrame') -> List[str]:
 def enforce_dtypes(
         df: 'pd.DataFrame',
         dtypes: Dict[str, str],
+        safe_copy: bool = True,
+        coerce_numeric: bool = True,
         debug: bool = False,
     ) -> 'pd.DataFrame':
     """
@@ -548,6 +556,14 @@ def enforce_dtypes(
     dtypes: Dict[str, str]
         The data types to attempt to enforce on the DataFrame.
 
+    safe_copy: bool, default True
+        If `True`, create a copy before comparing and modifying the dataframes.
+        Setting to `False` may mutate the DataFrames.
+        See `meerschaum.utils.dataframe.filter_unseen_df`.
+
+    coerce_numeric: bool, default True
+        If `True`, convert float and int collisions to numeric.
+
     debug: bool, default False
         Verbosity toggle.
 
@@ -569,6 +585,8 @@ def enforce_dtypes(
         is_dtype_numeric,
         attempt_cast_to_numeric,
     )
+    if safe_copy:
+        df = df.copy()
     df_dtypes = {c: str(t) for c, t in df.dtypes.items()}
     if len(df_dtypes) == 0:
         if debug:
@@ -674,7 +692,7 @@ def enforce_dtypes(
             explicitly_numeric
             or col in df_numeric_cols
             or (mixed_numeric_types and not explicitly_float)
-        )
+        ) and coerce_numeric
         if cast_to_numeric:
             common_dtypes[col] = attempt_cast_to_numeric
             common_diff_dtypes[col] = attempt_cast_to_numeric
@@ -860,3 +878,160 @@ def get_first_valid_dask_partition(ddf: 'dask.dataframe.DataFrame') -> Union['pd
         if len(pdf) > 0:
             return pdf
     return ddf.compute()
+
+
+def query_df(
+        df: 'pd.DataFrame',
+        params: Optional[Dict[str, Any]] = None,
+        begin: Union[datetime, int, None] = None,
+        end: Union[datetime, int, None] = None,
+        datetime_column: Optional[str] = None,
+        select_columns: Optional[List[str]] = None,
+        omit_columns: Optional[List[str]] = None,
+        inplace: bool = False,
+        reset_index: bool = False,
+        debug: bool = False,
+    ) -> 'pd.DataFrame':
+    """
+    Query the dataframe with the params dictionary.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        The DataFrame to query against.
+
+    params: Optional[Dict[str, Any]], default None
+        The parameters dictionary to use for the query.
+
+    begin: Union[datetime, int, None], default None
+        If `begin` and `datetime_column` are provided, only return rows with a timestamp
+        greater than or equal to this value.
+
+    end: Union[datetime, int, None], default None
+        If `begin` and `datetime_column` are provided, only return rows with a timestamp
+        less than this value.
+
+    datetime_column: Optional[str], default None
+        A `datetime_column` must be provided to use `begin` and `end`.
+
+    select_columns: Optional[List[str]], default None
+        If provided, only return these columns.
+
+    omit_columns: Optional[List[str]], default None
+        If provided, do not include these columns in the result.
+
+    inplace: bool, default False
+        If `True`, modify the DataFrame inplace rather than creating a new DataFrame.
+
+    reset_index: bool, default True
+        If `True`, reset the index in the resulting DataFrame.
+
+    Returns
+    -------
+    A Pandas DataFrame query result.
+    """
+    if not params and not begin and not end:
+        return df
+
+    import json
+    import meerschaum as mrsm
+    from meerschaum.utils.debug import dprint
+    from meerschaum.utils.misc import get_in_ex_params
+    from meerschaum.utils.warnings import warn
+
+    dtypes = {col: str(typ) for col, typ in df.dtypes.items()}
+
+    if begin or end:
+        if not datetime_column or datetime_column not in df.columns:
+            warn(
+                f"The datetime column '{datetime_column}' is not present in the Dataframe, "
+                + "ignoring begin and end...",
+            )
+            begin, end = None, None
+
+    if debug:
+        dprint(f"Querying dataframe:\n{params=} {begin=} {end=} {datetime_column=}")
+
+    in_ex_params = get_in_ex_params(params)
+
+    def serialize(x: Any) -> str:
+        if isinstance(x, (dict, list, tuple)):
+            return json.dumps(x, sort_keys=True, separators=(',', ':'), default=str)
+        if hasattr(x, 'isoformat'):
+            return x.isoformat()
+        return str(x)
+
+    masks = [
+        (
+            (df[datetime_column] >= begin)
+            if begin is not None and datetime_column
+            else True
+        ) & (
+            (df[datetime_column] < end)
+            if end is not None and datetime_column
+            else True
+        )
+    ]
+
+    masks.extend([
+        (
+            (
+                df[col].apply(serialize).isin(
+                    [
+                        serialize(_in_val)
+                        for _in_val in in_vals
+                    ]
+                ) if in_vals else True
+            ) & (
+                ~df[col].apply(serialize).isin(
+                    [
+                        serialize(_ex_val)
+                        for _ex_val in ex_vals
+                    ]
+                ) if ex_vals else True
+            )
+        )
+        for col, (in_vals, ex_vals) in in_ex_params.items()
+        if col in df.columns
+    ])
+    query_mask = masks[0]
+    for mask in masks:
+        query_mask = query_mask & mask
+
+    if inplace:
+        df.where(query_mask, inplace=inplace)
+        df.dropna(how='all', inplace=inplace)
+        result_df = df
+    else:
+        result_df = df.where(query_mask).dropna(how='all')
+
+    if reset_index:
+        result_df.reset_index(drop=True, inplace=True)
+
+    result_df = enforce_dtypes(
+        result_df,
+        dtypes,
+        safe_copy = (not inplace),
+        debug = debug,
+        coerce_numeric = False,
+    )
+
+    if select_columns == ['*']:
+        select_columns = None
+
+    if not select_columns and not omit_columns:
+        return result_df
+
+    if select_columns:
+        for col in list(result_df.columns):
+            if col not in select_columns:
+                del result_df[col]
+        return result_df
+
+    if omit_columns:
+        for col in list(result_df.columns):
+            if col in omit_columns:
+                del result_df[col]
+    if debug:
+        dprint(f"{dtypes=}")
+    return result_df