meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc8__py3-none-any.whl

meerschaum/utils/daemon/RotatingFile.py

@@ -14,12 +14,13 @@ import traceback
 import sys
 import atexit
 from datetime import datetime, timezone
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, Callable
 from meerschaum.config import get_config
 from meerschaum.utils.warnings import warn
 from meerschaum.utils.daemon.FileDescriptorInterceptor import FileDescriptorInterceptor
 from meerschaum.utils.threading import Thread
 import meerschaum as mrsm
+import threading
 daemon = mrsm.attempt_import('daemon')
 
 class RotatingFile(io.IOBase):
@@ -38,6 +39,7 @@ class RotatingFile(io.IOBase):
         redirect_streams: bool = False,
         write_timestamps: bool = False,
         timestamp_format: Optional[str] = None,
+        write_callback: Optional[Callable[[str], None]] = None,
     ):
         """
         Create a file-like object which manages other files.
@@ -66,6 +68,9 @@ class RotatingFile(io.IOBase):
         timestamp_format: str, default None
             If `write_timestamps` is `True`, use this format for the timestamps.
             Defaults to `'%Y-%m-%d %H:%M'`.
+
+        write_callback: Optional[Callable[[str], None]], default None
+            If provided, execute this callback with the data to be written.
         """
         self.file_path = pathlib.Path(file_path)
         if num_files_to_keep is None:
@@ -74,17 +79,18 @@ class RotatingFile(io.IOBase):
             max_file_size = get_config('jobs', 'logs', 'max_file_size')
         if timestamp_format is None:
             timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format')
-        if num_files_to_keep < 2:
-            raise ValueError("At least 2 files must be kept.")
-        if max_file_size < 1:
-            raise ValueError("Subfiles must contain at least one byte.")
+        if num_files_to_keep < 1:
+            raise ValueError("At least 1 file must be kept.")
+        if max_file_size < 100:
+            raise ValueError("Subfiles must contain at least 100 bytes.")
 
         self.num_files_to_keep = num_files_to_keep
         self.max_file_size = max_file_size
         self.redirect_streams = redirect_streams
         self.write_timestamps = write_timestamps
         self.timestamp_format = timestamp_format
-        self.subfile_regex_pattern = re.compile(r'(.*)\.log(?:\.\d+)?$')
+        self.write_callback = write_callback
+        self.subfile_regex_pattern = re.compile(r'(.*)\.log(?:\.\d+)?')
 
         ### When subfiles are opened, map from their index to the file objects.
         self.subfile_objects = {}
@@ -186,7 +192,7 @@ class RotatingFile(io.IOBase):
         """
         try:
             return int(subfile_name.replace(self.file_path.name + '.', ''))
-        except Exception as e:
+        except Exception:
             return -1
 
 
@@ -272,7 +278,7 @@ class RotatingFile(io.IOBase):
                 try:
                     daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
                     daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
-                except OSError as e:
+                except OSError:
                     warn(
                         f"Encountered an issue when redirecting streams:\n{traceback.format_exc()}"
                     )
@@ -287,30 +293,36 @@ class RotatingFile(io.IOBase):
             self.is_subfile_too_large(latest_subfile_index, potential_new_len)
         )
         if create_new_file:
-            old_subfile_index = latest_subfile_index
-            new_subfile_index = old_subfile_index + 1
-            new_file_path = self.get_subfile_path_from_index(new_subfile_index)
-            self._previous_file_obj = self._current_file_obj
-            self._current_file_obj = open(new_file_path, 'a+', encoding='utf-8')
-            self.subfile_objects[new_subfile_index] = self._current_file_obj
-            self.flush()
-
-            if self._previous_file_obj is not None:
-                if self.redirect_streams:
-                    self._redirected_subfile_objects[old_subfile_index] = self._previous_file_obj
-                    daemon.daemon.redirect_stream(self._previous_file_obj, self._current_file_obj)
-                    daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
-                    daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
-                self.close(unused_only=True)
+            self.increment_subfiles()
 
-            ### Sanity check in case writing somehow fails.
-            if self._previous_file_obj is self._current_file_obj:
-                self._previous_file_obj = None
+        return self._current_file_obj
 
-            self.delete(unused_only=True)
+    def increment_subfiles(self, increment_by: int = 1):
+        """
+        Create a new subfile and switch the file pointer over.
+        """
+        latest_subfile_index = self.get_latest_subfile_index()
+        old_subfile_index = latest_subfile_index
+        new_subfile_index = old_subfile_index + increment_by
+        new_file_path = self.get_subfile_path_from_index(new_subfile_index)
+        self._previous_file_obj = self._current_file_obj
+        self._current_file_obj = open(new_file_path, 'a+', encoding='utf-8')
+        self.subfile_objects[new_subfile_index] = self._current_file_obj
+        self.flush()
 
-        return self._current_file_obj
+        if self.redirect_streams:
+            if self._previous_file_obj is not None:
+                self._redirected_subfile_objects[old_subfile_index] = self._previous_file_obj
+                daemon.daemon.redirect_stream(self._previous_file_obj, self._current_file_obj)
+            daemon.daemon.redirect_stream(sys.stdout, self._current_file_obj)
+            daemon.daemon.redirect_stream(sys.stderr, self._current_file_obj)
+        self.close(unused_only=True)
+
+        ### Sanity check in case writing somehow fails.
+        if self._previous_file_obj is self._current_file_obj:
+            self._previous_file_obj = None
 
+        self.delete(unused_only=True)
 
     def close(self, unused_only: bool = False) -> None:
         """
@@ -330,7 +342,7 @@ class RotatingFile(io.IOBase):
             try:
                 if not subfile_object.closed:
                     subfile_object.close()
-            except Exception as e:
+            except Exception:
                 warn(f"Failed to close an open subfile:\n{traceback.format_exc()}")
 
             _ = self.subfile_objects.pop(subfile_index, None)
@@ -359,6 +371,12 @@ class RotatingFile(io.IOBase):
         As such, if data is larger than max_file_size, then the corresponding subfile
         may exceed this limit.
         """
+        try:
+            if callable(self.write_callback):
+                self.write_callback(data)
+        except Exception:
+            warn(f"Failed to execute write callback:\n{traceback.format_exc()}")
+
         try:
             self.file_path.parent.mkdir(exist_ok=True, parents=True)
             if isinstance(data, bytes):
@@ -379,7 +397,7 @@ class RotatingFile(io.IOBase):
             except BrokenPipeError:
                 warn("BrokenPipeError encountered. The daemon may have been terminated.")
                 return
-            except Exception as e:
+            except Exception:
                 warn(f"Failed to write to subfile:\n{traceback.format_exc()}")
             self.flush()
             self.delete(unused_only=True)
@@ -410,11 +428,10 @@ class RotatingFile(io.IOBase):
         )
         for subfile_path_to_delete in existing_subfile_paths[0:end_ix]:
             subfile_index = self.get_index_from_subfile_name(subfile_path_to_delete.name)
-            subfile_object = self.subfile_objects.get(subfile_index, None)
 
             try:
                 subfile_path_to_delete.unlink()
-            except Exception as e:
+            except Exception:
                 warn(
                     f"Unable to delete subfile '{subfile_path_to_delete}':\n"
                     + f"{traceback.format_exc()}"
@@ -586,20 +603,21 @@ class RotatingFile(io.IOBase):
             if not subfile_object.closed:
                 try:
                     subfile_object.flush()
-                except Exception as e:
+                except Exception:
                     warn(f"Failed to flush subfile {subfile_index}:\n{traceback.format_exc()}")
+
         if self.redirect_streams:
             try:
                 sys.stdout.flush()
             except BrokenPipeError:
                 pass
-            except Exception as e:
+            except Exception:
                 warn(f"Failed to flush STDOUT:\n{traceback.format_exc()}")
             try:
                 sys.stderr.flush()
             except BrokenPipeError:
                 pass
-            except Exception as e:
+            except Exception:
                 warn(f"Failed to flush STDERR:\n{traceback.format_exc()}")
 
 
@@ -665,10 +683,19 @@ class RotatingFile(io.IOBase):
         for thread in interceptor_threads[:end_ix]:
             try:
                 thread.join()
-            except Exception as e:
+            except Exception:
                 warn(f"Failed to join interceptor threads:\n{traceback.format_exc()}")
         del interceptor_threads[:end_ix]
 
+    def touch(self):
+        """
+        Touch the latest subfile.
+        """
+        subfile_path = self.get_latest_subfile_path()
+        subfile_path.touch()
+
+    def isatty(self) -> bool:
+        return True
 
     def __repr__(self) -> str:
         """

meerschaum/utils/daemon/StdinFile.py

@@ -13,6 +13,7 @@ import os
 import selectors
 import traceback
 
+import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Union
 from meerschaum.utils.warnings import warn
 
@@ -25,6 +26,8 @@ class StdinFile(io.TextIOBase):
         self,
         file_path: Union[pathlib.Path, str],
         lock_file_path: Optional[pathlib.Path] = None,
+        decode: bool = True,
+        refresh_seconds: Union[int, float, None] = None,
     ):
         if isinstance(file_path, str):
             file_path = pathlib.Path(file_path)
@@ -38,6 +41,13 @@ class StdinFile(io.TextIOBase):
         self._file_handler = None
         self._fd = None
         self.sel = selectors.DefaultSelector()
+        self.decode = decode
+        self._write_fp = None
+        self._refresh_seconds = refresh_seconds
+
+    @property
+    def encoding(self):
+        return 'utf-8'
 
     @property
     def file_handler(self):
@@ -47,11 +57,9 @@ class StdinFile(io.TextIOBase):
         if self._file_handler is not None:
             return self._file_handler
 
-        if self.file_path.exists():
-            self.file_path.unlink()
-
-        self.file_path.parent.mkdir(parents=True, exist_ok=True)
-        os.mkfifo(self.file_path.as_posix(), mode=0o600)
+        if not self.file_path.exists():
+            self.file_path.parent.mkdir(parents=True, exist_ok=True)
+            os.mkfifo(self.file_path.as_posix(), mode=0o600)
 
         self._fd = os.open(self.file_path, os.O_RDONLY | os.O_NONBLOCK)
         self._file_handler = os.fdopen(self._fd, 'rb', buffering=0)
@@ -59,11 +67,19 @@ class StdinFile(io.TextIOBase):
         return self._file_handler
 
     def write(self, data):
+        if self._write_fp is None:
+            self.file_path.parent.mkdir(parents=True, exist_ok=True)
+            if not self.file_path.exists():
+                os.mkfifo(self.file_path.as_posix(), mode=0o600)
+            self._write_fp = open(self.file_path, 'wb')
+
         if isinstance(data, str):
             data = data.encode('utf-8')
-
-        with open(self.file_path, 'wb') as f:
-            f.write(data)
+        try:
+            self._write_fp.write(data)
+            self._write_fp.flush()
+        except BrokenPipeError:
+            pass
 
     def fileno(self):
         fileno = self.file_handler.fileno()
@@ -83,18 +99,19 @@ class StdinFile(io.TextIOBase):
                             self.blocking_file_path.unlink()
                     except Exception:
                         warn(traceback.format_exc())
-                    return data.decode('utf-8')
+                    return data.decode('utf-8') if self.decode else data
             except (OSError, EOFError):
                 pass
 
-            self.blocking_file_path.touch()
-            time.sleep(0.1)
+            if not self.blocking_file_path.exists():
+                self.blocking_file_path.touch()
+            time.sleep(self.refresh_seconds)
 
     def readline(self, size=-1):
-        line = ''
+        line = '' if self.decode else b''
         while True:
             data = self.read(1)
-            if not data or data == '\n':
+            if not data or ((data == '\n') if self.decode else (data == b'\n')):
                 break
             line += data
 
@@ -111,11 +128,34 @@ class StdinFile(io.TextIOBase):
             self._file_handler = None
             self._fd = None
 
+        if self._write_fp is not None:
+            try:
+                self._write_fp.close()
+            except BrokenPipeError:
+                pass
+            self._write_fp = None
+
+        try:
+            if self.blocking_file_path.exists():
+                self.blocking_file_path.unlink()
+        except Exception:
+            pass
         super().close()
 
     def is_open(self):
         return self._file_handler is not None
 
+    def isatty(self) -> bool:
+        return False
+
+    @property
+    def refresh_seconds(self) -> Union[int, float]:
+        """
+        How many seconds between checking for blocking functions.
+        """
+        if not self._refresh_seconds:
+            self._refresh_seconds = mrsm.get_config('system', 'cli', 'refresh_seconds')
+        return self._refresh_seconds
 
     def __str__(self) -> str:
         return f"StdinFile('{self.file_path}')"

meerschaum/utils/daemon/__init__.py

@@ -16,8 +16,7 @@ import datetime
 import threading
 import shlex
 
-from meerschaum.utils.typing import SuccessTuple, List, Optional, Callable, Any, Dict
-from meerschaum.config._paths import DAEMON_RESOURCES_PATH
+from meerschaum.utils.typing import SuccessTuple, List, Optional, Callable, Any, Dict, Union
 from meerschaum.utils.daemon.StdinFile import StdinFile
 from meerschaum.utils.daemon.Daemon import Daemon
 from meerschaum.utils.daemon.RotatingFile import RotatingFile
@@ -41,9 +40,10 @@ __all__ = (
     'StdinFile',
     'RotatingFile',
     'FileDescriptorInterceptor',
-    'DAEMON_RESOURCES_PATH',
 )
 
+_daemons = {}
+
 
 def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     """Parse sysargs and execute a Meerschaum action as a daemon.
@@ -211,6 +211,7 @@ def get_daemon_ids() -> List[str]:
     """
     Return the IDs of all daemons on disk.
     """
+    from meerschaum.config._paths import DAEMON_RESOURCES_PATH
     return [
         daemon_dir
         for daemon_dir in sorted(os.listdir(DAEMON_RESOURCES_PATH))
@@ -298,8 +299,6 @@ def get_filtered_daemons(
             if warn:
                 _warn(f"Daemon '{d_id}' does not exist.", stack=False)
             continue
-        if d.hidden:
-            pass
         daemons.append(d)
     return daemons
 
@@ -311,3 +310,17 @@ def running_in_daemon() -> bool:
     from meerschaum._internal.static import STATIC_CONFIG
     daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
     return daemon_env_var in os.environ
+
+
+def get_current_daemon() -> Union[Daemon, None]:
+    """
+    If running withing a daemon context, return the corresponding `Daemon`.
+    Otherwise return `None`.
+    """
+    from meerschaum._internal.static import STATIC_CONFIG
+    daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
+    daemon_id = os.environ.get(daemon_env_var, None)
+    if daemon_id is None:
+        return None
+
+    return _daemons.get(daemon_id, Daemon(daemon_id=daemon_id))

meerschaum/utils/daemon/_names.py

@@ -7,9 +7,11 @@ Generate random names for jobs.
 """
 
 from __future__ import annotations
-import os, random
-from meerschaum.utils.typing import Dict, List, Tuple
-from meerschaum.config._paths import DAEMON_RESOURCES_PATH
+
+import os
+import random
+
+from meerschaum.utils.typing import Dict, List
 
 _bank: Dict[str, Dict[str, List[str]]] = {
     'adjectives': {
@@ -116,6 +118,7 @@ def get_new_daemon_name() -> str:
     Generate a new random name until a unique one is found
     (up to ~6000 maximum possibilities).
     """
+    from meerschaum.config._paths import DAEMON_RESOURCES_PATH
     existing_names = (
         os.listdir(DAEMON_RESOURCES_PATH)
         if DAEMON_RESOURCES_PATH.exists()

meerschaum/utils/debug.py

@@ -7,11 +7,31 @@ Functions to handle debug statements
 """
 
 from __future__ import annotations
+
+from datetime import datetime, timezone
+import meerschaum as mrsm
 from meerschaum.utils.typing import Union, Optional, List
 
+
+_rich_text = None
+def _import_rich_text_for_dprint():
+    """
+    Avoid calling `attempt_import()` on every dprint.
+    """
+    global _rich_text
+    if _rich_text is not None:
+        return _rich_text
+
+    from meerschaum.utils.packages import import_rich, attempt_import
+    _ = import_rich()
+    _rich_text = attempt_import('rich.text', lazy=False)
+    return _rich_text
+
+
 def dprint(
     msg: str,
     leader: bool = True,
+    timestamp: bool = True,
     package: bool = True,
     color: Optional[Union[str, List[str]]] = None,
     attrs: Optional[List[str]] = None,
@@ -35,6 +55,12 @@ def dprint(
     else:
         CHARSET, ANSI, colored, _color, cf = 'ascii', False, None, None, None
 
+    if timestamp:
+        from meerschaum.utils.dtypes import get_current_timestamp
+        now = get_current_timestamp('ms').replace(tzinfo=None)
+    else:
+        now = None
+
     import logging, sys, inspect
     logging.basicConfig(format='%(message)s')
     log = logging.getLogger(__name__)
@@ -46,8 +72,13 @@ def dprint(
     parent_package = parent_globals['__name__']
     msg = str(msg)
     premsg = ""
+
+    if now:
+        premsg = now.isoformat().split('T', maxsplit=1)[-1][:-3] + (' | ' if package else ':')
     if package:
-        premsg = parent_package + ':' + str(parent_lineno) + '\n'
+        premsg = premsg + parent_package + ':' + str(parent_lineno)
+    if premsg:
+        premsg += "\n"
     if leader and cf is not None:
         try:
             debug_leader = cf['formatting']['debug'][CHARSET]['icon'] if cf is not None else ''
@@ -75,10 +106,9 @@ def dprint(
                 _color = {}
         if colored is not None:
             premsg = colored(premsg, **_color)
+
     if _progress is not None:
-        from meerschaum.utils.packages import import_rich, attempt_import
-        rich = import_rich()
-        rich_text = attempt_import('rich.text')
+        rich_text = _import_rich_text_for_dprint()
         text = rich_text.Text.from_ansi(premsg + msg)
         _progress.console.log(text)
     else:

meerschaum/utils/dtypes/__init__.py

@@ -1054,7 +1054,7 @@ def get_current_timestamp(
     return ts_val.as_unit(MRSM_PRECISION_UNITS_ABBREVIATIONS[true_precision_unit])
 
 
-def dtype_is_special(type_: str) -> bool:
+def is_dtype_special(type_: str) -> bool:
     """
     Return whether a dtype should be treated as a special Meerschaum dtype.
     This is not the same as a Meerschaum alias.
@@ -1069,6 +1069,7 @@ def dtype_is_special(type_: str) -> bool:
         'geometry',
         'geography',
         'date',
+        'bool',
     ):
         return True
 
@@ -1081,6 +1082,9 @@ def dtype_is_special(type_: str) -> bool:
     if true_type.startswith('numeric'):
         return True
 
+    if true_type.startswith('bool'):
+        return True
+
     if true_type.startswith('geometry'):
         return True
 

meerschaum/utils/formatting/__init__.py

@@ -209,6 +209,8 @@ def get_console():
     try:
         console = rich_console.Console(force_terminal=True, color_system='truecolor')
     except Exception:
+        import traceback
+        traceback.print_exc()
         console = None
     return console
 
@@ -488,8 +490,9 @@ def fill_ansi(string: str, style: str = '') -> str:
         msg = Text.from_ansi(string)
     except AttributeError:
         import traceback
-        traceback.print_stack()
+        traceback.print_exc()
         msg = ''
+
     plain_indices = []
     for left_span, right_span in iterate_chunks(msg.spans, 2, fillvalue=len(msg)):
         left = left_span.end

meerschaum/utils/formatting/_jobs.py

@@ -97,7 +97,7 @@ def pprint_jobs(
             msg = '\n'.join(line.lstrip().rstrip() for line in lines)
             success_tuple = success, msg
             success_tuple_str = (
-                format_success_tuple(success_tuple, left_padding=1)
+                format_success_tuple(success_tuple, left_padding=0)
                 if success_tuple is not None
                 else None
             )