meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

meerschaum/utils/daemon/StdinFile.py

@@ -13,6 +13,7 @@ import os
 import selectors
 import traceback
 
+import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Union
 from meerschaum.utils.warnings import warn
 
@@ -25,6 +26,8 @@ class StdinFile(io.TextIOBase):
         self,
         file_path: Union[pathlib.Path, str],
         lock_file_path: Optional[pathlib.Path] = None,
+        decode: bool = True,
+        refresh_seconds: Union[int, float, None] = None,
     ):
         if isinstance(file_path, str):
             file_path = pathlib.Path(file_path)
@@ -38,6 +41,13 @@ class StdinFile(io.TextIOBase):
         self._file_handler = None
         self._fd = None
         self.sel = selectors.DefaultSelector()
+        self.decode = decode
+        self._write_fp = None
+        self._refresh_seconds = refresh_seconds
+
+    @property
+    def encoding(self):
+        return 'utf-8'
 
     @property
     def file_handler(self):
@@ -47,11 +57,9 @@ class StdinFile(io.TextIOBase):
         if self._file_handler is not None:
             return self._file_handler
 
-        if self.file_path.exists():
-            self.file_path.unlink()
-
-        self.file_path.parent.mkdir(parents=True, exist_ok=True)
-        os.mkfifo(self.file_path.as_posix(), mode=0o600)
+        if not self.file_path.exists():
+            self.file_path.parent.mkdir(parents=True, exist_ok=True)
+            os.mkfifo(self.file_path.as_posix(), mode=0o600)
 
         self._fd = os.open(self.file_path, os.O_RDONLY | os.O_NONBLOCK)
         self._file_handler = os.fdopen(self._fd, 'rb', buffering=0)
@@ -59,11 +67,19 @@ class StdinFile(io.TextIOBase):
         return self._file_handler
 
     def write(self, data):
+        if self._write_fp is None:
+            self.file_path.parent.mkdir(parents=True, exist_ok=True)
+            if not self.file_path.exists():
+                os.mkfifo(self.file_path.as_posix(), mode=0o600)
+            self._write_fp = open(self.file_path, 'wb')
+
         if isinstance(data, str):
             data = data.encode('utf-8')
-
-        with open(self.file_path, 'wb') as f:
-            f.write(data)
+        try:
+            self._write_fp.write(data)
+            self._write_fp.flush()
+        except BrokenPipeError:
+            pass
 
     def fileno(self):
         fileno = self.file_handler.fileno()
@@ -83,18 +99,19 @@ class StdinFile(io.TextIOBase):
                             self.blocking_file_path.unlink()
                     except Exception:
                         warn(traceback.format_exc())
-                    return data.decode('utf-8')
+                    return data.decode('utf-8') if self.decode else data
             except (OSError, EOFError):
                 pass
 
-            self.blocking_file_path.touch()
-            time.sleep(0.1)
+            if not self.blocking_file_path.exists():
+                self.blocking_file_path.touch()
+            time.sleep(self.refresh_seconds)
 
     def readline(self, size=-1):
-        line = ''
+        line = '' if self.decode else b''
         while True:
             data = self.read(1)
-            if not data or data == '\n':
+            if not data or ((data == '\n') if self.decode else (data == b'\n')):
                 break
             line += data
 
@@ -111,11 +128,34 @@ class StdinFile(io.TextIOBase):
             self._file_handler = None
             self._fd = None
 
+        if self._write_fp is not None:
+            try:
+                self._write_fp.close()
+            except BrokenPipeError:
+                pass
+            self._write_fp = None
+
+        try:
+            if self.blocking_file_path.exists():
+                self.blocking_file_path.unlink()
+        except Exception:
+            pass
         super().close()
 
     def is_open(self):
         return self._file_handler is not None
 
+    def isatty(self) -> bool:
+        return False
+
+    @property
+    def refresh_seconds(self) -> Union[int, float]:
+        """
+        How many seconds between checking for blocking functions.
+        """
+        if not self._refresh_seconds:
+            self._refresh_seconds = mrsm.get_config('system', 'cli', 'refresh_seconds')
+        return self._refresh_seconds
 
     def __str__(self) -> str:
         return f"StdinFile('{self.file_path}')"

meerschaum/utils/daemon/__init__.py

@@ -16,8 +16,7 @@ import datetime
 import threading
 import shlex
 
-from meerschaum.utils.typing import SuccessTuple, List, Optional, Callable, Any, Dict
-from meerschaum.config._paths import DAEMON_RESOURCES_PATH
+from meerschaum.utils.typing import SuccessTuple, List, Optional, Callable, Any, Dict, Union
 from meerschaum.utils.daemon.StdinFile import StdinFile
 from meerschaum.utils.daemon.Daemon import Daemon
 from meerschaum.utils.daemon.RotatingFile import RotatingFile
@@ -41,9 +40,10 @@ __all__ = (
     'StdinFile',
     'RotatingFile',
     'FileDescriptorInterceptor',
-    'DAEMON_RESOURCES_PATH',
 )
 
+_daemons = {}
+
 
 def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     """Parse sysargs and execute a Meerschaum action as a daemon.
@@ -211,6 +211,7 @@ def get_daemon_ids() -> List[str]:
     """
     Return the IDs of all daemons on disk.
     """
+    from meerschaum.config._paths import DAEMON_RESOURCES_PATH
     return [
         daemon_dir
         for daemon_dir in sorted(os.listdir(DAEMON_RESOURCES_PATH))
@@ -298,8 +299,6 @@ def get_filtered_daemons(
             if warn:
                 _warn(f"Daemon '{d_id}' does not exist.", stack=False)
             continue
-        if d.hidden:
-            pass
         daemons.append(d)
     return daemons
 
@@ -311,3 +310,17 @@ def running_in_daemon() -> bool:
     from meerschaum._internal.static import STATIC_CONFIG
     daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
     return daemon_env_var in os.environ
+
+
+def get_current_daemon() -> Union[Daemon, None]:
+    """
+    If running withing a daemon context, return the corresponding `Daemon`.
+    Otherwise return `None`.
+    """
+    from meerschaum._internal.static import STATIC_CONFIG
+    daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
+    daemon_id = os.environ.get(daemon_env_var, None)
+    if daemon_id is None:
+        return None
+
+    return _daemons.get(daemon_id, Daemon(daemon_id=daemon_id))

meerschaum/utils/daemon/_names.py

@@ -7,9 +7,11 @@ Generate random names for jobs.
 """
 
 from __future__ import annotations
-import os, random
-from meerschaum.utils.typing import Dict, List, Tuple
-from meerschaum.config._paths import DAEMON_RESOURCES_PATH
+
+import os
+import random
+
+from meerschaum.utils.typing import Dict, List
 
 _bank: Dict[str, Dict[str, List[str]]] = {
     'adjectives': {
@@ -116,6 +118,7 @@ def get_new_daemon_name() -> str:
     Generate a new random name until a unique one is found
     (up to ~6000 maximum possibilities).
     """
+    from meerschaum.config._paths import DAEMON_RESOURCES_PATH
     existing_names = (
         os.listdir(DAEMON_RESOURCES_PATH)
         if DAEMON_RESOURCES_PATH.exists()

meerschaum/utils/debug.py

@@ -7,11 +7,31 @@ Functions to handle debug statements
 """
 
 from __future__ import annotations
+
+from datetime import datetime, timezone
+import meerschaum as mrsm
 from meerschaum.utils.typing import Union, Optional, List
 
+
+_rich_text = None
+def _import_rich_text_for_dprint():
+    """
+    Avoid calling `attempt_import()` on every dprint.
+    """
+    global _rich_text
+    if _rich_text is not None:
+        return _rich_text
+
+    from meerschaum.utils.packages import import_rich, attempt_import
+    _ = import_rich()
+    _rich_text = attempt_import('rich.text', lazy=False)
+    return _rich_text
+
+
 def dprint(
     msg: str,
     leader: bool = True,
+    timestamp: bool = True,
     package: bool = True,
     color: Optional[Union[str, List[str]]] = None,
     attrs: Optional[List[str]] = None,
@@ -35,6 +55,12 @@ def dprint(
     else:
         CHARSET, ANSI, colored, _color, cf = 'ascii', False, None, None, None
 
+    if timestamp:
+        from meerschaum.utils.dtypes import get_current_timestamp
+        now = get_current_timestamp('ms').replace(tzinfo=None)
+    else:
+        now = None
+
     import logging, sys, inspect
     logging.basicConfig(format='%(message)s')
     log = logging.getLogger(__name__)
@@ -46,8 +72,13 @@ def dprint(
     parent_package = parent_globals['__name__']
     msg = str(msg)
     premsg = ""
+
+    if now:
+        premsg = now.isoformat().split('T', maxsplit=1)[-1][:-3] + (' | ' if package else ':')
     if package:
-        premsg = parent_package + ':' + str(parent_lineno) + '\n'
+        premsg = premsg + parent_package + ':' + str(parent_lineno)
+    if premsg:
+        premsg += "\n"
     if leader and cf is not None:
         try:
             debug_leader = cf['formatting']['debug'][CHARSET]['icon'] if cf is not None else ''
@@ -75,10 +106,9 @@ def dprint(
                 _color = {}
         if colored is not None:
             premsg = colored(premsg, **_color)
+
     if _progress is not None:
-        from meerschaum.utils.packages import import_rich, attempt_import
-        rich = import_rich()
-        rich_text = attempt_import('rich.text')
+        rich_text = _import_rich_text_for_dprint()
         text = rich_text.Text.from_ansi(premsg + msg)
         _progress.console.log(text)
     else:

meerschaum/utils/dtypes/__init__.py

@@ -1054,7 +1054,7 @@ def get_current_timestamp(
     return ts_val.as_unit(MRSM_PRECISION_UNITS_ABBREVIATIONS[true_precision_unit])
 
 
-def dtype_is_special(type_: str) -> bool:
+def is_dtype_special(type_: str) -> bool:
     """
     Return whether a dtype should be treated as a special Meerschaum dtype.
     This is not the same as a Meerschaum alias.
@@ -1069,6 +1069,7 @@ def dtype_is_special(type_: str) -> bool:
         'geometry',
         'geography',
         'date',
+        'bool',
     ):
         return True
 
@@ -1081,6 +1082,9 @@ def dtype_is_special(type_: str) -> bool:
     if true_type.startswith('numeric'):
         return True
 
+    if true_type.startswith('bool'):
+        return True
+
     if true_type.startswith('geometry'):
         return True
 

meerschaum/utils/formatting/__init__.py

@@ -209,6 +209,8 @@ def get_console():
     try:
         console = rich_console.Console(force_terminal=True, color_system='truecolor')
     except Exception:
+        import traceback
+        traceback.print_exc()
         console = None
     return console
 
@@ -488,8 +490,9 @@ def fill_ansi(string: str, style: str = '') -> str:
         msg = Text.from_ansi(string)
     except AttributeError:
         import traceback
-        traceback.print_stack()
+        traceback.print_exc()
         msg = ''
+
     plain_indices = []
     for left_span, right_span in iterate_chunks(msg.spans, 2, fillvalue=len(msg)):
         left = left_span.end

meerschaum/utils/formatting/_jobs.py

@@ -97,7 +97,7 @@ def pprint_jobs(
             msg = '\n'.join(line.lstrip().rstrip() for line in lines)
             success_tuple = success, msg
             success_tuple_str = (
-                format_success_tuple(success_tuple, left_padding=1)
+                format_success_tuple(success_tuple, left_padding=0)
                 if success_tuple is not None
                 else None
             )

meerschaum/utils/formatting/_pipes.py

@@ -277,18 +277,31 @@ def pprint_pipe_columns(
 
 
 def pipe_repr(
-    pipe: mrsm.Pipe,
+    pipe: Union[mrsm.Pipe, Dict[str, Any]],
     as_rich_text: bool = False,
     ansi: Optional[bool] = None,
 ) -> Union[str, 'rich.text.Text']:
     """
     Return a formatted string for representing a `meerschaum.Pipe`.
     """
-    from meerschaum.utils.formatting import UNICODE, ANSI, CHARSET, colored, rich_text_to_str
+    from meerschaum.utils.formatting import ANSI, colored, rich_text_to_str
     from meerschaum.utils.packages import import_rich, attempt_import
-    rich = import_rich()
+    import meerschaum as mrsm
+
+    _ = import_rich()
     Text = attempt_import('rich.text').Text
 
+    if isinstance(pipe, mrsm.Pipe):
+        connector_keys = pipe.connector_keys
+        metric_key = pipe.metric_key
+        location_key = pipe.location_key
+        instance_keys = pipe.instance_keys
+    else:
+        connector_keys = pipe.get('connector_keys')
+        metric_key = pipe.get('metric_key')
+        location_key = pipe.get('location_key')
+        instance_keys = pipe.get('instance_keys', get_config('meerschaum', 'instance'))
+
     styles = get_config('formatting', 'pipes', '__repr__', 'ansi', 'styles')
     if not ANSI or (ansi is False):
         styles = {k: '' for k in styles}
@@ -298,26 +311,26 @@ def pipe_repr(
     )
     text_obj = (
         Text.from_markup(_pipe_style_prefix + "Pipe(" + _pipe_style_suffix)
-        + colored(("'" + pipe.connector_keys + "'"), style=styles['connector'], as_rich_text=True)
+        + colored(("'" + connector_keys + "'"), style=styles['connector'], as_rich_text=True)
         + Text.from_markup(_pipe_style_prefix + ", " + _pipe_style_suffix)
-        + colored(("'" + pipe.metric_key + "'"), style=styles['metric'], as_rich_text=True)
+        + colored(("'" + metric_key + "'"), style=styles['metric'], as_rich_text=True)
         + (
             (
                 colored(', ', style=styles['punctuation'], as_rich_text=True)
                 + colored(
-                    ("'" + pipe.location_key + "'"),
+                    ("'" + location_key + "'"),
                     style=styles['location'], as_rich_text=True
                 )
-            ) if pipe.location_key is not None
+            ) if location_key is not None
             else colored('', style='', as_rich_text=True)
         ) + (
             ( ### Add the `instance=` argument.
                 colored(', instance=', style=styles['punctuation'], as_rich_text=True)
                 + colored(
-                    ("'" + pipe.instance_keys + "'"),
+                    ("'" + instance_keys + "'"),
                     style=styles['instance'], as_rich_text=True
                 )
-            ) if pipe.instance_keys != get_config('meerschaum', 'instance')
+            ) if instance_keys != get_config('meerschaum', 'instance')
             else colored('', style='', as_rich_text=True)
         )
         + Text.from_markup(_pipe_style_prefix + ")" + _pipe_style_suffix)
@@ -327,6 +340,7 @@ def pipe_repr(
     return rich_text_to_str(text_obj).replace('\n', '')
 
 
+
 def highlight_pipes(message: str) -> str:
     """
     Add syntax highlighting to an info message containing stringified `meerschaum.Pipe` objects.
@@ -334,48 +348,35 @@ def highlight_pipes(message: str) -> str:
     if 'Pipe(' not in message:
         return message
 
-    from meerschaum import Pipe
+    from meerschaum.utils.misc import parse_arguments_str
     segments = message.split('Pipe(')
     msg = ''
-    _d = {}
     for i, segment in enumerate(segments):
-        comma_index = segment.find(',')
-        paren_index = segment.find(')')
-        single_quote_index = segment.find("'")
-        double_quote_index = segment.find('"')
-
-        has_comma = comma_index != -1
-        has_paren = paren_index != -1
-        has_single_quote = single_quote_index != -1
-        has_double_quote = double_quote_index != -1
-        has_quote = has_single_quote or has_double_quote
-        quote_index = (
-            min(single_quote_index, double_quote_index)
-            if has_double_quote and has_single_quote
-            else (single_quote_index if has_single_quote else double_quote_index)
-        )
-
-        has_pipe = (
-            has_comma
-            and
-            has_paren
-            and
-            has_quote
-            and not
-            (comma_index > paren_index or quote_index > paren_index)
-        )
+        if i == 0:
+            msg += segment
+            continue
 
-        if has_pipe:
-            code = "_d['pipe'] = Pipe(" + segment[:paren_index + 1]
-            try:
-                exec(code)
-                _to_add = pipe_repr(_d['pipe']) + segment[paren_index + 1:]
-                _ = _d.pop('pipe', None)
-            except Exception as e:
-                _to_add = 'Pipe(' + segment
-            msg += _to_add
+        paren_index = segment.find(')')
+        if paren_index == -1:
+            msg += 'Pipe(' + segment
             continue
-        msg += segment
+        
+        pipe_args_str = segment[:paren_index]
+        try:
+            args, kwargs = parse_arguments_str(pipe_args_str)
+            pipe_dict = {
+                'connector_keys': args[0],
+                'metric_key': args[1],
+            }
+            if len(args) > 2:
+                pipe_dict['location_key'] = args[2]
+            if 'instance' in kwargs:
+                pipe_dict['instance_keys'] = kwargs['instance']
+            
+            _to_add = pipe_repr(pipe_dict) + segment[paren_index + 1:]
+        except Exception:
+            _to_add = 'Pipe(' + segment
+        msg += _to_add
     return msg
 
 

meerschaum/utils/formatting/_shell.py

@@ -6,7 +6,6 @@
 Formatting functions for the interactive shell
 """
 
-from re import sub
 from meerschaum.utils.threading import Lock
 _locks = {'_tried_clear_command': Lock()}
 
@@ -51,28 +50,39 @@ def clear_screen(debug: bool = False) -> bool:
     from meerschaum.utils.debug import dprint
     from meerschaum.config import get_config
     from meerschaum.utils.daemon import running_in_daemon
+    from meerschaum._internal.static import STATIC_CONFIG
     global _tried_clear_command
+    clear_string = '\033[2J'
+    reset_string = '\033[0m'
+    clear_token = STATIC_CONFIG['jobs']['clear_token']
 
-    if running_in_daemon():
+    if not get_config('shell', 'clear_screen'):
         return True
 
-    if not get_config('shell', 'clear_screen'):
+    if running_in_daemon():
+        if debug:
+            dprint("Skip printing clear token.")
+            print('\n', end='', flush=True)
+            return True
+        print(clear_token, flush=True)
         return True
 
+
     print("", end="", flush=True)
     if debug:
         dprint("Skipping screen clear.")
         return True
+
     if ANSI and platform.system() != 'Windows':
         if get_console() is not None:
             get_console().clear()
             print("", end="", flush=True)
             return True
-        clear_string = '\033[2J'
-        reset_string = '\033[0m'
+
         print(clear_string + reset_string, end="")
         print("", end="", flush=True)
         return True
+
     ### ANSI support is disabled, try system level instead
     if _tried_clear_command is not None:
         return os.system(_tried_clear_command) == 0
@@ -81,7 +91,7 @@ def clear_screen(debug: bool = False) -> bool:
     command = 'clear' if platform.system() != 'Windows' else 'cls'
     try:
         rc = os.system(command)
-    except Exception as e:
+    except Exception:
         pass
     if rc == 0:
         with _locks['_tried_clear_command']:

meerschaum/utils/misc.py

@@ -6,9 +6,11 @@ Miscellaneous functions go here
 """
 
 from __future__ import annotations
+
 import sys
 import functools
-from datetime import timedelta, datetime, timezone
+from datetime import timedelta, datetime
+
 from meerschaum.utils.typing import (
     Union,
     Any,
@@ -20,13 +22,8 @@ from meerschaum.utils.typing import (
     Iterable,
     PipesDict,
     Tuple,
-    InstanceConnector,
-    Hashable,
-    Generator,
-    Iterator,
     TYPE_CHECKING,
 )
-import meerschaum as mrsm
 if TYPE_CHECKING:
     import collections
 
@@ -43,6 +40,9 @@ __pdoc__: Dict[str, bool] = {
     'df_is_chunk_generator': False,
     'choices_docstring': False,
     '_get_subaction_names': False,
+    'is_pipe_registered': False,
+    'replace_pipes_in_dict': False,
+    'round_time': False,
 }
 
 
@@ -347,37 +347,6 @@ def edit_file(
     return rc == 0
 
 
-def is_pipe_registered(
-    pipe: mrsm.Pipe,
-    pipes: PipesDict,
-    debug: bool = False
-) -> bool:
-    """
-    Check if a Pipe is inside the pipes dictionary.
-
-    Parameters
-    ----------
-    pipe: meerschaum.Pipe
-        The pipe to see if it's in the dictionary.
-
-    pipes: PipesDict
-        The dictionary to search inside.
-
-    debug: bool, default False
-        Verbosity toggle.
-
-    Returns
-    -------
-    A bool indicating whether the pipe is inside the dictionary.
-    """
-    from meerschaum.utils.debug import dprint
-    ck, mk, lk = pipe.connector_keys, pipe.metric_key, pipe.location_key
-    if debug:
-        dprint(f'{ck}, {mk}, {lk}')
-        dprint(f'{pipe}, {pipes}')
-    return ck in pipes and mk in pipes[ck] and lk in pipes[ck][mk]
-
-
 def get_cols_lines(default_cols: int = 100, default_lines: int = 120) -> Tuple[int, int]:
     """
     Determine the columns and lines in the terminal.
@@ -1263,11 +1232,13 @@ def is_systemd_available() -> bool:
     import subprocess
     try:
         has_systemctl = subprocess.call(
-            ['systemctl', '-h'],
+            ['systemctl', 'whoami'],
             stdout=subprocess.DEVNULL,
             stderr=subprocess.STDOUT,
         ) == 0
     except Exception:
+        import traceback
+        traceback.print_exc()
         has_systemctl = False
     return has_systemctl
 
@@ -1844,6 +1815,15 @@ def replace_pipes_in_dict(*args, **kwargs):
     return replace_pipes_in_dict(*args, **kwargs)
 
 
+def is_pipe_registered(*args, **kwargs):
+    """
+    Placeholder function to prevent breaking legacy behavior.
+    See `meerschaum.utils.pipes.is_pipe_registered`.
+    """
+    from meerschaum.utils.pipes import is_pipe_registered
+    return is_pipe_registered(*args, **kwargs)
+
+
 def round_time(*args, **kwargs):
     """
     Placeholder function to prevent breaking legacy behavior.