meerschaum 2.2.5.dev3__py3-none-any.whl → 2.2.6__py3-none-any.whl

meerschaum/config/__init__.py

@@ -26,7 +26,14 @@ from meerschaum.config._paths import (
 from meerschaum.config._patch import (
     apply_patch_to_config,
 )
-__all__ = ('get_plugin_config', 'write_plugin_config', 'get_config', 'write_config', 'set_config',)
+__all__ = (
+    'get_plugin_config',
+    'write_plugin_config',
+    'get_config',
+    'write_config',
+    'set_config',
+    'paths',
+)
 __pdoc__ = {'static': False, 'resources': False, 'stack': False, }
 _locks = {'config': RLock()}
 

meerschaum/config/_paths.py

@@ -34,12 +34,29 @@ if ENVIRONMENT_ROOT_DIR in os.environ:
             f"Invalid root directory '{str(_ROOT_DIR_PATH)}' set for " +
             f"environment variable '{ENVIRONMENT_ROOT_DIR}'.\n" +
             f"Please enter a valid path for {ENVIRONMENT_ROOT_DIR}.",
-            file = sys.stderr,
+            file=sys.stderr,
         )
         sys.exit(1)
 else:
     _ROOT_DIR_PATH = DEFAULT_ROOT_DIR_PATH
 
+
+ENVIRONMENT_CONFIG_DIR = STATIC_CONFIG['environment']['config_dir']
+if ENVIRONMENT_CONFIG_DIR in os.environ:
+    _CONFIG_DIR_PATH = Path(os.environ[ENVIRONMENT_CONFIG_DIR]).resolve()
+    if not _CONFIG_DIR_PATH.exists():
+        print(
+            (
+                f"Invalid configuration directory '{_CONFIG_DIR_PATH}' set"
+                + f" for environment variable '{ENVIRONMENT_CONFIG_DIR}'\n"
+                + f"Please enter a valid path for {ENVIRONMENT_CONFIG_DIR}."
+            ),
+            file=sys.stderr,
+        )
+        sys.exit(1)
+else:
+    _CONFIG_DIR_PATH = _ROOT_DIR_PATH / 'config'
+
 ENVIRONMENT_PLUGINS_DIR = STATIC_CONFIG['environment']['plugins']
 if ENVIRONMENT_PLUGINS_DIR in os.environ:
     try:
@@ -100,7 +117,7 @@ paths = {
     'PACKAGE_ROOT_PATH'              : Path(__file__).parent.parent.resolve().as_posix(),
     'ROOT_DIR_PATH'                  : _ROOT_DIR_PATH.as_posix(),
     'VIRTENV_RESOURCES_PATH'         : _VENVS_DIR_PATH.as_posix(),
-    'CONFIG_DIR_PATH'                : ('{ROOT_DIR_PATH}', 'config'),
+    'CONFIG_DIR_PATH'                : _CONFIG_DIR_PATH.as_posix(),
     'DEFAULT_CONFIG_DIR_PATH'        : ('{ROOT_DIR_PATH}', 'default_config'),
     'PATCH_DIR_PATH'                 : ('{ROOT_DIR_PATH}', 'patch_config'),
     'PERMANENT_PATCH_DIR_PATH'       : ('{ROOT_DIR_PATH}', 'permanent_patch_config'),
@@ -168,6 +185,7 @@ def set_root(root: Union[Path, str]):
         if isinstance(path_parts, tuple) and path_parts[0] == '{ROOT_DIR_PATH}':
             globals()[path_name] = __getattr__(path_name)
 
+
 def __getattr__(name: str) -> Path:
     if name not in paths:
         if name not in globals():

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.2.5.dev3"
+__version__ = "2.2.6"

meerschaum/config/paths.py

@@ -6,5 +6,24 @@
 External API for importing Meerschaum paths.
 """
 
-from meerschaum.config._paths import __getattr__, paths
-__all__ = tuple(paths.keys())
+import pathlib
+import inspect
+import meerschaum.config._paths as _paths
+
+
+def __getattr__(*args, **kwargs):
+    return _paths.__getattr__(*args, **kwargs)
+
+
+_globals_dict = inspect.getmembers(
+    _paths,
+    lambda member: not inspect.isroutine(member)
+)
+_all_caps_globals = [
+    name
+    for name, value in _globals_dict
+    if ('PATH' in name or 'FILE' in name) and not name.startswith('_')
+    and isinstance(value, pathlib.Path)
+]
+
+__all__ = tuple(_all_caps_globals + list(_paths.paths.keys()))

meerschaum/config/static/__init__.py

@@ -46,6 +46,7 @@ STATIC_CONFIG: Dict[str, Any] = {
     },
     'environment': {
         'config': 'MRSM_CONFIG',
+        'config_dir': 'MRSM_CONFIG_DIR',
         'patch': 'MRSM_PATCH',
         'root': 'MRSM_ROOT_DIR',
         'plugins': 'MRSM_PLUGINS_DIR',

meerschaum/connectors/Connector.py

@@ -18,7 +18,7 @@ class InvalidAttributesError(Exception):
 
 class Connector(metaclass=abc.ABCMeta):
     """
-    The base connector class to hold connection attributes,
+    The base connector class to hold connection attributes.
     """
     def __init__(
             self,
@@ -27,6 +27,8 @@ class Connector(metaclass=abc.ABCMeta):
             **kw: Any
         ):
         """
+        Set the given keyword arguments as attributes.
+
         Parameters
         ----------
         type: str
@@ -35,9 +37,12 @@ class Connector(metaclass=abc.ABCMeta):
         label: str
             The `label` for the connector.
 
+
+        Examples
+        --------
         Run `mrsm edit config` and to edit connectors in the YAML file:
 
-        ```
+        ```yaml
         meerschaum:
             connections:
                 {type}:

meerschaum/connectors/__init__.py

@@ -24,6 +24,7 @@ from meerschaum.connectors.api.APIConnector import APIConnector
 from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
 
 __all__ = (
+    "make_connector",
     "Connector",
     "SQLConnector",
     "APIConnector",
@@ -290,13 +291,14 @@ def make_connector(
     --------
     >>> import meerschaum as mrsm
     >>> from meerschaum.connectors import make_connector, Connector
+    >>> 
+    >>> @make_connector
     >>> class FooConnector(Connector):
-    ...     def __init__(self, label: str, **kw):
-    ...         super().__init__('foo', label, **kw)
+    ...     REQUIRED_ATTRIBUTES: list[str] = ['username', 'password']
     ... 
-    >>> make_connector(FooConnector)
-    >>> mrsm.get_connector('foo', 'bar')
-    foo:bar
+    >>> conn = mrsm.get_connector('foo:bar', username='dog', password='cat')
+    >>> print(conn.username, conn.password)
+    dog cat
     >>> 
     """
     import re

meerschaum/connectors/api/APIConnector.py

@@ -83,12 +83,16 @@ class APIConnector(Connector):
         if 'uri' in kw:
             from_uri_params = self.from_uri(kw['uri'], as_dict=True)
             label = label or from_uri_params.get('label', None)
-            from_uri_params.pop('label', None)
+            _ = from_uri_params.pop('label', None)
             kw.update(from_uri_params)
 
         super().__init__('api', label=label, **kw)
         if 'protocol' not in self.__dict__:
-            self.protocol = 'http'
+            self.protocol = (
+                'https' if self.__dict__.get('uri', '').startswith('https')
+                else 'http'
+            )
+
         if 'uri' not in self.__dict__:
             self.verify_attributes(required_attributes)
         else:
@@ -97,6 +101,7 @@ class APIConnector(Connector):
             if 'host' not in conn_attrs:
                 raise Exception(f"Invalid URI for '{self}'.")
             self.__dict__.update(conn_attrs)
+
         self.url = (
             self.protocol + '://' +
             self.host

meerschaum/connectors/api/_actions.py

@@ -9,44 +9,31 @@ Functions to interact with /mrsm/actions
 from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Optional, List
 
-def get_actions(
-        self,
-    ) -> list:
-    """Get available actions from the API server"""
+def get_actions(self) -> list:
+    """Get available actions from the API instance."""
     from meerschaum.config.static import STATIC_CONFIG
     return self.get(STATIC_CONFIG['api']['endpoints']['actions'])
 
 
 def do_action(
-        self,
-        action: Optional[List[str]] = None,
-        sysargs: Optional[List[str]] = None,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    self,
+    action: Optional[List[str]] = None,
+    sysargs: Optional[List[str]] = None,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """Execute a Meerschaum action remotely.
-    
-    If sysargs is provided, parse those instead. Otherwise infer everything from keyword arguments.
-    
-    NOTE: The first index of `action` should NOT be removed!
-    Example: action = ['show', 'config']
-    
-    Returns: tuple (succeeded : bool, message : str)
 
-    Parameters
-    ----------
-    action: Optional[List[str]] :
-         (Default value = None)
-    sysargs: Optional[List[str]] :
-         (Default value = None)
-    debug: bool :
-         (Default value = False)
-    **kw :
-        
-
-    Returns
-    -------
+    If `sysargs` are provided, parse those instead.
+    Otherwise infer everything from keyword arguments.
 
+    Examples
+    --------
+    >>> conn = mrsm.get_connector('api:main')
+    >>> conn.do_action(['show', 'pipes'])
+    (True, "Success")
+    >>> conn.do_action(['show', 'arguments'], name='test')
+    (True, "Success")
     """
     import sys, json
     from meerschaum.utils.debug import dprint
@@ -63,7 +50,12 @@ def do_action(
     else:
         json_dict = kw
         json_dict['action'] = action
-        json_dict['debug'] = debug
+        if 'noask' not in kw:
+            json_dict['noask'] = True
+        if 'yes' not in kw:
+            json_dict['yes'] = True
+        if debug:
+            json_dict['debug'] = debug
 
     root_action = json_dict['action'][0]
     del json_dict['action'][0]

meerschaum/connectors/api/_uri.py

@@ -11,11 +11,11 @@ from meerschaum.utils.warnings import warn, error
 
 @classmethod
 def from_uri(
-        cls,
-        uri: str,
-        label: Optional[str] = None,
-        as_dict: bool = False,
-    ) -> Union[
+    cls,
+    uri: str,
+    label: Optional[str] = None,
+    as_dict: bool = False,
+) -> Union[
         'meerschaum.connectors.APIConnector',
         Dict[str, Union[str, int]],
     ]:

meerschaum/core/Pipe/__init__.py

@@ -50,6 +50,7 @@ with correct credentials, as well as a network connection and valid permissions.
 """
 
 from __future__ import annotations
+import sys
 import copy
 from meerschaum.utils.typing import Optional, Dict, Any, Union, InstanceConnector, List
 from meerschaum.utils.formatting._pipes import pipe_repr
@@ -433,13 +434,16 @@ class Pipe:
             + str(self.instance_keys) + sep
         )
 
-    def __repr__(self, **kw) -> str:
-        return pipe_repr(self, **kw)
+    def __repr__(self, ansi: bool=True, **kw) -> str:
+        if not hasattr(sys, 'ps1'):
+            ansi = False
+
+        return pipe_repr(self, ansi=ansi, **kw)
 
     def __pt_repr__(self):
         from meerschaum.utils.packages import attempt_import
         prompt_toolkit_formatted_text = attempt_import('prompt_toolkit.formatted_text', lazy=False)
-        return prompt_toolkit_formatted_text.ANSI(self.__repr__())
+        return prompt_toolkit_formatted_text.ANSI(pipe_repr(self, ansi=True))
 
     def __getstate__(self) -> Dict[str, Any]:
         """

meerschaum/core/Pipe/_data.py

@@ -8,24 +8,32 @@ Retrieve Pipes' data from instances.
 
 from __future__ import annotations
 from datetime import datetime, timedelta
-from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator, List, Tuple, Iterator
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import (
+    Optional, Dict, Any, Union, List, Tuple, Iterator, TYPE_CHECKING,
+)
 from meerschaum.config import get_config
 
+if TYPE_CHECKING:
+    pd = mrsm.attempt_import('pandas')
+
+
 def get_data(
-        self,
-        select_columns: Optional[List[str]] = None,
-        omit_columns: Optional[List[str]] = None,
-        begin: Union[datetime, int, None] = None,
-        end: Union[datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        as_iterator: bool = False,
-        as_chunks: bool = False,
-        as_dask: bool = False,
-        chunk_interval: Union[timedelta, int, None] = None,
-        fresh: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union['pd.DataFrame', Generator['pd.DataFrame'], None]:
+    self,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    as_iterator: bool = False,
+    as_chunks: bool = False,
+    as_dask: bool = False,
+    chunk_interval: Union[timedelta, int, None] = None,
+    fresh: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Union['pd.DataFrame', Iterator['pd.DataFrame'], None]:
     """
     Get a pipe's data from the instance connector.
 

meerschaum/core/Pipe/_deduplicate.py

@@ -8,7 +8,7 @@ Delete duplicate rows within a pipe's table.
 
 from __future__ import annotations
 from datetime import datetime, timedelta
-from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Tuple
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Tuple, Union
 
 
 def deduplicate(

meerschaum/core/Pipe/_dtypes.py

@@ -8,7 +8,12 @@ Enforce data types for a pipe's underlying table.
 
 from __future__ import annotations
 from io import StringIO
+import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Any, Optional
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    pd = mrsm.attempt_import('pandas')
 
 def enforce_dtypes(
         self,

meerschaum/core/Pipe/_fetch.py

@@ -8,11 +8,14 @@ Functions for fetching new data into the Pipe
 
 from __future__ import annotations
 from datetime import timedelta, datetime
+
 import meerschaum as mrsm
-from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator
+from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator, TYPE_CHECKING
 from meerschaum.config import get_config
 from meerschaum.utils.warnings import warn
-from meerschaum.utils.misc import filter_keywords
+
+if TYPE_CHECKING:
+    pd = mrsm.attempt_import('pandas')
 
 def fetch(
         self,
@@ -55,6 +58,7 @@ def fetch(
 
     from meerschaum.connectors import custom_types, get_connector_plugin
     from meerschaum.utils.debug import dprint, _checkpoint
+    from meerschaum.utils.misc import filter_arguments
 
     _chunk_hook = kw.pop('chunk_hook', None)
     kw['workers'] = self.get_num_workers(kw.get('workers', None))
@@ -72,24 +76,22 @@ def fetch(
                 chunk_message = '\n' + chunk_label + '\n' + chunk_message
             return chunk_success, chunk_message
 
-
     with mrsm.Venv(get_connector_plugin(self.connector)):
-        df = self.connector.fetch(
+        _args, _kwargs = filter_arguments(
+            self.connector.fetch,
             self,
-            **filter_keywords(
-                self.connector.fetch,
-                begin=_determine_begin(
-                    self,
-                    begin,
-                    check_existing=check_existing,
-                    debug=debug,
-                ),
-                end=end,
-                chunk_hook=_chunk_hook,
+            begin=_determine_begin(
+                self,
+                begin,
+                check_existing=check_existing,
                 debug=debug,
-                **kw
-            )
+            ),
+            end=end,
+            chunk_hook=_chunk_hook,
+            debug=debug,
+            **kw
         )
+        df = self.connector.fetch(*_args, **_kwargs)
     return df
 
 

meerschaum/core/Pipe/_sync.py

@@ -14,7 +14,9 @@ import threading
 import multiprocessing
 import functools
 from datetime import datetime, timedelta
+from typing import TYPE_CHECKING
 
+import meerschaum as mrsm
 from meerschaum.utils.typing import (
     Union,
     Optional,
@@ -26,13 +28,16 @@ from meerschaum.utils.typing import (
     List,
     Iterable,
     Generator,
-    Iterator,
 )
 from meerschaum.utils.warnings import warn, error
 
+if TYPE_CHECKING:
+    pd = mrsm.attempt_import('pandas')
+
 class InferFetch:
     MRSM_INFER_FETCH: bool = True
 
+
 def sync(
     self,
     df: Union[
@@ -125,7 +130,7 @@ def sync(
     from meerschaum.utils.formatting import get_console
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
-    from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords
+    from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords, filter_arguments
     from meerschaum.utils.pool import get_pool
     from meerschaum.config import get_config
 
@@ -210,28 +215,28 @@ def sync(
             ):
                 with Venv(get_connector_plugin(self.instance_connector)):
                     p._exists = None
-                    return self.instance_connector.sync_pipe_inplace(
+                    _args, _kwargs = filter_arguments(
+                        p.instance_connector.sync_pipe_inplace,
                         p,
-                        **filter_keywords(
-                            p.instance_connector.sync_pipe_inplace,
-                            debug=debug,
-                            **kw
-                        )
+                        debug=debug,
+                        **kw
+                    )
+                    return self.instance_connector.sync_pipe_inplace(
+                        *_args,
+                        **_kwargs
                     )
-
 
             ### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods.
             try:
                 if getattr(p.connector, 'sync', None) is not None:
                     with Venv(get_connector_plugin(p.connector), debug=debug):
-                        return_tuple = p.connector.sync(
+                        _args, _kwargs = filter_arguments(
+                            p.connector.sync,
                             p,
-                            **filter_keywords(
-                                p.connector.sync,
-                                debug=debug,
-                                **kw
-                            )
+                            debug=debug,
+                            **kw
                         )
+                        return_tuple = p.connector.sync(*_args, **_kwargs)
                     p._exists = None
                     if not isinstance(return_tuple, tuple):
                         return_tuple = (

meerschaum/plugins/_Plugin.py

@@ -654,17 +654,18 @@ class Plugin:
             import ast, re
             ### NOTE: This technically would break 
             ### if `required` was the very first line of the file.
-            req_start_match = re.search(r'\nrequired(\s?)=', text)
+            req_start_match = re.search(r'required(:\s*)?.*=', text)
             if not req_start_match:
                 return []
             req_start = req_start_match.start()
+            equals_sign = req_start + text[req_start:].find('=')
 
             ### Dependencies may have brackets within the strings, so push back the index.
-            first_opening_brace = req_start + 1 + text[req_start:].find('[')
+            first_opening_brace = equals_sign + 1 + text[equals_sign:].find('[')
             if first_opening_brace == -1:
                 return []
 
-            next_closing_brace = req_start + 1 + text[req_start:].find(']')
+            next_closing_brace = equals_sign + 1 + text[equals_sign:].find(']')
             if next_closing_brace == -1:
                 return []
 
@@ -681,12 +682,11 @@ class Plugin:
 
             req_end = end_ix + 1
             req_text = (
-                text[req_start:req_end]
-                .lstrip()
-                .replace('required', '', 1)
+                text[(first_opening_brace-1):req_end]
                 .lstrip()
                 .replace('=', '', 1)
                 .lstrip()
+                .rstrip()
             )
             try:
                 required = ast.literal_eval(req_text)

meerschaum/plugins/__init__.py

@@ -27,7 +27,7 @@ _locks = {
     'PLUGINS_INTERNAL_LOCK_PATH': RLock(),
 }
 __all__ = (
-    "Plugin", "make_action", "api_plugin", "import_plugins",
+    "Plugin", "make_action", "api_plugin", "dash_plugin", "import_plugins",
     "reload_plugins", "get_plugins", "get_data_plugins", "add_plugin_argument",
     "pre_sync_hook", "post_sync_hook",
 )

meerschaum/utils/daemon/RotatingFile.py

@@ -32,14 +32,14 @@ class RotatingFile(io.IOBase):
     SEEK_BACK_ATTEMPTS: int = 5
 
     def __init__(
-            self,
-            file_path: pathlib.Path,
-            num_files_to_keep: Optional[int] = None,
-            max_file_size: Optional[int] = None,
-            redirect_streams: bool = False,
-            write_timestamps: bool = False,
-            timestamp_format: str = '%Y-%m-%d %H:%M',
-        ):
+        self,
+        file_path: pathlib.Path,
+        num_files_to_keep: Optional[int] = None,
+        max_file_size: Optional[int] = None,
+        redirect_streams: bool = False,
+        write_timestamps: bool = False,
+        timestamp_format: str = '%Y-%m-%d %H:%M',
+    ):
         """
         Create a file-like object which manages other files.
 
@@ -79,11 +79,7 @@ class RotatingFile(io.IOBase):
         self.redirect_streams = redirect_streams
         self.write_timestamps = write_timestamps
         self.timestamp_format = timestamp_format
-        self.subfile_regex_pattern = re.compile(
-            r'^'
-            + self.file_path.name
-            + r'(?:\.\d+)?$'
-        )
+        self.subfile_regex_pattern = re.compile(r'(.*)\.log(?:\.\d+)?$')
 
         ### When subfiles are opened, map from their index to the file objects.
         self.subfile_objects = {}
@@ -173,7 +169,7 @@ class RotatingFile(io.IOBase):
         latest_index = (
             self.get_index_from_subfile_name(existing_subfile_paths[-1].name)
             if existing_subfile_paths
-            else -1
+            else 0
         )
         return latest_index
 
@@ -222,9 +218,12 @@ class RotatingFile(io.IOBase):
             [
                 (file_name, self.get_index_from_subfile_name(file_name))
                 for file_name in os.listdir(self.file_path.parent)
-                if re.match(self.subfile_regex_pattern, file_name)
+                if (
+                    file_name.startswith(self.file_path.name)
+                    and re.match(self.subfile_regex_pattern, file_name)
+                )
             ],
-            key = lambda x: x[1],
+            key=lambda x: x[1],
         )
         return [
             (self.file_path.parent / file_name)