meerschaum 2.1.2__py3-none-any.whl → 2.1.3.dev3__py3-none-any.whl

meerschaum/actions/show.py

@@ -7,7 +7,8 @@ This module contains functions for printing elements.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Union, Sequence, Any, Optional, List, Dict
+import meerschaum as mrsm
+from meerschaum.utils.typing import SuccessTuple, Union, Sequence, Any, Optional, List, Dict, Tuple
 
 def show(
         action: Optional[List[str]] = None,
@@ -39,6 +40,7 @@ def show(
         'users'      : _show_users,
         'jobs'       : _show_jobs,
         'logs'       : _show_logs,
+        'tags'       : _show_tags,
     }
     return choose_subaction(action, show_options, **kw)
 
@@ -735,6 +737,75 @@ def _show_environment(
     return True, "Success"
 
 
+def _show_tags(
+        tags: Optional[List[str]] = None,
+        workers: Optional[int] = None,
+        nopretty: bool = False,
+        **kwargs
+    ) -> SuccessTuple:
+    """
+    Show the existing tags and their associated pipes.
+    """
+    import json
+    from collections import defaultdict
+    import meerschaum as mrsm
+    from meerschaum.utils.formatting import pipe_repr, UNICODE, ANSI
+    from meerschaum.utils.pool import get_pool
+    from meerschaum.config import get_config
+    rich_tree, rich_panel, rich_text, rich_console, rich_columns = (
+        mrsm.attempt_import('rich.tree', 'rich.panel', 'rich.text', 'rich.console', 'rich.columns')
+    )
+    panel = rich_panel.Panel.fit('Tags')
+    tree = rich_tree.Tree(panel)
+    pipes = mrsm.get_pipes(as_list=True, **kwargs)
+    pool = get_pool(workers=workers)
+    tag_prefix = get_config('formatting', 'pipes', 'unicode', 'icons', 'tag') if UNICODE else ''
+    tag_style = get_config('formatting', 'pipes', 'ansi', 'styles', 'tags') if ANSI else None
+
+    tags_pipes = defaultdict(lambda: [])
+    gather_pipe_tags = lambda pipe: (pipe, (pipe.tags or []))
+
+    pipes_tags = dict(pool.map(gather_pipe_tags, pipes))
+
+    for pipe, tags in pipes_tags.items():
+        for tag in tags:
+            tags_pipes[tag].append(pipe)
+
+    columns = []
+    sorted_tags = sorted([tag for tag in tags_pipes])
+    for tag in sorted_tags:
+        _pipes = tags_pipes[tag]
+        tag_text = (
+            rich_text.Text(tag_prefix)
+            + rich_text.Text(
+                tag,
+                style = tag_style,
+            )
+        )
+        pipes_texts = [
+            pipe_repr(pipe, as_rich_text=True)
+            for pipe in _pipes
+        ]
+        tag_group = rich_console.Group(*pipes_texts)
+        tag_panel = rich_panel.Panel(tag_group, title=tag_text, title_align='left')
+        columns.append(tag_panel)
+
+    if not nopretty:
+        mrsm.pprint(
+            rich_columns.Columns(
+                columns,
+                equal = True,
+            ),
+        )
+    else:
+        for tag, _pipes in tags_pipes.items():
+            print(tag)
+            for pipe in _pipes:
+                print(json.dumps(pipe.meta))
+
+    return True, "Success"
+
+
 
 ### NOTE: This must be the final statement of the module.
 ###       Any subactions added below these lines will not

meerschaum/actions/sync.py

@@ -10,6 +10,7 @@ NOTE: `sync` required a SQL connection and is not intended for client use
 
 from __future__ import annotations
 from datetime import timedelta
+import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Tuple, Union
 
 def sync(
@@ -400,33 +401,60 @@ def _wrap_pipe(
     """
     Wrapper function for handling exceptions.
     """
+    import time
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.venv import Venv
+    from meerschaum.plugins import _pre_sync_hooks, _post_sync_hooks
+    from meerschaum.utils.misc import filter_keywords
+
+    sync_start = time.perf_counter()
+    sync_kwargs = {k: v for k, v in kw.items() if k != 'blocking'}
+    sync_kwargs.update({
+        'blocking': (not unblock),
+        'force': force,
+        'debug': debug,
+        'min_seconds': min_seconds,
+        'workers': workers,
+        'bounded': 'bounded',
+        'chunk_interval': chunk_interval,
+    })
+    if not verify and not deduplicate:
+        sync_method = pipe.sync
+    elif not verify and deduplicate:
+        sync_method = pipe.deduplicate
+    else:
+        sync_method = pipe.verify
+        sync_kwargs['deduplicate'] = deduplicate
+
+    for module_name, pre_sync_hooks in _pre_sync_hooks.items():
+        plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
+        plugin = mrsm.Plugin(plugin_name) if plugin_name else None
+        with Venv(plugin):
+            for pre_sync_hook in pre_sync_hooks:
+                _ = pre_sync_hook(pipe, **filter_keywords(pre_sync_hook, **sync_kwargs))
+   
     try:
         with Venv(get_connector_plugin(pipe.connector), debug=debug):
-            if not verify and not deduplicate:
-                sync_method = pipe.sync
-            elif not verify and deduplicate:
-                sync_method = pipe.deduplicate
-            else:
-                sync_method = pipe.verify
-                kw['deduplicate'] = deduplicate
-            return_tuple = sync_method(
-                blocking = (not unblock),
-                force = force,
-                debug = debug,
-                min_seconds = min_seconds,
-                workers = workers,
-                bounded = bounded,
-                chunk_interval = chunk_interval,
-                **{k: v for k, v in kw.items() if k != 'blocking'}
-            )
+            return_tuple = sync_method(**sync_kwargs)
     except Exception as e:
         import traceback
         traceback.print_exception(type(e), e, e.__traceback__)
         print("Error: " + str(e))
         return_tuple = (False, f"Failed to sync {pipe} with exception:" + "\n" + str(e))
 
+    duration = time.perf_counter() - sync_start
+    sync_kwargs['duration'] = duration
+    for module_name, post_sync_hooks in _post_sync_hooks.items():
+        plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
+        plugin = mrsm.Plugin(plugin_name) if plugin_name else None
+        with Venv(plugin):
+            for post_sync_hook in post_sync_hooks:
+                _ = post_sync_hook(
+                    pipe,
+                    return_tuple,
+                    **filter_keywords(post_sync_hook, **sync_kwargs)
+                )
+ 
     return return_tuple
 
 

meerschaum/config/_edit.py

@@ -6,9 +6,9 @@
 Functions for editing the configuration file
 """
 
-#  from meerschaum.utils.debug import dprint
 from __future__ import annotations
 import sys
+import pathlib
 from meerschaum.utils.typing import Optional, Any, SuccessTuple, Mapping, Dict, List, Union
 
 def edit_config(
@@ -21,7 +21,7 @@ def edit_config(
     from meerschaum.config import get_config, config
     from meerschaum.config._read_config import get_keyfile_path
     from meerschaum.config._paths import CONFIG_DIR_PATH
-    from meerschaum.utils.packages import reload_package
+    from meerschaum.utils.packages import reload_meerschaum
     from meerschaum.utils.misc import edit_file
     from meerschaum.utils.debug import dprint
 
@@ -35,10 +35,7 @@ def edit_config(
         get_config(k, write_missing=True, warn=False)
         edit_file(get_keyfile_path(k, create_new=True))
 
-    if debug:
-        dprint("Reloading configuration...")
-    reload_package('meerschaum', debug=debug, **kw)
-
+    reload_meerschaum(debug=debug)
     return (True, "Success")
 
 
@@ -78,7 +75,7 @@ def write_config(
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.yaml import yaml
     from meerschaum.utils.misc import filter_keywords
-    import json, os, pathlib
+    import json, os
     if config_dict is None:
         from meerschaum.config import _config
         cf = _config()
@@ -162,7 +159,6 @@ def general_write_yaml_config(
     """
 
     from meerschaum.utils.debug import dprint
-    from pathlib import Path
 
     if files is None:
         files = {}
@@ -172,7 +168,7 @@ def general_write_yaml_config(
         header = None
         if isinstance(value, tuple):
             config, header = value
-        path = Path(fp)
+        path = pathlib.Path(fp)
         path.parent.mkdir(parents=True, exist_ok=True)
         path.touch(exist_ok=True)
         with open(path, 'w+') as f:
@@ -193,32 +189,12 @@ def general_write_yaml_config(
     return True
 
 def general_edit_config(
-        action : Optional[List[str]] = None,
-        files : Optional[Dict[str, Union[str, pathlib.Path]]] = None,
-        default : str = None,
-        debug : bool = False
+        action: Optional[List[str]] = None,
+        files: Optional[Dict[str, Union[str, pathlib.Path]]] = None,
+        default: Optional[str] = None,
+        debug: bool = False
     ):
-    """Edit any config files
-
-    Parameters
-    ----------
-    action : Optional[List[str]] :
-         (Default value = None)
-    files : Optional[Dict[str :
-        
-    Union[str :
-        
-    pathlib.Path]]] :
-         (Default value = None)
-    default : str :
-         (Default value = None)
-    debug : bool :
-         (Default value = False)
-
-    Returns
-    -------
-
-    """
+    """Prompt the user to edit any config files."""
     if default is None:
         raise Exception("Provide a default choice for which file to edit")
     import os

meerschaum/config/_formatting.py

@@ -35,6 +35,7 @@ default_formatting_config = {
         'running'          : '🟢',
         'paused'           : '🟡',
         'stopped'          : '🔴',
+        'tag'              : '🏷️',
     },
     'pipes'                : {
         'unicode'          : {
@@ -43,6 +44,7 @@ default_formatting_config = {
                 'metric'   : 'MRSM{formatting:emoji:metric} ',
                 'location' : 'MRSM{formatting:emoji:location} ',
                 'key'      : 'MRSM{formatting:emoji:key} ',
+                'tag'      : 'MRSM{formatting:emoji:tag} ',
             },
         },
         'ascii'            : {
@@ -51,6 +53,7 @@ default_formatting_config = {
                 'metric'   : '',
                 'location' : '',
                 'key'      : '',
+                'tag'      : '',
             },
         },
         'ansi'             : {
@@ -61,6 +64,7 @@ default_formatting_config = {
                 'key'      : '',
                 'guide'    : 'dim',
                 'none'     : 'black on magenta',
+                'tags'     : 'bold yellow underline',
             },
         },
         '__repr__'         : {

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.1.2"
+__version__ = "2.1.3.dev3"

meerschaum/config/static/__init__.py

@@ -56,6 +56,8 @@ STATIC_CONFIG: Dict[str, Any] = {
         'dep_group': 'MRSM_DEP_GROUP',
         'home': 'MRSM_HOME',
         'src': 'MRSM_SRC',
+        'uid': 'MRSM_UID',
+        'gid': 'MRSM_GID',
         'noask': 'MRSM_NOASK',
         'id': 'MRSM_SERVER_ID',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',

meerschaum/connectors/sql/_pipes.py

@@ -171,7 +171,7 @@ def fetch_pipes_keys(
     """
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.misc import separate_negation_values
+    from meerschaum.utils.misc import separate_negation_values, flatten_list
     from meerschaum.utils.sql import OMIT_NULLSFIRST_FLAVORS, table_exists
     from meerschaum.config.static import STATIC_CONFIG
     import json
@@ -254,25 +254,31 @@ def fetch_pipes_keys(
         q = q.where(coalesce(pipes_tbl.c[c], 'None').not_in(_ex_vals)) if _ex_vals else q
 
     ### Finally, parse tags.
-    _in_tags, _ex_tags = separate_negation_values(tags)
-    ors = []
-    for nt in _in_tags:
-        ors.append(
-            sqlalchemy.cast(
-                pipes_tbl.c['parameters'],
-                sqlalchemy.String,
-            ).like(f'%"tags":%"{nt}"%')
-        )
-    q = q.where(sqlalchemy.and_(sqlalchemy.or_(*ors).self_group())) if ors else q
-    ors = []
-    for xt in _ex_tags:
-        ors.append(
-            sqlalchemy.cast(
-                pipes_tbl.c['parameters'],
-                sqlalchemy.String,
-            ).not_like(f'%"tags":%"{xt}"%')
-        )
-    q = q.where(sqlalchemy.and_(sqlalchemy.or_(*ors).self_group())) if ors else q
+    tag_groups = [tag.split(',') for tag in tags]
+    in_ex_tag_groups = [separate_negation_values(tag_group) for tag_group in tag_groups]
+
+    ors, nands = [], []
+    for _in_tags, _ex_tags in in_ex_tag_groups:
+        sub_ands = []
+        for nt in _in_tags:
+            sub_ands.append(
+                sqlalchemy.cast(
+                    pipes_tbl.c['parameters'],
+                    sqlalchemy.String,
+                ).like(f'%"tags":%"{nt}"%')
+            )
+        ors.append(sqlalchemy.and_(*sub_ands))
+
+        for xt in _ex_tags:
+            nands.append(
+                sqlalchemy.cast(
+                    pipes_tbl.c['parameters'],
+                    sqlalchemy.String,
+                ).not_like(f'%"tags":%"{xt}"%')
+            )
+
+    q = q.where(sqlalchemy.and_(*nands)) if nands else q
+    q = q.where(sqlalchemy.or_(*ors)) if ors else q
     loc_asc = sqlalchemy.asc(pipes_tbl.c['location_key'])
     if self.flavor not in OMIT_NULLSFIRST_FLAVORS:
         loc_asc = sqlalchemy.nullsfirst(loc_asc)

meerschaum/core/Pipe/_data.py

@@ -8,7 +8,7 @@ Retrieve Pipes' data from instances.
 
 from __future__ import annotations
 from datetime import datetime, timedelta
-from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator
+from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator, List, Tuple
 from meerschaum.config import get_config
 
 def get_data(
@@ -350,20 +350,20 @@ def get_backtrack_data(
         If begin is `None` (default), use the most recent observed datetime
         (AKA sync_time).
 
+        ```
+        E.g. begin = 02:00
+
+        Search this region.           Ignore this, even if there's data.
+        /  /  /  /  /  /  /  /  /  |
+        -----|----------|----------|----------|----------|----------|
+        00:00      01:00      02:00      03:00      04:00      05:00
+
+        ```
+
     params: Optional[Dict[str, Any]], default None
         The standard Meerschaum `params` query dictionary.
         
         
-    ```
-    E.g. begin = 02:00
-
-    Search this region.           Ignore this, even if there's data.
-    /  /  /  /  /  /  /  /  /  |
-    -----|----------|----------|----------|----------|----------|
-    00:00      01:00      02:00      03:00      04:00      05:00
-
-    ```
-
     fresh: bool, default False
         If `True`, Ignore local cache and pull directly from the instance connector.
         Only comes into effect if a pipe was created with `cache=True`.

meerschaum/core/Pipe/_fetch.py

@@ -15,8 +15,8 @@ from meerschaum.utils.warnings import warn
 
 def fetch(
         self,
-        begin: Optional[datetime.datetime, str] = '',
-        end: Optional[datetime.datetime] = None,
+        begin: Union[datetime, str, None] = '',
+        end: Optional[datetime] = None,
         check_existing: bool = True,
         sync_chunks: bool = False,
         debug: bool = False,
@@ -27,10 +27,10 @@ def fetch(
 
     Parameters
     ----------
-    begin: Optional[datetime.datetime, str], default '':
+    begin: Union[datetime, str, None], default '':
         If provided, only fetch data newer than or equal to `begin`.
 
-    end: Optional[datetime.datetime], default None:
+    end: Optional[datetime], default None:
         If provided, only fetch data older than or equal to `end`.
 
     check_existing: bool, default True

meerschaum/plugins/_Plugin.py

@@ -283,8 +283,8 @@ class Plugin:
         import tarfile
         import re
         import ast
-        from meerschaum.plugins import reload_plugins, sync_plugins_symlinks
-        from meerschaum.utils.packages import attempt_import, determine_version, reload_package
+        from meerschaum.plugins import sync_plugins_symlinks
+        from meerschaum.utils.packages import attempt_import, determine_version, reload_meerschaum
         from meerschaum.utils.venv import init_venv
         from meerschaum.utils.misc import safely_extract_tar
         old_cwd = os.getcwd()
@@ -415,8 +415,7 @@ class Plugin:
         if '_module' in self.__dict__:
             del self.__dict__['_module']
         init_venv(venv=self.name, force=True, debug=debug)
-        reload_package('meerschaum')
-        reload_plugins([self.name], debug=debug)
+        reload_meerschaum(debug=debug)
 
         ### if we've already failed, return here
         if not success or abort:
@@ -493,8 +492,8 @@ class Plugin:
         """
         Remove a plugin, its virtual environment, and archive file.
         """
-        from meerschaum.utils.packages import reload_package
-        from meerschaum.plugins import reload_plugins, sync_plugins_symlinks
+        from meerschaum.utils.packages import reload_meerschaum
+        from meerschaum.plugins import sync_plugins_symlinks
         from meerschaum.utils.warnings import warn, info
         warnings_thrown_count: int = 0
         max_warnings: int = 3
@@ -529,8 +528,7 @@ class Plugin:
         success = warnings_thrown_count < max_warnings
         sync_plugins_symlinks(debug=debug)
         self.deactivate_venv(force=True, debug=debug)
-        reload_package('meerschaum')
-        reload_plugins(debug=debug)
+        reload_meerschaum(debug=debug)
         return success, (
             f"Successfully uninstalled plugin '{self}'." if success
             else f"Failed to uninstall plugin '{self}'."

meerschaum/plugins/__init__.py

@@ -12,8 +12,12 @@ from meerschaum.utils.threading import Lock, RLock
 from meerschaum.plugins._Plugin import Plugin
 
 _api_plugins: Dict[str, List[Callable[['fastapi.App'], Any]]] = {}
+_pre_sync_hooks: Dict[str, List[Callable[[Any], Any]]] = {}
+_post_sync_hooks: Dict[str, List[Callable[[Any], Any]]] = {}
 _locks = {
     '_api_plugins': RLock(),
+    '_pre_sync_hooks': RLock(),
+    '_post_sync_hooks': RLock(),
     '__path__': RLock(),
     'sys.path': RLock(),
     'internal_plugins': RLock(),
@@ -23,6 +27,7 @@ _locks = {
 __all__ = (
     "Plugin", "make_action", "api_plugin", "import_plugins",
     "reload_plugins", "get_plugins", "get_data_plugins", "add_plugin_argument",
+    "pre_sync_hook", "post_sync_hook",
 )
 __pdoc__ = {
     'venvs': False, 'data': False, 'stack': False, 'plugins': False,
@@ -81,6 +86,76 @@ def make_action(
     return function
 
 
+def pre_sync_hook(
+        function: Callable[[Any], Any],
+    ) -> Callable[[Any], Any]:
+    """
+    Register a function as a sync hook to be executed right before sync.
+    
+    Parameters
+    ----------
+    function: Callable[[Any], Any]
+        The function to execute right before a sync.
+        
+    Returns
+    -------
+    Another function (this is a decorator function).
+
+    Examples
+    --------
+    >>> from meerschaum.plugins import pre_sync_hook
+    >>>
+    >>> @pre_sync_hook
+    ... def log_sync(pipe, **kwargs):
+    ...     print(f"About to sync {pipe} with kwargs:\n{kwargs}.")
+    >>>
+    """
+    with _locks['_pre_sync_hooks']:
+        try:
+            if function.__module__ not in _pre_sync_hooks:
+                _pre_sync_hooks[function.__module__] = []
+            _pre_sync_hooks[function.__module__].append(function)
+        except Exception as e:
+            from meerschaum.utils.warnings import warn
+            warn(e)
+    return function
+
+
+def post_sync_hook(
+        function: Callable[[Any], Any],
+    ) -> Callable[[Any], Any]:
+    """
+    Register a function as a sync hook to be executed upon completion of a sync.
+    
+    Parameters
+    ----------
+    function: Callable[[Any], Any]
+        The function to execute upon completion of a sync.
+        
+    Returns
+    -------
+    Another function (this is a decorator function).
+
+    Examples
+    --------
+    >>> from meerschaum.plugins import post_sync_hook
+    >>>
+    >>> @post_sync_hook
+    ... def log_sync(pipe, success_tuple, duration=None, **kwargs):
+    ...     print(f"It took {round(duration, 2)} seconds to sync {pipe}.")
+    >>>
+    """
+    with _locks['_post_sync_hooks']:
+        try:
+            if function.__module__ not in _post_sync_hooks:
+                _post_sync_hooks[function.__module__] = []
+            _post_sync_hooks[function.__module__].append(function)
+        except Exception as e:
+            from meerschaum.utils.warnings import warn
+            warn(e)
+    return function
+
+
 def api_plugin(function: Callable[[Any], Any]) -> Callable[[Any], Any]:
     """
     Execute the function when initializing the Meerschaum API module.
@@ -106,7 +181,7 @@ def api_plugin(function: Callable[[Any], Any]) -> Callable[[Any], Any]:
     >>> def initialize_plugin(app):
     ...     @app.get('/my/new/path')
     ...     def new_path():
-    ...         return {'message' : 'It works!'}
+    ...         return {'message': 'It works!'}
     >>>
     """
     with _locks['_api_plugins']:

meerschaum/utils/__init__.py

@@ -8,10 +8,25 @@ These include tools from primary utilities (get_pipes)
 to miscellaneous helper functions.
 """
 
-### import_children will be depreciated in a future release.
-### Explicit is better than implicit (mostly).
-
-# from meerschaum.utils.packages import import_children
-# import_children()
-
+__all__ = (
+    'daemon',
+    'dataframe',
+    'debug',
+    'dtypes',
+    'formatting',
+    'interactive',
+    'misc',
+    'networking',
+    'packages',
+    'pool',
+    'process',
+    'prompt',
+    'schedule',
+    'sql',
+    'threading',
+    'typing',
+    'venv',
+    'warnings',
+    'yaml',
+)
 from meerschaum.utils.get_pipes import get_pipes

meerschaum/utils/daemon/__init__.py

@@ -3,7 +3,7 @@
 # vim:fenc=utf-8
 
 """
-Daemonize processes via daemoniker.
+Manage Daemons via the `Daemon` class.
 """
 
 from __future__ import annotations

meerschaum/utils/formatting/__init__.py

@@ -19,6 +19,7 @@ from meerschaum.utils.formatting._pipes import (
     format_pipe_success_tuple,
     print_pipes_results,
     extract_stats_from_message,
+    pipe_repr,
 )
 from meerschaum.utils.threading import Lock, RLock
 
@@ -39,6 +40,9 @@ __all__ = sorted([
     'highlight_pipes',
     'pprint_pipes',
     'make_header',
+    'pipe_repr',
+    'print_pipes_results',
+    'extract_stats_from_message',
 ])
 __pdoc__ = {}
 _locks = {

meerschaum/utils/formatting/_pipes.py

@@ -276,8 +276,8 @@ def pprint_pipe_columns(
 
 
 def pipe_repr(
-        pipe: 'meerschaum.Pipe',
-        as_rich_text: bool=False,
+        pipe: mrsm.Pipe,
+        as_rich_text: bool = False,
         ansi: Optional[bool] = None,
     ) -> Union[str, 'rich.text.Text']:
     """

meerschaum/utils/get_pipes.py

@@ -18,7 +18,7 @@ def get_pipes(
         connector_keys: Union[str, List[str], None] = None,
         metric_keys: Union[str, List[str], None] = None,
         location_keys: Union[str, List[str], None] = None,
-        tags: Optional[List[str], None] = None,
+        tags: Optional[List[str]] = None,
         params: Optional[Dict[str, Any]] = None,
         mrsm_instance: Union[str, InstanceConnector, None] = None,
         instance: Union[str, InstanceConnector, None] = None,