meerschaum 2.9.5__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

meerschaum/models/pipes.py

@@ -0,0 +1,247 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Pydantic model for a pipe's keys.
+"""
+
+from __future__ import annotations
+
+from typing import Optional, List, Tuple, Dict, Any
+
+import meerschaum as mrsm
+
+pydantic = mrsm.attempt_import('pydantic', lazy=False)
+from pydantic import (
+    BaseModel,
+    RootModel,
+    field_validator,
+    ValidationInfo,
+    ConfigDict,
+)
+
+
+class ConnectorKeysModel(RootModel[str]):
+    """
+    Validate the connector keys:
+        - May not begin with an underscore (_).
+        - Must be at least one character long.
+        - May contain one or zero colons (:).
+    """
+    model_config = ConfigDict(
+        frozen=True,
+        json_schema_extra={
+            'example': 'sql:main',
+        },
+    )
+    @field_validator('root')
+    @classmethod
+    def validate_connector_keys(cls, v: str, info: ValidationInfo) -> str:
+        """Validate the connector keys."""
+        if not v:
+            raise ValueError("Connector keys must be at least one character long.")
+        if v.startswith('_'):
+            raise ValueError("Connector keys may not begin with an underscore.")
+        if v.count(':') > 1:
+            raise ValueError("Connector keys may contain at most one colon.")
+        return v
+
+
+class MetricKeyModel(RootModel[str]):
+    """
+    Validate the metric key:
+
+        - May not begin with an underscore (_).
+        - Must be at least one character long.
+    """
+    model_config = ConfigDict(
+        frozen=True,
+        json_schema_extra={
+            'example': 'weather',
+        },
+    )
+    @field_validator('root')
+    @classmethod
+    def validate_metric_key(cls, v: str, info: ValidationInfo) -> str:
+        """Validate the metric key."""
+        if not v:
+            raise ValueError("Metric key must be at least one character long.")
+        if v.startswith('_'):
+            raise ValueError("Metric key may not begin with an underscore.")
+        return v
+
+
+class LocationKeyModel(RootModel[Optional[str]]):
+    """
+    Validate the location key:
+
+        - May not begin with an underscore (_).
+        - May be null (`None`).
+        - If not null, must be at least one character long.
+    """
+    model_config = ConfigDict(
+        frozen=True,
+        json_schema_extra={
+            'example': 'us.co.denver',
+        },
+    )
+    @field_validator('root')
+    @classmethod
+    def validate_location_key(cls, v: Optional[str], info: ValidationInfo) -> Optional[str]:
+        """Validate the location key."""
+        if v is None:
+            return v
+        if not v:
+            raise ValueError("Location key must be at least one character long if not null.")
+        if v.startswith('_'):
+            raise ValueError("Location key may not begin with an underscore.")
+        return v
+
+
+class InstanceKeysModel(RootModel[str]):
+    """
+    Validate the instance keys. The instance keys are connector keys which must have one colon
+    (e.g. `'sql:main'`).
+    """
+    model_config = ConfigDict(
+        frozen=True,
+        json_schema_extra={
+            'example': 'sql:main',
+        },
+    )
+    @field_validator('root')
+    @classmethod
+    def validate_instance_keys(cls, v: str, info: ValidationInfo) -> str:
+        """Validate the instance keys."""
+        if not v:
+            raise ValueError("Instance keys must be at least one character long.")
+        if v.startswith('_'):
+            raise ValueError("Instance keys may not begin with an underscore.")
+        if v.count(':') != 1:
+            raise ValueError("Instance keys must contain exactly one colon.")
+        return v
+
+
+class PipeModel(BaseModel):
+    """
+    Define the four components to uniquely identify a pipe.
+    """
+    connector_keys: ConnectorKeysModel
+    metric_key: MetricKeyModel
+    location_key: Optional[LocationKeyModel] = None
+    instance_keys: Optional[InstanceKeysModel] = None
+    model_config = ConfigDict(
+        json_schema_extra={
+            'example': {
+                'connector_keys': 'sql:main',
+                'metric_key': 'weather',
+                'location_key': 'atl',
+                'instance_keys': 'sql:main',
+            },
+        },
+    )
+
+
+class PipeWithParametersModel(PipeModel):
+    """
+    A Pipe's model including its parameters dictionary.
+    """
+    parameters: Dict[str, Any]
+    model_config = ConfigDict(
+        json_schema_extra={
+            'example': {
+                'connector_keys': 'sql:main',
+                'metric_key': 'weather',
+                'location_key': 'atl',
+                'instance_keys': 'sql:main',
+                'parameters': {
+                    'columns': {
+                        'datetime': 'dt',
+                        'id': 'id',
+                    },
+                },
+            },
+        },
+    )
+
+class PipesWithParametersDictModel(
+    RootModel[
+        Dict[
+            str,
+            Dict[
+                str,
+                Dict[
+                    str,
+                    PipeWithParametersModel
+                ]
+            ]
+        ]
+    ]
+):
+    """
+    A dictionary of pipes' parameters, nested by connector, metric, and location keys.
+    """
+    @field_validator('root')
+    @classmethod
+    def validate_keys(cls, v: dict, info: ValidationInfo) -> dict:
+        """
+        Validate the dictionary keys.
+        """
+        for c_key, metrics in v.items():
+            ConnectorKeysModel(root=c_key)
+            for m_key, locations in metrics.items():
+                MetricKeyModel(root=m_key)
+                for l_key in locations:
+                    LocationKeyModel(root=l_key if l_key not in ('None', 'null', '[None]') else None)
+        return v
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            'example': {
+                "plugin:noaa": {
+                    "weather": {
+                        "None": {
+                            "pipe_id": 1,
+                            "connector_keys": "plugin:noaa",
+                            "metric_key": "weather",
+                            "location_key": "clemson",
+                            "parameters": {
+                                "columns": {
+                                    "datetime": "timestamp",
+                                    "id": "station"
+                                },
+                                "noaa": {
+                                    "stations": {
+                                        "KCEU": {
+                                            "name": "Clemson, Clemson-Oconee County Airport",
+                                            "geometry": {
+                                            "type": "Point",
+                                                "coordinates": [
+                                                    -82.88139,
+                                                    34.67222
+                                                ]
+                                            }
+                                        }
+                                    }
+                                },
+                                "verify": {
+                                    "chunk_minutes": 20160
+                                },
+                                "target": "plugin_noaa_weather",
+                                "tags": [],
+                                "dtypes": {
+                                    "timestamp": "datetime",
+                                    "geometry": "json",
+                                    "cloudLayers": "json",
+                                    "presentWeather": "json"
+                                },
+                                "static": False,
+                                "enforce": True
+                            }
+                        }
+                    }
+                }
+            },
+        },
+    )

meerschaum/models/tokens.py

@@ -0,0 +1,38 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define Pydantic models for tokens.
+"""
+
+import uuid
+from typing import Optional, List, Union
+from datetime import datetime
+
+import meerschaum as mrsm
+
+pydantic = mrsm.attempt_import('pydantic', lazy=False)
+from pydantic import (
+    BaseModel,
+    RootModel,
+    field_validator,
+    ValidationInfo,
+    ConfigDict,
+    Field,
+)
+from meerschaum.models.users import UserModel
+from meerschaum._internal.static import STATIC_CONFIG
+
+
+class TokenModel(BaseModel):
+    """Pydantic model for a Meerschaum token."""
+    model_config = ConfigDict(from_attributes=True)
+
+    id: Optional[uuid.UUID] = Field(default=None)
+    creation: Optional[datetime] = Field(default=None)
+    expiration: Optional[datetime] = Field(default=None)
+    label: Optional[str] = Field(default=None)
+    user_id: Optional[Union[int, uuid.UUID]] = Field(default=None)
+    secret_hash: Optional[str] = Field(default=None)
+    scopes: Optional[List[str]] = Field(default=list(STATIC_CONFIG['tokens']['scopes']))
+    is_valid: bool = Field(default=True)

meerschaum/models/users.py

@@ -0,0 +1,26 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define Pydantic models for users.
+"""
+
+from __future__ import annotations
+from typing import Optional, Dict, Any, Union
+import uuid
+
+import meerschaum as mrsm
+
+pydantic = mrsm.attempt_import('pydantic', lazy=False)
+from pydantic import BaseModel, Field, ConfigDict
+
+
+class UserModel(BaseModel):
+    """Pydantic model for a Meerschaum User."""
+    model_config = ConfigDict(from_attributes=True)
+
+    user_id: Optional[Union[int, uuid.UUID]] = Field(default=None)
+    username: str
+    email: Optional[str] = Field(default=None)
+    type: Optional[str] = Field(default=None)
+    attributes: Optional[Dict[str, Any]] = Field(default=None)

meerschaum/plugins/__init__.py

@@ -7,11 +7,13 @@ Expose plugin management APIs from the `meerschaum.plugins` module.
 """
 
 from __future__ import annotations
+
 import functools
+
 import meerschaum as mrsm
 from meerschaum.utils.typing import Callable, Any, Union, Optional, Dict, List, Tuple
 from meerschaum.utils.threading import Lock, RLock
-from meerschaum.plugins._Plugin import Plugin
+from meerschaum.core.Plugin import Plugin
 
 _api_plugins: Dict[str, List[Callable[['fastapi.App'], Any]]] = {}
 _pre_sync_hooks: Dict[str, List[Callable[[Any], Any]]] = {}
@@ -28,15 +30,28 @@ _locks = {
     'PLUGINS_INTERNAL_LOCK_PATH': RLock(),
 }
 __all__ = (
-    "Plugin", "make_action", "api_plugin", "dash_plugin", "web_page",
-    "import_plugins", "from_plugin_import",
-    "reload_plugins", "get_plugins", "get_data_plugins", "add_plugin_argument",
-    "pre_sync_hook", "post_sync_hook",
+    "Plugin",
+    "make_action",
+    "api_plugin",
+    "dash_plugin",
+    "web_page",
+    "import_plugins",
+    "from_plugin_import",
+    "reload_plugins",
+    "get_plugins",
+    "get_data_plugins",
+    "add_plugin_argument",
+    "pre_sync_hook",
+    "post_sync_hook",
 )
 __pdoc__ = {
-    'venvs': False, 'data': False, 'stack': False, 'plugins': False,
+    'venvs': False,
+    'data': False,
+    'stack': False,
+    'plugins': False,
 }
 
+
 def make_action(
     function: Callable[[Any], Any],
     shell: bool = False,
@@ -298,7 +313,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
     import importlib.util
     from meerschaum.utils.misc import flatten_list, make_symlink, is_symlink
     from meerschaum.utils.warnings import error, warn as _warn
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.venv import Venv, activate_venv, deactivate_venv, is_venv_active
     from meerschaum.config._paths import (
         PLUGINS_RESOURCES_PATH,

meerschaum/plugins/bootstrap.py

@@ -129,9 +129,10 @@ FEATURE_LINES: Dict[str, str] = {
         "    import dash.html as html\n"
         "    import dash.dcc as dcc\n"
         "    from dash import Input, Output, State, no_update\n"
+        "    from dash.exceptions import PreventUpdate\n"
         "    import dash_bootstrap_components as dbc\n\n"
         "    # Create a new page at the path `/dash/{plugin_name}`.\n"
-        "    @web_page('{plugin_name}', login_required=False)\n"
+        "    @web_page('{plugin_name}', page_group='{plugin_name}', login_required=False)\n"
         "    def page_layout():\n"
         "        \"\"\"Return the layout objects for this page.\"\"\"\n"
         "        return dbc.Container([\n"

meerschaum/utils/_get_pipes.py

@@ -10,11 +10,21 @@ from __future__ import annotations
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import (
-    Sequence, Optional, Union, Mapping, Any, InstanceConnector, PipesDict, List, Dict, Tuple
+    Sequence,
+    Optional,
+    Union,
+    Mapping,
+    Any,
+    PipesDict,
+    List,
+    Dict,
+    Tuple,
+    InstanceConnector,
 )
 
 __pdoc__ = {'get_pipes': True, 'fetch_pipes_keys': True}
 
+
 def get_pipes(
     connector_keys: Union[str, List[str], None] = None,
     metric_keys: Union[str, List[str], None] = None,
@@ -24,10 +34,12 @@ def get_pipes(
     mrsm_instance: Union[str, InstanceConnector, None] = None,
     instance: Union[str, InstanceConnector, None] = None,
     as_list: bool = False,
+    as_tags_dict: bool = False,
     method: str = 'registered',
+    workers: Optional[int] = None,
     debug: bool = False,
     **kw: Any
-) -> Union[PipesDict, List[mrsm.Pipe]]:
+) -> Union[PipesDict, List[mrsm.Pipe], Dict[str, mrsm.Pipe]]:
     """
     Return a dictionary or list of `meerschaum.Pipe` objects.
 
@@ -62,6 +74,10 @@ def get_pipes(
         `False` : `{connector_keys: {metric_key: {location_key: Pipe}}}`
         `True`  : `[Pipe]`
 
+    as_tags_dict: bool, default False
+        If `True`, return a dictionary mapping tags to pipes.
+        Pipes with multiple tags will be repeated.
+
     method: str, default 'registered'
         Available options: `['registered', 'explicit', 'all']`
         If `'registered'` (default), create pipes based on registered keys in the connector's pipes table
@@ -71,6 +87,11 @@ def get_pipes(
         If `'all'`, create pipes from predefined metrics and locations. Required `connector_keys`.
         **NOTE:** Method `'all'` is not implemented!
 
+    workers: Optional[int], default None
+        If provided (and `as_tags_dict` is `True`), set the number of workers for the pool
+        to fetch tags.
+        Only takes effect if the instance connector supports multi-threading
+
     **kw: Any:
         Keyword arguments to pass to the `meerschaum.Pipe` constructor.
         
@@ -80,6 +101,7 @@ def get_pipes(
     A dictionary of dictionaries and `meerschaum.Pipe` objects
     in the connector, metric, location hierarchy.
     If `as_list` is `True`, return a list of `meerschaum.Pipe` objects.
+    If `as_tags_dict` is `True`, return a dictionary mapping tags to pipes.
 
     Examples
     --------
@@ -100,14 +122,17 @@ def get_pipes(
     >>> pipes['sql:main']['weather'][None]
     >>> ### Return a list instead:
     >>> get_pipes(as_list=True)
-    [sql_main_weather]
-    >>> 
+    [Pipe('sql:main', 'weather')]
+    >>> get_pipes(as_tags_dict=True)
+    {'gvl': Pipe('sql:main', 'weather')}
     ```
     """
 
     from meerschaum.config import get_config
     from meerschaum.utils.warnings import error
     from meerschaum.utils.misc import filter_keywords
+    from meerschaum.utils.pool import get_pool
+    from collections import defaultdict
 
     if connector_keys is None:
         connector_keys = []
@@ -159,6 +184,7 @@ def get_pipes(
         location_keys = location_keys,
         tags = tags,
         params = params,
+        workers = workers,
         debug = debug
     )
     if result is None:
@@ -182,17 +208,32 @@ def get_pipes(
             **filter_keywords(Pipe, **kw)
         )
 
-    if not as_list:
+    if not as_list and not as_tags_dict:
         return pipes
+
     from meerschaum.utils.misc import flatten_pipes_dict
-    return flatten_pipes_dict(pipes)
+    pipes_list = flatten_pipes_dict(pipes)
+    if as_list:
+        return pipes_list
+
+    pool = get_pool(workers=(workers if connector.IS_THREAD_SAFE else 1))
+    def gather_pipe_tags(pipe: mrsm.Pipe) -> Tuple[mrsm.Pipe, List[str]]:
+        return pipe, (pipe.tags or [])
+
+    tags_pipes = defaultdict(lambda: [])
+    pipes_tags = dict(pool.map(gather_pipe_tags, pipes_list))
+    for pipe, tags in pipes_tags.items():
+        for tag in (tags or []):
+            tags_pipes[tag].append(pipe)
+
+    return dict(tags_pipes)
 
 
 def fetch_pipes_keys(
-        method: str,
-        connector: 'meerschaum.connectors.Connector',
-        **kw: Any
-    ) -> List[Tuple[str, str, str]]:
+    method: str,
+    connector: 'mrsm.connectors.InstanceConnector',
+    **kw: Any
+) -> List[Tuple[str, str, str]]:
     """
     Fetch keys for pipes according to a method.
 
@@ -201,7 +242,7 @@ def fetch_pipes_keys(
     method: str
         The method by which to fetch the keys. See `get_pipes()` above.
 
-    connector: meerschaum.connectors.Connector
+    connector: meerschaum.connectors.InstanceConnector
         The connector to use to fetch the keys.
         Must be of type `meerschaum.connectors.sql.SQLConnector.SQLConnector`
         or `meerschaum.connectors.api.APIConnector.APIConnector`.
@@ -234,14 +275,14 @@ def fetch_pipes_keys(
     from meerschaum.utils.warnings import error
 
     def _registered(
-            connector_keys: Optional[List[str]] = None,
-            metric_keys: Optional[List[str]] = None,
-            location_keys: Optional[List[str]] = None,
-            tags: Optional[List[str]] = None,
-            params: Optional[Dict[str, Any]] = None,
-            debug: bool = False,
-            **kw
-        ) -> List[Tuple[str, str, str]]:
+        connector_keys: Optional[List[str]] = None,
+        metric_keys: Optional[List[str]] = None,
+        location_keys: Optional[List[str]] = None,
+        tags: Optional[List[str]] = None,
+        params: Optional[Dict[str, Any]] = None,
+        debug: bool = False,
+        **kw
+    ) -> List[Tuple[str, str, str]]:
         """
         Get keys from the pipes table or the API directly.
         Builds query or URL based on provided keys and parameters.
@@ -269,14 +310,14 @@ def fetch_pipes_keys(
         )
 
     def _explicit(
-            connector_keys: Optional[List[str]] = None,
-            metric_keys: Optional[List[str]] = None,
-            location_keys: Optional[List[str]] = None,
-            params: Optional[Dict[str, Any]] = None,
-            tags: Optional[List[str]] = None,
-            debug: bool = False,
-            **kw
-        ) -> List[Tuple[str, str, str]]:
+        connector_keys: Optional[List[str]] = None,
+        metric_keys: Optional[List[str]] = None,
+        location_keys: Optional[List[str]] = None,
+        params: Optional[Dict[str, Any]] = None,
+        tags: Optional[List[str]] = None,
+        debug: bool = False,
+        **kw
+    ) -> List[Tuple[str, str, str]]:
         """
         Explicitly build Pipes based on provided keys.
         Raises an error if `connector_keys` or `metric_keys` is empty,

meerschaum/utils/daemon/Daemon.py

@@ -24,7 +24,7 @@ from meerschaum.utils.typing import (
     is_success_tuple, Tuple,
 )
 from meerschaum.config import get_config
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 from meerschaum.config._paths import (
     DAEMON_RESOURCES_PATH, LOGS_RESOURCES_PATH, DAEMON_ERROR_LOG_PATH,
 )
@@ -338,6 +338,7 @@ class Daemon:
                     result = False, str(e)
                 finally:
                     _results[self.daemon_id] = result
+                    self.properties['result'] = result
 
                     if keep_daemon_output:
                         self._capture_process_timestamp('ended')

meerschaum/utils/daemon/__init__.py

@@ -7,7 +7,15 @@ Manage Daemons via the `Daemon` class.
 """
 
 from __future__ import annotations
-import os, pathlib, shutil, json, datetime, threading, shlex
+
+import os
+import pathlib
+import shutil
+import json
+import datetime
+import threading
+import shlex
+
 from meerschaum.utils.typing import SuccessTuple, List, Optional, Callable, Any, Dict
 from meerschaum.config._paths import DAEMON_RESOURCES_PATH
 from meerschaum.utils.daemon.StdinFile import StdinFile
@@ -17,6 +25,26 @@ from meerschaum.utils.daemon.FileDescriptorInterceptor import FileDescriptorInte
 from meerschaum.utils.daemon._names import get_new_daemon_name
 
 
+__all__ = (
+    'daemon_action',
+    'daemon_entry',
+    'get_daemons',
+    'get_daemon_ids',
+    'get_running_daemons',
+    'get_stopped_daemons',
+    'get_paused_daemons',
+    'get_filtered_daemons',
+    'get_new_daemon_name',
+    'run_daemon',
+    'running_in_daemon',
+    'Daemon',
+    'StdinFile',
+    'RotatingFile',
+    'FileDescriptorInterceptor',
+    'DAEMON_RESOURCES_PATH',
+)
+
+
 def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     """Parse sysargs and execute a Meerschaum action as a daemon.
 
@@ -280,6 +308,6 @@ def running_in_daemon() -> bool:
     """
     Return whether the current thread is running in a Daemon context.
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     daemon_env_var = STATIC_CONFIG['environment']['daemon_id']
     return daemon_env_var in os.environ