meerschaum 2.9.5__py3-none-any.whl → 3.0.0__py3-none-any.whl

meerschaum/config/environment.py

@@ -0,0 +1,262 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Patch the runtime configuration from environment variables.
+"""
+
+import os
+import re
+import json
+import contextlib
+import copy
+import pathlib
+
+from meerschaum.utils.typing import List, Union, Dict, Any, Optional
+from meerschaum._internal.static import STATIC_CONFIG
+
+
+def apply_environment_patches(env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Apply patches defined in `MRSM_CONFIG` and `MRSM_PATCH`.
+    """
+    config_var = STATIC_CONFIG['environment']['config']
+    patch_var = STATIC_CONFIG['environment']['patch']
+    apply_environment_config(config_var, env=env)
+    apply_environment_config(patch_var, env=env)
+
+
+def apply_environment_config(env_var: str, env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Parse a dictionary (simple or JSON) from an environment variable
+    and apply it to the current configuration.
+    """
+    from meerschaum.config import get_config, set_config, _config
+    from meerschaum.config._patch import apply_patch_to_config
+
+    env = env if env is not None else os.environ
+
+    if env_var not in env:
+        return
+
+    from meerschaum.utils.misc import string_to_dict
+    try:
+        _patch = string_to_dict(str(os.environ[env_var]).lstrip())
+    except Exception:
+        _patch = None
+
+    error_msg = (
+        f"Environment variable {env_var} is set but cannot be parsed.\n"
+        f"Unset {env_var} or change to JSON or simplified dictionary format "
+        "(see --help, under params for formatting)\n"
+        f"{env_var} is set to:\n{os.environ[env_var]}\n"
+        f"Skipping patching os environment into config..."
+    )
+
+    if not isinstance(_patch, dict):
+        print(error_msg)
+        return
+
+    valids = []
+
+    def load_key(key: str) -> Union[Dict[str, Any], None]:
+        try:
+            c = get_config(key, warn=False)
+        except Exception:
+            c = None
+        return c
+
+    ### This was multi-threaded, but I ran into all sorts of locking issues.
+    keys = list(_patch.keys())
+    for key in keys:
+        _ = load_key(key)
+
+    ### Load and patch config files.
+    set_config(
+        apply_patch_to_config(
+            _config(),
+            _patch,
+        )
+    )
+
+
+def apply_environment_uris(env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Patch temporary connectors defined in environment variables which start with
+    `MRSM_SQL_` or `MRSM_API_`.
+    """
+    for env_var in get_connector_env_vars(env=env):
+        apply_connector_uri(env_var, env=env)
+
+
+def get_connector_env_regex() -> str:
+    """
+    Return the regex pattern for valid environment variable names for instance connectors.
+    """
+    return STATIC_CONFIG['environment']['uri_regex']
+
+
+def get_connector_env_vars(env: Optional[Dict[str, Any]] = None) -> List[str]:
+    """
+    Get the names of the environment variables which match the Meerschaum connector regex.
+
+    Examples
+    --------
+    >>> get_connector_environment_vars()
+    ['MRSM_SQL_FOO']
+    """
+    uri_regex = get_connector_env_regex()
+    env_vars = []
+
+    env = env if env is not None else os.environ
+
+    for env_var in env:
+        matched = re.match(uri_regex, env_var)
+        if matched is None:
+            continue
+        if env_var in STATIC_CONFIG['environment'].values():
+            continue
+        env_vars.append(env_var)
+
+    return env_vars
+
+
+def apply_connector_uri(env_var: str, env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Parse and validate a URI obtained from an environment variable.
+    """
+    from meerschaum.config import get_config, set_config, _config
+    from meerschaum.config._patch import apply_patch_to_config
+    from meerschaum.config._read_config import search_and_substitute_config
+    from meerschaum.utils.warnings import warn
+
+    env = env if env is not None else os.environ
+
+    if env_var not in env:
+        return
+
+    uri_regex = get_connector_env_regex()
+    matched = re.match(uri_regex, env_var)
+    groups = matched.groups()
+    typ, label = groups[0].lower(), groups[1].lower()
+    if not typ or not label:
+        return
+
+    uri = env[env_var]
+
+    if uri.lstrip().startswith('{') and uri.rstrip().endswith('}'):
+        try:
+            conn_attrs = json.loads(uri)
+        except Exception:
+            warn(f"Unable to parse JSON for environment connector '{typ}:{label}'.")
+            conn_attrs = {'uri': uri}
+    else:
+        conn_attrs = {'uri': uri}
+
+    set_config(
+        apply_patch_to_config(
+            {'meerschaum': get_config('meerschaum')},
+            {'meerschaum': {'connectors': {typ: {label: conn_attrs}}}},
+        )
+    )
+
+
+def get_env_vars(env: Optional[Dict[str, Any]] = None) -> List[str]:
+    """
+    Return all environment variables which begin with `'MRSM_'`.
+    """
+    prefix = STATIC_CONFIG['environment']['prefix']
+    env = env if env is not None else os.environ
+    return sorted([env_var for env_var in env if env_var.startswith(prefix)])
+
+
+@contextlib.contextmanager
+def replace_env(env: Union[Dict[str, Any], None]):
+    """
+    Temporarily replace environment variables and current configuration.
+
+    Parameters
+    ----------
+    env: Dict[str, Any]
+        The new environment dictionary to be patched on `os.environ`.
+    """
+    if env is None:
+        try:
+            yield
+        finally:
+            return
+
+    from meerschaum.config import _config, set_config
+    from meerschaum.config.paths import (
+        set_root,
+        set_plugins_dir_paths,
+        set_venvs_dir_path,
+        set_config_dir_path,
+        ROOT_DIR_PATH,
+        PLUGINS_DIR_PATHS,
+        VIRTENV_RESOURCES_PATH,
+        CONFIG_DIR_PATH,
+    )
+
+    old_environ = dict(os.environ)
+    old_config = copy.deepcopy(_config())
+    old_root_dir_path = ROOT_DIR_PATH
+    old_plugins_dir_paths = PLUGINS_DIR_PATHS
+    old_venvs_dir_path = VIRTENV_RESOURCES_PATH
+    old_config_dir_path = CONFIG_DIR_PATH
+
+    os.environ.update(env)
+
+    root_dir_env_var = STATIC_CONFIG['environment']['root']
+    plugins_dir_env_var = STATIC_CONFIG['environment']['plugins']
+    config_dir_env_var = STATIC_CONFIG['environment']['config_dir']
+    venvs_dir_env_var = STATIC_CONFIG['environment']['venvs']
+
+    replaced_root = False
+    if root_dir_env_var in env:
+        root_dir_path = pathlib.Path(env[root_dir_env_var])
+        set_root(root_dir_path)
+        replaced_root = True
+
+    replaced_plugins = False
+    if plugins_dir_env_var in env:
+        plugins_dir_paths = env[plugins_dir_env_var]
+        set_plugins_dir_paths(plugins_dir_paths)
+        replaced_plugins = True
+
+    replaced_venvs = False
+    if venvs_dir_env_var in env:
+        venv_dir_path = pathlib.Path(env[venvs_dir_env_var])
+        set_venvs_dir_path(venv_dir_path)
+        replaced_venvs = True
+
+    replaced_config_dir = False
+    if config_dir_env_var in env:
+        config_dir_path = pathlib.Path(env[config_dir_env_var])
+        set_config_dir_path(config_dir_path)
+        replaced_config_dir = True
+
+    apply_environment_patches(env)
+    apply_environment_uris(env)
+
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(old_environ)
+
+        if replaced_root:
+            set_root(old_root_dir_path)
+
+        if replaced_plugins:
+            set_plugins_dir_paths(old_plugins_dir_paths)
+
+        if replaced_venvs:
+            set_venvs_dir_path(old_venvs_dir_path)
+
+        if replaced_config_dir:
+            set_config_dir_path(old_config_dir_path)
+
+        _config().clear()
+        set_config(old_config)

meerschaum/config/stack/__init__.py

@@ -14,6 +14,8 @@ import json
 from meerschaum.config._paths import (
     GRAFANA_DATASOURCE_PATH,
     GRAFANA_DASHBOARD_PATH,
+    DB_INIT_RESOURCES_PATH,
+    DB_CREATE_EXTENSIONS_PATH,
     ROOT_DIR_PATH,
 )
 from meerschaum.config._paths import STACK_COMPOSE_FILENAME, STACK_ENV_FILENAME
@@ -39,7 +41,7 @@ valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
 
 env_dict = {
     'COMPOSE_PROJECT_NAME': 'mrsm',
-    'TIMESCALEDB_VERSION': 'latest-pg16',
+    'TIMESCALEDB_VERSION': 'pg17',
     'POSTGRES_USER': db_user,
     'POSTGRES_PASSWORD': db_pass,
     'POSTGRES_DB': db_base,
@@ -54,19 +56,22 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
     {
         'meerschaum': 'MRSM{!meerschaum}',
         'system': 'MRSM{!system}',
+        'api': 'MRSM{!api}',
     },
     indent=4,
 ).replace(
     '"MRSM{!system}"', 'MRSM{!system}'
 ).replace(
-    '"MRSM{!meerschaum}"', 'MRSM{!meerschaum}',
+    '"MRSM{!meerschaum}"', 'MRSM{!meerschaum}'
+).replace(
+    '"MRSM{!api}"', 'MRSM{!api}'
 )
 
 volumes = {
     'api_root': '/meerschaum',
-    'meerschaum_db_data': '/var/lib/postgresql/data',
+    'meerschaum_db_data': '/home/postgres/pgdata',
     'grafana_storage': '/var/lib/grafana',
-    'valkey_data': '/bitnami/valkey/data',
+    'valkey_data': '/valkey/data',
 }
 networks = {
     'frontend': None,
@@ -121,7 +126,7 @@ default_docker_compose_config = {
                 'POSTGRES_PASSWORD': '<DOLLAR>POSTGRES_PASSWORD',
                 'ALLOW_IP_RANGE': env_dict['ALLOW_IP_RANGE'],
             },
-            'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB',
+            'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100',
             'healthcheck': {
                 'test': [
                     'CMD-SHELL', 'pg_isready -d <DOLLAR>POSTGRES_DB -U <DOLLAR>POSTGRES_USER',
@@ -131,13 +136,14 @@ default_docker_compose_config = {
                 'retries': 5
             },
             'restart': 'always',
-            'image': 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
+            'image': 'timescale/timescaledb-ha:' + env_dict['TIMESCALEDB_VERSION'],
             'ports': [
                 f'{db_port}:5432',
             ],
             'hostname': db_hostname,
             'volumes': [
                 'meerschaum_db_data:' + volumes['meerschaum_db_data'],
+                f'{DB_INIT_RESOURCES_PATH.as_posix()}:/docker-entrypoint-initdb.d:z,ro',
             ],
             'shm_size': '1024m',
             'networks': [
@@ -180,7 +186,7 @@ default_docker_compose_config = {
             ],
         },
         'valkey': {
-            'image': 'bitnami/valkey:latest',
+            'image': 'valkey/valkey:latest',
             'restart': 'always',
             'environment': {
                 'VALKEY_PASSWORD': '<DOLLAR>VALKEY_PASSWORD',
@@ -224,8 +230,8 @@ default_docker_compose_config = {
             'volumes': [
                 'grafana_storage' + ':' + volumes['grafana_storage'],
                 ### NOTE: Mount with the 'z' option for SELinux.
-                f'{GRAFANA_DATASOURCE_PATH.parent}:/etc/grafana/provisioning/datasources:z,ro',
-                f'{GRAFANA_DASHBOARD_PATH.parent}:/etc/grafana/provisioning/dashboards:z,ro',
+                f'{GRAFANA_DATASOURCE_PATH.parent.as_posix()}:/etc/grafana/provisioning/datasources:z,ro',
+                f'{GRAFANA_DASHBOARD_PATH.parent.as_posix()}:/etc/grafana/provisioning/dashboards:z,ro',
             ],
             'environment': {
                 'GF_SECURITY_ALLOW_EMBEDDING': 'true',
@@ -273,6 +279,21 @@ def _sync_stack_files():
         substitute = True,
     )
 
+    _write_initdb()
+
+def _write_initdb():
+    create_postgis_text = (
+        "CREATE EXTENSION IF NOT EXISTS timescaledb;\n"
+        "CREATE EXTENSION IF NOT EXISTS postgis;\n"
+        "CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit;\n"
+        "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;\n"
+    )
+    if DB_CREATE_EXTENSIONS_PATH.exists():
+        return
+
+    with open(DB_CREATE_EXTENSIONS_PATH, 'w+', encoding='utf-8') as f:
+        f.write(create_postgis_text)
+
 NECESSARY_FILES = [STACK_COMPOSE_PATH, GRAFANA_DATASOURCE_PATH, GRAFANA_DASHBOARD_PATH]
 def get_necessary_files():
     from meerschaum.config import get_config
@@ -286,19 +307,20 @@ def get_necessary_files():
 
 
 def write_stack(
-        debug: bool = False 
-    ):
+    debug: bool = False 
+):
     """Write Docker Compose configuration files."""
     from meerschaum.config._edit import general_write_yaml_config
     from meerschaum.config._sync import sync_files
     general_write_yaml_config(get_necessary_files(), debug=debug)
     return sync_files(['stack'])
    
+
 def edit_stack(
-        action: Optional[List[str]] = None,
-        debug: bool = False,
-        **kw
-    ):
+    action: Optional[List[str]] = None,
+    debug: bool = False,
+    **kw
+):
     """Open docker-compose.yaml or .env for editing."""
     from meerschaum.config._edit import general_edit_config
     if action is None:

meerschaum/config/static.py

@@ -0,0 +1,18 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Alias import for the internal static configuration dictionary.
+"""
+
+from meerschaum._internal.static import SERVER_ID, STATIC_CONFIG
+
+__all__ = ('STATIC_CONFIG',)
+
+
+def _static_config():
+    """
+    Alias function for the global `STATIC_CONFIG` dictionary.
+    """
+    return STATIC_CONFIG

meerschaum/connectors/_Connector.py

@@ -7,10 +7,12 @@ Define the parent `Connector` class.
 """
 
 from __future__ import annotations
+
 import abc
 import copy
 from meerschaum.utils.typing import Iterable, Optional, Any, Union, List, Dict
 
+
 class InvalidAttributesError(Exception):
     """
     Raised when the incorrect attributes are set in the Connector.
@@ -20,6 +22,9 @@ class Connector(metaclass=abc.ABCMeta):
     """
     The base connector class to hold connection attributes.
     """
+
+    IS_INSTANCE: bool = False
+
     def __init__(
         self,
         type: Optional[str] = None,
@@ -70,7 +75,7 @@ class Connector(metaclass=abc.ABCMeta):
         inherit_default: bool = True,
         **kw: Any
     ):
-        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum._internal.static import STATIC_CONFIG
         from meerschaum.utils.warnings import error
 
         self._attributes = {}
@@ -147,11 +152,10 @@ class Connector(metaclass=abc.ABCMeta):
         ------
         An error if any of the required attributes are missing.
         """
-        from meerschaum.utils.warnings import error, warn
-        from meerschaum.utils.debug import dprint
+        from meerschaum.utils.warnings import error
         from meerschaum.utils.misc import items_str
         if required_attributes is None:
-            required_attributes = ['label']
+            required_attributes = ['type', 'label']
 
         missing_attributes = set()
         for a in required_attributes:
@@ -213,6 +217,8 @@ class Connector(metaclass=abc.ABCMeta):
                 else r'executor$'
             )
             _type = re.sub(suffix_regex, '', self.__class__.__name__.lower())
+            if not _type or _type.lower() == 'instance':
+                raise ValueError("No type could be determined for this connector.")
             self.__dict__['type'] = _type
         return _type
 
@@ -224,8 +230,7 @@ class Connector(metaclass=abc.ABCMeta):
         """
         _label = self.__dict__.get('label', None)
         if _label is None:
-            from meerschaum.config.static import STATIC_CONFIG
+            from meerschaum._internal.static import STATIC_CONFIG
             _label = STATIC_CONFIG['connectors']['default_label']
             self.__dict__['label'] = _label
         return _label
-

meerschaum/connectors/__init__.py

@@ -14,18 +14,19 @@ For ease of use, you can also import from the root `meerschaum` module:
 from __future__ import annotations
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Any, Union, List, Dict
+from meerschaum.utils.typing import Any, Union, List, Dict, Optional
 from meerschaum.utils.threading import RLock
 from meerschaum.utils.warnings import warn
 
 from meerschaum.connectors._Connector import Connector, InvalidAttributesError
+from meerschaum.connectors.instance._InstanceConnector import InstanceConnector
 from meerschaum.connectors.sql._SQLConnector import SQLConnector
 from meerschaum.connectors.api._APIConnector import APIConnector
-from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
 
 __all__ = (
     "make_connector",
     "Connector",
+    "InstanceConnector",
     "SQLConnector",
     "APIConnector",
     "get_connector",
@@ -50,30 +51,16 @@ _locks: Dict[str, RLock] = {
     'connectors'               : RLock(),
     'types'                    : RLock(),
     'custom_types'             : RLock(),
+    'plugins_types'            : RLock(),
     '_loaded_plugin_connectors': RLock(),
     'instance_types'           : RLock(),
 }
-attributes: Dict[str, Dict[str, Any]] = {
-    'api': {
-        'required': [
-            'host',
-            'username',
-            'password',
-        ],
-        'optional': [
-            'port',
-        ],
-        'default': {
-            'protocol': 'http',
-        },
-    },
-    'sql': {
-        'flavors': sql_flavor_configs,
-    },
-}
+
 ### Fill this with objects only when connectors are first referenced.
 types: Dict[str, Any] = {}
 custom_types: set = set()
+plugins_types: Dict[str, List[str]] = {}
+_known_custom_types: set = set()
 _loaded_plugin_connectors: bool = False
 
 
@@ -88,7 +75,6 @@ def get_connector(
     Return existing connector or create new connection and store for reuse.
     
     You can create new connectors if enough parameters are provided for the given type and flavor.
-    
 
     Parameters
     ----------
@@ -130,7 +116,7 @@ def get_connector(
     """
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum.config import get_config
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
     global _loaded_plugin_connectors
     if isinstance(type, str) and not label and ':' in type:
@@ -313,16 +299,23 @@ def make_connector(cls, _is_executor: bool = False):
     >>> 
     """
     import re
+    from meerschaum.plugins import _get_parent_plugin
     suffix_regex = (
         r'connector$'
         if not _is_executor
         else r'executor$'
     )
+    plugin_name = _get_parent_plugin(2)
     typ = re.sub(suffix_regex, '', cls.__name__.lower())
     with _locks['types']:
         types[typ] = cls
     with _locks['custom_types']:
         custom_types.add(typ)
+    if plugin_name:
+        with _locks['plugins_types']:
+            if plugin_name not in plugins_types:
+                plugins_types[plugin_name] = []
+            plugins_types[plugin_name].append(typ)
     with _locks['connectors']:
         if typ not in connectors:
             connectors[typ] = {}
@@ -353,6 +346,30 @@ def load_plugin_connectors():
     import_plugins(*to_import)
 
 
+def unload_plugin_connectors(
+    plugin_names: Optional[List[str]] = None,
+    debug: bool = False,
+) -> None:
+    """
+    Unload custom connectors added by plugins.
+    """
+    from meerschaum.plugins import get_plugins_names
+    global custom_types, _known_custom_types, types, plugins_types, connectors
+
+    plugin_names = plugin_names or get_plugins_names()
+
+    for plugin_name in plugin_names:
+        plugin_types = plugins_types.get(plugin_name, [])
+        for typ in plugin_types:
+            _ = types.pop(typ, None)
+            _ = connectors.pop(typ, None)
+            if typ in instance_types:
+                instance_types.remove(typ)
+
+    custom_types.clear()
+    custom_types.update(_known_custom_types)
+
+
 def get_connector_plugin(
     connector: Connector,
 ) -> Union[str, None, mrsm.Plugin]:
@@ -389,3 +406,5 @@ def _load_builtin_custom_connectors():
     """
     import meerschaum.jobs.systemd
     import meerschaum.connectors.valkey
+    _known_custom_types.add('valkey')
+    _known_custom_types.add('systemd')

meerschaum/connectors/api/_APIConnector.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 
 from datetime import datetime, timedelta, timezone
 from meerschaum.utils.typing import Optional, List, Union
-from meerschaum.connectors import Connector
+from meerschaum.connectors import InstanceConnector
 from meerschaum.utils.warnings import warn, error
 from meerschaum.utils.packages import attempt_import
 
@@ -18,15 +18,13 @@ required_attributes = {
     'host',
 }
 
-class APIConnector(Connector):
+class APIConnector(InstanceConnector):
     """
     Connect to a Meerschaum API instance.
     """
 
-    IS_INSTANCE: bool = True
     IS_THREAD_SAFE: bool = False
-
-    OPTIONAL_ATTRIBUTES: List[str] = ['port']
+    OPTIONAL_ATTRIBUTES: List[str] = ['port', 'client_secret', 'client_id', 'api_key']
 
     from ._request import (
         make_request,
@@ -82,6 +80,16 @@ class APIConnector(Connector):
         get_user_type,
         get_user_attributes,
     )
+    from ._tokens import (
+        register_token,
+        get_token_model,
+        get_tokens,
+        edit_token,
+        invalidate_token,
+        get_token_scopes,
+        token_exists,
+        delete_token,
+    )
     from ._uri import from_uri
     from ._jobs import (
         get_jobs,
@@ -154,9 +162,15 @@ class APIConnector(Connector):
         """
         Return the fully qualified URI.
         """
+        import urllib.parse
         username = self.__dict__.get('username', None)
         password = self.__dict__.get('password', None)
+        client_id = self.__dict__.get('client_id', None)
+        client_secret = self.__dict__.get('client_secret', None)
+        api_key = self.__dict__.get('api_key', None)
         creds = (username + ':' + password + '@') if username and password else ''
+        params = {}
+        params_str = ('?' + urllib.parse.urlencode(params)) if params else ''
         return (
             self.protocol
             + '://'
@@ -167,9 +181,9 @@ class APIConnector(Connector):
                 if self.__dict__.get('port', None)
                 else ''
             )
+            + params_str
         )
 
-
     @property
     def session(self):
         if self._session is None:
@@ -206,3 +220,17 @@ class APIConnector(Connector):
         Return the instance keys to be sent alongside pipe requests.
         """
         return self._instance_keys
+
+    @property
+    def login_scheme(self) -> str:
+        """
+        Return the login scheme to use based on the configured credentials.
+        """
+        if 'username' in self.__dict__:
+            return 'password'
+        if 'client_id' in self.__dict__:
+            return 'client_credentials'
+        elif 'api_key' in self.__dict__:
+            return 'api_key'
+
+        return 'password'