meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

meerschaum/config/environment.py

@@ -0,0 +1,262 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Patch the runtime configuration from environment variables.
+"""
+
+import os
+import re
+import json
+import contextlib
+import copy
+import pathlib
+
+from meerschaum.utils.typing import List, Union, Dict, Any, Optional
+from meerschaum._internal.static import STATIC_CONFIG
+
+
+def apply_environment_patches(env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Apply patches defined in `MRSM_CONFIG` and `MRSM_PATCH`.
+    """
+    config_var = STATIC_CONFIG['environment']['config']
+    patch_var = STATIC_CONFIG['environment']['patch']
+    apply_environment_config(config_var, env=env)
+    apply_environment_config(patch_var, env=env)
+
+
+def apply_environment_config(env_var: str, env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Parse a dictionary (simple or JSON) from an environment variable
+    and apply it to the current configuration.
+    """
+    from meerschaum.config import get_config, set_config, _config
+    from meerschaum.config._patch import apply_patch_to_config
+
+    env = env if env is not None else os.environ
+
+    if env_var not in env:
+        return
+
+    from meerschaum.utils.misc import string_to_dict
+    try:
+        _patch = string_to_dict(str(os.environ[env_var]).lstrip())
+    except Exception:
+        _patch = None
+
+    error_msg = (
+        f"Environment variable {env_var} is set but cannot be parsed.\n"
+        f"Unset {env_var} or change to JSON or simplified dictionary format "
+        "(see --help, under params for formatting)\n"
+        f"{env_var} is set to:\n{os.environ[env_var]}\n"
+        f"Skipping patching os environment into config..."
+    )
+
+    if not isinstance(_patch, dict):
+        print(error_msg)
+        return
+
+    valids = []
+
+    def load_key(key: str) -> Union[Dict[str, Any], None]:
+        try:
+            c = get_config(key, warn=False)
+        except Exception:
+            c = None
+        return c
+
+    ### This was multi-threaded, but I ran into all sorts of locking issues.
+    keys = list(_patch.keys())
+    for key in keys:
+        _ = load_key(key)
+
+    ### Load and patch config files.
+    set_config(
+        apply_patch_to_config(
+            _config(),
+            _patch,
+        )
+    )
+
+
+def apply_environment_uris(env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Patch temporary connectors defined in environment variables which start with
+    `MRSM_SQL_` or `MRSM_API_`.
+    """
+    for env_var in get_connector_env_vars(env=env):
+        apply_connector_uri(env_var, env=env)
+
+
+def get_connector_env_regex() -> str:
+    """
+    Return the regex pattern for valid environment variable names for instance connectors.
+    """
+    return STATIC_CONFIG['environment']['uri_regex']
+
+
+def get_connector_env_vars(env: Optional[Dict[str, Any]] = None) -> List[str]:
+    """
+    Get the names of the environment variables which match the Meerschaum connector regex.
+
+    Examples
+    --------
+    >>> get_connector_environment_vars()
+    ['MRSM_SQL_FOO']
+    """
+    uri_regex = get_connector_env_regex()
+    env_vars = []
+
+    env = env if env is not None else os.environ
+
+    for env_var in env:
+        matched = re.match(uri_regex, env_var)
+        if matched is None:
+            continue
+        if env_var in STATIC_CONFIG['environment'].values():
+            continue
+        env_vars.append(env_var)
+
+    return env_vars
+
+
+def apply_connector_uri(env_var: str, env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Parse and validate a URI obtained from an environment variable.
+    """
+    from meerschaum.config import get_config, set_config, _config
+    from meerschaum.config._patch import apply_patch_to_config
+    from meerschaum.config._read_config import search_and_substitute_config
+    from meerschaum.utils.warnings import warn
+
+    env = env if env is not None else os.environ
+
+    if env_var not in env:
+        return
+
+    uri_regex = get_connector_env_regex()
+    matched = re.match(uri_regex, env_var)
+    groups = matched.groups()
+    typ, label = groups[0].lower(), groups[1].lower()
+    if not typ or not label:
+        return
+
+    uri = env[env_var]
+
+    if uri.lstrip().startswith('{') and uri.rstrip().endswith('}'):
+        try:
+            conn_attrs = json.loads(uri)
+        except Exception:
+            warn(f"Unable to parse JSON for environment connector '{typ}:{label}'.")
+            conn_attrs = {'uri': uri}
+    else:
+        conn_attrs = {'uri': uri}
+
+    set_config(
+        apply_patch_to_config(
+            {'meerschaum': get_config('meerschaum')},
+            {'meerschaum': {'connectors': {typ: {label: conn_attrs}}}},
+        )
+    )
+
+
+def get_env_vars(env: Optional[Dict[str, Any]] = None) -> List[str]:
+    """
+    Return all environment variables which begin with `'MRSM_'`.
+    """
+    prefix = STATIC_CONFIG['environment']['prefix']
+    env = env if env is not None else os.environ
+    return sorted([env_var for env_var in env if env_var.startswith(prefix)])
+
+
+@contextlib.contextmanager
+def replace_env(env: Union[Dict[str, Any], None]):
+    """
+    Temporarily replace environment variables and current configuration.
+
+    Parameters
+    ----------
+    env: Dict[str, Any]
+        The new environment dictionary to be patched on `os.environ`.
+    """
+    if env is None:
+        try:
+            yield
+        finally:
+            return
+
+    from meerschaum.config import _config, set_config
+    from meerschaum.config.paths import (
+        set_root,
+        set_plugins_dir_paths,
+        set_venvs_dir_path,
+        set_config_dir_path,
+        ROOT_DIR_PATH,
+        PLUGINS_DIR_PATHS,
+        VIRTENV_RESOURCES_PATH,
+        CONFIG_DIR_PATH,
+    )
+
+    old_environ = dict(os.environ)
+    old_config = copy.deepcopy(_config())
+    old_root_dir_path = ROOT_DIR_PATH
+    old_plugins_dir_paths = PLUGINS_DIR_PATHS
+    old_venvs_dir_path = VIRTENV_RESOURCES_PATH
+    old_config_dir_path = CONFIG_DIR_PATH
+
+    os.environ.update(env)
+
+    root_dir_env_var = STATIC_CONFIG['environment']['root']
+    plugins_dir_env_var = STATIC_CONFIG['environment']['plugins']
+    config_dir_env_var = STATIC_CONFIG['environment']['config_dir']
+    venvs_dir_env_var = STATIC_CONFIG['environment']['venvs']
+
+    replaced_root = False
+    if root_dir_env_var in env:
+        root_dir_path = pathlib.Path(env[root_dir_env_var])
+        set_root(root_dir_path)
+        replaced_root = True
+
+    replaced_plugins = False
+    if plugins_dir_env_var in env:
+        plugins_dir_paths = env[plugins_dir_env_var]
+        set_plugins_dir_paths(plugins_dir_paths)
+        replaced_plugins = True
+
+    replaced_venvs = False
+    if venvs_dir_env_var in env:
+        venv_dir_path = pathlib.Path(env[venvs_dir_env_var])
+        set_venvs_dir_path(venv_dir_path)
+        replaced_venvs = True
+
+    replaced_config_dir = False
+    if config_dir_env_var in env:
+        config_dir_path = pathlib.Path(env[config_dir_env_var])
+        set_config_dir_path(config_dir_path)
+        replaced_config_dir = True
+
+    apply_environment_patches(env)
+    apply_environment_uris(env)
+
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(old_environ)
+
+        if replaced_root:
+            set_root(old_root_dir_path)
+
+        if replaced_plugins:
+            set_plugins_dir_paths(old_plugins_dir_paths)
+
+        if replaced_venvs:
+            set_venvs_dir_path(old_venvs_dir_path)
+
+        if replaced_config_dir:
+            set_config_dir_path(old_config_dir_path)
+
+        _config().clear()
+        set_config(old_config)

meerschaum/config/stack/__init__.py

@@ -56,19 +56,22 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
     {
         'meerschaum': 'MRSM{!meerschaum}',
         'system': 'MRSM{!system}',
+        'api': 'MRSM{!api}',
     },
     indent=4,
 ).replace(
     '"MRSM{!system}"', 'MRSM{!system}'
 ).replace(
-    '"MRSM{!meerschaum}"', 'MRSM{!meerschaum}',
+    '"MRSM{!meerschaum}"', 'MRSM{!meerschaum}'
+).replace(
+    '"MRSM{!api}"', 'MRSM{!api}'
 )
 
 volumes = {
     'api_root': '/meerschaum',
-    'meerschaum_db_data': '/var/lib/postgresql/data',
+    'meerschaum_db_data': '/home/postgres/pgdata',
     'grafana_storage': '/var/lib/grafana',
-    'valkey_data': '/bitnami/valkey/data',
+    'valkey_data': '/valkey/data',
 }
 networks = {
     'frontend': None,
@@ -122,7 +125,6 @@ default_docker_compose_config = {
                 'POSTGRES_DB': '<DOLLAR>POSTGRES_DB',
                 'POSTGRES_PASSWORD': '<DOLLAR>POSTGRES_PASSWORD',
                 'ALLOW_IP_RANGE': env_dict['ALLOW_IP_RANGE'],
-                #  'POSTGRES_INITDB_ARGS': '-c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100'
             },
             'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100',
             'healthcheck': {
@@ -184,7 +186,7 @@ default_docker_compose_config = {
             ],
         },
         'valkey': {
-            'image': 'bitnami/valkey:latest',
+            'image': 'valkey/valkey:latest',
             'restart': 'always',
             'environment': {
                 'VALKEY_PASSWORD': '<DOLLAR>VALKEY_PASSWORD',

meerschaum/connectors/_Connector.py

@@ -152,8 +152,7 @@ class Connector(metaclass=abc.ABCMeta):
         ------
         An error if any of the required attributes are missing.
         """
-        from meerschaum.utils.warnings import error, warn
-        from meerschaum.utils.debug import dprint
+        from meerschaum.utils.warnings import error
         from meerschaum.utils.misc import items_str
         if required_attributes is None:
             required_attributes = ['type', 'label']

meerschaum/connectors/__init__.py

@@ -14,7 +14,7 @@ For ease of use, you can also import from the root `meerschaum` module:
 from __future__ import annotations
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Any, Union, List, Dict
+from meerschaum.utils.typing import Any, Union, List, Dict, Optional
 from meerschaum.utils.threading import RLock
 from meerschaum.utils.warnings import warn
 
@@ -51,6 +51,7 @@ _locks: Dict[str, RLock] = {
     'connectors'               : RLock(),
     'types'                    : RLock(),
     'custom_types'             : RLock(),
+    'plugins_types'            : RLock(),
     '_loaded_plugin_connectors': RLock(),
     'instance_types'           : RLock(),
 }
@@ -58,6 +59,8 @@ _locks: Dict[str, RLock] = {
 ### Fill this with objects only when connectors are first referenced.
 types: Dict[str, Any] = {}
 custom_types: set = set()
+plugins_types: Dict[str, List[str]] = {}
+_known_custom_types: set = set()
 _loaded_plugin_connectors: bool = False
 
 
@@ -72,7 +75,6 @@ def get_connector(
     Return existing connector or create new connection and store for reuse.
     
     You can create new connectors if enough parameters are provided for the given type and flavor.
-    
 
     Parameters
     ----------
@@ -297,16 +299,23 @@ def make_connector(cls, _is_executor: bool = False):
     >>> 
     """
     import re
+    from meerschaum.plugins import _get_parent_plugin
     suffix_regex = (
         r'connector$'
         if not _is_executor
         else r'executor$'
     )
+    plugin_name = _get_parent_plugin(2)
     typ = re.sub(suffix_regex, '', cls.__name__.lower())
     with _locks['types']:
         types[typ] = cls
     with _locks['custom_types']:
         custom_types.add(typ)
+    if plugin_name:
+        with _locks['plugins_types']:
+            if plugin_name not in plugins_types:
+                plugins_types[plugin_name] = []
+            plugins_types[plugin_name].append(typ)
     with _locks['connectors']:
         if typ not in connectors:
             connectors[typ] = {}
@@ -337,6 +346,30 @@ def load_plugin_connectors():
     import_plugins(*to_import)
 
 
+def unload_plugin_connectors(
+    plugin_names: Optional[List[str]] = None,
+    debug: bool = False,
+) -> None:
+    """
+    Unload custom connectors added by plugins.
+    """
+    from meerschaum.plugins import get_plugins_names
+    global custom_types, _known_custom_types, types, plugins_types, connectors
+
+    plugin_names = plugin_names or get_plugins_names()
+
+    for plugin_name in plugin_names:
+        plugin_types = plugins_types.get(plugin_name, [])
+        for typ in plugin_types:
+            _ = types.pop(typ, None)
+            _ = connectors.pop(typ, None)
+            if typ in instance_types:
+                instance_types.remove(typ)
+
+    custom_types.clear()
+    custom_types.update(_known_custom_types)
+
+
 def get_connector_plugin(
     connector: Connector,
 ) -> Union[str, None, mrsm.Plugin]:
@@ -373,3 +406,5 @@ def _load_builtin_custom_connectors():
     """
     import meerschaum.jobs.systemd
     import meerschaum.connectors.valkey
+    _known_custom_types.add('valkey')
+    _known_custom_types.add('systemd')

meerschaum/connectors/api/_APIConnector.py

@@ -233,4 +233,4 @@ class APIConnector(InstanceConnector):
         elif 'api_key' in self.__dict__:
             return 'api_key'
 
-        raise ValueError(f"Could not determine the login scheme for '{self}'.")
+        return 'password'

meerschaum/connectors/api/_jobs.py

@@ -380,3 +380,14 @@ def get_job_is_blocking_on_stdin(self, name: str, debug: bool = False) -> bool:
         return False
 
     return response.json()
+
+
+def get_job_prompt_kwargs(self, name: str, debug: bool = False) -> Dict[str, Any]:
+    """
+    Return the kwargs to the blocking `prompt()`, if available.
+    """
+    response = self.get(JOBS_ENDPOINT + f'/{name}/prompt_kwargs', debug=debug)
+    if not response:
+        return {}
+
+    return response.json()

meerschaum/connectors/api/_pipes.py

@@ -112,7 +112,13 @@ def fetch_pipes_keys(
     tags: Optional[List[str]] = None,
     params: Optional[Dict[str, Any]] = None,
     debug: bool = False
-) -> Union[List[Tuple[str, str, Union[str, None]]]]:
+) -> List[
+        Union[
+            Tuple[str, str, Union[str, None]],
+            Tuple[str, str, Union[str, None], List[str]],
+            Tuple[str, str, Union[str, None], Dict[str, Any]]
+        ]
+    ]:
     """
     Fetch registered Pipes' keys from the API.
     

meerschaum/connectors/instance/_plugins.py

@@ -16,13 +16,20 @@ def get_plugins_pipe(self) -> 'mrsm.Pipe':
     """
     Return the internal pipe for syncing plugins metadata.
     """
+    if '_plugins_pipe' in self.__dict__:
+        return self._plugins_pipe
+
+    cache_connector = self.__dict__.get('_cache_connector', None)
     users_pipe = self.get_users_pipe()
     user_id_dtype = users_pipe.dtypes.get('user_id', 'uuid')
-    return mrsm.Pipe(
+
+    self._plugins_pipe = mrsm.Pipe(
         'mrsm', 'plugins',
         instance=self,
         target='mrsm_plugins',
         temporary=True,
+        cache=True,
+        cache_connector_keys=cache_connector,
         static=True,
         null_indices=False,
         columns={
@@ -36,6 +43,7 @@ def get_plugins_pipe(self) -> 'mrsm.Pipe':
             'version': 'string',
         },
     )
+    return self._plugins_pipe
 
 
 def register_plugin(self, plugin: Plugin, debug: bool = False) -> mrsm.SuccessTuple:

meerschaum/connectors/instance/_tokens.py

@@ -15,23 +15,33 @@ import meerschaum as mrsm
 from meerschaum.core import Token, User
 from meerschaum.core.User import hash_password
 from meerschaum._internal.static import STATIC_CONFIG
+from meerschaum.utils.warnings import dprint
 
 
 def get_tokens_pipe(self) -> mrsm.Pipe:
     """
     Return the internal pipe for tokens management.
     """
+    if '_tokens_pipe' in self.__dict__:
+        return self._tokens_pipe
+
     users_pipe = self.get_users_pipe()
-    user_id_dtype = users_pipe.dtypes.get('user_id', 'uuid')
+    user_id_dtype = (
+        users_pipe._attributes.get('parameters', {}).get('dtypes', {}).get('user_id', 'uuid')
+    )
+
+    cache_connector = self.__dict__.get('_cache_connector', None)
 
-    return mrsm.Pipe(
+    self._tokens_pipe = mrsm.Pipe(
         'mrsm', 'tokens',
         instance=self,
         target='mrsm_tokens',
         temporary=True,
+        cache=True,
+        cache_connector_keys=cache_connector,
         static=True,
         autotime=True,
-        null_indices=True,
+        null_indices=False,
         columns={
             'datetime': 'creation',
             'primary': 'id',
@@ -51,6 +61,7 @@ def get_tokens_pipe(self) -> mrsm.Pipe:
             'secret_hash': 'string',
         },
     )
+    return self._tokens_pipe
 
 
 def register_token(
@@ -200,10 +211,16 @@ def get_tokens(
     if ids:
         params['id'] = ids
         
+    if debug:
+        dprint(f"Getting tokens with {user_id=}, {params=}")
+
     tokens_df = tokens_pipe.get_data(params=params, debug=debug)
     if tokens_df is None:
         return []
 
+    if debug:
+        dprint(f"Retrieved tokens dataframe:\n{tokens_df}")
+
     tokens_docs = tokens_df.to_dict(orient='records')
     return [
         Token(

meerschaum/connectors/instance/_users.py

@@ -18,11 +18,17 @@ def get_users_pipe(self) -> 'mrsm.Pipe':
     """
     Return the pipe used for users registration.
     """
-    return mrsm.Pipe(
+    if '_users_pipe' in self.__dict__:
+        return self._users_pipe
+
+    cache_connector = self.__dict__.get('_cache_connector', None)
+    self._users_pipe = mrsm.Pipe(
         'mrsm', 'users',
         instance=self,
         target='mrsm_users',
         temporary=True,
+        cache=True,
+        cache_connector_keys=cache_connector,
         static=True,
         null_indices=False,
         columns={
@@ -40,6 +46,7 @@ def get_users_pipe(self) -> 'mrsm.Pipe':
             'unique': 'username',
         },
     )
+    return self._users_pipe
 
 
 def register_user(

meerschaum/connectors/parse.py

@@ -111,7 +111,7 @@ def parse_repo_keys(keys: Optional[str] = None, **kw):
     """Parse the Meerschaum repository value into an APIConnector."""
     from meerschaum.config import get_config
     if keys is None:
-        keys = get_config('meerschaum', 'default_repository', patch=True)
+        keys = get_config('meerschaum', 'repository', patch=True)
     keys = str(keys)
     if ':' not in keys:
         keys = 'api:' + keys

meerschaum/connectors/sql/_create_engine.py

@@ -78,6 +78,9 @@ def create_engine(
     _host = self.__dict__.get('host', None)
     _port = self.__dict__.get('port', None)
     _database = self.__dict__.get('database', None)
+    if _database == '{SQLITE_DB_PATH}':
+        from meerschaum.config.paths import SQLITE_DB_PATH
+        _database = SQLITE_DB_PATH.as_posix()
     _options = self.__dict__.get('options', {})
     if isinstance(_options, str):
         _options = dict(urllib.parse.parse_qsl(_options))