meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc8__py3-none-any.whl

meerschaum/config/_paths.py

@@ -9,10 +9,17 @@ Define file paths
 from __future__ import annotations
 
 from pathlib import Path
-import os, platform, sys, json
-from meerschaum.utils.typing import Union
+import contextlib
+import os
+import platform
+import sys
+import json
+
+from typing import Union, List
+
 from meerschaum._internal.static import STATIC_CONFIG
 
+
 DOT_CONFIG_DIR_PATH = Path(
     os.environ.get('XDG_CONFIG_HOME', Path.home() / '.config')
     if platform.system() != 'Windows'
@@ -71,7 +78,7 @@ if ENVIRONMENT_PLUGINS_DIR in os.environ:
                 if path_str
             ]
         )
-    except Exception as e:
+    except Exception:
         PLUGINS_DIR_PATHS = []
 
     if not PLUGINS_DIR_PATHS:
@@ -82,20 +89,23 @@ if ENVIRONMENT_PLUGINS_DIR in os.environ:
             f"`export {ENVIRONMENT_PLUGINS_DIR}=./plugins:/another/path/to/plugins`\n\n"
             "or a JSON-encoded path list:\n\n"
             f"`export {ENVIRONMENT_PLUGINS_DIR}=" + "'[\"./plugins\", \"/another/path/to/plugins\"]'`"
-            f"",
+            "",
         )
         sys.exit(1)
 else:
     PLUGINS_DIR_PATHS = [_ROOT_DIR_PATH / 'plugins']
 
 ### Remove duplicate plugins paths.
-_seen_plugins_paths, _plugins_paths_to_remove = set(), set()
-for _plugin_path in PLUGINS_DIR_PATHS:
-    if _plugin_path in _seen_plugins_paths:
-        _plugins_paths_to_remove.add(_plugin_path)
-    _seen_plugins_paths.add(_plugin_path)
-for _plugin_path in _plugins_paths_to_remove:
-    PLUGINS_DIR_PATHS.remove(_plugin_path)
+def _process_plugins_dir_paths(plugins_dir_paths: List[Path]):
+    _seen_plugins_paths, _plugins_paths_to_remove = set(), set()
+    for _plugin_path in plugins_dir_paths:
+        if _plugin_path in _seen_plugins_paths:
+            _plugins_paths_to_remove.add(_plugin_path)
+        _seen_plugins_paths.add(_plugin_path)
+    for _plugin_path in _plugins_paths_to_remove:
+        plugins_dir_paths.remove(_plugin_path)
+_process_plugins_dir_paths(PLUGINS_DIR_PATHS)
+
 
 ENVIRONMENT_VENVS_DIR = STATIC_CONFIG['environment']['venvs']
 if ENVIRONMENT_VENVS_DIR in os.environ:
@@ -144,6 +154,8 @@ paths = {
     'API_SECRET_KEY_PATH'            : ('{API_CONFIG_RESOURCES_PATH}', '.api_secret_key'),
     'API_UVICORN_RESOURCES_PATH'     : ('{API_CONFIG_RESOURCES_PATH}', 'uvicorn'),
     'API_UVICORN_CONFIG_PATH'        : ('{API_UVICORN_RESOURCES_PATH}', '.thread_config.json'),
+    
+    'WEBTERM_INTERNAL_RESOURCES_PATH': ('{INTERNAL_RESOURCES_PATH}', 'webterm'),
 
     'CACHE_RESOURCES_PATH'           : ('{ROOT_DIR_PATH}', '.cache'),
     'PIPES_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'pipes'),
@@ -151,12 +163,16 @@ paths = {
     'VENVS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'venvs'),
     'SQL_CONN_CACHE_RESOURCES_PATH'  : ('{CACHE_RESOURCES_PATH}', 'sql'),
 
+    'CLI_RESOURCES_PATH'             : ('{INTERNAL_RESOURCES_PATH}', 'cli'),
+    'CLI_LOGS_RESOURCES_PATH'        : ('{CLI_RESOURCES_PATH}', 'logs'),
+
     'PLUGINS_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
     'PLUGINS_INTERNAL_LOCK_PATH'     : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),
     'PLUGINS_PACKAGES_INTERNAL_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'packaged_plugins'),
     'PLUGINS_ARCHIVES_RESOURCES_PATH': ('{PLUGINS_RESOURCES_PATH}', '.archives'),
     'PLUGINS_TEMP_RESOURCES_PATH'    : ('{PLUGINS_RESOURCES_PATH}', '.tmp'),
     'PLUGINS_INIT_PATH'              : ('{PLUGINS_RESOURCES_PATH}', '__init__.py'),
+    'PLUGINS_INJECTED_RESOURCES_PATH': ('{PLUGINS_RESOURCES_PATH}', '.injected'),
 
     'DB_RESOURCES_PATH'              : ('{ROOT_DIR_PATH}', 'db'),
     'DB_INIT_RESOURCES_PATH'         : ('{DB_RESOURCES_PATH}', 'initdb'),
@@ -197,11 +213,110 @@ paths = {
 
 def set_root(root: Union[Path, str]):
     """Modify the value of `ROOT_DIR_PATH`."""
-    paths['ROOT_DIR_PATH'] = Path(root).resolve()
+    paths['ROOT_DIR_PATH'] = Path(root).resolve().as_posix()
     for path_name, path_parts in paths.items():
         if isinstance(path_parts, tuple) and path_parts[0] == '{ROOT_DIR_PATH}':
             globals()[path_name] = __getattr__(path_name)
 
+def set_plugins_dir_paths(plugins_dir_paths: Union[List[Path], Path, str]):
+    """Modify the value of `PLUGINS_DIR_PATHS`."""
+    global PLUGINS_DIR_PATHS
+    if isinstance(plugins_dir_paths, str):
+        if plugins_dir_paths.strip() and plugins_dir_paths.lstrip()[0] == '[':
+            plugins_dir_paths = json.loads(plugins_dir_paths)
+        else:
+            plugins_dir_paths = plugins_dir_paths.split(':')
+        plugins_dir_paths = [Path(_path).resolve() for _path in plugins_dir_paths]
+    if isinstance(plugins_dir_paths, Path):
+        plugins_dir_paths = [plugins_dir_paths.resolve()]
+
+    PLUGINS_DIR_PATHS = plugins_dir_paths
+    _process_plugins_dir_paths(plugins_dir_paths)
+
+def set_venvs_dir_path(venvs_dir_path: Union[str, Path]):
+    """Modify the value of `VIRTENV_RESOURCES_PATH`."""
+    paths['VIRTENV_RESOURCES_PATH'] = Path(venvs_dir_path).resolve().as_posix()
+
+def set_config_dir_path(config_dir_path: Union[str, Path]):
+    paths['CONFIG_DIR_PATH'] = Path(config_dir_path).resolve().as_posix()
+
+
+@contextlib.contextmanager
+def replace_root_dir(root_dir_path: Union[str, Path, None]):
+    """
+    Temporarily replace the root directory path.
+    """
+    if root_dir_path is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_root = paths.get('ROOT_DIR_PATH', _ROOT_DIR_PATH)
+    set_root(root_dir_path)
+
+    try:
+        yield
+    finally:
+        set_root(old_root)
+
+@contextlib.contextmanager
+def replace_plugins_dir_paths(plugins_dir_paths: Union[List[Path], None]):
+    """
+    Temporarily replace the plugins directory paths.
+    """
+    if plugins_dir_paths is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_plugins_dir_paths = PLUGINS_DIR_PATHS
+    set_plugins_dir_paths(plugins_dir_paths)
+
+    try:
+        yield
+    finally:
+        set_plugins_dir_paths(old_plugins_dir_paths)
+
+@contextlib.contextmanager
+def replace_venvs_dir_path(venvs_dir_path: Union[Path, None]):
+    """
+    Temporarily replace the virtual environments directory path.
+    """
+    if venvs_dir_path is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_venvs_dir_path = paths.get('VIRTENV_RESOURCES_PATH', _VENVS_DIR_PATH)
+    set_venvs_dir_path(venvs_dir_path)
+
+    try:
+        yield
+    finally:
+        set_venvs_dir_path(old_venvs_dir_path)
+
+@contextlib.contextmanager
+def replace_config_dir_path(config_dir_path: Union[Path, None]):
+    """
+    Temporarily replace the config directory path.
+    """
+    if config_dir_path is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_config_dir_path = paths.get('CONFIG_DIR_PATH', _CONFIG_DIR_PATH)
+    set_config_dir_path(config_dir_path)
+
+    try:
+        yield
+    finally:
+        set_config_dir_path(old_config_dir_path)
+
 
 def __getattr__(name: str) -> Path:
     if name not in paths:

meerschaum/config/_read_config.py

@@ -8,7 +8,9 @@ Import the config yaml file
 from __future__ import annotations
 import pathlib
 
+import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Dict, Any, List, Tuple, Union
+from meerschaum._internal.static import STATIC_CONFIG
 
 
 def read_config(
@@ -53,7 +55,6 @@ def read_config(
     import itertools
     from meerschaum.utils.yaml import yaml, _yaml
     from meerschaum.config._paths import CONFIG_DIR_PATH
-    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.config._patch import apply_patch_to_config
     if directory is None:
         directory = CONFIG_DIR_PATH
@@ -106,20 +107,25 @@ def read_config(
             ### If default config contains symlinks, add them to the config to write.
             try:
                 _default_symlinks = _default_dict[symlinks_key][mk]
-            except Exception:
+            except KeyError:
                 _default_symlinks = {}
+
             config[mk] = _default_dict[mk]
             if _default_symlinks:
                 if symlinks_key not in config:
                     config[symlinks_key] = {}
+                if symlinks_key not in config_to_write:
+                    config_to_write[symlinks_key] = {}
+
                 if mk not in config[symlinks_key]:
                     config[symlinks_key][mk] = {}
+                if mk not in config_to_write[symlinks_key]:
+                    config_to_write[symlinks_key][mk] = {}
+
                 config[symlinks_key][mk] = apply_patch_to_config(
                     config[symlinks_key][mk], 
                     _default_symlinks
                 )
-                if symlinks_key not in config_to_write:
-                    config_to_write[symlinks_key] = {}
                 config_to_write[symlinks_key][mk] = config[symlinks_key][mk]
 
             ### Write the default key.
@@ -387,10 +393,16 @@ def search_and_substitute_config(
             s[_keys[-1]] = _pattern
 
         from meerschaum.config._patch import apply_patch_to_config
-        from meerschaum._internal.static import STATIC_CONFIG
         symlinks_key = STATIC_CONFIG['config']['symlinks_key']
-        if symlinks_key in parsed_config:
-            parsed_config[symlinks_key] = apply_patch_to_config(parsed_config[symlinks_key], symlinks)
+        if symlinks:
+            if symlinks_key not in parsed_config:
+                parsed_config[symlinks_key] = symlinks
+            else:
+                parsed_config[symlinks_key] = apply_patch_to_config(
+                    parsed_config[symlinks_key],
+                    symlinks,
+                    warn=False,
+                )
 
     return parsed_config
 
@@ -409,15 +421,24 @@ def revert_symlinks_config(config: Dict[str, Any]) -> Dict[str, Any]:
     -------
     A configuration dictionary with `_symlinks` re-applied.
     """
-    from meerschaum.config import apply_patch_to_config
+    import copy
     from meerschaum._internal.static import STATIC_CONFIG
+
     symlinks_key = STATIC_CONFIG['config']['symlinks_key']
     if symlinks_key not in config:
         return config
 
-    symlinks_config = config[symlinks_key]
-    reverted_config = apply_patch_to_config(config, symlinks_config)
-    _ = reverted_config.pop(symlinks_key, None)
+    reverted_config = copy.deepcopy(config)
+    symlinks_config = reverted_config.pop(symlinks_key)
+
+    def deep_patch(target_dict, patch_dict):
+        for key, value in patch_dict.items():
+            if isinstance(value, dict) and key in target_dict and isinstance(target_dict[key], dict):
+                deep_patch(target_dict[key], value)
+            else:
+                target_dict[key] = value
+
+    deep_patch(reverted_config, symlinks_config)
     return reverted_config
 
 
@@ -462,7 +483,6 @@ def get_keyfile_path(
         )
     except IndexError:
         if create_new:
-            from meerschaum._internal.static import STATIC_CONFIG
             default_filetype = STATIC_CONFIG['config']['default_filetype']
             return pathlib.Path(os.path.join(directory, key + '.' + default_filetype))
         return None

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "3.0.0rc4"
+__version__ = "3.0.0rc8"

meerschaum/config/environment.py

@@ -0,0 +1,262 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Patch the runtime configuration from environment variables.
+"""
+
+import os
+import re
+import json
+import contextlib
+import copy
+import pathlib
+
+from meerschaum.utils.typing import List, Union, Dict, Any, Optional
+from meerschaum._internal.static import STATIC_CONFIG
+
+
+def apply_environment_patches(env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Apply patches defined in `MRSM_CONFIG` and `MRSM_PATCH`.
+    """
+    config_var = STATIC_CONFIG['environment']['config']
+    patch_var = STATIC_CONFIG['environment']['patch']
+    apply_environment_config(config_var, env=env)
+    apply_environment_config(patch_var, env=env)
+
+
+def apply_environment_config(env_var: str, env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Parse a dictionary (simple or JSON) from an environment variable
+    and apply it to the current configuration.
+    """
+    from meerschaum.config import get_config, set_config, _config
+    from meerschaum.config._patch import apply_patch_to_config
+
+    env = env if env is not None else os.environ
+
+    if env_var not in env:
+        return
+
+    from meerschaum.utils.misc import string_to_dict
+    try:
+        _patch = string_to_dict(str(os.environ[env_var]).lstrip())
+    except Exception:
+        _patch = None
+
+    error_msg = (
+        f"Environment variable {env_var} is set but cannot be parsed.\n"
+        f"Unset {env_var} or change to JSON or simplified dictionary format "
+        "(see --help, under params for formatting)\n"
+        f"{env_var} is set to:\n{os.environ[env_var]}\n"
+        f"Skipping patching os environment into config..."
+    )
+
+    if not isinstance(_patch, dict):
+        print(error_msg)
+        return
+
+    valids = []
+
+    def load_key(key: str) -> Union[Dict[str, Any], None]:
+        try:
+            c = get_config(key, warn=False)
+        except Exception:
+            c = None
+        return c
+
+    ### This was multi-threaded, but I ran into all sorts of locking issues.
+    keys = list(_patch.keys())
+    for key in keys:
+        _ = load_key(key)
+
+    ### Load and patch config files.
+    set_config(
+        apply_patch_to_config(
+            _config(),
+            _patch,
+        )
+    )
+
+
+def apply_environment_uris(env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Patch temporary connectors defined in environment variables which start with
+    `MRSM_SQL_` or `MRSM_API_`.
+    """
+    for env_var in get_connector_env_vars(env=env):
+        apply_connector_uri(env_var, env=env)
+
+
+def get_connector_env_regex() -> str:
+    """
+    Return the regex pattern for valid environment variable names for instance connectors.
+    """
+    return STATIC_CONFIG['environment']['uri_regex']
+
+
+def get_connector_env_vars(env: Optional[Dict[str, Any]] = None) -> List[str]:
+    """
+    Get the names of the environment variables which match the Meerschaum connector regex.
+
+    Examples
+    --------
+    >>> get_connector_environment_vars()
+    ['MRSM_SQL_FOO']
+    """
+    uri_regex = get_connector_env_regex()
+    env_vars = []
+
+    env = env if env is not None else os.environ
+
+    for env_var in env:
+        matched = re.match(uri_regex, env_var)
+        if matched is None:
+            continue
+        if env_var in STATIC_CONFIG['environment'].values():
+            continue
+        env_vars.append(env_var)
+
+    return env_vars
+
+
+def apply_connector_uri(env_var: str, env: Optional[Dict[str, Any]] = None) -> None:
+    """
+    Parse and validate a URI obtained from an environment variable.
+    """
+    from meerschaum.config import get_config, set_config, _config
+    from meerschaum.config._patch import apply_patch_to_config
+    from meerschaum.config._read_config import search_and_substitute_config
+    from meerschaum.utils.warnings import warn
+
+    env = env if env is not None else os.environ
+
+    if env_var not in env:
+        return
+
+    uri_regex = get_connector_env_regex()
+    matched = re.match(uri_regex, env_var)
+    groups = matched.groups()
+    typ, label = groups[0].lower(), groups[1].lower()
+    if not typ or not label:
+        return
+
+    uri = env[env_var]
+
+    if uri.lstrip().startswith('{') and uri.rstrip().endswith('}'):
+        try:
+            conn_attrs = json.loads(uri)
+        except Exception:
+            warn(f"Unable to parse JSON for environment connector '{typ}:{label}'.")
+            conn_attrs = {'uri': uri}
+    else:
+        conn_attrs = {'uri': uri}
+
+    set_config(
+        apply_patch_to_config(
+            {'meerschaum': get_config('meerschaum')},
+            {'meerschaum': {'connectors': {typ: {label: conn_attrs}}}},
+        )
+    )
+
+
+def get_env_vars(env: Optional[Dict[str, Any]] = None) -> List[str]:
+    """
+    Return all environment variables which begin with `'MRSM_'`.
+    """
+    prefix = STATIC_CONFIG['environment']['prefix']
+    env = env if env is not None else os.environ
+    return sorted([env_var for env_var in env if env_var.startswith(prefix)])
+
+
+@contextlib.contextmanager
+def replace_env(env: Union[Dict[str, Any], None]):
+    """
+    Temporarily replace environment variables and current configuration.
+
+    Parameters
+    ----------
+    env: Dict[str, Any]
+        The new environment dictionary to be patched on `os.environ`.
+    """
+    if env is None:
+        try:
+            yield
+        finally:
+            return
+
+    from meerschaum.config import _config, set_config
+    from meerschaum.config.paths import (
+        set_root,
+        set_plugins_dir_paths,
+        set_venvs_dir_path,
+        set_config_dir_path,
+        ROOT_DIR_PATH,
+        PLUGINS_DIR_PATHS,
+        VIRTENV_RESOURCES_PATH,
+        CONFIG_DIR_PATH,
+    )
+
+    old_environ = dict(os.environ)
+    old_config = copy.deepcopy(_config())
+    old_root_dir_path = ROOT_DIR_PATH
+    old_plugins_dir_paths = PLUGINS_DIR_PATHS
+    old_venvs_dir_path = VIRTENV_RESOURCES_PATH
+    old_config_dir_path = CONFIG_DIR_PATH
+
+    os.environ.update(env)
+
+    root_dir_env_var = STATIC_CONFIG['environment']['root']
+    plugins_dir_env_var = STATIC_CONFIG['environment']['plugins']
+    config_dir_env_var = STATIC_CONFIG['environment']['config_dir']
+    venvs_dir_env_var = STATIC_CONFIG['environment']['venvs']
+
+    replaced_root = False
+    if root_dir_env_var in env:
+        root_dir_path = pathlib.Path(env[root_dir_env_var])
+        set_root(root_dir_path)
+        replaced_root = True
+
+    replaced_plugins = False
+    if plugins_dir_env_var in env:
+        plugins_dir_paths = env[plugins_dir_env_var]
+        set_plugins_dir_paths(plugins_dir_paths)
+        replaced_plugins = True
+
+    replaced_venvs = False
+    if venvs_dir_env_var in env:
+        venv_dir_path = pathlib.Path(env[venvs_dir_env_var])
+        set_venvs_dir_path(venv_dir_path)
+        replaced_venvs = True
+
+    replaced_config_dir = False
+    if config_dir_env_var in env:
+        config_dir_path = pathlib.Path(env[config_dir_env_var])
+        set_config_dir_path(config_dir_path)
+        replaced_config_dir = True
+
+    apply_environment_patches(env)
+    apply_environment_uris(env)
+
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(old_environ)
+
+        if replaced_root:
+            set_root(old_root_dir_path)
+
+        if replaced_plugins:
+            set_plugins_dir_paths(old_plugins_dir_paths)
+
+        if replaced_venvs:
+            set_venvs_dir_path(old_venvs_dir_path)
+
+        if replaced_config_dir:
+            set_config_dir_path(old_config_dir_path)
+
+        _config().clear()
+        set_config(old_config)

meerschaum/config/stack/__init__.py

@@ -56,19 +56,22 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
     {
         'meerschaum': 'MRSM{!meerschaum}',
         'system': 'MRSM{!system}',
+        'api': 'MRSM{!api}',
     },
     indent=4,
 ).replace(
     '"MRSM{!system}"', 'MRSM{!system}'
 ).replace(
-    '"MRSM{!meerschaum}"', 'MRSM{!meerschaum}',
+    '"MRSM{!meerschaum}"', 'MRSM{!meerschaum}'
+).replace(
+    '"MRSM{!api}"', 'MRSM{!api}'
 )
 
 volumes = {
     'api_root': '/meerschaum',
-    'meerschaum_db_data': '/var/lib/postgresql/data',
+    'meerschaum_db_data': '/home/postgres/pgdata',
     'grafana_storage': '/var/lib/grafana',
-    'valkey_data': '/bitnami/valkey/data',
+    'valkey_data': '/valkey/data',
 }
 networks = {
     'frontend': None,
@@ -122,7 +125,6 @@ default_docker_compose_config = {
                 'POSTGRES_DB': '<DOLLAR>POSTGRES_DB',
                 'POSTGRES_PASSWORD': '<DOLLAR>POSTGRES_PASSWORD',
                 'ALLOW_IP_RANGE': env_dict['ALLOW_IP_RANGE'],
-                #  'POSTGRES_INITDB_ARGS': '-c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100'
             },
             'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100',
             'healthcheck': {
@@ -184,7 +186,7 @@ default_docker_compose_config = {
             ],
         },
         'valkey': {
-            'image': 'bitnami/valkey:latest',
+            'image': 'valkey/valkey:latest',
             'restart': 'always',
             'environment': {
                 'VALKEY_PASSWORD': '<DOLLAR>VALKEY_PASSWORD',

meerschaum/connectors/_Connector.py

@@ -152,8 +152,7 @@ class Connector(metaclass=abc.ABCMeta):
         ------
         An error if any of the required attributes are missing.
         """
-        from meerschaum.utils.warnings import error, warn
-        from meerschaum.utils.debug import dprint
+        from meerschaum.utils.warnings import error
         from meerschaum.utils.misc import items_str
         if required_attributes is None:
             required_attributes = ['type', 'label']