meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

meerschaum/config/_formatting.py

@@ -28,6 +28,8 @@ default_formatting_config = {
         'connector'        : '🔌',
         'metric'           : '📊',
         'location'         : '📍',
+        'locked'           : '🔒',
+        'unlocked'         : '🔓',
         'key'              : '🔑',
         'idea'             : '💡',
         'connected'        : '🟢',

meerschaum/config/_patch.py

@@ -9,12 +9,13 @@ Functions for patching the configuration dictionary
 import sys
 import copy
 from meerschaum.utils.typing import Dict, Any
-from meerschaum.utils.warnings import warn
+from meerschaum.utils.warnings import warn as _warn
 
 def apply_patch_to_config(
-        config: Dict[str, Any],
-        patch: Dict[str, Any],
-    ) -> Dict[str, Any]:
+    config: Dict[str, Any],
+    patch: Dict[str, Any],
+    warn: bool = False,
+) -> Dict[str, Any]:
     """Patch the config dict with a new dict (cascade patching)."""
     _base = copy.deepcopy(config) if isinstance(config, dict) else {}
     if not isinstance(patch, dict):
@@ -24,7 +25,8 @@ def apply_patch_to_config(
         if base is None:
             return {}
         if not isinstance(base, dict):
-            warn(f"Overwriting the value {base} with a dictionary:\n{patch}")
+            if warn:
+                _warn(f"Overwriting the value {base} with a dictionary:\n{patch}")
             base = {}
         for key, value in patch.items():
             if isinstance(value, dict):
@@ -37,9 +39,9 @@ def apply_patch_to_config(
 
 
 def write_patch(
-        patch: Dict[str, Any],
-        debug: bool = False
-    ):
+    patch: Dict[str, Any],
+    debug: bool = False
+) -> None:
     """Write patch dict to yaml."""
     from meerschaum.utils.debug import dprint
     from meerschaum.config._paths import PATCH_DIR_PATH

meerschaum/config/_paths.py

@@ -9,9 +9,16 @@ Define file paths
 from __future__ import annotations
 
 from pathlib import Path
-import os, platform, sys, json
-from meerschaum.utils.typing import Union
-from meerschaum.config.static import STATIC_CONFIG
+import contextlib
+import os
+import platform
+import sys
+import json
+
+from typing import Union, List
+
+from meerschaum._internal.static import STATIC_CONFIG
+
 
 DOT_CONFIG_DIR_PATH = Path(
     os.environ.get('XDG_CONFIG_HOME', Path.home() / '.config')
@@ -71,7 +78,7 @@ if ENVIRONMENT_PLUGINS_DIR in os.environ:
                 if path_str
             ]
         )
-    except Exception as e:
+    except Exception:
         PLUGINS_DIR_PATHS = []
 
     if not PLUGINS_DIR_PATHS:
@@ -82,20 +89,23 @@ if ENVIRONMENT_PLUGINS_DIR in os.environ:
             f"`export {ENVIRONMENT_PLUGINS_DIR}=./plugins:/another/path/to/plugins`\n\n"
             "or a JSON-encoded path list:\n\n"
             f"`export {ENVIRONMENT_PLUGINS_DIR}=" + "'[\"./plugins\", \"/another/path/to/plugins\"]'`"
-            f"",
+            "",
         )
         sys.exit(1)
 else:
     PLUGINS_DIR_PATHS = [_ROOT_DIR_PATH / 'plugins']
 
 ### Remove duplicate plugins paths.
-_seen_plugins_paths, _plugins_paths_to_remove = set(), set()
-for _plugin_path in PLUGINS_DIR_PATHS:
-    if _plugin_path in _seen_plugins_paths:
-        _plugins_paths_to_remove.add(_plugin_path)
-    _seen_plugins_paths.add(_plugin_path)
-for _plugin_path in _plugins_paths_to_remove:
-    PLUGINS_DIR_PATHS.remove(_plugin_path)
+def _process_plugins_dir_paths(plugins_dir_paths: List[Path]):
+    _seen_plugins_paths, _plugins_paths_to_remove = set(), set()
+    for _plugin_path in plugins_dir_paths:
+        if _plugin_path in _seen_plugins_paths:
+            _plugins_paths_to_remove.add(_plugin_path)
+        _seen_plugins_paths.add(_plugin_path)
+    for _plugin_path in _plugins_paths_to_remove:
+        plugins_dir_paths.remove(_plugin_path)
+_process_plugins_dir_paths(PLUGINS_DIR_PATHS)
+
 
 ENVIRONMENT_VENVS_DIR = STATIC_CONFIG['environment']['venvs']
 if ENVIRONMENT_VENVS_DIR in os.environ:
@@ -144,11 +154,17 @@ paths = {
     'API_SECRET_KEY_PATH'            : ('{API_CONFIG_RESOURCES_PATH}', '.api_secret_key'),
     'API_UVICORN_RESOURCES_PATH'     : ('{API_CONFIG_RESOURCES_PATH}', 'uvicorn'),
     'API_UVICORN_CONFIG_PATH'        : ('{API_UVICORN_RESOURCES_PATH}', '.thread_config.json'),
+    
+    'WEBTERM_INTERNAL_RESOURCES_PATH': ('{INTERNAL_RESOURCES_PATH}', 'webterm'),
 
     'CACHE_RESOURCES_PATH'           : ('{ROOT_DIR_PATH}', '.cache'),
     'PIPES_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'pipes'),
     'USERS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'users'),
     'VENVS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'venvs'),
+    'SQL_CONN_CACHE_RESOURCES_PATH'  : ('{CACHE_RESOURCES_PATH}', 'sql'),
+
+    'CLI_RESOURCES_PATH'             : ('{INTERNAL_RESOURCES_PATH}', 'cli'),
+    'CLI_LOGS_RESOURCES_PATH'        : ('{CLI_RESOURCES_PATH}', 'logs'),
 
     'PLUGINS_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
     'PLUGINS_INTERNAL_LOCK_PATH'     : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),
@@ -156,6 +172,11 @@ paths = {
     'PLUGINS_ARCHIVES_RESOURCES_PATH': ('{PLUGINS_RESOURCES_PATH}', '.archives'),
     'PLUGINS_TEMP_RESOURCES_PATH'    : ('{PLUGINS_RESOURCES_PATH}', '.tmp'),
     'PLUGINS_INIT_PATH'              : ('{PLUGINS_RESOURCES_PATH}', '__init__.py'),
+    'PLUGINS_INJECTED_RESOURCES_PATH': ('{PLUGINS_RESOURCES_PATH}', '.injected'),
+
+    'DB_RESOURCES_PATH'              : ('{ROOT_DIR_PATH}', 'db'),
+    'DB_INIT_RESOURCES_PATH'         : ('{DB_RESOURCES_PATH}', 'initdb'),
+    'DB_CREATE_EXTENSIONS_PATH'      : ('{DB_INIT_RESOURCES_PATH}', 'create-extensions.sql'),
 
     'SQLITE_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'sqlite'),
     'SQLITE_DB_PATH'                 : ('{SQLITE_RESOURCES_PATH}', 'mrsm_local.db'),
@@ -192,11 +213,110 @@ paths = {
 
 def set_root(root: Union[Path, str]):
     """Modify the value of `ROOT_DIR_PATH`."""
-    paths['ROOT_DIR_PATH'] = Path(root).resolve()
+    paths['ROOT_DIR_PATH'] = Path(root).resolve().as_posix()
     for path_name, path_parts in paths.items():
         if isinstance(path_parts, tuple) and path_parts[0] == '{ROOT_DIR_PATH}':
             globals()[path_name] = __getattr__(path_name)
 
+def set_plugins_dir_paths(plugins_dir_paths: Union[List[Path], Path, str]):
+    """Modify the value of `PLUGINS_DIR_PATHS`."""
+    global PLUGINS_DIR_PATHS
+    if isinstance(plugins_dir_paths, str):
+        if plugins_dir_paths.strip() and plugins_dir_paths.lstrip()[0] == '[':
+            plugins_dir_paths = json.loads(plugins_dir_paths)
+        else:
+            plugins_dir_paths = plugins_dir_paths.split(':')
+        plugins_dir_paths = [Path(_path).resolve() for _path in plugins_dir_paths]
+    if isinstance(plugins_dir_paths, Path):
+        plugins_dir_paths = [plugins_dir_paths.resolve()]
+
+    PLUGINS_DIR_PATHS = plugins_dir_paths
+    _process_plugins_dir_paths(plugins_dir_paths)
+
+def set_venvs_dir_path(venvs_dir_path: Union[str, Path]):
+    """Modify the value of `VIRTENV_RESOURCES_PATH`."""
+    paths['VIRTENV_RESOURCES_PATH'] = Path(venvs_dir_path).resolve().as_posix()
+
+def set_config_dir_path(config_dir_path: Union[str, Path]):
+    paths['CONFIG_DIR_PATH'] = Path(config_dir_path).resolve().as_posix()
+
+
+@contextlib.contextmanager
+def replace_root_dir(root_dir_path: Union[str, Path, None]):
+    """
+    Temporarily replace the root directory path.
+    """
+    if root_dir_path is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_root = paths.get('ROOT_DIR_PATH', _ROOT_DIR_PATH)
+    set_root(root_dir_path)
+
+    try:
+        yield
+    finally:
+        set_root(old_root)
+
+@contextlib.contextmanager
+def replace_plugins_dir_paths(plugins_dir_paths: Union[List[Path], None]):
+    """
+    Temporarily replace the plugins directory paths.
+    """
+    if plugins_dir_paths is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_plugins_dir_paths = PLUGINS_DIR_PATHS
+    set_plugins_dir_paths(plugins_dir_paths)
+
+    try:
+        yield
+    finally:
+        set_plugins_dir_paths(old_plugins_dir_paths)
+
+@contextlib.contextmanager
+def replace_venvs_dir_path(venvs_dir_path: Union[Path, None]):
+    """
+    Temporarily replace the virtual environments directory path.
+    """
+    if venvs_dir_path is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_venvs_dir_path = paths.get('VIRTENV_RESOURCES_PATH', _VENVS_DIR_PATH)
+    set_venvs_dir_path(venvs_dir_path)
+
+    try:
+        yield
+    finally:
+        set_venvs_dir_path(old_venvs_dir_path)
+
+@contextlib.contextmanager
+def replace_config_dir_path(config_dir_path: Union[Path, None]):
+    """
+    Temporarily replace the config directory path.
+    """
+    if config_dir_path is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_config_dir_path = paths.get('CONFIG_DIR_PATH', _CONFIG_DIR_PATH)
+    set_config_dir_path(config_dir_path)
+
+    try:
+        yield
+    finally:
+        set_config_dir_path(old_config_dir_path)
+
 
 def __getattr__(name: str) -> Path:
     if name not in paths:

meerschaum/config/_read_config.py

@@ -8,8 +8,9 @@ Import the config yaml file
 from __future__ import annotations
 import pathlib
 
+import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Dict, Any, List, Tuple, Union
-from meerschaum.config import get_config
+from meerschaum._internal.static import STATIC_CONFIG
 
 
 def read_config(
@@ -49,11 +50,11 @@ def read_config(
     >>> read_config(keys=['meerschaum'], with_filename=True)
     >>> ({...}, ['meerschaum.yaml'])
     """
-    import sys, shutil, os, json, itertools
-    from meerschaum.utils.packages import attempt_import
+    import os
+    import json
+    import itertools
     from meerschaum.utils.yaml import yaml, _yaml
     from meerschaum.config._paths import CONFIG_DIR_PATH
-    from meerschaum.config.static import STATIC_CONFIG
     from meerschaum.config._patch import apply_patch_to_config
     if directory is None:
         directory = CONFIG_DIR_PATH
@@ -106,20 +107,25 @@ def read_config(
             ### If default config contains symlinks, add them to the config to write.
             try:
                 _default_symlinks = _default_dict[symlinks_key][mk]
-            except Exception as e:
+            except KeyError:
                 _default_symlinks = {}
+
             config[mk] = _default_dict[mk]
             if _default_symlinks:
                 if symlinks_key not in config:
                     config[symlinks_key] = {}
+                if symlinks_key not in config_to_write:
+                    config_to_write[symlinks_key] = {}
+
                 if mk not in config[symlinks_key]:
                     config[symlinks_key][mk] = {}
+                if mk not in config_to_write[symlinks_key]:
+                    config_to_write[symlinks_key][mk] = {}
+
                 config[symlinks_key][mk] = apply_patch_to_config(
                     config[symlinks_key][mk], 
                     _default_symlinks
                 )
-                if symlinks_key not in config_to_write:
-                    config_to_write[symlinks_key] = {}
                 config_to_write[symlinks_key][mk] = config[symlinks_key][mk]
 
             ### Write the default key.
@@ -179,6 +185,7 @@ def read_config(
                         import traceback
                         traceback.print_exc()
                         _config_key = {}
+                substitute = False
                 _single_key_config = (
                     search_and_substitute_config({key: _config_key}) if substitute
                     else {key: _config_key}
@@ -208,15 +215,16 @@ def read_config(
 
 
 def search_and_substitute_config(
-        config: Dict[str, Any],
-        leading_key: str = "MRSM",
-        delimiter: str = ":",
-        begin_key: str = "{",
-        end_key: str = "}",
-        literal_key: str = '!',
-        keep_symlinks: bool = True,
-    ) -> Dict[str, Any]:
-    """Search the config for Meerschaum substitution syntax and substite with value of keys.
+    config: Dict[str, Any],
+    leading_key: str = "MRSM",
+    delimiter: str = ":",
+    begin_key: str = "{",
+    end_key: str = "}",
+    literal_key: str = '!',
+    keep_symlinks: bool = True,
+) -> Dict[str, Any]:
+    """
+    Search the config for Meerschaum substitution syntax and substite with value of keys.
 
     Parameters
     ----------
@@ -242,7 +250,7 @@ def search_and_substitute_config(
         - ' MRSM{a:b:c} '  => ' "{\'d\': 1}"' : not isolated
         - ' MRSM{!a:b:c} ' => ' {"d": 1}'     : literal
 
-    keep_symlinks :
+    keep_symlinks: bool, default True
         If True, include the symlinks under the top-level key '_symlinks' (never written to a file).
         Defaults to True.
         
@@ -251,7 +259,13 @@ def search_and_substitute_config(
         ```
         MRSM{meerschaum:connectors:main:host} => cf['meerschaum']['connectors']['main']['host']
         ``` 
+
+    Returns
+    -------
+    The configuration dictionary with `MRSM{}` symlinks replaced with
+    the values from the current configuration.
     """
+    from meerschaum.config import get_config
 
     _links = []
     def _find_symlinks(d, _keys: Optional[List[str]] = None):
@@ -270,9 +284,6 @@ def search_and_substitute_config(
     import json
     needle = leading_key + begin_key
     haystack = json.dumps(config, separators=(',', ':'))
-    mod_haystack = list(str(haystack))
-    buff = str(needle)
-    max_index = len(haystack) - len(buff)
 
     patterns = {}
     isolated_patterns = {}
@@ -329,7 +340,7 @@ def search_and_substitute_config(
                 write_missing=False,
                 sync_files=False,
             )
-        except Exception as e:
+        except Exception:
             import traceback
             traceback.print_exc()
             valid = False
@@ -382,15 +393,55 @@ def search_and_substitute_config(
             s[_keys[-1]] = _pattern
 
         from meerschaum.config._patch import apply_patch_to_config
-        from meerschaum.config.static import STATIC_CONFIG
         symlinks_key = STATIC_CONFIG['config']['symlinks_key']
-        if symlinks_key not in parsed_config:
-            parsed_config[symlinks_key] = {}
-        parsed_config[symlinks_key] = apply_patch_to_config(parsed_config[symlinks_key], symlinks)
+        if symlinks:
+            if symlinks_key not in parsed_config:
+                parsed_config[symlinks_key] = symlinks
+            else:
+                parsed_config[symlinks_key] = apply_patch_to_config(
+                    parsed_config[symlinks_key],
+                    symlinks,
+                    warn=False,
+                )
 
     return parsed_config
 
 
+def revert_symlinks_config(config: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Given a configuration dictionary, re-apply the values from the
+    accompanying `_symlinks` dictionary.
+
+    Parameters
+    ----------
+    config: Dict[str, Any]
+        The configuration dictionary containing a `_symlinks` dictionary.
+
+    Returns
+    -------
+    A configuration dictionary with `_symlinks` re-applied.
+    """
+    import copy
+    from meerschaum._internal.static import STATIC_CONFIG
+
+    symlinks_key = STATIC_CONFIG['config']['symlinks_key']
+    if symlinks_key not in config:
+        return config
+
+    reverted_config = copy.deepcopy(config)
+    symlinks_config = reverted_config.pop(symlinks_key)
+
+    def deep_patch(target_dict, patch_dict):
+        for key, value in patch_dict.items():
+            if isinstance(value, dict) and key in target_dict and isinstance(target_dict[key], dict):
+                deep_patch(target_dict[key], value)
+            else:
+                target_dict[key] = value
+
+    deep_patch(reverted_config, symlinks_config)
+    return reverted_config
+
+
 def get_possible_keys() -> List[str]:
     """
     Return a list of possible top-level keys.
@@ -407,12 +458,13 @@ def get_possible_keys() -> List[str]:
 
 
 def get_keyfile_path(
-        key: str,
-        create_new: bool = False,
-        directory: Union[pathlib.Path, str, None] = None,
-    ) -> Union[pathlib.Path, None]:
+    key: str,
+    create_new: bool = False,
+    directory: Union[pathlib.Path, str, None] = None,
+) -> Union[pathlib.Path, None]:
     """Determine a key's file path."""
-    import os, pathlib
+    import os
+    import pathlib
     if directory is None:
         from meerschaum.config._paths import CONFIG_DIR_PATH
         directory = CONFIG_DIR_PATH
@@ -422,16 +474,15 @@ def get_keyfile_path(
             os.path.join(
                 directory,
                 read_config(
-                    keys = [key],
-                    with_filenames = True,
-                    write_missing = False,
-                    substitute = False,
+                    keys=[key],
+                    with_filenames=True,
+                    write_missing=False,
+                    substitute=False,
                 )[1][0]
             )
         )
-    except IndexError as e:
+    except IndexError:
         if create_new:
-            from meerschaum.config.static import STATIC_CONFIG
             default_filetype = STATIC_CONFIG['config']['default_filetype']
             return pathlib.Path(os.path.join(directory, key + '.' + default_filetype))
         return None

meerschaum/config/_sync.py

@@ -40,10 +40,11 @@ def sync_yaml_configs(
         If provided, iterate through a list of tuples,
         replacing the old string (index 0) with the new string (index 1).
     """
-    import os, sys
+    import os
+    import sys
     try:
         from meerschaum.utils.yaml import yaml, _yaml
-    except Exception as e:
+    except Exception:
         return
     from meerschaum.config._patch import apply_patch_to_config
     import meerschaum.config
@@ -110,7 +111,8 @@ def sync_files(keys: Optional[List[str]] = None):
             GRAFANA_DATASOURCE_PATH,
             GRAFANA_DASHBOARD_PATH,
         )
-        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum._internal.static import STATIC_CONFIG
+        from meerschaum.config.stack import _write_initdb
 
         sync_yaml_configs(
             ['stack', STACK_COMPOSE_FILENAME],
@@ -131,6 +133,7 @@ def sync_files(keys: Optional[List[str]] = None):
             GRAFANA_DASHBOARD_PATH,
             substitute=True,
         )
+        _write_initdb()
 
     key_functions = {
         'stack': _stack,

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.9.4"
+__version__ = "3.0.0"