meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

meerschaum/config/_default.py

@@ -36,8 +36,7 @@ CONNECTOR_ATTRIBUTES: Dict[str, Dict[str, Any]] = {
 default_meerschaum_config = {
     'instance': 'sql:main',
     'api_instance': 'MRSM{meerschaum:instance}',
-    'web_instance': 'MRSM{meerschaum:instance}',
-    'default_repository': 'api:mrsm',
+    'repository': 'api:mrsm',
     'connectors': {
         'sql': {
             'default': {},
@@ -51,7 +50,7 @@ default_meerschaum_config = {
             },
             'local': {
                 'flavor': 'sqlite',
-                'database': SQLITE_DB_PATH.as_posix(),
+                'database': "{SQLITE_DB_PATH}",
             },
             'memory': {
                 'flavor': 'sqlite',
@@ -117,75 +116,107 @@ default_system_config = {
                 'connect_args': {},
             },
         },
-
         'api': {
         },
     },
-    ### not to be confused with system_config['connectors']['api'], this is the configuration
-    ### for the API server itself.
-    'api': {
-        'uvicorn': {
-            'app': 'meerschaum.api:app',
-            'port': 8000,
-            'host': '0.0.0.0',
-            'workers': max(int(multiprocessing.cpu_count() / 2), 1),
-            'proxy_headers': True,
-            'forwarded_allow_ips': '*',
-        },
-        'cache': {
-            'connector': 'valkey:main',
-            'session_expires_minutes': 43200,
+    'api': 'MRSM{api}',
+    'webterm': 'MRSM{api:webterm}',
+    'cli': {
+        'max_daemons': (multiprocessing.cpu_count() * 3),
+        'refresh_seconds': 0.1,
+        'allowed_prefixes': ['*'],
+        'disallowed_prefixes': [
+            'edit',
+            'start daemon',
+            'start job',
+            'stop daemon',
+            'show daemon',
+            'restart daemon',
+            'reload',
+            'start worker',
+            'start job',
+            'python',
+            'login',
+            'executor',
+            'os ',
+            'sh ',
+            'start api',
+            'start webterm',
+            'stack',
+            'instance',
+            'debug',
+            'bootstrap',
+            'daemon',
+        ],
+    },
+    'experimental': {
+        'fetch': False,
+        'cache': True,
+        'space': False,
+        'join_fetch': False,
+        'inplace_sync': True,
+        'uv_pip': True,
+        'systemd_healthcheck': False,
+        'valkey_session_cache': True,
+        'cli_daemon': True,
+    },
+}
+
+default_api_config = {
+    'uvicorn': {
+        'app': 'meerschaum.api:app',
+        'port': 8000,
+        'host': '0.0.0.0',
+        'workers': max(int(multiprocessing.cpu_count() / 2), 1),
+        'proxy_headers': True,
+        'forwarded_allow_ips': '*',
+    },
+    'cache': {
+        'connector': 'valkey:main',
+        'session_expires_minutes': 43200,
+    },
+    'data': {
+        'max_response_row_limit': 100_000,
+        'chunks': {
+            'ttl_seconds': 1800,
         },
-        'data': {
-            'max_response_row_limit': 100_000,
-            'chunks': {
-                'ttl_seconds': 1800,
-            },
+    },
+    'endpoints': {
+        'docs_in_production': True,
+    },
+    'tokens': {
+        'valid_refresh_minutes': 60,
+        'default_expiration_days': 366,
+    },
+    'permissions':       {
+        'registration': {
+            'users': True,
+            'pipes': True,
+            'plugins': True,
         },
-        'endpoints': {
-            'docs_in_production': True,
+        'actions': {
+            'non_admin': True,
         },
-        'tokens': {
-            'valid_refresh_minutes': 60,
-            'default_expiration_days': 366,
+        'chaining': {
+            'insecure_parent_instance': False,
+            'child_apis': False,
         },
-        'permissions':       {
-            'registration': {
-                'users': True,
-                'pipes': True,
-                'plugins': True,
-            },
-            'actions': {
-                'non_admin': True,
-            },
-            'chaining': {
-                'insecure_parent_instance': False,
-                'child_apis': False,
-            },
-            'instances': {
-                'allow_multiple_instances': True,
-                'allowed_instance_keys': ['*']
-            },
+        'instances': {
+            'allow_multiple_instances': True,
+            'allowed_instance_keys': ['*']
         },
-        'protocol': default_meerschaum_config['connectors']['api']['default']['protocol'],
     },
+    'protocol': default_meerschaum_config['connectors']['api']['default']['protocol'],
     'webterm': {
         'tmux': {
             'enabled': True,
             'session_suffix': '_mrsm',
         },
-    },
-    'experimental': {
-        'fetch': False,
-        'cache': True,
-        'space': False,
-        'join_fetch': False,
-        'inplace_sync': True,
-        'uv_pip': True,
-        'systemd_healthcheck': False,
-        'valkey_session_cache': True,
+        'host': '127.0.0.1',
+        'port': 8765,
     },
 }
+
 default_pipes_config = {
     'parameters': {
         'columns': {
@@ -221,15 +252,12 @@ default_pipes_config = {
     },
 }
 default_plugins_config = {}
-default_experimental_config = {
-    'venv': True,
-}
-
 
 ### build default config dictionary
 default_config = {}
 default_config['meerschaum'] = default_meerschaum_config
 default_config['system'] = default_system_config
+default_config['api'] = default_api_config
 from meerschaum.config._formatting import default_formatting_config
 default_config['formatting'] = default_formatting_config
 from meerschaum.config._shell import default_shell_config
@@ -241,7 +269,7 @@ default_config['jobs'] = default_jobs_config
 ### add configs from other packages
 try:
     import meerschaum.config.stack
-except ImportError as e:
+except ImportError:
     pass
 finally:
     from meerschaum.config.stack import default_stack_config

meerschaum/config/_edit.py

@@ -103,19 +103,23 @@ def write_config(
     if directory is None:
         from meerschaum.config._paths import CONFIG_DIR_PATH
         directory = CONFIG_DIR_PATH
+
+    from meerschaum.config import _allow_write_missing
     from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.config._default import default_header_comment
-    from meerschaum.config._patch import apply_patch_to_config
-    from meerschaum.config._read_config import get_keyfile_path
-    from meerschaum.utils.debug import dprint
+    from meerschaum.config._read_config import get_keyfile_path, revert_symlinks_config
     from meerschaum.utils.yaml import yaml
     from meerschaum.utils.misc import filter_keywords
-    import json, os
+    import json
+    import os
     if config_dict is None:
         from meerschaum.config import _config
-        cf = _config()
+        cf = _config(allow_replaced=False)
         config_dict = cf
 
+    if not _allow_write_missing:
+        return False
+
     default_filetype = STATIC_CONFIG['config']['default_filetype']
     filetype_dumpers = {
         'yml' : yaml.dump,
@@ -123,9 +127,7 @@ def write_config(
         'json' : json.dump,
     }
 
-    symlinks_key = STATIC_CONFIG['config']['symlinks_key']
-    symlinks = config_dict.pop(symlinks_key) if symlinks_key in config_dict else {}
-    config_dict = apply_patch_to_config(config_dict, symlinks)
+    config_dict = revert_symlinks_config(config_dict)
 
     def determine_filetype(k, v):
         if k == 'meerschaum':

meerschaum/config/_formatting.py

@@ -28,6 +28,8 @@ default_formatting_config = {
         'connector'        : '🔌',
         'metric'           : '📊',
         'location'         : '📍',
+        'locked'           : '🔒',
+        'unlocked'         : '🔓',
         'key'              : '🔑',
         'idea'             : '💡',
         'connected'        : '🟢',

meerschaum/config/_patch.py

@@ -9,11 +9,12 @@ Functions for patching the configuration dictionary
 import sys
 import copy
 from meerschaum.utils.typing import Dict, Any
-from meerschaum.utils.warnings import warn
+from meerschaum.utils.warnings import warn as _warn
 
 def apply_patch_to_config(
     config: Dict[str, Any],
     patch: Dict[str, Any],
+    warn: bool = False,
 ) -> Dict[str, Any]:
     """Patch the config dict with a new dict (cascade patching)."""
     _base = copy.deepcopy(config) if isinstance(config, dict) else {}
@@ -24,7 +25,8 @@ def apply_patch_to_config(
         if base is None:
             return {}
         if not isinstance(base, dict):
-            warn(f"Overwriting the value {base} with a dictionary:\n{patch}")
+            if warn:
+                _warn(f"Overwriting the value {base} with a dictionary:\n{patch}")
             base = {}
         for key, value in patch.items():
             if isinstance(value, dict):

meerschaum/config/_paths.py

@@ -9,10 +9,17 @@ Define file paths
 from __future__ import annotations
 
 from pathlib import Path
-import os, platform, sys, json
-from meerschaum.utils.typing import Union
+import contextlib
+import os
+import platform
+import sys
+import json
+
+from typing import Union, List
+
 from meerschaum._internal.static import STATIC_CONFIG
 
+
 DOT_CONFIG_DIR_PATH = Path(
     os.environ.get('XDG_CONFIG_HOME', Path.home() / '.config')
     if platform.system() != 'Windows'
@@ -71,7 +78,7 @@ if ENVIRONMENT_PLUGINS_DIR in os.environ:
                 if path_str
             ]
         )
-    except Exception as e:
+    except Exception:
         PLUGINS_DIR_PATHS = []
 
     if not PLUGINS_DIR_PATHS:
@@ -82,20 +89,23 @@ if ENVIRONMENT_PLUGINS_DIR in os.environ:
             f"`export {ENVIRONMENT_PLUGINS_DIR}=./plugins:/another/path/to/plugins`\n\n"
             "or a JSON-encoded path list:\n\n"
             f"`export {ENVIRONMENT_PLUGINS_DIR}=" + "'[\"./plugins\", \"/another/path/to/plugins\"]'`"
-            f"",
+            "",
         )
         sys.exit(1)
 else:
     PLUGINS_DIR_PATHS = [_ROOT_DIR_PATH / 'plugins']
 
 ### Remove duplicate plugins paths.
-_seen_plugins_paths, _plugins_paths_to_remove = set(), set()
-for _plugin_path in PLUGINS_DIR_PATHS:
-    if _plugin_path in _seen_plugins_paths:
-        _plugins_paths_to_remove.add(_plugin_path)
-    _seen_plugins_paths.add(_plugin_path)
-for _plugin_path in _plugins_paths_to_remove:
-    PLUGINS_DIR_PATHS.remove(_plugin_path)
+def _process_plugins_dir_paths(plugins_dir_paths: List[Path]):
+    _seen_plugins_paths, _plugins_paths_to_remove = set(), set()
+    for _plugin_path in plugins_dir_paths:
+        if _plugin_path in _seen_plugins_paths:
+            _plugins_paths_to_remove.add(_plugin_path)
+        _seen_plugins_paths.add(_plugin_path)
+    for _plugin_path in _plugins_paths_to_remove:
+        plugins_dir_paths.remove(_plugin_path)
+_process_plugins_dir_paths(PLUGINS_DIR_PATHS)
+
 
 ENVIRONMENT_VENVS_DIR = STATIC_CONFIG['environment']['venvs']
 if ENVIRONMENT_VENVS_DIR in os.environ:
@@ -144,6 +154,8 @@ paths = {
     'API_SECRET_KEY_PATH'            : ('{API_CONFIG_RESOURCES_PATH}', '.api_secret_key'),
     'API_UVICORN_RESOURCES_PATH'     : ('{API_CONFIG_RESOURCES_PATH}', 'uvicorn'),
     'API_UVICORN_CONFIG_PATH'        : ('{API_UVICORN_RESOURCES_PATH}', '.thread_config.json'),
+    
+    'WEBTERM_INTERNAL_RESOURCES_PATH': ('{INTERNAL_RESOURCES_PATH}', 'webterm'),
 
     'CACHE_RESOURCES_PATH'           : ('{ROOT_DIR_PATH}', '.cache'),
     'PIPES_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'pipes'),
@@ -151,12 +163,16 @@ paths = {
     'VENVS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'venvs'),
     'SQL_CONN_CACHE_RESOURCES_PATH'  : ('{CACHE_RESOURCES_PATH}', 'sql'),
 
+    'CLI_RESOURCES_PATH'             : ('{INTERNAL_RESOURCES_PATH}', 'cli'),
+    'CLI_LOGS_RESOURCES_PATH'        : ('{CLI_RESOURCES_PATH}', 'logs'),
+
     'PLUGINS_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
     'PLUGINS_INTERNAL_LOCK_PATH'     : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),
     'PLUGINS_PACKAGES_INTERNAL_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'packaged_plugins'),
     'PLUGINS_ARCHIVES_RESOURCES_PATH': ('{PLUGINS_RESOURCES_PATH}', '.archives'),
     'PLUGINS_TEMP_RESOURCES_PATH'    : ('{PLUGINS_RESOURCES_PATH}', '.tmp'),
     'PLUGINS_INIT_PATH'              : ('{PLUGINS_RESOURCES_PATH}', '__init__.py'),
+    'PLUGINS_INJECTED_RESOURCES_PATH': ('{PLUGINS_RESOURCES_PATH}', '.injected'),
 
     'DB_RESOURCES_PATH'              : ('{ROOT_DIR_PATH}', 'db'),
     'DB_INIT_RESOURCES_PATH'         : ('{DB_RESOURCES_PATH}', 'initdb'),
@@ -197,11 +213,110 @@ paths = {
 
 def set_root(root: Union[Path, str]):
     """Modify the value of `ROOT_DIR_PATH`."""
-    paths['ROOT_DIR_PATH'] = Path(root).resolve()
+    paths['ROOT_DIR_PATH'] = Path(root).resolve().as_posix()
     for path_name, path_parts in paths.items():
         if isinstance(path_parts, tuple) and path_parts[0] == '{ROOT_DIR_PATH}':
             globals()[path_name] = __getattr__(path_name)
 
+def set_plugins_dir_paths(plugins_dir_paths: Union[List[Path], Path, str]):
+    """Modify the value of `PLUGINS_DIR_PATHS`."""
+    global PLUGINS_DIR_PATHS
+    if isinstance(plugins_dir_paths, str):
+        if plugins_dir_paths.strip() and plugins_dir_paths.lstrip()[0] == '[':
+            plugins_dir_paths = json.loads(plugins_dir_paths)
+        else:
+            plugins_dir_paths = plugins_dir_paths.split(':')
+        plugins_dir_paths = [Path(_path).resolve() for _path in plugins_dir_paths]
+    if isinstance(plugins_dir_paths, Path):
+        plugins_dir_paths = [plugins_dir_paths.resolve()]
+
+    PLUGINS_DIR_PATHS = plugins_dir_paths
+    _process_plugins_dir_paths(plugins_dir_paths)
+
+def set_venvs_dir_path(venvs_dir_path: Union[str, Path]):
+    """Modify the value of `VIRTENV_RESOURCES_PATH`."""
+    paths['VIRTENV_RESOURCES_PATH'] = Path(venvs_dir_path).resolve().as_posix()
+
+def set_config_dir_path(config_dir_path: Union[str, Path]):
+    paths['CONFIG_DIR_PATH'] = Path(config_dir_path).resolve().as_posix()
+
+
+@contextlib.contextmanager
+def replace_root_dir(root_dir_path: Union[str, Path, None]):
+    """
+    Temporarily replace the root directory path.
+    """
+    if root_dir_path is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_root = paths.get('ROOT_DIR_PATH', _ROOT_DIR_PATH)
+    set_root(root_dir_path)
+
+    try:
+        yield
+    finally:
+        set_root(old_root)
+
+@contextlib.contextmanager
+def replace_plugins_dir_paths(plugins_dir_paths: Union[List[Path], None]):
+    """
+    Temporarily replace the plugins directory paths.
+    """
+    if plugins_dir_paths is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_plugins_dir_paths = PLUGINS_DIR_PATHS
+    set_plugins_dir_paths(plugins_dir_paths)
+
+    try:
+        yield
+    finally:
+        set_plugins_dir_paths(old_plugins_dir_paths)
+
+@contextlib.contextmanager
+def replace_venvs_dir_path(venvs_dir_path: Union[Path, None]):
+    """
+    Temporarily replace the virtual environments directory path.
+    """
+    if venvs_dir_path is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_venvs_dir_path = paths.get('VIRTENV_RESOURCES_PATH', _VENVS_DIR_PATH)
+    set_venvs_dir_path(venvs_dir_path)
+
+    try:
+        yield
+    finally:
+        set_venvs_dir_path(old_venvs_dir_path)
+
+@contextlib.contextmanager
+def replace_config_dir_path(config_dir_path: Union[Path, None]):
+    """
+    Temporarily replace the config directory path.
+    """
+    if config_dir_path is None:
+        try:
+            yield
+        finally:
+            return
+
+    old_config_dir_path = paths.get('CONFIG_DIR_PATH', _CONFIG_DIR_PATH)
+    set_config_dir_path(config_dir_path)
+
+    try:
+        yield
+    finally:
+        set_config_dir_path(old_config_dir_path)
+
 
 def __getattr__(name: str) -> Path:
     if name not in paths:

meerschaum/config/_read_config.py

@@ -8,7 +8,9 @@ Import the config yaml file
 from __future__ import annotations
 import pathlib
 
+import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Dict, Any, List, Tuple, Union
+from meerschaum._internal.static import STATIC_CONFIG
 
 
 def read_config(
@@ -53,7 +55,6 @@ def read_config(
     import itertools
     from meerschaum.utils.yaml import yaml, _yaml
     from meerschaum.config._paths import CONFIG_DIR_PATH
-    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.config._patch import apply_patch_to_config
     if directory is None:
         directory = CONFIG_DIR_PATH
@@ -106,20 +107,25 @@ def read_config(
             ### If default config contains symlinks, add them to the config to write.
             try:
                 _default_symlinks = _default_dict[symlinks_key][mk]
-            except Exception:
+            except KeyError:
                 _default_symlinks = {}
+
             config[mk] = _default_dict[mk]
             if _default_symlinks:
                 if symlinks_key not in config:
                     config[symlinks_key] = {}
+                if symlinks_key not in config_to_write:
+                    config_to_write[symlinks_key] = {}
+
                 if mk not in config[symlinks_key]:
                     config[symlinks_key][mk] = {}
+                if mk not in config_to_write[symlinks_key]:
+                    config_to_write[symlinks_key][mk] = {}
+
                 config[symlinks_key][mk] = apply_patch_to_config(
                     config[symlinks_key][mk], 
                     _default_symlinks
                 )
-                if symlinks_key not in config_to_write:
-                    config_to_write[symlinks_key] = {}
                 config_to_write[symlinks_key][mk] = config[symlinks_key][mk]
 
             ### Write the default key.
@@ -387,10 +393,16 @@ def search_and_substitute_config(
             s[_keys[-1]] = _pattern
 
         from meerschaum.config._patch import apply_patch_to_config
-        from meerschaum._internal.static import STATIC_CONFIG
         symlinks_key = STATIC_CONFIG['config']['symlinks_key']
-        if symlinks_key in parsed_config:
-            parsed_config[symlinks_key] = apply_patch_to_config(parsed_config[symlinks_key], symlinks)
+        if symlinks:
+            if symlinks_key not in parsed_config:
+                parsed_config[symlinks_key] = symlinks
+            else:
+                parsed_config[symlinks_key] = apply_patch_to_config(
+                    parsed_config[symlinks_key],
+                    symlinks,
+                    warn=False,
+                )
 
     return parsed_config
 
@@ -410,13 +422,12 @@ def revert_symlinks_config(config: Dict[str, Any]) -> Dict[str, Any]:
     A configuration dictionary with `_symlinks` re-applied.
     """
     from meerschaum.config import apply_patch_to_config
-    from meerschaum._internal.static import STATIC_CONFIG
     symlinks_key = STATIC_CONFIG['config']['symlinks_key']
     if symlinks_key not in config:
         return config
 
     symlinks_config = config[symlinks_key]
-    reverted_config = apply_patch_to_config(config, symlinks_config)
+    reverted_config = apply_patch_to_config(symlinks_config, config, warn=False)
     _ = reverted_config.pop(symlinks_key, None)
     return reverted_config
 
@@ -462,7 +473,6 @@ def get_keyfile_path(
         )
     except IndexError:
         if create_new:
-            from meerschaum._internal.static import STATIC_CONFIG
             default_filetype = STATIC_CONFIG['config']['default_filetype']
             return pathlib.Path(os.path.join(directory, key + '.' + default_filetype))
         return None

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "3.0.0rc4"
+__version__ = "3.0.0rc7"