meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

meerschaum/config/__init__.py

@@ -9,14 +9,25 @@ and if interactive, print the welcome message.
 
 from __future__ import annotations
 
-import os, shutil, sys, pathlib, copy
+import os
+import shutil
+import sys
+import copy
+import contextlib
+
 from meerschaum.utils.typing import Any, Dict, Optional, Union
 from meerschaum.utils.threading import RLock
-from meerschaum.utils.warnings import warn, error
 
 from meerschaum.config._version import __version__
 from meerschaum.config._edit import edit_config, write_config
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum.config._read_config import (
+    search_and_substitute_config,
+    revert_symlinks_config,
+    get_possible_keys,
+    get_keyfile_path,
+    read_config,
+)
+from meerschaum._internal.static import STATIC_CONFIG
 
 from meerschaum.config._paths import (
     PERMANENT_PATCH_DIR_PATH,
@@ -31,22 +42,40 @@ __all__ = (
     'write_plugin_config',
     'get_config',
     'write_config',
+    'edit_config',
     'set_config',
+    'replace_config',
+    'search_and_substitute_config',
+    'revert_symlinks_config',
+    'get_possible_keys',
+    'get_keyfile_path',
+    'apply_patch_to_config',
+    'read_config',
     'paths',
+    'STATIC_CONFIG',
 )
 __pdoc__ = {'static': False, 'resources': False, 'stack': False, }
 _locks = {'config': RLock()}
 
 ### apply config preprocessing (e.g. main to meta)
 config = {}
+_backup_config = None
+_allow_write_missing: bool = True
+
+
 def _config(
-        *keys: str, reload: bool = False, substitute: bool = True,
-        sync_files: bool = True, write_missing: bool = True,
-    ) -> Dict[str, Any]:
+    *keys: str,
+    reload: bool = False,
+    substitute: bool = True,
+    sync_files: bool = True,
+    allow_replaced: bool = True,
+    write_missing: bool = True,
+) -> Dict[str, Any]:
     """
     Read and process the configuration file.
     """
-    global config
+    global config, _backup_config
+
     if config is None or reload:
         with _locks['config']:
             config = {}
@@ -56,12 +85,16 @@ def _config(
         key_config = read_config(
             keys = [keys[0]],
             substitute = substitute,
-            write_missing = write_missing,
+            write_missing = write_missing and _allow_write_missing,
         )
         if keys[0] in key_config:
             config[keys[0]] = key_config[keys[0]]
-            if sync_files:
+            if sync_files and _allow_write_missing:
                 _sync_files(keys=[keys[0] if keys else None])
+
+    if not allow_replaced:
+        return _backup_config if _backup_config is not None else config
+
     return config
 
 
@@ -137,7 +170,11 @@ def get_config(
         dprint(f"Indexing keys: {keys}", color=False)
 
     if len(keys) == 0:
-        _rc = _config(substitute=substitute, sync_files=sync_files, write_missing=write_missing)
+        _rc = _config(
+            substitute=substitute,
+            sync_files=sync_files,
+            write_missing=(write_missing and _allow_write_missing),
+        )
         if as_tuple:
             return True, _rc 
         return _rc
@@ -153,18 +190,18 @@ def get_config(
     ):
         try:
             _subbed = search_and_substitute_config({keys[0]: config[keys[0]]})
-        except Exception as e:
+        except Exception:
             import traceback
             traceback.print_exc()
+            _subbed = {keys[0]: config[keys[0]]}
+
         config[keys[0]] = _subbed[keys[0]]
         if symlinks_key in _subbed:
             if symlinks_key not in config:
                 config[symlinks_key] = {}
-            if keys[0] not in config[symlinks_key]:
-                config[symlinks_key][keys[0]] = {}
-            config[symlinks_key][keys[0]] = apply_patch_to_config(
-                _subbed,
-                config[symlinks_key][keys[0]]
+            config[symlinks_key] = apply_patch_to_config(
+                _subbed.get(symlinks_key, {}),
+                config.get(symlinks_key, {}),
             )
 
     from meerschaum.config._sync import sync_files as _sync_files
@@ -193,7 +230,7 @@ def get_config(
         for k in keys:
             try:
                 c = c[k]
-            except Exception as e:
+            except Exception:
                 invalid_keys = True
                 break
         if invalid_keys:
@@ -208,7 +245,7 @@ def get_config(
             for k in keys:
                 try:
                     _c = _c[k]
-                except Exception as e:
+                except Exception:
                     in_default = False
             if in_default:
                 c = _c
@@ -219,7 +256,7 @@ def get_config(
                     if warn:
                         from meerschaum.utils.warnings import warn as _warn
                         _warn(warning_msg, stacklevel=3, color=False)
-                except Exception as e:
+                except Exception:
                     if warn:
                         print(warning_msg)
                 if as_tuple:
@@ -247,14 +284,15 @@ def get_config(
 
 
 def get_plugin_config(
-        *keys: str,
-        warn: bool = False,
-        **kw: Any
-    ) -> Optional[Any]:
+    *keys: str,
+    warn: bool = False,
+    **kw: Any
+) -> Optional[Any]:
     """
     This may only be called from within a Meerschaum plugin.
     See `meerschaum.config.get_config` for arguments.
     """
+    from meerschaum.utils.warnings import error
     from meerschaum.plugins import _get_parent_plugin
     parent_plugin_name = _get_parent_plugin(2)
     if parent_plugin_name is None:
@@ -271,16 +309,17 @@ def get_plugin_config(
 
 
 def write_plugin_config(
-        config_dict: Dict[str, Any],
-        **kw : Any
-    ):
+    config_dict: Dict[str, Any],
+    **kw: Any
+):
     """
     Write a plugin's configuration dictionary.
     """
+    from meerschaum.utils.warnings import error
     from meerschaum.plugins import _get_parent_plugin
     parent_plugin_name = _get_parent_plugin(2)
     if parent_plugin_name is None:
-        error(f"You may only call `get_plugin_config()` from within a Meerschaum plugin.")
+        error("You may only call `get_plugin_config()` from within a Meerschaum plugin.")
     plugins_cf = get_config('plugins', warn=False)
     if plugins_cf is None:
         plugins_cf = {}
@@ -289,13 +328,44 @@ def write_plugin_config(
     return write_config(cf, **kw)
 
 
+@contextlib.contextmanager
+def replace_config(config_: Union[Dict[str, Any], None]):
+    """
+    Temporarily override the Meerschaum config dictionary.
+
+    Parameters
+    ----------
+    config_: Dict[str, Any]
+        The new config dictionary to temporarily replace the canonical `config`.
+    """
+    if config_ is None:
+        try:
+            yield
+        finally:
+            return
+
+    global _backup_config, _allow_write_missing
+
+    _backup_config = _config()
+    _allow_write_missing = False
+    set_config(config_)
+
+    try:
+        yield
+    finally:
+        set_config(_backup_config)
+        _allow_write_missing = True
+
 ### This need to be below get_config to avoid a circular import.
 from meerschaum.config._read_config import read_config
 
 ### If environment variable MRSM_CONFIG or MRSM_PATCH is set, patch config before anything else.
-from meerschaum.config._environment import apply_environment_patches, apply_environment_uris
-apply_environment_uris()
-apply_environment_patches()
+from meerschaum.config.environment import (
+    apply_environment_patches as _apply_environment_patches,
+    apply_environment_uris as _apply_environment_uris,
+)
+_apply_environment_uris()
+_apply_environment_patches()
 
 
 from meerschaum.config._paths import PATCH_DIR_PATH, PERMANENT_PATCH_DIR_PATH

meerschaum/config/_default.py

@@ -6,30 +6,51 @@
 The default configuration values to write to config.yaml.
 """
 
-import sys, os, multiprocessing
+import multiprocessing
+from typing import Dict, Any
 
-from meerschaum.connectors import attributes as connector_attributes
 from meerschaum.config._paths import SQLITE_DB_PATH
+from meerschaum._internal.static import STATIC_CONFIG
 
+CONNECTOR_ATTRIBUTES: Dict[str, Dict[str, Any]] = {
+    'api': {
+        'required': [
+            'host',
+        ],
+        'optional': [
+            'port',
+            'username',
+            'password',
+            'client_id',
+            'client_secret',
+            'api_key',
+        ],
+        'default': {
+            'protocol': 'http',
+        },
+    },
+    'sql': {
+        'flavors': STATIC_CONFIG['sql']['create_engine_flavors'],
+    },
+}
 default_meerschaum_config = {
     'instance': 'sql:main',
     'api_instance': 'MRSM{meerschaum:instance}',
-    'web_instance': 'MRSM{meerschaum:instance}',
-    'default_repository': 'api:mrsm',
+    'repository': 'api:mrsm',
     'connectors': {
         'sql': {
             'default': {},
             'main': {
                 'username': 'mrsm',
                 'password': 'mrsm',
-                'flavor': 'timescaledb',
+                'flavor': 'timescaledb-ha',
                 'host': 'localhost',
                 'database': 'meerschaum',
                 'port': 5432,
             },
             'local': {
                 'flavor': 'sqlite',
-                'database': str(SQLITE_DB_PATH),
+                'database': "{SQLITE_DB_PATH}",
             },
             'memory': {
                 'flavor': 'sqlite',
@@ -37,7 +58,7 @@ default_meerschaum_config = {
             },
         },
         'api': {
-            'default': connector_attributes['api']['default'],
+            'default': CONNECTOR_ATTRIBUTES['api']['default'],
             'main': {
                 'host': 'localhost',
                 'port': 8000,
@@ -73,9 +94,11 @@ default_system_config = {
                 'postgis': True,
                 'citus': True,
                 'timescaledb': True,
+                'timescaledb-ha': True,
                 'mssql': True,
             },
             'instance': {
+                'create_metadata_cache_minutes': 14400,
                 'stale_temporary_tables_minutes': 1440,
                 'temporary_target': {
                     'prefix': '_',
@@ -93,59 +116,40 @@ default_system_config = {
                 'connect_args': {},
             },
         },
-
         'api': {
         },
     },
-    ### not to be confused with system_config['connectors']['api'], this is the configuration
-    ### for the API server itself.
-    'api': {
-        'uvicorn': {
-            'app': 'meerschaum.api:app',
-            'port': 8000,
-            'host': '0.0.0.0',
-            'workers': max(int(multiprocessing.cpu_count() / 2), 1),
-            'proxy_headers': True,
-            'forwarded_allow_ips': '*',
-        },
-        'cache': {
-            'connector': 'valkey:main',
-            'session_expires_minutes': 43200,
-        },
-        'data': {
-            'max_response_row_limit': 100_000,
-            'chunks': {
-                'ttl_seconds': 1800,
-            },
-        },
-        'endpoints': {
-            'docs_in_production': True,
-        },
-        'permissions':       {
-            'registration': {
-                'users': True,
-                'pipes': True,
-                'plugins': True,
-            },
-            'actions': {
-                'non_admin': True,
-            },
-            'chaining': {
-                'insecure_parent_instance': False,
-                'child_apis': False,
-            },
-            'instances': {
-                'allow_multiple_instances': True,
-                'allowed_instance_keys': ['*']
-            },
-        },
-        'protocol': default_meerschaum_config['connectors']['api']['default']['protocol'],
-    },
-    'webterm': {
-        'tmux': {
-            'enabled': True,
-            'session_suffix': '_mrsm',
-        },
+    'api': 'MRSM{api}',
+    'webterm': 'MRSM{api:webterm}',
+    'cli': {
+        'max_daemons': (multiprocessing.cpu_count() * 3),
+        'refresh_seconds': 0.1,
+        'allowed_prefixes': ['*'],
+        'disallowed_prefixes': [
+            'edit',
+            'start daemon',
+            'start job',
+            'stop job',
+            'delete job',
+            'stop daemon',
+            'show daemon',
+            'restart daemon',
+            'reload',
+            'start worker',
+            'show log',
+            'python',
+            'login',
+            'executor',
+            'os ',
+            'sh ',
+            'start api',
+            'start webterm',
+            'stack',
+            'instance',
+            'debug',
+            'bootstrap',
+            'daemon',
+        ],
     },
     'experimental': {
         'fetch': False,
@@ -156,8 +160,65 @@ default_system_config = {
         'uv_pip': True,
         'systemd_healthcheck': False,
         'valkey_session_cache': True,
+        'cli_daemon': True,
     },
 }
+
+default_api_config = {
+    'uvicorn': {
+        'app': 'meerschaum.api:app',
+        'port': 8000,
+        'host': '0.0.0.0',
+        'workers': max(int(multiprocessing.cpu_count() / 2), 1),
+        'proxy_headers': True,
+        'forwarded_allow_ips': '*',
+    },
+    'cache': {
+        'connector': 'valkey:main',
+        'session_expires_minutes': 43200,
+    },
+    'data': {
+        'max_response_row_limit': 100_000,
+        'chunks': {
+            'ttl_seconds': 1800,
+        },
+    },
+    'endpoints': {
+        'docs_in_production': True,
+    },
+    'tokens': {
+        'valid_refresh_minutes': 60,
+        'default_expiration_days': 366,
+    },
+    'permissions':       {
+        'registration': {
+            'users': True,
+            'pipes': True,
+            'plugins': True,
+        },
+        'actions': {
+            'non_admin': True,
+        },
+        'chaining': {
+            'insecure_parent_instance': False,
+            'child_apis': False,
+        },
+        'instances': {
+            'allow_multiple_instances': True,
+            'allowed_instance_keys': ['*']
+        },
+    },
+    'protocol': default_meerschaum_config['connectors']['api']['default']['protocol'],
+    'webterm': {
+        'tmux': {
+            'enabled': True,
+            'session_suffix': '_mrsm',
+        },
+        'host': '127.0.0.1',
+        'port': 8765,
+    },
+}
+
 default_pipes_config = {
     'parameters': {
         'columns': {
@@ -172,26 +233,33 @@ default_pipes_config = {
         },
     },
     'attributes': {
-        'local_cache_timeout_seconds': 60,
+        'local_cache_timeout_seconds': 600.0,
     },
     'sync': {
         'filter_params_index_limit': 250,
+        'exists_cache_seconds': 60.0,
     },
     'verify': {
         'max_chunks_syncs': 3,
     },
+    'autotime': {
+        'column_name_if_datetime_missing': 'ts',
+    },
+    'dtypes': {
+        'min_ratio_columns_changed_for_full_astype': 0.5,
+        'columns_types_cache_seconds': 60.0,
+    },
+    'static': {
+        'static_schema_cache_seconds': 3600.0,
+    },
 }
 default_plugins_config = {}
-default_experimental_config = {
-    'venv': True,
-}
-
-
 
 ### build default config dictionary
 default_config = {}
 default_config['meerschaum'] = default_meerschaum_config
 default_config['system'] = default_system_config
+default_config['api'] = default_api_config
 from meerschaum.config._formatting import default_formatting_config
 default_config['formatting'] = default_formatting_config
 from meerschaum.config._shell import default_shell_config
@@ -203,7 +271,7 @@ default_config['jobs'] = default_jobs_config
 ### add configs from other packages
 try:
     import meerschaum.config.stack
-except ImportError as e:
+except ImportError:
     pass
 finally:
     from meerschaum.config.stack import default_stack_config

meerschaum/config/_edit.py

@@ -7,8 +7,9 @@ Functions for editing the configuration file
 """
 
 from __future__ import annotations
+
 import pathlib
-from meerschaum.utils.typing import Optional, Any, SuccessTuple, Mapping, Dict, List, Union
+from meerschaum.utils.typing import Optional, Any, SuccessTuple, Dict, List, Union
 
 
 def edit_config(
@@ -19,20 +20,27 @@ def edit_config(
 ) -> SuccessTuple:
     """Edit the configuration files."""
     from meerschaum.config import get_config, config
-    from meerschaum.config._read_config import get_keyfile_path, read_config
+    from meerschaum.config._read_config import get_keyfile_path, read_config, revert_symlinks_config
     from meerschaum.config._paths import CONFIG_DIR_PATH
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.packages import reload_meerschaum
     from meerschaum.utils.misc import edit_file
-    from meerschaum.utils.warnings import warn, dprint
+    from meerschaum.utils.warnings import warn
     from meerschaum.utils.prompt import prompt
 
     if keys is None:
         keys = []
 
+    symlinks_key = STATIC_CONFIG['config']['symlinks_key']
     def _edit_key(key: str):
+        new_key_config = None
         while True:
             ### If defined in default, create the config file.
-            key_config = config.pop(key, None)
+            symlinks_key_config = config.get(symlinks_key, {}).get(key, {})
+            key_config = revert_symlinks_config({
+                key: config.pop(key, {}),
+                symlinks_key: {key: symlinks_key_config},
+            })
             keyfile_path = get_keyfile_path(key, create_new=True)
             get_config(key, write_missing=True, warn=False)
 
@@ -60,7 +68,7 @@ def edit_config(
         for k in keys:
             _edit_key(k)
     except KeyboardInterrupt:
-        return False, f""
+        return False, ""
 
     reload_meerschaum(debug=debug)
     return (True, "Success")
@@ -95,19 +103,23 @@ def write_config(
     if directory is None:
         from meerschaum.config._paths import CONFIG_DIR_PATH
         directory = CONFIG_DIR_PATH
-    from meerschaum.config.static import STATIC_CONFIG
+
+    from meerschaum.config import _allow_write_missing
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.config._default import default_header_comment
-    from meerschaum.config._patch import apply_patch_to_config
-    from meerschaum.config._read_config import get_keyfile_path
-    from meerschaum.utils.debug import dprint
+    from meerschaum.config._read_config import get_keyfile_path, revert_symlinks_config
     from meerschaum.utils.yaml import yaml
     from meerschaum.utils.misc import filter_keywords
-    import json, os
+    import json
+    import os
     if config_dict is None:
         from meerschaum.config import _config
-        cf = _config()
+        cf = _config(allow_replaced=False)
         config_dict = cf
 
+    if not _allow_write_missing:
+        return False
+
     default_filetype = STATIC_CONFIG['config']['default_filetype']
     filetype_dumpers = {
         'yml' : yaml.dump,
@@ -115,9 +127,7 @@ def write_config(
         'json' : json.dump,
     }
 
-    symlinks_key = STATIC_CONFIG['config']['symlinks_key']
-    symlinks = config_dict.pop(symlinks_key) if symlinks_key in config_dict else {}
-    config_dict = apply_patch_to_config(config_dict, symlinks)
+    config_dict = revert_symlinks_config(config_dict)
 
     def determine_filetype(k, v):
         if k == 'meerschaum':
@@ -153,7 +163,7 @@ def write_config(
                 success = True
             except Exception as e:
                 success = False
-                print(f"FAILED TO WRITE!")
+                print("FAILED TO WRITE!")
                 print(e)
                 print(filter_keywords(
                     filetype_dumpers[filetype],
@@ -165,16 +175,16 @@ def write_config(
                 try:
                     if os.path.exists(filepath):
                         os.remove(filepath)
-                except Exception as e:
+                except Exception:
                     print(f"Failed to write '{k}'")
                 return False
 
     return True
 
 def general_write_yaml_config(
-        files: Optional[Dict[pathlib.Path, Dict[str, Any]]] = None,
-        debug: bool = False
-    ):
+    files: Optional[Dict[pathlib.Path, Dict[str, Any]]] = None,
+    debug: bool = False
+):
     """
     Write configuration dictionaries to file paths with optional headers.
 
@@ -216,18 +226,15 @@ def general_write_yaml_config(
     return True
 
 def general_edit_config(
-        action: Optional[List[str]] = None,
-        files: Optional[Dict[str, Union[str, pathlib.Path]]] = None,
-        default: Optional[str] = None,
-        debug: bool = False
-    ):
+    action: Optional[List[str]] = None,
+    files: Optional[Dict[str, Union[str, pathlib.Path]]] = None,
+    default: Optional[str] = None,
+    debug: bool = False
+) -> SuccessTuple:
     """Prompt the user to edit any config files."""
     if default is None:
         raise Exception("Provide a default choice for which file to edit")
-    import os
-    from subprocess import call
     from meerschaum.utils.misc import edit_file
-    from meerschaum.utils.debug import dprint
 
     if files is None:
         files = {}
@@ -257,11 +264,10 @@ def copy_default_to_config(debug : bool = False):
     return True
 
 def write_default_config(
-        debug : bool = False,
-        **kw
-    ):
+    debug: bool = False,
+    **kw
+):
     """Write the default configuration files."""
-    import os
     from meerschaum.config._paths import DEFAULT_CONFIG_DIR_PATH
-    from meerschaum.config._default import default_config, default_header_comment
+    from meerschaum.config._default import default_config
     return write_config(default_config, directory=DEFAULT_CONFIG_DIR_PATH)