meerschaum 2.9.4__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

meerschaum/config/__init__.py

@@ -9,14 +9,24 @@ and if interactive, print the welcome message.
 
 from __future__ import annotations
 
-import os, shutil, sys, pathlib, copy
-from meerschaum.utils.typing import Any, Dict, Optional, Union
+import os
+import shutil
+import sys
+import copy
+
+from meerschaum.utils.typing import Any, Dict, Optional
 from meerschaum.utils.threading import RLock
-from meerschaum.utils.warnings import warn, error
 
 from meerschaum.config._version import __version__
 from meerschaum.config._edit import edit_config, write_config
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum.config._read_config import (
+    search_and_substitute_config,
+    revert_symlinks_config,
+    get_possible_keys,
+    get_keyfile_path,
+    read_config,
+)
+from meerschaum._internal.static import STATIC_CONFIG
 
 from meerschaum.config._paths import (
     PERMANENT_PATCH_DIR_PATH,
@@ -31,8 +41,16 @@ __all__ = (
     'write_plugin_config',
     'get_config',
     'write_config',
+    'edit_config',
     'set_config',
+    'search_and_substitute_config',
+    'revert_symlinks_config',
+    'get_possible_keys',
+    'get_keyfile_path',
+    'apply_patch_to_config',
+    'read_config',
     'paths',
+    'STATIC_CONFIG',
 )
 __pdoc__ = {'static': False, 'resources': False, 'stack': False, }
 _locks = {'config': RLock()}
@@ -40,9 +58,12 @@ _locks = {'config': RLock()}
 ### apply config preprocessing (e.g. main to meta)
 config = {}
 def _config(
-        *keys: str, reload: bool = False, substitute: bool = True,
-        sync_files: bool = True, write_missing: bool = True,
-    ) -> Dict[str, Any]:
+    *keys: str,
+    reload: bool = False,
+    substitute: bool = True,
+    sync_files: bool = True,
+    write_missing: bool = True,
+) -> Dict[str, Any]:
     """
     Read and process the configuration file.
     """
@@ -153,7 +174,7 @@ def get_config(
     ):
         try:
             _subbed = search_and_substitute_config({keys[0]: config[keys[0]]})
-        except Exception as e:
+        except Exception:
             import traceback
             traceback.print_exc()
         config[keys[0]] = _subbed[keys[0]]
@@ -164,7 +185,7 @@ def get_config(
                 config[symlinks_key][keys[0]] = {}
             config[symlinks_key][keys[0]] = apply_patch_to_config(
                 _subbed,
-                config[symlinks_key][keys[0]]
+                {symlinks_key: config[symlinks_key][keys[0]]}
             )
 
     from meerschaum.config._sync import sync_files as _sync_files
@@ -193,7 +214,7 @@ def get_config(
         for k in keys:
             try:
                 c = c[k]
-            except Exception as e:
+            except Exception:
                 invalid_keys = True
                 break
         if invalid_keys:
@@ -208,7 +229,7 @@ def get_config(
             for k in keys:
                 try:
                     _c = _c[k]
-                except Exception as e:
+                except Exception:
                     in_default = False
             if in_default:
                 c = _c
@@ -219,7 +240,7 @@ def get_config(
                     if warn:
                         from meerschaum.utils.warnings import warn as _warn
                         _warn(warning_msg, stacklevel=3, color=False)
-                except Exception as e:
+                except Exception:
                     if warn:
                         print(warning_msg)
                 if as_tuple:
@@ -247,14 +268,15 @@ def get_config(
 
 
 def get_plugin_config(
-        *keys: str,
-        warn: bool = False,
-        **kw: Any
-    ) -> Optional[Any]:
+    *keys: str,
+    warn: bool = False,
+    **kw: Any
+) -> Optional[Any]:
     """
     This may only be called from within a Meerschaum plugin.
     See `meerschaum.config.get_config` for arguments.
     """
+    from meerschaum.utils.warnings import error
     from meerschaum.plugins import _get_parent_plugin
     parent_plugin_name = _get_parent_plugin(2)
     if parent_plugin_name is None:
@@ -271,16 +293,17 @@ def get_plugin_config(
 
 
 def write_plugin_config(
-        config_dict: Dict[str, Any],
-        **kw : Any
-    ):
+    config_dict: Dict[str, Any],
+    **kw: Any
+):
     """
     Write a plugin's configuration dictionary.
     """
+    from meerschaum.utils.warnings import error
     from meerschaum.plugins import _get_parent_plugin
     parent_plugin_name = _get_parent_plugin(2)
     if parent_plugin_name is None:
-        error(f"You may only call `get_plugin_config()` from within a Meerschaum plugin.")
+        error("You may only call `get_plugin_config()` from within a Meerschaum plugin.")
     plugins_cf = get_config('plugins', warn=False)
     if plugins_cf is None:
         plugins_cf = {}

meerschaum/config/_default.py

@@ -6,11 +6,33 @@
 The default configuration values to write to config.yaml.
 """
 
-import sys, os, multiprocessing
+import multiprocessing
+from typing import Dict, Any
 
-from meerschaum.connectors import attributes as connector_attributes
 from meerschaum.config._paths import SQLITE_DB_PATH
+from meerschaum._internal.static import STATIC_CONFIG
 
+CONNECTOR_ATTRIBUTES: Dict[str, Dict[str, Any]] = {
+    'api': {
+        'required': [
+            'host',
+        ],
+        'optional': [
+            'port',
+            'username',
+            'password',
+            'client_id',
+            'client_secret',
+            'api_key',
+        ],
+        'default': {
+            'protocol': 'http',
+        },
+    },
+    'sql': {
+        'flavors': STATIC_CONFIG['sql']['create_engine_flavors'],
+    },
+}
 default_meerschaum_config = {
     'instance': 'sql:main',
     'api_instance': 'MRSM{meerschaum:instance}',
@@ -22,14 +44,14 @@ default_meerschaum_config = {
             'main': {
                 'username': 'mrsm',
                 'password': 'mrsm',
-                'flavor': 'timescaledb',
+                'flavor': 'timescaledb-ha',
                 'host': 'localhost',
                 'database': 'meerschaum',
                 'port': 5432,
             },
             'local': {
                 'flavor': 'sqlite',
-                'database': str(SQLITE_DB_PATH),
+                'database': SQLITE_DB_PATH.as_posix(),
             },
             'memory': {
                 'flavor': 'sqlite',
@@ -37,7 +59,7 @@ default_meerschaum_config = {
             },
         },
         'api': {
-            'default': connector_attributes['api']['default'],
+            'default': CONNECTOR_ATTRIBUTES['api']['default'],
             'main': {
                 'host': 'localhost',
                 'port': 8000,
@@ -73,6 +95,7 @@ default_system_config = {
                 'postgis': True,
                 'citus': True,
                 'timescaledb': True,
+                'timescaledb-ha': True,
                 'mssql': True,
             },
             'instance': {
@@ -121,6 +144,10 @@ default_system_config = {
         'endpoints': {
             'docs_in_production': True,
         },
+        'tokens': {
+            'valid_refresh_minutes': 60,
+            'default_expiration_days': 366,
+        },
         'permissions':       {
             'registration': {
                 'users': True,

meerschaum/config/_edit.py

@@ -7,8 +7,9 @@ Functions for editing the configuration file
 """
 
 from __future__ import annotations
+
 import pathlib
-from meerschaum.utils.typing import Optional, Any, SuccessTuple, Mapping, Dict, List, Union
+from meerschaum.utils.typing import Optional, Any, SuccessTuple, Dict, List, Union
 
 
 def edit_config(
@@ -19,20 +20,27 @@ def edit_config(
 ) -> SuccessTuple:
     """Edit the configuration files."""
     from meerschaum.config import get_config, config
-    from meerschaum.config._read_config import get_keyfile_path, read_config
+    from meerschaum.config._read_config import get_keyfile_path, read_config, revert_symlinks_config
     from meerschaum.config._paths import CONFIG_DIR_PATH
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.packages import reload_meerschaum
     from meerschaum.utils.misc import edit_file
-    from meerschaum.utils.warnings import warn, dprint
+    from meerschaum.utils.warnings import warn
     from meerschaum.utils.prompt import prompt
 
     if keys is None:
         keys = []
 
+    symlinks_key = STATIC_CONFIG['config']['symlinks_key']
     def _edit_key(key: str):
+        new_key_config = None
         while True:
             ### If defined in default, create the config file.
-            key_config = config.pop(key, None)
+            symlinks_key_config = config.get(symlinks_key, {}).get(key, {})
+            key_config = revert_symlinks_config({
+                key: config.pop(key, {}),
+                symlinks_key: {key: symlinks_key_config},
+            })
             keyfile_path = get_keyfile_path(key, create_new=True)
             get_config(key, write_missing=True, warn=False)
 
@@ -60,7 +68,7 @@ def edit_config(
         for k in keys:
             _edit_key(k)
     except KeyboardInterrupt:
-        return False, f""
+        return False, ""
 
     reload_meerschaum(debug=debug)
     return (True, "Success")
@@ -95,7 +103,7 @@ def write_config(
     if directory is None:
         from meerschaum.config._paths import CONFIG_DIR_PATH
         directory = CONFIG_DIR_PATH
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.config._default import default_header_comment
     from meerschaum.config._patch import apply_patch_to_config
     from meerschaum.config._read_config import get_keyfile_path
@@ -153,7 +161,7 @@ def write_config(
                 success = True
             except Exception as e:
                 success = False
-                print(f"FAILED TO WRITE!")
+                print("FAILED TO WRITE!")
                 print(e)
                 print(filter_keywords(
                     filetype_dumpers[filetype],
@@ -165,16 +173,16 @@ def write_config(
                 try:
                     if os.path.exists(filepath):
                         os.remove(filepath)
-                except Exception as e:
+                except Exception:
                     print(f"Failed to write '{k}'")
                 return False
 
     return True
 
 def general_write_yaml_config(
-        files: Optional[Dict[pathlib.Path, Dict[str, Any]]] = None,
-        debug: bool = False
-    ):
+    files: Optional[Dict[pathlib.Path, Dict[str, Any]]] = None,
+    debug: bool = False
+):
     """
     Write configuration dictionaries to file paths with optional headers.
 
@@ -216,18 +224,15 @@ def general_write_yaml_config(
     return True
 
 def general_edit_config(
-        action: Optional[List[str]] = None,
-        files: Optional[Dict[str, Union[str, pathlib.Path]]] = None,
-        default: Optional[str] = None,
-        debug: bool = False
-    ):
+    action: Optional[List[str]] = None,
+    files: Optional[Dict[str, Union[str, pathlib.Path]]] = None,
+    default: Optional[str] = None,
+    debug: bool = False
+) -> SuccessTuple:
     """Prompt the user to edit any config files."""
     if default is None:
         raise Exception("Provide a default choice for which file to edit")
-    import os
-    from subprocess import call
     from meerschaum.utils.misc import edit_file
-    from meerschaum.utils.debug import dprint
 
     if files is None:
         files = {}
@@ -257,11 +262,10 @@ def copy_default_to_config(debug : bool = False):
     return True
 
 def write_default_config(
-        debug : bool = False,
-        **kw
-    ):
+    debug: bool = False,
+    **kw
+):
     """Write the default configuration files."""
-    import os
     from meerschaum.config._paths import DEFAULT_CONFIG_DIR_PATH
-    from meerschaum.config._default import default_config, default_header_comment
+    from meerschaum.config._default import default_config
     return write_config(default_config, directory=DEFAULT_CONFIG_DIR_PATH)

meerschaum/config/_environment.py

@@ -10,7 +10,7 @@ import os
 import re
 import json
 from meerschaum.utils.typing import List, Union, Dict, Any
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 
 def apply_environment_patches() -> None:
     """

meerschaum/config/_patch.py

@@ -12,9 +12,9 @@ from meerschaum.utils.typing import Dict, Any
 from meerschaum.utils.warnings import warn
 
 def apply_patch_to_config(
-        config: Dict[str, Any],
-        patch: Dict[str, Any],
-    ) -> Dict[str, Any]:
+    config: Dict[str, Any],
+    patch: Dict[str, Any],
+) -> Dict[str, Any]:
     """Patch the config dict with a new dict (cascade patching)."""
     _base = copy.deepcopy(config) if isinstance(config, dict) else {}
     if not isinstance(patch, dict):
@@ -37,9 +37,9 @@ def apply_patch_to_config(
 
 
 def write_patch(
-        patch: Dict[str, Any],
-        debug: bool = False
-    ):
+    patch: Dict[str, Any],
+    debug: bool = False
+) -> None:
     """Write patch dict to yaml."""
     from meerschaum.utils.debug import dprint
     from meerschaum.config._paths import PATCH_DIR_PATH

meerschaum/config/_paths.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 from pathlib import Path
 import os, platform, sys, json
 from meerschaum.utils.typing import Union
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 
 DOT_CONFIG_DIR_PATH = Path(
     os.environ.get('XDG_CONFIG_HOME', Path.home() / '.config')
@@ -157,6 +157,10 @@ paths = {
     'PLUGINS_TEMP_RESOURCES_PATH'    : ('{PLUGINS_RESOURCES_PATH}', '.tmp'),
     'PLUGINS_INIT_PATH'              : ('{PLUGINS_RESOURCES_PATH}', '__init__.py'),
 
+    'DB_RESOURCES_PATH'              : ('{ROOT_DIR_PATH}', 'db'),
+    'DB_INIT_RESOURCES_PATH'         : ('{DB_RESOURCES_PATH}', 'initdb'),
+    'DB_CREATE_EXTENSIONS_PATH'      : ('{DB_INIT_RESOURCES_PATH}', 'create-extensions.sql'),
+
     'SQLITE_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'sqlite'),
     'SQLITE_DB_PATH'                 : ('{SQLITE_RESOURCES_PATH}', 'mrsm_local.db'),
 

meerschaum/config/_read_config.py

@@ -9,7 +9,6 @@ from __future__ import annotations
 import pathlib
 
 from meerschaum.utils.typing import Optional, Dict, Any, List, Tuple, Union
-from meerschaum.config import get_config
 
 
 def read_config(
@@ -49,11 +48,12 @@ def read_config(
     >>> read_config(keys=['meerschaum'], with_filename=True)
     >>> ({...}, ['meerschaum.yaml'])
     """
-    import sys, shutil, os, json, itertools
-    from meerschaum.utils.packages import attempt_import
+    import os
+    import json
+    import itertools
     from meerschaum.utils.yaml import yaml, _yaml
     from meerschaum.config._paths import CONFIG_DIR_PATH
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.config._patch import apply_patch_to_config
     if directory is None:
         directory = CONFIG_DIR_PATH
@@ -106,7 +106,7 @@ def read_config(
             ### If default config contains symlinks, add them to the config to write.
             try:
                 _default_symlinks = _default_dict[symlinks_key][mk]
-            except Exception as e:
+            except Exception:
                 _default_symlinks = {}
             config[mk] = _default_dict[mk]
             if _default_symlinks:
@@ -179,6 +179,7 @@ def read_config(
                         import traceback
                         traceback.print_exc()
                         _config_key = {}
+                substitute = False
                 _single_key_config = (
                     search_and_substitute_config({key: _config_key}) if substitute
                     else {key: _config_key}
@@ -208,15 +209,16 @@ def read_config(
 
 
 def search_and_substitute_config(
-        config: Dict[str, Any],
-        leading_key: str = "MRSM",
-        delimiter: str = ":",
-        begin_key: str = "{",
-        end_key: str = "}",
-        literal_key: str = '!',
-        keep_symlinks: bool = True,
-    ) -> Dict[str, Any]:
-    """Search the config for Meerschaum substitution syntax and substite with value of keys.
+    config: Dict[str, Any],
+    leading_key: str = "MRSM",
+    delimiter: str = ":",
+    begin_key: str = "{",
+    end_key: str = "}",
+    literal_key: str = '!',
+    keep_symlinks: bool = True,
+) -> Dict[str, Any]:
+    """
+    Search the config for Meerschaum substitution syntax and substite with value of keys.
 
     Parameters
     ----------
@@ -242,7 +244,7 @@ def search_and_substitute_config(
         - ' MRSM{a:b:c} '  => ' "{\'d\': 1}"' : not isolated
         - ' MRSM{!a:b:c} ' => ' {"d": 1}'     : literal
 
-    keep_symlinks :
+    keep_symlinks: bool, default True
         If True, include the symlinks under the top-level key '_symlinks' (never written to a file).
         Defaults to True.
         
@@ -251,7 +253,13 @@ def search_and_substitute_config(
         ```
         MRSM{meerschaum:connectors:main:host} => cf['meerschaum']['connectors']['main']['host']
         ``` 
+
+    Returns
+    -------
+    The configuration dictionary with `MRSM{}` symlinks replaced with
+    the values from the current configuration.
     """
+    from meerschaum.config import get_config
 
     _links = []
     def _find_symlinks(d, _keys: Optional[List[str]] = None):
@@ -270,9 +278,6 @@ def search_and_substitute_config(
     import json
     needle = leading_key + begin_key
     haystack = json.dumps(config, separators=(',', ':'))
-    mod_haystack = list(str(haystack))
-    buff = str(needle)
-    max_index = len(haystack) - len(buff)
 
     patterns = {}
     isolated_patterns = {}
@@ -329,7 +334,7 @@ def search_and_substitute_config(
                 write_missing=False,
                 sync_files=False,
             )
-        except Exception as e:
+        except Exception:
             import traceback
             traceback.print_exc()
             valid = False
@@ -382,15 +387,40 @@ def search_and_substitute_config(
             s[_keys[-1]] = _pattern
 
         from meerschaum.config._patch import apply_patch_to_config
-        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum._internal.static import STATIC_CONFIG
         symlinks_key = STATIC_CONFIG['config']['symlinks_key']
-        if symlinks_key not in parsed_config:
-            parsed_config[symlinks_key] = {}
-        parsed_config[symlinks_key] = apply_patch_to_config(parsed_config[symlinks_key], symlinks)
+        if symlinks_key in parsed_config:
+            parsed_config[symlinks_key] = apply_patch_to_config(parsed_config[symlinks_key], symlinks)
 
     return parsed_config
 
 
+def revert_symlinks_config(config: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Given a configuration dictionary, re-apply the values from the
+    accompanying `_symlinks` dictionary.
+
+    Parameters
+    ----------
+    config: Dict[str, Any]
+        The configuration dictionary containing a `_symlinks` dictionary.
+
+    Returns
+    -------
+    A configuration dictionary with `_symlinks` re-applied.
+    """
+    from meerschaum.config import apply_patch_to_config
+    from meerschaum._internal.static import STATIC_CONFIG
+    symlinks_key = STATIC_CONFIG['config']['symlinks_key']
+    if symlinks_key not in config:
+        return config
+
+    symlinks_config = config[symlinks_key]
+    reverted_config = apply_patch_to_config(config, symlinks_config)
+    _ = reverted_config.pop(symlinks_key, None)
+    return reverted_config
+
+
 def get_possible_keys() -> List[str]:
     """
     Return a list of possible top-level keys.
@@ -407,12 +437,13 @@ def get_possible_keys() -> List[str]:
 
 
 def get_keyfile_path(
-        key: str,
-        create_new: bool = False,
-        directory: Union[pathlib.Path, str, None] = None,
-    ) -> Union[pathlib.Path, None]:
+    key: str,
+    create_new: bool = False,
+    directory: Union[pathlib.Path, str, None] = None,
+) -> Union[pathlib.Path, None]:
     """Determine a key's file path."""
-    import os, pathlib
+    import os
+    import pathlib
     if directory is None:
         from meerschaum.config._paths import CONFIG_DIR_PATH
         directory = CONFIG_DIR_PATH
@@ -422,16 +453,16 @@ def get_keyfile_path(
             os.path.join(
                 directory,
                 read_config(
-                    keys = [key],
-                    with_filenames = True,
-                    write_missing = False,
-                    substitute = False,
+                    keys=[key],
+                    with_filenames=True,
+                    write_missing=False,
+                    substitute=False,
                 )[1][0]
             )
         )
-    except IndexError as e:
+    except IndexError:
         if create_new:
-            from meerschaum.config.static import STATIC_CONFIG
+            from meerschaum._internal.static import STATIC_CONFIG
             default_filetype = STATIC_CONFIG['config']['default_filetype']
             return pathlib.Path(os.path.join(directory, key + '.' + default_filetype))
         return None

meerschaum/config/_sync.py

@@ -40,10 +40,11 @@ def sync_yaml_configs(
         If provided, iterate through a list of tuples,
         replacing the old string (index 0) with the new string (index 1).
     """
-    import os, sys
+    import os
+    import sys
     try:
         from meerschaum.utils.yaml import yaml, _yaml
-    except Exception as e:
+    except Exception:
         return
     from meerschaum.config._patch import apply_patch_to_config
     import meerschaum.config
@@ -110,7 +111,8 @@ def sync_files(keys: Optional[List[str]] = None):
             GRAFANA_DATASOURCE_PATH,
             GRAFANA_DASHBOARD_PATH,
         )
-        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum._internal.static import STATIC_CONFIG
+        from meerschaum.config.stack import _write_initdb
 
         sync_yaml_configs(
             ['stack', STACK_COMPOSE_FILENAME],
@@ -131,6 +133,7 @@ def sync_files(keys: Optional[List[str]] = None):
             GRAFANA_DASHBOARD_PATH,
             substitute=True,
         )
+        _write_initdb()
 
     key_functions = {
         'stack': _stack,

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.9.4"
+__version__ = "3.0.0rc1"