meerschaum 1.7.1__py3-none-any.whl → 1.7.3__py3-none-any.whl

meerschaum/_internal/arguments/_parser.py

@@ -270,7 +270,7 @@ groups['api'].add_argument(
     help = "Start the API in secure mode, only allowing `admin` users to execute actions.",
 )
 groups['api'].add_argument(
-    '--no-auth', '--noauth', action='store_true',
+    '--no-auth', '--noauth', '--insecure', action='store_true',
     help = 'When starting the API, do not require authentication. WARNING: This is dangerous!',
 )
 groups['api'].add_argument(

meerschaum/actions/api.py

@@ -157,7 +157,6 @@ def _api_start(
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum.utils.pool import get_pool
     import shutil
-    import os
     from copy import deepcopy
 
     if action is None:
@@ -276,6 +275,7 @@ def _api_start(
     env_dict = {
         MRSM_SERVER_ID: SERVER_ID,
         MRSM_RUNTIME: 'api',
+        MRSM_CONFIG: json.loads(os.environ.get(MRSM_CONFIG, '{}')),
     }
     for env_var in get_env_vars():
         if env_var in env_dict:
@@ -319,7 +319,8 @@ def _api_start(
         ]
         for key, val in env_dict.items():
             gunicorn_args += [
-                '--env', key + '=' + (json.dumps(val) if isinstance(val, dict) else val)
+                '--env', key + "="
+                + (json.dumps(val) if isinstance(val, (dict, list)) else val)
             ]
         if workers is not None:
             gunicorn_args += ['--workers', str(workers)]

meerschaum/actions/stack.py

@@ -43,6 +43,7 @@ def stack(
         attempt_import, run_python_package, venv_contains_package,
         pip_install,
     )
+    from meerschaum.config._sync import sync_files
     from meerschaum.config import get_config
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn
@@ -73,9 +74,10 @@ def stack(
         if not path.exists():
             bootstrap = True
             break
-    ### if bootstrap flag was set, create files
     if bootstrap:
         write_stack(debug=debug)
+    else: 
+        sync_files(['stack'])
 
     compose_command = ['up']
     ### default: alias stack as docker-compose
@@ -125,14 +127,8 @@ def stack(
                 warn(f"Unable to install `pyyaml` into venv '{_compose_venv}'.")
 
     cmd_list = [
-        _arg for _arg in (
-            settings_list
-            + compose_command
-            + (
-                sysargs[2:] if len(sysargs) > 2 and not sub_args
-                else sub_args
-            )
-        )
+        _arg
+        for _arg in (settings_list + sysargs[1:])
         if _arg != '--debug'
     ]
     if debug:

meerschaum/api/dash/components.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 from meerschaum.utils.venv import Venv
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html
 from meerschaum.utils.typing import SuccessTuple, List
-from meerschaum.config.static import _static_config
+from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.misc import remove_ansi
 from meerschaum.actions import get_shell
 from meerschaum.api import endpoints, CHECK_UPDATE
@@ -154,7 +154,7 @@ def alert_from_success_tuple(success: SuccessTuple) -> dbc.Alert:
             id = 'success-alert',
             dismissable = True,
             fade = True,
-            is_open = not (success[1] in _static_config()['system']['success']['ignore']),
+            is_open = not (success[1] in STATIC_CONFIG['system']['success']['ignore']),
             color = 'success' if success[0] else 'danger',
         )
     )

meerschaum/api/routes/_misc.py

@@ -16,10 +16,14 @@ from meerschaum.api import (
     private, no_auth, manager
 )
 import fastapi
-from starlette.responses import FileResponse
+from starlette.responses import FileResponse, JSONResponse
+from meerschaum.connectors.poll import _wrap_retry_connect
 
 @app.get(endpoints['favicon'], tags=['Misc'])
 def get_favicon() -> Any:
+    """
+    Return the favicon file.
+    """
     from meerschaum.config._paths import API_STATIC_PATH
     return FileResponse(os.path.join(API_STATIC_PATH, 'ico', 'logo.ico'))
 
@@ -30,6 +34,9 @@ def get_chaining_status(
             fastapi.Depends(manager) if private else None
         ),
     ) -> bool:
+    """
+    Return whether this API instance may be chained.
+    """
     return check_allow_chaining()
 
 
@@ -39,6 +46,9 @@ def get_instance_info(
             fastapi.Depends(manager) if private else None
         ),
     ) -> Dict[str, Union[str, int]]:
+    """
+    Return information about this API instance.
+    """
     from meerschaum import __version__ as version
     num_plugins = len(get_api_connector().get_plugins(debug=debug))
     num_users = len(get_api_connector().get_users(debug=debug))
@@ -50,6 +60,22 @@ def get_instance_info(
         'num_pipes': num_pipes,
     }
 
+
+@app.get(endpoints['healthcheck'], tags=['Misc'])
+def get_healtheck() -> Dict[str, Any]:
+    """
+    Return a success message to confirm this API is reachable.
+    """
+    conn = get_api_connector()
+    success = _wrap_retry_connect(
+        conn.meta,
+        max_retries = 1, 
+    )
+    message = "Success" if success else "Failed to connect to instance connector."
+    status_code = 200 if success else 503
+    return JSONResponse({"success": success, "message": message}, status_code=status_code)
+
+
 if debug:
     @app.get('/id', tags=['Misc'])
     def get_ids(

meerschaum/config/__init__.py

@@ -45,12 +45,13 @@ def _config(
     if config is None or reload:
         with _locks['config']:
             config = {}
+
     if keys and keys[0] not in config:
         from meerschaum.config._sync import sync_files as _sync_files
         key_config = read_config(
-            keys=[keys[0]],
-            substitute=substitute,
-            write_missing=write_missing,
+            keys = [keys[0]],
+            substitute = substitute,
+            write_missing = write_missing,
         )
         if keys[0] in key_config:
             config[keys[0]] = key_config[keys[0]]
@@ -232,7 +233,6 @@ def get_config(
                 )
             )
             if patch and keys[0] != symlinks_key:
-                #  print("Updating configuration, please wait...")
                 if write_missing:
                     write_config(config, debug=debug)
 

meerschaum/config/_edit.py

@@ -71,7 +71,7 @@ def write_config(
     if directory is None:
         from meerschaum.config._paths import CONFIG_DIR_PATH
         directory = CONFIG_DIR_PATH
-    from meerschaum.config.static import _static_config
+    from meerschaum.config.static import STATIC_CONFIG
     from meerschaum.config._default import default_header_comment
     from meerschaum.config._patch import apply_patch_to_config
     from meerschaum.config._read_config import get_keyfile_path
@@ -84,14 +84,14 @@ def write_config(
         cf = _config()
         config_dict = cf
 
-    default_filetype = _static_config()['config']['default_filetype']
+    default_filetype = STATIC_CONFIG['config']['default_filetype']
     filetype_dumpers = {
         'yml' : yaml.dump,
         'yaml' : yaml.dump,
         'json' : json.dump,
     }
 
-    symlinks_key = _static_config()['config']['symlinks_key']
+    symlinks_key = STATIC_CONFIG['config']['symlinks_key']
     symlinks = config_dict.pop(symlinks_key) if symlinks_key in config_dict else {}
     config_dict = apply_patch_to_config(config_dict, symlinks)
 
@@ -148,27 +148,17 @@ def write_config(
     return True
 
 def general_write_yaml_config(
-        files : Optional[Dict[str, pathlib.Path]] = None,
-        debug : bool = False
+        files: Optional[Dict[pathlib.Path, Dict[str, Any]]] = None,
+        debug: bool = False
     ):
-    """Write configuration dictionaries to file paths with optional headers.
-    
-    files : dict
-        Dictionary of paths -> dict or tuple of format (dict, header).
-        If item is a tuple, the header will be written at the top of the file.
+    """
+    Write configuration dictionaries to file paths with optional headers.
 
     Parameters
     ----------
-    files : Optional[Dict[str :
-        
-    pathlib.Path]] :
-         (Default value = None)
-    debug : bool :
-         (Default value = False)
-
-    Returns
-    -------
-
+    files: Optional[Dict[str, pathlib.Path]], default None
+        Dictionary of paths -> dict or tuple of format (dict, header).
+        If item is a tuple, the header will be written at the top of the file.
     """
 
     from meerschaum.utils.debug import dprint

meerschaum/config/_read_config.py

@@ -46,7 +46,7 @@ def read_config(
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.yaml import yaml, _yaml
     from meerschaum.config._paths import CONFIG_DIR_PATH
-    from meerschaum.config.static import _static_config
+    from meerschaum.config.static import STATIC_CONFIG
     from meerschaum.config._patch import apply_patch_to_config
     if directory is None:
         directory = CONFIG_DIR_PATH
@@ -57,11 +57,11 @@ def read_config(
         return default_config
 
     ### Each key corresponds to a YAML or JSON file.
-    symlinks_key = _static_config()['config']['symlinks_key']
+    symlinks_key = STATIC_CONFIG['config']['symlinks_key']
     config = {}
     config_to_write = {}
 
-    default_filetype = _static_config()['config']['default_filetype']
+    default_filetype = STATIC_CONFIG['config']['default_filetype']
     filetype_loaders = {
         'yml' : yaml.load,
         'yaml' : yaml.load,
@@ -187,7 +187,6 @@ def read_config(
                 print(f"Unable to parse {filename}!")
                 import traceback
                 traceback.print_exc()
-                #  print(e)
                 input(f"Press [Enter] to open '{filename}' and fix formatting errors.")
                 from meerschaum.utils.misc import edit_file
                 edit_file(filepath)
@@ -363,8 +362,8 @@ def search_and_substitute_config(
             s[_keys[-1]] = _pattern
 
         from meerschaum.config._patch import apply_patch_to_config
-        from meerschaum.config.static import _static_config
-        symlinks_key = _static_config()['config']['symlinks_key']
+        from meerschaum.config.static import STATIC_CONFIG
+        symlinks_key = STATIC_CONFIG['config']['symlinks_key']
         if symlinks_key not in parsed_config:
             parsed_config[symlinks_key] = {}
         parsed_config[symlinks_key] = apply_patch_to_config(parsed_config[symlinks_key], symlinks)
@@ -403,16 +402,16 @@ def get_keyfile_path(
             os.path.join(
                 directory,
                 read_config(
-                    keys=[key],
-                    with_filenames=True,
-                    write_missing=False,
-                    substitute=False,
+                    keys = [key],
+                    with_filenames = True,
+                    write_missing = False,
+                    substitute = False,
                 )[1][0]
             )
         )
     except IndexError as e:
         if create_new:
-            from meerschaum.config.static import _static_config
-            default_filetype = _static_config()['config']['default_filetype']
+            from meerschaum.config.static import STATIC_CONFIG
+            default_filetype = STATIC_CONFIG['config']['default_filetype']
             return pathlib.Path(os.path.join(directory, key + '.' + default_filetype))
         return None

meerschaum/config/_sync.py

@@ -10,7 +10,6 @@ from __future__ import annotations
 from meerschaum.utils.typing import Optional, List, Tuple
 
 def sync_yaml_configs(
-        config_path: pathlib.Path,
         keys: List[str],
         sub_path: pathlib.Path,
         substitute: bool = True,
@@ -19,46 +18,24 @@ def sync_yaml_configs(
     ) -> None:
     """Synchronize sub-configuration with main configuration file.
     
-    NOTE: This function might need refactoring to work better with the new
-    `read_config` system.
-
     Parameters
     ----------
-    config_path :
-        Not sure if this is necessary.
-    keys :
+    keys: List[str]
         The config keys to read via `get_config()`.
-    sub_path :
+
+    sub_path: pathlib.Path
         The derivative file to write.
-    substitute :
+
+    substitute: bool, default True
         If `True`, parse `MRSM{}` syntax and substitute values.
         See `get_config()` for more information.
-        Defaults to `True`.
-    permissions :
+
+    permissions: Optional[int], default None
         If not `None`, set permissions of the derivative file.
-        Defaults to `None`.
-    replace_tuples :
+
+    replace_tuples: Optional[List[Tuple[str]]], default None
         If provided, iterate through a list of tuples,
         replacing the old string (index 0) with the new string (index 1).
-        Defaults to `None`.
-    config_path: pathlib.Path :
-        
-    keys: List[str] :
-        
-    sub_path: pathlib.Path :
-        
-    substitute: bool :
-         (Default value = True)
-    permissions: Optional[int] :
-         (Default value = None)
-    replace_tuples: Optional[List[Tuple[str :
-        
-    str]]] :
-         (Default value = None)
-
-    Returns
-    -------
-
     """
     import os, sys
     try:
@@ -68,22 +45,12 @@ def sync_yaml_configs(
     from meerschaum.config._patch import apply_patch_to_config
     import meerschaum.config
     from meerschaum.utils.packages import reload_package
-    from meerschaum.config._read_config import search_and_substitute_config
-    if not os.path.isfile(config_path) or not os.path.isfile(sub_path):
-        return
-
-    def _read_config(path):
-        """Read YAML file with header comment
-
-        Parameters
-        ----------
-        path :
-            
+    from meerschaum.config._read_config import search_and_substitute_config, get_keyfile_path
 
-        Returns
-        -------
-
-        """
+    def _read_yaml_config(path):
+        """Read YAML file with header comment."""
+        if not path.exists():
+            return "", {}
         header_comment = ""
         with open(path, 'r') as f:
             if _yaml is not None:
@@ -98,58 +65,60 @@ def sync_yaml_configs(
                 header_comment += line
         return header_comment, config
 
-    config_header, config = _read_config(config_path)
-    sub_header, sub_config = _read_config(sub_path)
-
     from meerschaum.config import get_config
+    config_path = get_keyfile_path(keys[0], create_new=True)
     c = get_config(*keys, substitute=substitute, sync_files=False)
+
+    config_header, config = _read_yaml_config(config_path)
+    sub_header, sub_config = _read_yaml_config(sub_path)
+
     if substitute:
         sub_config = search_and_substitute_config(sub_config)
 
-    if c != sub_config:
-        config_time = os.path.getmtime(config_path)
-        sub_time = os.path.getmtime(sub_path)
-
-        sub_config = c
-        new_config_text = yaml.dump(c, sort_keys=False)
-        if replace_tuples:
-            for replace_tuple in replace_tuples:
-                new_config_text = new_config_text.replace(replace_tuple[0], replace_tuple[1])
-        new_header = sub_header
-        new_path = sub_path
-
-        ### write changes
-        with open(new_path, 'w+') as f:
-            f.write(new_header)
-            f.write(new_config_text)
+    sub_config = c
+    new_config_text = yaml.dump(c, sort_keys=False)
+    if replace_tuples:
+        for replace_tuple in replace_tuples:
+            new_config_text = new_config_text.replace(replace_tuple[0], replace_tuple[1])
+    new_header = sub_header
+    new_path = sub_path
+
+    ### write changes
+    with open(new_path, 'w+') as f:
+        f.write(new_header)
+        f.write(new_config_text)
     if permissions is not None:
         os.chmod(new_path, permissions)
 
 def sync_files(keys: Optional[List[str]] = None):
     if keys is None:
         keys = []
+
     def _stack():
         import os
-        from meerschaum.config._paths import CONFIG_DIR_PATH, STACK_ENV_PATH, STACK_COMPOSE_PATH
-        from meerschaum.config._paths import STACK_COMPOSE_FILENAME, STACK_ENV_FILENAME
-        from meerschaum.config._paths import GRAFANA_DATASOURCE_PATH, GRAFANA_DASHBOARD_PATH
-        from meerschaum.config._paths import MOSQUITTO_CONFIG_PATH
+        from meerschaum.config._paths import (
+            CONFIG_DIR_PATH,
+            STACK_ENV_PATH,
+            STACK_COMPOSE_PATH,
+            STACK_COMPOSE_FILENAME,
+            STACK_ENV_FILENAME,
+            GRAFANA_DATASOURCE_PATH,
+            GRAFANA_DASHBOARD_PATH,
+        )
+        from meerschaum.config.static import STATIC_CONFIG
 
         sync_yaml_configs(
-            CONFIG_DIR_PATH / 'stack.yaml',
             ['stack', STACK_COMPOSE_FILENAME],
             STACK_COMPOSE_PATH,
             substitute = True,
-            replace_tuples = [('$', '$$')],
+            replace_tuples = [('$', '$$'), ('<DOLLAR>', '$')],
         )
         sync_yaml_configs(
-            CONFIG_DIR_PATH / 'stack.yaml',
             ['stack', 'grafana', 'datasource'],
             GRAFANA_DATASOURCE_PATH,
             substitute = True,
         )
         sync_yaml_configs(
-            CONFIG_DIR_PATH / 'stack.yaml',
             ['stack', 'grafana', 'dashboard'],
             GRAFANA_DASHBOARD_PATH,
             substitute = True,
@@ -157,7 +126,7 @@ def sync_files(keys: Optional[List[str]] = None):
 
 
     key_functions = {
-        'stack' : _stack,
+        'stack': _stack,
     }
     if keys is None:
         keys = list(key_functions.keys())

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "1.7.1"
+__version__ = "1.7.3"

meerschaum/config/stack/__init__.py

@@ -29,11 +29,11 @@ db_base = "MRSM{meerschaum:connectors:sql:main:database}"
 db_hostname = "db"
 db_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:db:hostname}'
 api_port = "MRSM{meerschaum:connectors:api:main:port}"
-api_host = "meerschaum_api"
+api_host = "api"
 
 env_dict = {
     'COMPOSE_PROJECT_NAME' : 'mrsm',
-    'TIMESCALEDB_VERSION' : 'latest-pg14-oss',
+    'TIMESCALEDB_VERSION' : 'latest-pg15-oss',
     'POSTGRES_USER' : f'{db_user}',
     'POSTGRES_PASSWORD' : f'{db_pass}',
     'POSTGRES_DB' : f'{db_base}',
@@ -54,13 +54,25 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
     '"MRSM{!meerschaum}"', 'MRSM{!meerschaum}',
 )
 
+volumes = {
+    'api_root': '/meerschaum',
+    'meerschaum_db_data': '/var/lib/postgresql/data',
+    'grafana_storage': '/var/lib/grafana',
+}
+networks = {
+    'frontend': None,
+    'backend': None,
+}
 env_dict['MEERSCHAUM_API_PATCH'] = json.dumps(
     {
-        'meerschaum' : {
-            'connectors' : {
-                'sql' : {
-                    'main' : {
-                        'host' : db_host,
+        'meerschaum': {
+            'connectors': {
+                'sql': {
+                    'main': {
+                        'host': db_host,
+                    },
+                    'local': {
+                        'database': volumes['api_root'] + '/sqlite/mrsm_local.db'
                     },
                 },
             },
@@ -82,15 +94,6 @@ compose_header = """
 ##############################################################
 """
 
-volumes = {
-    'api_root' : '/meerschaum',
-    'meerschaum_db_data' : '/var/lib/postgresql/data',
-    'grafana_storage' : '/var/lib/grafana',
-}
-networks = {
-    'frontend' : None,
-    'backend' : None,
-}
 
 default_docker_compose_config = {
     'version': '3.9',
@@ -98,16 +101,15 @@ default_docker_compose_config = {
         'db': {
             'environment': {
                 'TIMESCALEDB_TELEMETRY': 'off',
-                'POSTGRES_USER': '$POSTGRES_USER',
-                'POSTGRES_DB': '$POSTGRES_DB',
-                'POSTGRES_PASSWORD': '$POSTGRES_PASSWORD',
+                'POSTGRES_USER': '<DOLLAR>POSTGRES_USER',
+                'POSTGRES_DB': '<DOLLAR>POSTGRES_DB',
+                'POSTGRES_PASSWORD': '<DOLLAR>POSTGRES_PASSWORD',
                 'ALLOW_IP_RANGE': env_dict['ALLOW_IP_RANGE'],
             },
             'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB',
             'healthcheck': {
                 'test': [
-                    'CMD-SHELL', 'pg_isready',
-                    '-U', '$POSTGRES_USER', '-d', '$POSTGRES_DB',
+                    'CMD-SHELL', 'pg_isready -d <DOLLAR>POSTGRES_DB -U <DOLLAR>POSTGRES_USER',
                 ],
                 'interval': '5s',
                 'timeout': '3s',
@@ -136,6 +138,14 @@ default_docker_compose_config = {
                 'backend',
             ],
             'command': f'start api --production --port {api_port}',
+            'healthcheck': {
+                'test': [
+                    'CMD', 'curl', '--fail', f'http://localhost:{api_port}/healthcheck',
+                ],
+                'interval': '5s',
+                'timeout': '3s',
+                'retries': 3
+            },
             'environment': {
                 'MRSM_CONFIG': env_dict['MEERSCHAUM_API_CONFIG'],
                 'MRSM_PATCH': env_dict['MEERSCHAUM_API_PATCH'],

meerschaum/config/static/__init__.py

@@ -8,12 +8,13 @@ Insert non-user-editable configuration files here.
 
 import os
 import uuid
+from typing import Dict, Any
 from meerschaum.utils.misc import generate_password
 
 __all__ = ['STATIC_CONFIG']
 
-SERVER_ID = os.environ.get('MRSM_SERVER_ID', generate_password(6))
-STATIC_CONFIG = {
+SERVER_ID: str = os.environ.get('MRSM_SERVER_ID', generate_password(6))
+STATIC_CONFIG: Dict[str, Any] = {
     'api': {
         'endpoints': {
             'index': '/',
@@ -32,6 +33,7 @@ STATIC_CONFIG = {
             'webterm': '/webterm',
             'webterm_websocket': '/websocket',
             'info': '/info',
+            'healthcheck': '/healthcheck',
         },
         'oauth': {
             'token_expires_minutes': 720,
@@ -83,6 +85,9 @@ STATIC_CONFIG = {
     'connectors': {
         'default_label': 'main',
     },
+    'stack': {
+        'dollar_standin': '<DOLLAR>', ### Temporary replacement for docker-compose.yaml.
+    },
     'users': {
         'password_hash': {
             'schemes': [

meerschaum/connectors/api/_actions.py

@@ -50,7 +50,7 @@ def do_action(
     """
     import sys, json
     from meerschaum.utils.debug import dprint
-    from meerschaum.config.static import _static_config
+    from meerschaum.config.static import STATIC_CONFIG
     from meerschaum.utils.misc import json_serialize_datetime
     if action is None:
         action = []
@@ -67,7 +67,7 @@ def do_action(
 
     root_action = json_dict['action'][0]
     del json_dict['action'][0]
-    r_url = f"{_static_config()['api']['endpoints']['actions']}/{root_action}"
+    r_url = f"{STATIC_CONFIG['api']['endpoints']['actions']}/{root_action}"
     
     if debug:
         from meerschaum.utils.formatting import pprint

meerschaum/connectors/api/_login.py

@@ -18,7 +18,7 @@ def login(
     """Log in and set the session token."""
     from meerschaum.utils.warnings import warn as _warn, info, error
     from meerschaum.core import User
-    from meerschaum.config.static import _static_config
+    from meerschaum.config.static import STATIC_CONFIG
     import json, datetime
     try:
         login_data = {
@@ -28,7 +28,7 @@ def login(
     except AttributeError:
         return False, f"Please login with the command `login {self}`."
     response = self.post(
-        _static_config()['api']['endpoints']['login'],
+        STATIC_CONFIG['api']['endpoints']['login'],
         data = login_data,
         use_token = False,
         debug = debug