meerschaum 2.9.5__py3-none-any.whl → 3.0.0rc2__py3-none-any.whl

meerschaum/config/stack/__init__.py

@@ -14,6 +14,8 @@ import json
 from meerschaum.config._paths import (
     GRAFANA_DATASOURCE_PATH,
     GRAFANA_DASHBOARD_PATH,
+    DB_INIT_RESOURCES_PATH,
+    DB_CREATE_EXTENSIONS_PATH,
     ROOT_DIR_PATH,
 )
 from meerschaum.config._paths import STACK_COMPOSE_FILENAME, STACK_ENV_FILENAME
@@ -39,7 +41,7 @@ valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
 
 env_dict = {
     'COMPOSE_PROJECT_NAME': 'mrsm',
-    'TIMESCALEDB_VERSION': 'latest-pg16',
+    'TIMESCALEDB_VERSION': 'pg17',
     'POSTGRES_USER': db_user,
     'POSTGRES_PASSWORD': db_pass,
     'POSTGRES_DB': db_base,
@@ -120,8 +122,9 @@ default_docker_compose_config = {
                 'POSTGRES_DB': '<DOLLAR>POSTGRES_DB',
                 'POSTGRES_PASSWORD': '<DOLLAR>POSTGRES_PASSWORD',
                 'ALLOW_IP_RANGE': env_dict['ALLOW_IP_RANGE'],
+                #  'POSTGRES_INITDB_ARGS': '-c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100'
             },
-            'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB',
+            'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100',
             'healthcheck': {
                 'test': [
                     'CMD-SHELL', 'pg_isready -d <DOLLAR>POSTGRES_DB -U <DOLLAR>POSTGRES_USER',
@@ -131,13 +134,14 @@ default_docker_compose_config = {
                 'retries': 5
             },
             'restart': 'always',
-            'image': 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
+            'image': 'timescale/timescaledb-ha:' + env_dict['TIMESCALEDB_VERSION'],
             'ports': [
                 f'{db_port}:5432',
             ],
             'hostname': db_hostname,
             'volumes': [
                 'meerschaum_db_data:' + volumes['meerschaum_db_data'],
+                f'{DB_INIT_RESOURCES_PATH.as_posix()}:/docker-entrypoint-initdb.d:z,ro',
             ],
             'shm_size': '1024m',
             'networks': [
@@ -224,8 +228,8 @@ default_docker_compose_config = {
             'volumes': [
                 'grafana_storage' + ':' + volumes['grafana_storage'],
                 ### NOTE: Mount with the 'z' option for SELinux.
-                f'{GRAFANA_DATASOURCE_PATH.parent}:/etc/grafana/provisioning/datasources:z,ro',
-                f'{GRAFANA_DASHBOARD_PATH.parent}:/etc/grafana/provisioning/dashboards:z,ro',
+                f'{GRAFANA_DATASOURCE_PATH.parent.as_posix()}:/etc/grafana/provisioning/datasources:z,ro',
+                f'{GRAFANA_DASHBOARD_PATH.parent.as_posix()}:/etc/grafana/provisioning/dashboards:z,ro',
             ],
             'environment': {
                 'GF_SECURITY_ALLOW_EMBEDDING': 'true',
@@ -273,6 +277,21 @@ def _sync_stack_files():
         substitute = True,
     )
 
+    _write_initdb()
+
+def _write_initdb():
+    create_postgis_text = (
+        "CREATE EXTENSION IF NOT EXISTS timescaledb;\n"
+        "CREATE EXTENSION IF NOT EXISTS postgis;\n"
+        "CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit;\n"
+        "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;\n"
+    )
+    if DB_CREATE_EXTENSIONS_PATH.exists():
+        return
+
+    with open(DB_CREATE_EXTENSIONS_PATH, 'w+', encoding='utf-8') as f:
+        f.write(create_postgis_text)
+
 NECESSARY_FILES = [STACK_COMPOSE_PATH, GRAFANA_DATASOURCE_PATH, GRAFANA_DASHBOARD_PATH]
 def get_necessary_files():
     from meerschaum.config import get_config
@@ -286,19 +305,20 @@ def get_necessary_files():
 
 
 def write_stack(
-        debug: bool = False 
-    ):
+    debug: bool = False 
+):
     """Write Docker Compose configuration files."""
     from meerschaum.config._edit import general_write_yaml_config
     from meerschaum.config._sync import sync_files
     general_write_yaml_config(get_necessary_files(), debug=debug)
     return sync_files(['stack'])
    
+
 def edit_stack(
-        action: Optional[List[str]] = None,
-        debug: bool = False,
-        **kw
-    ):
+    action: Optional[List[str]] = None,
+    debug: bool = False,
+    **kw
+):
     """Open docker-compose.yaml or .env for editing."""
     from meerschaum.config._edit import general_edit_config
     if action is None:

meerschaum/config/static.py

@@ -0,0 +1,18 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Alias import for the internal static configuration dictionary.
+"""
+
+from meerschaum._internal.static import SERVER_ID, STATIC_CONFIG
+
+__all__ = ('STATIC_CONFIG',)
+
+
+def _static_config():
+    """
+    Alias function for the global `STATIC_CONFIG` dictionary.
+    """
+    return STATIC_CONFIG

meerschaum/connectors/_Connector.py

@@ -7,10 +7,12 @@ Define the parent `Connector` class.
 """
 
 from __future__ import annotations
+
 import abc
 import copy
 from meerschaum.utils.typing import Iterable, Optional, Any, Union, List, Dict
 
+
 class InvalidAttributesError(Exception):
     """
     Raised when the incorrect attributes are set in the Connector.
@@ -20,6 +22,9 @@ class Connector(metaclass=abc.ABCMeta):
     """
     The base connector class to hold connection attributes.
     """
+
+    IS_INSTANCE: bool = False
+
     def __init__(
         self,
         type: Optional[str] = None,
@@ -70,7 +75,7 @@ class Connector(metaclass=abc.ABCMeta):
         inherit_default: bool = True,
         **kw: Any
     ):
-        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum._internal.static import STATIC_CONFIG
         from meerschaum.utils.warnings import error
 
         self._attributes = {}
@@ -151,7 +156,7 @@ class Connector(metaclass=abc.ABCMeta):
         from meerschaum.utils.debug import dprint
         from meerschaum.utils.misc import items_str
         if required_attributes is None:
-            required_attributes = ['label']
+            required_attributes = ['type', 'label']
 
         missing_attributes = set()
         for a in required_attributes:
@@ -213,6 +218,8 @@ class Connector(metaclass=abc.ABCMeta):
                 else r'executor$'
             )
             _type = re.sub(suffix_regex, '', self.__class__.__name__.lower())
+            if not _type or _type.lower() == 'instance':
+                raise ValueError("No type could be determined for this connector.")
             self.__dict__['type'] = _type
         return _type
 
@@ -224,8 +231,7 @@ class Connector(metaclass=abc.ABCMeta):
         """
         _label = self.__dict__.get('label', None)
         if _label is None:
-            from meerschaum.config.static import STATIC_CONFIG
+            from meerschaum._internal.static import STATIC_CONFIG
             _label = STATIC_CONFIG['connectors']['default_label']
             self.__dict__['label'] = _label
         return _label
-

meerschaum/connectors/__init__.py

@@ -19,13 +19,14 @@ from meerschaum.utils.threading import RLock
 from meerschaum.utils.warnings import warn
 
 from meerschaum.connectors._Connector import Connector, InvalidAttributesError
+from meerschaum.connectors.instance._InstanceConnector import InstanceConnector
 from meerschaum.connectors.sql._SQLConnector import SQLConnector
 from meerschaum.connectors.api._APIConnector import APIConnector
-from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
 
 __all__ = (
     "make_connector",
     "Connector",
+    "InstanceConnector",
     "SQLConnector",
     "APIConnector",
     "get_connector",
@@ -53,24 +54,7 @@ _locks: Dict[str, RLock] = {
     '_loaded_plugin_connectors': RLock(),
     'instance_types'           : RLock(),
 }
-attributes: Dict[str, Dict[str, Any]] = {
-    'api': {
-        'required': [
-            'host',
-            'username',
-            'password',
-        ],
-        'optional': [
-            'port',
-        ],
-        'default': {
-            'protocol': 'http',
-        },
-    },
-    'sql': {
-        'flavors': sql_flavor_configs,
-    },
-}
+
 ### Fill this with objects only when connectors are first referenced.
 types: Dict[str, Any] = {}
 custom_types: set = set()
@@ -130,7 +114,7 @@ def get_connector(
     """
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum.config import get_config
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
     global _loaded_plugin_connectors
     if isinstance(type, str) and not label and ':' in type:

meerschaum/connectors/api/_APIConnector.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 
 from datetime import datetime, timedelta, timezone
 from meerschaum.utils.typing import Optional, List, Union
-from meerschaum.connectors import Connector
+from meerschaum.connectors import InstanceConnector
 from meerschaum.utils.warnings import warn, error
 from meerschaum.utils.packages import attempt_import
 
@@ -18,15 +18,13 @@ required_attributes = {
     'host',
 }
 
-class APIConnector(Connector):
+class APIConnector(InstanceConnector):
     """
     Connect to a Meerschaum API instance.
     """
 
-    IS_INSTANCE: bool = True
     IS_THREAD_SAFE: bool = False
-
-    OPTIONAL_ATTRIBUTES: List[str] = ['port']
+    OPTIONAL_ATTRIBUTES: List[str] = ['port', 'client_secret', 'client_id', 'api_key']
 
     from ._request import (
         make_request,
@@ -82,6 +80,16 @@ class APIConnector(Connector):
         get_user_type,
         get_user_attributes,
     )
+    from ._tokens import (
+        register_token,
+        get_token_model,
+        get_tokens,
+        edit_token,
+        invalidate_token,
+        get_token_scopes,
+        token_exists,
+        delete_token,
+    )
     from ._uri import from_uri
     from ._jobs import (
         get_jobs,
@@ -154,9 +162,15 @@ class APIConnector(Connector):
         """
         Return the fully qualified URI.
         """
+        import urllib.parse
         username = self.__dict__.get('username', None)
         password = self.__dict__.get('password', None)
+        client_id = self.__dict__.get('client_id', None)
+        client_secret = self.__dict__.get('client_secret', None)
+        api_key = self.__dict__.get('api_key', None)
         creds = (username + ':' + password + '@') if username and password else ''
+        params = {}
+        params_str = ('?' + urllib.parse.urlencode(params)) if params else ''
         return (
             self.protocol
             + '://'
@@ -167,9 +181,9 @@ class APIConnector(Connector):
                 if self.__dict__.get('port', None)
                 else ''
             )
+            + params_str
         )
 
-
     @property
     def session(self):
         if self._session is None:
@@ -206,3 +220,17 @@ class APIConnector(Connector):
         Return the instance keys to be sent alongside pipe requests.
         """
         return self._instance_keys
+
+    @property
+    def login_scheme(self) -> str:
+        """
+        Return the login scheme to use based on the configured credentials.
+        """
+        if 'username' in self.__dict__:
+            return 'password'
+        if 'client_id' in self.__dict__:
+            return 'client_credentials'
+        elif 'api_key' in self.__dict__:
+            return 'api_key'
+
+        raise ValueError(f"Could not determine the login scheme for '{self}'.")

meerschaum/connectors/api/_actions.py

@@ -14,7 +14,7 @@ from functools import partial
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, List, Callable, Optional
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 
 ACTIONS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['actions']
 TEMP_PREFIX: str = STATIC_CONFIG['api']['jobs']['temp_prefix']
@@ -88,7 +88,7 @@ def do_action_legacy(
     """
     import sys, json
     from meerschaum.utils.debug import dprint
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.misc import json_serialize_datetime
     if action is None:
         action = []

meerschaum/connectors/api/_jobs.py

@@ -14,7 +14,7 @@ from datetime import datetime
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable, Optional
 from meerschaum.jobs import Job
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 from meerschaum.utils.warnings import warn, dprint
 
 JOBS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['jobs']

meerschaum/connectors/api/_login.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 import json
 import datetime
 from meerschaum.utils.typing import SuccessTuple, Any, Union
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 from meerschaum.utils.warnings import warn as _warn
 
 
@@ -22,14 +22,40 @@ def login(
     **kw: Any
 ) -> SuccessTuple:
     """Log in and set the session token."""
+    if self.login_scheme == 'api_key':
+        validate_response = self.post(
+            STATIC_CONFIG['api']['endpoints']['tokens'] + '/validate',
+            headers={'Authorization': f'Bearer {self.api_key}'},
+            use_token=False,
+            debug=debug,
+        )
+        if not validate_response:
+            return False, "API key is not valid."
+        return True, "API key is valid."
+
     try:
-        login_data = {
-            'username': self.username,
-            'password': self.password,
-        }
+        if self.login_scheme == 'password':
+            login_data = {
+                'username': self.username,
+                'password': self.password,
+            }
+        elif self.login_scheme == 'client_credentials':
+            login_data = {
+                'client_id': self.client_id,
+                'client_secret': self.client_secret,
+            }
     except AttributeError:
+        login_data = {}
+
+    if not login_data:
         return False, f"Please login with the command `login {self}`."
 
+    login_scheme_msg = (
+        f" as user '{login_data['username']}'"
+        if self.login_scheme == 'username'
+        else ''
+    )
+
     response = self.post(
         STATIC_CONFIG['api']['endpoints']['login'],
         data=login_data,
@@ -37,7 +63,7 @@ def login(
         debug=debug,
     )
     if response:
-        msg = f"Successfully logged into '{self}' as user '{login_data['username']}'."
+        msg = f"Successfully logged into '{self}'{login_scheme_msg}'."
         self._token = json.loads(response.text)['access_token']
         self._expires = datetime.datetime.strptime(
             json.loads(response.text)['expires'], 
@@ -45,7 +71,7 @@ def login(
         )
     else:
         msg = (
-            f"Failed to log into '{self}' as user '{login_data['username']}'.\n" +
+            f"Failed to log into '{self}'{login_scheme_msg}.\n" +
             f"    Please verify login details for connector '{self}'."
         )
         if warn and not self.__dict__.get('_emitted_warning', False):

meerschaum/connectors/api/_misc.py

@@ -13,7 +13,7 @@ def get_mrsm_version(self, **kw) -> Optional[str]:
     """
     Return the Meerschaum version of the API instance.
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     try:
         j = self.get(
             STATIC_CONFIG['api']['endpoints']['version'] + '/mrsm',
@@ -31,7 +31,7 @@ def get_chaining_status(self, **kw) -> Optional[bool]:
     """
     Fetch the chaining status of the API instance.
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     try:
         response = self.get(
             STATIC_CONFIG['api']['endpoints']['chaining'],

meerschaum/connectors/api/_pipes.py

@@ -21,7 +21,7 @@ def pipe_r_url(
     pipe: mrsm.Pipe
 ) -> str:
     """Return a relative URL path from a Pipe's keys."""
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     location_key = pipe.location_key
     if location_key is None:
         location_key = '[None]'
@@ -87,7 +87,7 @@ def edit_pipe(
     response = self.patch(
         r_url + '/edit',
         params={'patch': patch, 'instance_keys': self.get_pipe_instance_keys(pipe)},
-        json=pipe.parameters,
+        json=pipe.get_parameters(apply_symlinks=False),
         debug=debug,
     )
     if debug:
@@ -142,7 +142,7 @@ def fetch_pipes_keys(
     -------
     A list of tuples containing pipes' keys.
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     if connector_keys is None:
         connector_keys = []
     if metric_keys is None:
@@ -167,6 +167,8 @@ def fetch_pipes_keys(
             debug=debug
         ).json()
     except Exception as e:
+        import traceback
+        traceback.print_exc()
         error(str(e))
 
     if 'detail' in j:
@@ -185,10 +187,11 @@ def sync_pipe(
     """Sync a DataFrame into a Pipe."""
     from decimal import Decimal
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.misc import json_serialize_datetime, items_str, interval_str
+    from meerschaum.utils.dtypes import json_serialize_value
+    from meerschaum.utils.misc import items_str, interval_str
     from meerschaum.config import get_config
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.dataframe import get_numeric_cols, to_json
+    from meerschaum.utils.dataframe import get_special_cols, to_json
     begin = time.perf_counter()
     more_itertools = attempt_import('more_itertools')
     if df is None:
@@ -197,8 +200,10 @@ def sync_pipe(
 
     def get_json_str(c):
         ### allow syncing dict or JSON without needing to import pandas (for IOT devices)
-        if isinstance(c, (dict, list)):
-            return json.dumps(c, default=json_serialize_datetime)
+        if isinstance(c, str):
+            return c
+        if isinstance(c, (dict, list, tuple)):
+            return json.dumps(c, default=json_serialize_value)
         return to_json(c, orient='columns')
 
     df = json.loads(df) if isinstance(df, str) else df
@@ -218,26 +223,6 @@ def sync_pipe(
             else [partition.compute() for partition in df.partitions]
         )
 
-        numeric_cols = get_numeric_cols(df)
-        if numeric_cols:
-            for col in numeric_cols:
-                df[col] = df[col].apply(lambda x: f'{x:f}' if isinstance(x, Decimal) else x)
-            pipe_dtypes = pipe.dtypes
-            new_numeric_cols = [
-                col
-                for col in numeric_cols
-                if pipe_dtypes.get(col, None) != 'numeric'
-            ]
-            pipe.dtypes.update({
-                col: 'numeric'
-                for col in new_numeric_cols
-            })
-            edit_success, edit_msg = pipe.edit(debug=debug)
-            if not edit_success:
-                warn(
-                    "Failed to update new numeric columns "
-                    + f"{items_str(new_numeric_cols)}:\n{edit_msg}"
-                )
     elif isinstance(df, dict):
         ### `_chunks` is a dict of lists of dicts.
         ### e.g. {'a' : [ {'a':[1, 2]}, {'a':[3, 4]} ] }
@@ -313,7 +298,7 @@ def sync_pipe(
 
     success_tuple = True, (
         f"It took {interval_str(timedelta(seconds=(time.perf_counter() - begin)))} "
-        + "to sync {rowcount:,} row"
+        + f"to sync {rowcount:,} row"
         + ('s' if rowcount != 1 else '')
         + f" across {num_success_chunks:,} chunk" + ('s' if num_success_chunks != 1 else '') +
         f" to {pipe}."
@@ -324,7 +309,7 @@ def sync_pipe(
 def delete_pipe(
     self,
     pipe: Optional[mrsm.Pipe] = None,
-    debug: bool = None,        
+    debug: bool = False,
 ) -> SuccessTuple:
     """Delete a Pipe and drop its table."""
     if pipe is None:
@@ -576,7 +561,7 @@ def create_metadata(
     A bool indicating success.
     """
     from meerschaum.utils.debug import dprint
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     r_url = STATIC_CONFIG['api']['endpoints']['metadata']
     response = self.post(r_url, debug=debug)
     if debug:
@@ -663,7 +648,7 @@ def drop_pipe(
     from meerschaum.utils.warnings import error
     from meerschaum.utils.debug import dprint
     if pipe is None:
-        error(f"Pipe cannot be None.")
+        error("Pipe cannot be None.")
     r_url = pipe_r_url(pipe)
     response = self.delete(
         r_url + '/drop',

meerschaum/connectors/api/_plugins.py

@@ -16,7 +16,7 @@ def plugin_r_url(
     plugin: Union[mrsm.core.Plugin, str],
 ) -> str:
     """Generate a relative URL path from a Plugin."""
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     return f"{STATIC_CONFIG['api']['endpoints']['plugins']}/{plugin}"
 
 
@@ -111,7 +111,7 @@ def get_plugins(
     """
     import json
     from meerschaum.utils.warnings import error
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     response = self.get(
         STATIC_CONFIG['api']['endpoints']['plugins'],
         params = {'user_id': user_id, 'search_term': search_term},

meerschaum/connectors/api/_request.py

@@ -11,7 +11,7 @@ import urllib.parse
 import pathlib
 from meerschaum.utils.typing import Any, Optional, Dict, Union
 from meerschaum.utils.debug import dprint
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 
 METHODS = {
     'GET',