meerschaum 2.3.6__py3-none-any.whl → 2.4.0.dev1__py3-none-any.whl

meerschaum/config/stack/__init__.py

@@ -31,15 +31,23 @@ db_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:db:hostname}'
 api_port = "MRSM{meerschaum:connectors:api:main:port}"
 api_host = "api"
 
+valkey_hostname = "valkey"
+valkey_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:valkey:hostname}'
+valkey_port = "MRSM{meerschaum:connectors:valkey:main:port}"
+valkey_username = 'MRSM{meerschaum:connectors:valkey:main:username}'
+valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
+
 env_dict = {
-    'COMPOSE_PROJECT_NAME' : 'mrsm',
-    'TIMESCALEDB_VERSION' : 'latest-pg16-oss',
-    'POSTGRES_USER' : f'{db_user}',
-    'POSTGRES_PASSWORD' : f'{db_pass}',
-    'POSTGRES_DB' : f'{db_base}',
-    'MEERSCHAUM_API_HOSTNAME' : f'{api_host}',
-    'ALLOW_IP_RANGE' : '0.0.0.0/0',
-    'MEERSCHAUM_API_CONFIG_RESOURCES' : '/meerschaum',
+    'COMPOSE_PROJECT_NAME': 'mrsm',
+    'TIMESCALEDB_VERSION': 'latest-pg16-oss',
+    'POSTGRES_USER': db_user,
+    'POSTGRES_PASSWORD': db_pass,
+    'POSTGRES_DB': db_base,
+    'VALKEY_USERNAME': valkey_username,
+    'VALKEY_PASSWORD': valkey_password,
+    'MEERSCHAUM_API_HOSTNAME': api_host,
+    'ALLOW_IP_RANGE': '0.0.0.0/0',
+    'MEERSCHAUM_API_CONFIG_RESOURCES': '/meerschaum',
 }
 ### apply patch to host config to change hostname to the Docker service name
 env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
@@ -58,6 +66,7 @@ volumes = {
     'api_root': '/meerschaum',
     'meerschaum_db_data': '/var/lib/postgresql/data',
     'grafana_storage': '/var/lib/grafana',
+    'valkey_data': '/bitnami/valkey/data',
 }
 networks = {
     'frontend': None,
@@ -76,10 +85,14 @@ env_dict['MEERSCHAUM_API_PATCH'] = json.dumps(
                         'database': volumes['api_root'] + '/sqlite/mrsm_local.db'
                     },
                 },
+                'valkey': {
+                    'host': valkey_host,
+                    'port': 6379,
+                },
             },
         },
     },
-    indent = 4,
+    indent=4,
 )
 
 compose_header = """
@@ -113,19 +126,19 @@ default_docker_compose_config = {
                 ],
                 'interval': '5s',
                 'timeout': '3s',
-                'retries': 3
+                'retries': 5
             },
             'restart': 'always',
-            'image' : 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
-            'ports' : [
+            'image': 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
+            'ports': [
                 f'{db_port}:5432',
             ],
-            'hostname' : f'{db_hostname}',
-            'volumes' : [
+            'hostname': f'{db_hostname}',
+            'volumes': [
                 'meerschaum_db_data:' + volumes['meerschaum_db_data'],
             ],
             'shm_size': '1024m',
-            'networks' : [
+            'networks': [
                 'backend',
             ],
         },
@@ -156,11 +169,41 @@ default_docker_compose_config = {
                 'db': {
                     'condition': 'service_healthy',
                 },
+                'valkey': {
+                    'condition': 'service_healthy',
+                },
             },
-            'volumes' : [
+            'volumes': [
                 'api_root:' + volumes['api_root'],
             ],
         },
+        'valkey': {
+            'image': 'bitnami/valkey',
+            'restart': 'always',
+            'environment': {
+                'VALKEY_PASSWORD': '<DOLLAR>VALKEY_PASSWORD',
+                'VALKEY_RDB_POLICY_DISABLED': 'no',
+                'VALKEY_RDB_POLICY': '900#1 600#5 300#10 120#50 60#1000 30#10000',
+            },
+            'hostname': valkey_hostname,
+            'ports': [
+                f'{valkey_port}:6379',
+            ],
+            'volumes': [
+                'valkey_data:' + volumes['valkey_data'],
+            ],
+            'healthcheck': {
+                'test': [
+                    'CMD', 'valkey-cli', 'ping',
+                ],
+                'interval': '5s',
+                'timeout': '3s',
+                'retries': 5,
+            },
+            'networks': [
+                'backend',
+            ],
+        },
         'grafana': {
             'image': 'grafana/grafana:latest',
             'ports': [

meerschaum/connectors/Connector.py

@@ -53,17 +53,23 @@ class Connector(metaclass=abc.ABCMeta):
         """
         self._original_dict = copy.deepcopy(self.__dict__)
         self._set_attributes(type=type, label=label, **kw)
-        self.verify_attributes(getattr(self, 'REQUIRED_ATTRIBUTES', None))
+
+        ### NOTE: Override `REQUIRED_ATTRIBUTES` if `uri` is set.
+        self.verify_attributes(
+            ['uri']
+            if 'uri' in self.__dict__
+            else getattr(self, 'REQUIRED_ATTRIBUTES', None)
+        )
 
     def _reset_attributes(self):
         self.__dict__ = self._original_dict
 
     def _set_attributes(
-            self,
-            *args,
-            inherit_default: bool = True,
-            **kw: Any
-        ):
+        self,
+        *args,
+        inherit_default: bool = True,
+        **kw: Any
+    ):
         from meerschaum.config.static import STATIC_CONFIG
         from meerschaum.utils.warnings import error
 
@@ -114,12 +120,11 @@ class Connector(metaclass=abc.ABCMeta):
         ### finally, update __dict__ with _attributes.
         self.__dict__.update(self._attributes)
 
-
     def verify_attributes(
-            self,
-            required_attributes: Optional[List[str]] = None,
-            debug: bool = False
-        ) -> None:
+        self,
+        required_attributes: Optional[List[str]] = None,
+        debug: bool = False,
+    ) -> None:
         """
         Ensure that the required attributes have been met.
         
@@ -147,6 +152,7 @@ class Connector(metaclass=abc.ABCMeta):
         from meerschaum.utils.misc import items_str
         if required_attributes is None:
             required_attributes = ['label']
+
         missing_attributes = set()
         for a in required_attributes:
             if a not in self.__dict__:
@@ -158,8 +164,8 @@ class Connector(metaclass=abc.ABCMeta):
                     + f"for connector '{self.type}:{self.label}'."
                 ),
                 InvalidAttributesError,
-                silent = True,
-                stack = False
+                silent=True,
+                stack=False
             )
 
 

meerschaum/connectors/__init__.py

@@ -39,6 +39,7 @@ connectors: Dict[str, Dict[str, Connector]] = {
     'api'    : {},
     'sql'    : {},
     'plugin' : {},
+    'valkey' : {},
 }
 instance_types: List[str] = ['sql', 'api']
 _locks: Dict[str, RLock] = {
@@ -164,13 +165,15 @@ def get_connector(
 
     if 'sql' not in types:
         from meerschaum.connectors.plugin import PluginConnector
+        from meerschaum.connectors.valkey import ValkeyConnector
         with _locks['types']:
             types.update({
-                'api'   : APIConnector,
-                'sql'   : SQLConnector,
+                'api': APIConnector,
+                'sql': SQLConnector,
                 'plugin': PluginConnector,
+                'valkey': ValkeyConnector,
             })
-    
+
     ### determine if we need to call the constructor
     if not refresh:
         ### see if any user-supplied arguments differ from the existing instance
@@ -273,7 +276,7 @@ def is_connected(keys: str, **kw) -> bool:
         with warnings.catch_warnings():
             warnings.filterwarnings('ignore')
             return conn.test_connection(**kw)
-    except Exception as e:
+    except Exception:
         return False
 
 
@@ -340,7 +343,7 @@ def load_plugin_connectors():
             to_import.append(plugin.name)
     if not to_import:
         return
-    import_plugins(*to_import) 
+    import_plugins(*to_import)
 
 
 def get_connector_plugin(
@@ -378,3 +381,4 @@ def _load_builtin_custom_connectors():
     Import custom connectors decorated with `@make_connector` or `@make_executor`.
     """
     import meerschaum.jobs.systemd
+    import meerschaum.connectors.valkey

meerschaum/connectors/poll.py

@@ -9,16 +9,16 @@ Poll database and API connections.
 from meerschaum.utils.typing import InstanceConnector, Union, Optional, Dict, Any
 
 def retry_connect(
-        connector: Union[InstanceConnector, None] = None,
-        max_retries: int = 50,
-        retry_wait: int = 3,
-        workers: int = 1,
-        warn: bool = True,
-        print_on_connect: bool = False,
-        enforce_chaining: bool = True,
-        enforce_login: bool = True,
-        debug: bool = False,
-    ) -> bool:
+    connector: Union[InstanceConnector, None] = None,
+    max_retries: int = 50,
+    retry_wait: int = 3,
+    workers: int = 1,
+    warn: bool = True,
+    print_on_connect: bool = False,
+    enforce_chaining: bool = True,
+    enforce_login: bool = True,
+    debug: bool = False,
+) -> bool:
     """
     Keep trying to connect to the database.
 
@@ -85,16 +85,16 @@ def retry_connect(
 
 
 def _wrap_retry_connect(
-        connector_meta: Dict[str, Any],
-        max_retries: int = 50,
-        retry_wait: int = 3,
-        workers: int = 1,
-        print_on_connect: bool = False,
-        warn: bool = True,
-        enforce_chaining: bool = True,
-        enforce_login: bool = True,
-        debug: bool = False,
-    ) -> bool:
+    connector_meta: Dict[str, Any],
+    max_retries: int = 50,
+    retry_wait: int = 3,
+    workers: int = 1,
+    print_on_connect: bool = False,
+    warn: bool = True,
+    enforce_chaining: bool = True,
+    enforce_login: bool = True,
+    debug: bool = False,
+) -> bool:
     """
     Keep trying to connect to the database.
 
@@ -144,8 +144,6 @@ def _wrap_retry_connect(
     import time
 
     connector = get_connector(**connector_meta)
-    if connector.type not in instance_types:
-        return None
 
     if not hasattr(connector, 'test_connection'):
         return True
@@ -157,7 +155,15 @@ def _wrap_retry_connect(
             dprint(f"Trying to connect to '{connector}'...")
             dprint(f"Attempt ({retries + 1} / {max_retries})")
 
-        if connector.type == 'sql':
+        if connector.type not in ('sql', 'api'):
+            try:
+                connected = connector.test_connection()
+            except Exception as e:
+                if warn:
+                    print(e)
+                connected = False
+
+        elif connector.type == 'sql':
 
             def _connect(_connector):
                 ### Test queries like `SELECT 1`.
@@ -185,7 +191,7 @@ def _wrap_retry_connect(
                             _warn(
                                 f"Meerschaum instance '{connector}' does not allow chaining " +
                                 "and cannot be used as the parent for this instance.",
-                                stack = False
+                                stack=False,
                             )
                         return False