meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0.dev0__py3-none-any.whl

meerschaum/config/stack/__init__.py

@@ -31,15 +31,23 @@ db_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:db:hostname}'
 api_port = "MRSM{meerschaum:connectors:api:main:port}"
 api_host = "api"
 
+valkey_hostname = "valkey"
+valkey_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:valkey:hostname}'
+valkey_port = "MRSM{meerschaum:connectors:valkey:main:port}"
+valkey_username = 'MRSM{meerschaum:connectors:valkey:main:username}'
+valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
+
 env_dict = {
-    'COMPOSE_PROJECT_NAME' : 'mrsm',
-    'TIMESCALEDB_VERSION' : 'latest-pg16-oss',
-    'POSTGRES_USER' : f'{db_user}',
-    'POSTGRES_PASSWORD' : f'{db_pass}',
-    'POSTGRES_DB' : f'{db_base}',
-    'MEERSCHAUM_API_HOSTNAME' : f'{api_host}',
-    'ALLOW_IP_RANGE' : '0.0.0.0/0',
-    'MEERSCHAUM_API_CONFIG_RESOURCES' : '/meerschaum',
+    'COMPOSE_PROJECT_NAME': 'mrsm',
+    'TIMESCALEDB_VERSION': 'latest-pg16-oss',
+    'POSTGRES_USER': db_user,
+    'POSTGRES_PASSWORD': db_pass,
+    'POSTGRES_DB': db_base,
+    'VALKEY_USERNAME': valkey_username,
+    'VALKEY_PASSWORD': valkey_password,
+    'MEERSCHAUM_API_HOSTNAME': api_host,
+    'ALLOW_IP_RANGE': '0.0.0.0/0',
+    'MEERSCHAUM_API_CONFIG_RESOURCES': '/meerschaum',
 }
 ### apply patch to host config to change hostname to the Docker service name
 env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
@@ -56,9 +64,9 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
 
 volumes = {
     'api_root': '/meerschaum',
-    'api_user_local': '/home/meerschaum/.local',
     'meerschaum_db_data': '/var/lib/postgresql/data',
     'grafana_storage': '/var/lib/grafana',
+    'valkey_data': '/bitnami/valkey/data',
 }
 networks = {
     'frontend': None,
@@ -77,10 +85,14 @@ env_dict['MEERSCHAUM_API_PATCH'] = json.dumps(
                         'database': volumes['api_root'] + '/sqlite/mrsm_local.db'
                     },
                 },
+                'valkey': {
+                    'host': valkey_host,
+                    'port': 6379,
+                },
             },
         },
     },
-    indent = 4,
+    indent=4,
 )
 
 compose_header = """
@@ -114,19 +126,19 @@ default_docker_compose_config = {
                 ],
                 'interval': '5s',
                 'timeout': '3s',
-                'retries': 3
+                'retries': 5
             },
             'restart': 'always',
-            'image' : 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
-            'ports' : [
+            'image': 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
+            'ports': [
                 f'{db_port}:5432',
             ],
-            'hostname' : f'{db_hostname}',
-            'volumes' : [
+            'hostname': f'{db_hostname}',
+            'volumes': [
                 'meerschaum_db_data:' + volumes['meerschaum_db_data'],
             ],
             'shm_size': '1024m',
-            'networks' : [
+            'networks': [
                 'backend',
             ],
         },
@@ -157,10 +169,39 @@ default_docker_compose_config = {
                 'db': {
                     'condition': 'service_healthy',
                 },
+                'valkey': {
+                    'condition': 'service_healthy',
+                },
             },
-            'volumes' : [
+            'volumes': [
                 'api_root:' + volumes['api_root'],
-                'api_user_local:' + volumes['api_user_local'],
+            ],
+        },
+        'valkey': {
+            'image': 'bitnami/valkey',
+            'restart': 'always',
+            'environment': {
+                'VALKEY_PASSWORD': '<DOLLAR>VALKEY_PASSWORD',
+                'VALKEY_RDB_POLICY_DISABLED': 'no',
+                'VALKEY_RDB_POLICY': '900#1 600#5 300#10 120#50 60#1000 30#10000',
+            },
+            'hostname': valkey_hostname,
+            'ports': [
+                f'{valkey_port}:6379',
+            ],
+            'volumes': [
+                'valkey_data:' + volumes['valkey_data'],
+            ],
+            'healthcheck': {
+                'test': [
+                    'CMD', 'valkey-cli', 'ping',
+                ],
+                'interval': '5s',
+                'timeout': '3s',
+                'retries': 5,
+            },
+            'networks': [
+                'backend',
             ],
         },
         'grafana': {

meerschaum/config/static/__init__.py

@@ -46,6 +46,7 @@ STATIC_CONFIG: Dict[str, Any] = {
             'stdin_message': 'MRSM_STDIN',
             'stop_message': 'MRSM_STOP',
             'metadata_cache_seconds': 5,
+            'temp_prefix': '.api-temp-',
         },
     },
     'sql': {
@@ -73,6 +74,7 @@ STATIC_CONFIG: Dict[str, Any] = {
         'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
         'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
         'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
+        'systemd_delete_job': 'MRSM_SYSTEMD_DELETE_JOB',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },

meerschaum/connectors/Connector.py

@@ -53,17 +53,23 @@ class Connector(metaclass=abc.ABCMeta):
         """
         self._original_dict = copy.deepcopy(self.__dict__)
         self._set_attributes(type=type, label=label, **kw)
-        self.verify_attributes(getattr(self, 'REQUIRED_ATTRIBUTES', None))
+
+        ### NOTE: Override `REQUIRED_ATTRIBUTES` if `uri` is set.
+        self.verify_attributes(
+            ['uri']
+            if 'uri' in self.__dict__
+            else getattr(self, 'REQUIRED_ATTRIBUTES', None)
+        )
 
     def _reset_attributes(self):
         self.__dict__ = self._original_dict
 
     def _set_attributes(
-            self,
-            *args,
-            inherit_default: bool = True,
-            **kw: Any
-        ):
+        self,
+        *args,
+        inherit_default: bool = True,
+        **kw: Any
+    ):
         from meerschaum.config.static import STATIC_CONFIG
         from meerschaum.utils.warnings import error
 
@@ -114,12 +120,11 @@ class Connector(metaclass=abc.ABCMeta):
         ### finally, update __dict__ with _attributes.
         self.__dict__.update(self._attributes)
 
-
     def verify_attributes(
-            self,
-            required_attributes: Optional[List[str]] = None,
-            debug: bool = False
-        ) -> None:
+        self,
+        required_attributes: Optional[List[str]] = None,
+        debug: bool = False,
+    ) -> None:
         """
         Ensure that the required attributes have been met.
         
@@ -147,6 +152,7 @@ class Connector(metaclass=abc.ABCMeta):
         from meerschaum.utils.misc import items_str
         if required_attributes is None:
             required_attributes = ['label']
+
         missing_attributes = set()
         for a in required_attributes:
             if a not in self.__dict__:
@@ -158,8 +164,8 @@ class Connector(metaclass=abc.ABCMeta):
                     + f"for connector '{self.type}:{self.label}'."
                 ),
                 InvalidAttributesError,
-                silent = True,
-                stack = False
+                silent=True,
+                stack=False
             )
 
 

meerschaum/connectors/__init__.py

@@ -39,6 +39,7 @@ connectors: Dict[str, Dict[str, Connector]] = {
     'api'    : {},
     'sql'    : {},
     'plugin' : {},
+    'valkey' : {},
 }
 instance_types: List[str] = ['sql', 'api']
 _locks: Dict[str, RLock] = {
@@ -164,13 +165,15 @@ def get_connector(
 
     if 'sql' not in types:
         from meerschaum.connectors.plugin import PluginConnector
+        from meerschaum.connectors.valkey import ValkeyConnector
         with _locks['types']:
             types.update({
-                'api'   : APIConnector,
-                'sql'   : SQLConnector,
+                'api': APIConnector,
+                'sql': SQLConnector,
                 'plugin': PluginConnector,
+                'valkey': ValkeyConnector,
             })
-    
+
     ### determine if we need to call the constructor
     if not refresh:
         ### see if any user-supplied arguments differ from the existing instance
@@ -273,7 +276,7 @@ def is_connected(keys: str, **kw) -> bool:
         with warnings.catch_warnings():
             warnings.filterwarnings('ignore')
             return conn.test_connection(**kw)
-    except Exception as e:
+    except Exception:
         return False
 
 
@@ -340,7 +343,7 @@ def load_plugin_connectors():
             to_import.append(plugin.name)
     if not to_import:
         return
-    import_plugins(*to_import) 
+    import_plugins(*to_import)
 
 
 def get_connector_plugin(
@@ -378,3 +381,4 @@ def _load_builtin_custom_connectors():
     Import custom connectors decorated with `@make_connector` or `@make_executor`.
     """
     import meerschaum.jobs.systemd
+    import meerschaum.connectors.valkey

meerschaum/connectors/api/_actions.py

@@ -17,6 +17,7 @@ from meerschaum.utils.typing import SuccessTuple, List, Callable, Optional
 from meerschaum.config.static import STATIC_CONFIG
 
 ACTIONS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['actions']
+TEMP_PREFIX: str = STATIC_CONFIG['api']['jobs']['temp_prefix']
 
 
 def get_actions(self):
@@ -37,43 +38,28 @@ async def do_action_async(
     callback_function: Callable[[str], None] = partial(print, end=''),
 ) -> SuccessTuple:
     """
-    Monitor a job's log files and await a callback with the changes.
+    Execute an action as a temporary remote job.
     """
-    websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
-    protocol = 'ws' if self.URI.startswith('http://') else 'wss'
-    port = self.port if 'port' in self.__dict__ else ''
-    uri = f"{protocol}://{self.host}:{port}{ACTIONS_ENDPOINT}/ws"
-    if sysargs and sysargs[0] == 'api' and len(sysargs) > 2:
-        sysargs = sysargs[2:]
-
-    sysargs_str = json.dumps(sysargs)
-
-    async with websockets.connect(uri) as websocket:
-        try:
-            await websocket.send(self.token or 'no-login')
-            response = await websocket.recv()
-            init_data = json.loads(response)
-            if not init_data.get('is_authenticated'):
-                return False, "Cannot authenticate with actions endpoint."
-
-            await websocket.send(sysargs_str)
-        except websockets_exceptions.ConnectionClosedOK:
-            return False, "Connection was closed."
-
-        while True:
-            try:
-                line = await websocket.recv()
-                if asyncio.iscoroutinefunction(callback_function):
-                    await callback_function(line)
-                else:
-                    callback_function(line)
-            except KeyboardInterrupt:
-                await websocket.close()
-                break
-            except websockets_exceptions.ConnectionClosedOK:
-                break
-
-    return True, "Success"
+    from meerschaum._internal.arguments import remove_api_executor_keys
+    from meerschaum.utils.misc import generate_password
+    sysargs = remove_api_executor_keys(sysargs)
+
+    job_name = TEMP_PREFIX + generate_password(12)
+    job = mrsm.Job(job_name, sysargs, executor_keys=str(self))
+
+    start_success, start_msg = job.start()
+    if not start_success:
+        return start_success, start_msg
+
+    await job.monitor_logs_async(
+        callback_function=callback_function,
+        stop_on_exit=True,
+        strip_timestamps=True,
+    )
+
+    success, msg = job.result
+    job.delete()
+    return success, msg
 
 
 def do_action_legacy(

meerschaum/connectors/api/_jobs.py

@@ -277,6 +277,7 @@ async def monitor_logs_async(
     """
     Monitor a job's log files and await a callback with the changes.
     """
+    import traceback
     from meerschaum.jobs import StopMonitoringLogs
     from meerschaum.utils.formatting._jobs import strip_timestamp_from_line
 

meerschaum/connectors/poll.py

@@ -9,16 +9,16 @@ Poll database and API connections.
 from meerschaum.utils.typing import InstanceConnector, Union, Optional, Dict, Any
 
 def retry_connect(
-        connector: Union[InstanceConnector, None] = None,
-        max_retries: int = 50,
-        retry_wait: int = 3,
-        workers: int = 1,
-        warn: bool = True,
-        print_on_connect: bool = False,
-        enforce_chaining: bool = True,
-        enforce_login: bool = True,
-        debug: bool = False,
-    ) -> bool:
+    connector: Union[InstanceConnector, None] = None,
+    max_retries: int = 50,
+    retry_wait: int = 3,
+    workers: int = 1,
+    warn: bool = True,
+    print_on_connect: bool = False,
+    enforce_chaining: bool = True,
+    enforce_login: bool = True,
+    debug: bool = False,
+) -> bool:
     """
     Keep trying to connect to the database.
 
@@ -85,16 +85,16 @@ def retry_connect(
 
 
 def _wrap_retry_connect(
-        connector_meta: Dict[str, Any],
-        max_retries: int = 50,
-        retry_wait: int = 3,
-        workers: int = 1,
-        print_on_connect: bool = False,
-        warn: bool = True,
-        enforce_chaining: bool = True,
-        enforce_login: bool = True,
-        debug: bool = False,
-    ) -> bool:
+    connector_meta: Dict[str, Any],
+    max_retries: int = 50,
+    retry_wait: int = 3,
+    workers: int = 1,
+    print_on_connect: bool = False,
+    warn: bool = True,
+    enforce_chaining: bool = True,
+    enforce_login: bool = True,
+    debug: bool = False,
+) -> bool:
     """
     Keep trying to connect to the database.
 
@@ -144,8 +144,6 @@ def _wrap_retry_connect(
     import time
 
     connector = get_connector(**connector_meta)
-    if connector.type not in instance_types:
-        return None
 
     if not hasattr(connector, 'test_connection'):
         return True
@@ -157,7 +155,15 @@ def _wrap_retry_connect(
             dprint(f"Trying to connect to '{connector}'...")
             dprint(f"Attempt ({retries + 1} / {max_retries})")
 
-        if connector.type == 'sql':
+        if connector.type not in ('sql', 'api'):
+            try:
+                connected = connector.test_connection()
+            except Exception as e:
+                if warn:
+                    print(e)
+                connected = False
+
+        elif connector.type == 'sql':
 
             def _connect(_connector):
                 ### Test queries like `SELECT 1`.
@@ -185,7 +191,7 @@ def _wrap_retry_connect(
                             _warn(
                                 f"Meerschaum instance '{connector}' does not allow chaining " +
                                 "and cannot be used as the parent for this instance.",
-                                stack = False
+                                stack=False,
                             )
                         return False