meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0__py3-none-any.whl

meerschaum/api/routes/_actions.py

@@ -7,28 +7,13 @@ Execute Meerschaum Actions via the API
 """
 
 from __future__ import annotations
-import asyncio
-import traceback
-import shlex
-from functools import partial
-from datetime import datetime, timezone
-
-from fastapi import WebSocket, WebSocketDisconnect
-from websockets.exceptions import ConnectionClosedError
-
-from meerschaum.utils.misc import generate_password
-from meerschaum.jobs import Job
-from meerschaum.utils.warnings import warn
-from meerschaum.utils.typing import SuccessTuple, Union, List, Dict, Any
+
+from meerschaum.utils.typing import SuccessTuple, List, Dict, Any
 from meerschaum.api import (
     fastapi, app, endpoints, get_api_connector, debug, manager, private, no_auth
 )
 from meerschaum.actions import actions
-import meerschaum.core
 from meerschaum.config import get_config
-from meerschaum._internal.arguments._parse_arguments import parse_dict_to_sysargs, parse_arguments
-from meerschaum.api.routes._jobs import clean_sysargs
-from meerschaum.jobs._Job import StopMonitoringLogs
 
 actions_endpoint = endpoints['actions']
 
@@ -67,104 +52,6 @@ def get_actions(
     return list(actions)
 
 
-async def notify_client(client, content: str):
-    """
-    Send a line of text to a client.
-    """
-    try:
-        await client.send_text(content)
-    except (RuntimeError, WebSocketDisconnect, Exception):
-        raise StopMonitoringLogs
-
-_temp_jobs = {}
-@app.websocket(actions_endpoint + '/ws')
-async def do_action_websocket(websocket: WebSocket):
-    """
-    Execute an action and stream the output to the client.
-    """
-    await websocket.accept()
-
-    stop_event = asyncio.Event()
-    job_name = '.' + generate_password(12)
-    job = None
-
-    async def monitor_logs(job):
-        success, msg = job.start()
-        await job.monitor_logs_async(
-            partial(notify_client, websocket),
-            stop_event=stop_event,
-            stop_on_exit=True,
-        )
-
-    try:
-        token = await websocket.receive_text()
-        user = await manager.get_current_user(token) if not no_auth else None
-        if user is None and not no_auth:
-            stop_event.set()
-            if job is not None:
-                job.delete()
-                _ = _temp_jobs.pop(job_name, None)
-            raise fastapi.HTTPException(
-                status_code=401,
-                detail="Invalid credentials.",
-            )
-
-        auth_success, auth_msg = (
-            is_user_allowed_to_execute(user)
-            if not no_auth
-            else (True, "Success")
-        )
-        auth_payload = {
-            'is_authenticated': auth_success,
-            'timestamp': datetime.now(timezone.utc).isoformat(),
-        }
-        await websocket.send_json(auth_payload)
-        if not auth_success:
-            stop_event.set()
-            job.stop()
-            await websocket.close()
-
-        sysargs = clean_sysargs(await websocket.receive_json())
-        job = Job(
-            job_name,
-            sysargs,
-            executor_keys='local',
-            _properties={
-                'logs': {
-                    'write_timestamps': False,
-                },
-            },
-        )
-        _temp_jobs[job_name] = job
-        monitor_task = asyncio.create_task(monitor_logs(job))
-
-        ### NOTE: Await incoming text to trigger `WebSocketDisconnect`.
-        while True:
-            await websocket.receive_text()
-
-    except fastapi.HTTPException:
-        await websocket.send_text("Invalid credentials.")
-        await websocket.close()
-    except (WebSocketDisconnect, asyncio.CancelledError):
-        stop_event.set()
-        job.stop()
-    except Exception:
-        stop_event.set()
-        job.stop()
-        warn(f"Error in logs websocket:\n{traceback.format_exc()}")
-    finally:
-        stop_event.set()
-        job.stop()
-        monitor_task.cancel()
-        if job is not None:
-            job.delete()
-        _ = _temp_jobs.pop(job_name, None)
-        try:
-            await websocket.close()
-        except RuntimeError:
-            pass
-
-
 @app.post(actions_endpoint + "/{action}", tags=['Actions'])
 def do_action_legacy(
     action: str,
@@ -180,7 +67,7 @@ def do_action_legacy(
     ----------
     action: str
         The action to perform.
-        
+
     keywords: Dict[str, Any]
         The keywords dictionary to pass to the action.
 
@@ -189,7 +76,6 @@ def do_action_legacy(
     A `SuccessTuple`.
     """
     if curr_user is not None and curr_user.type != 'admin':
-        from meerschaum.config import get_config
         allow_non_admin = get_config(
             'system', 'api', 'permissions', 'actions', 'non_admin', patch=True
         )
@@ -201,7 +87,7 @@ def do_action_legacy(
                 + "and search for 'permissions'. "
                 + "\nUnder the keys 'api:permissions:actions', "
                 + "you can allow non-admin users to perform actions."
-        )
+            )
 
     if action not in actions:
         return False, f"Invalid action '{action}'."

meerschaum/api/routes/_jobs.py

@@ -9,7 +9,6 @@ Manage jobs via the Meerschaum API.
 from __future__ import annotations
 
 import os
-import select
 import asyncio
 import traceback
 from datetime import datetime
@@ -18,12 +17,10 @@ from functools import partial
 
 from fastapi import WebSocket, WebSocketDisconnect
 
-from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
+from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union
 from meerschaum.jobs import (
     get_jobs as _get_jobs,
     Job,
-    StopMonitoringLogs,
-    get_executor_keys_from_context,
 )
 from meerschaum.utils.warnings import warn
 
@@ -32,15 +29,23 @@ from meerschaum.api import (
     app,
     endpoints,
     manager,
-    debug,
     no_auth,
-    private,
 )
 from meerschaum.config.static import STATIC_CONFIG
 
 JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
 JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
-EXECUTOR_KEYS: str = get_executor_keys_from_context()
+NONINTERACTIVE_ENV: str = STATIC_CONFIG['environment']['noninteractive']
+EXECUTOR_KEYS: str = 'local'
+
+
+def _get_job(name: str):
+    systemd_job = Job(name, executor_keys='systemd')
+    if systemd_job.exists():
+        return systemd_job
+
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    return job
 
 
 @app.get(endpoints['jobs'], tags=['Jobs'])
@@ -52,7 +57,7 @@ def get_jobs(
     """
     Return metadata about the current jobs.
     """
-    jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=False)
+    jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=True)
     return {
         name: {
             'sysargs': job.sysargs,
@@ -83,7 +88,7 @@ def get_job(
     """
     Return metadata for a single job.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -142,6 +147,10 @@ def create_job(
         name,
         clean_sysargs(sysargs),
         executor_keys=EXECUTOR_KEYS,
+        env={
+            NONINTERACTIVE_ENV: '1',
+            **dict(os.environ)
+        },
         _properties=properties,
     )
     if job.exists():
@@ -163,7 +172,7 @@ def delete_job(
     """
     Delete a job.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.delete()
 
 
@@ -177,7 +186,7 @@ def get_job_exists(
     """
     Return whether a job exists.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.exists()
 
 
@@ -192,7 +201,7 @@ def get_logs(
     Return a job's log text.
     To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -212,7 +221,7 @@ def start_job(
     """
     Start a job if stopped.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -231,7 +240,7 @@ def stop_job(
     """
     Stop a job if running.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -250,7 +259,7 @@ def pause_job(
     """
     Pause a job if running.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -269,7 +278,7 @@ def get_stop_time(
     """
     Get the timestamp when the job was manually stopped.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.stop_time
 
 
@@ -283,12 +292,13 @@ def get_is_blocking_on_stdin(
     """
     Return whether a job is blocking on stdin.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.is_blocking_on_stdin()
 
 
 _job_clients = defaultdict(lambda: [])
 _job_stop_events = defaultdict(lambda: asyncio.Event())
+_job_queues = defaultdict(lambda: asyncio.Queue())
 async def notify_clients(name: str, websocket: WebSocket, content: str):
     """
     Write the given content to all connected clients.
@@ -315,12 +325,16 @@ async def get_input_from_clients(name: str, websocket: WebSocket) -> str:
     async def _read_client(client):
         try:
             await client.send_text(JOBS_STDIN_MESSAGE)
-            data = await client.receive_text()
+            data = await _job_queues[name].get()
         except WebSocketDisconnect:
             if client in _job_clients[name]:
                 _job_clients[name].remove(client)
+            if not _job_clients[name]:
+                _job_stop_events[name].set()
         except Exception:
             pass
+        finally:
+            _job_queues[name].task_done()
         return data
 
     read_tasks = [
@@ -357,10 +371,12 @@ async def logs_websocket(name: str, websocket: WebSocket):
     Stream logs from a job over a websocket.
     """
     await websocket.accept()
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     _job_clients[name].append(websocket)
 
+    _task = None
     async def monitor_logs():
+        nonlocal _task
         try:
             callback_function = partial(
                 notify_clients,
@@ -377,16 +393,17 @@ async def logs_websocket(name: str, websocket: WebSocket):
                 name,
                 websocket,
             )
-            await job.monitor_logs_async(
+            _task = asyncio.create_task(job.monitor_logs_async(
                 callback_function=callback_function,
                 input_callback_function=input_callback_function,
                 stop_callback_function=stop_callback_function,
                 stop_event=_job_stop_events[name],
                 stop_on_exit=True,
                 accept_input=True,
-            )
+            ))
         except Exception:
             warn(traceback.format_exc())
+            _task.cancel()
 
     try:
         token = await websocket.receive_text()
@@ -397,13 +414,17 @@ async def logs_websocket(name: str, websocket: WebSocket):
                 detail="Invalid credentials.",
             )
         monitor_task = asyncio.create_task(monitor_logs())
-        await monitor_task
+        while True:
+            text = await websocket.receive_text()
+            await _job_queues[name].put(text)
+
     except fastapi.HTTPException:
         await websocket.send_text("Invalid credentials.")
         await websocket.close()
     except WebSocketDisconnect:
-        _job_stop_events[name].set()
-        monitor_task.cancel()
+        if not _job_clients[name]:
+            _job_stop_events[name].set()
+            monitor_task.cancel()
     except asyncio.CancelledError:
         pass
     except Exception:

meerschaum/api/routes/_login.py

@@ -23,8 +23,8 @@ from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
 
 @manager.user_loader()
 def load_user(
-        username: str
-    ) -> User:
+    username: str
+) -> User:
     """
     Create the `meerschaum.core.User` object from the username.
     """
@@ -33,8 +33,8 @@ def load_user(
 
 @app.post(endpoints['login'], tags=['Users'])
 def login(
-        data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
-    ) -> Dict[str, Any]:
+    data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
+) -> Dict[str, Any]:
     """
     Login and set the session token.
     """

meerschaum/api/routes/_pipes.py

@@ -474,9 +474,9 @@ def get_pipe_data(
         df[col] = df[col].apply(lambda x: f'{x:f}' if isinstance(x, Decimal) else x)
 
     json_content = df.to_json(
-        date_format = 'iso',
-        orient = 'records',
-        date_unit = 'us',
+        date_format='iso',
+        orient='records',
+        date_unit='us',
     )
 
     return fastapi.Response(

meerschaum/api/routes/_webterm.py

@@ -8,10 +8,9 @@ Routes to the Webterm proxy.
 
 import asyncio
 from meerschaum.utils.typing import Optional
-from meerschaum.api import app, no_auth, manager, endpoints
+from meerschaum.api import app, endpoints
 from meerschaum.utils.packages import attempt_import
-from meerschaum.api.dash import active_sessions
-from meerschaum.api.dash.users import is_session_authenticated
+from meerschaum.api.dash.sessions import is_session_authenticated
 fastapi, fastapi_responses = attempt_import('fastapi', 'fastapi.responses')
 import starlette
 httpx = attempt_import('httpx')
@@ -25,9 +24,9 @@ PlainTextResponse = fastapi_responses.PlainTextResponse
 
 @app.get(endpoints['webterm'], tags=["Webterm"])
 async def get_webterm(
-        request: Request,
-        s: Optional[str] = None,
-    ) -> HTMLResponse:
+    request: Request,
+    s: Optional[str] = None,
+) -> HTMLResponse:
     """
     Get the main HTML template for the Webterm.
     """

meerschaum/config/_default.py

@@ -16,7 +16,6 @@ default_meerschaum_config = {
     'api_instance': 'MRSM{meerschaum:instance}',
     'web_instance': 'MRSM{meerschaum:instance}',
     'default_repository': 'api:mrsm',
-    #  'default_executor': 'local',
     'connectors': {
         'sql': {
             'default': {},
@@ -53,6 +52,14 @@ default_meerschaum_config = {
                 'protocol': 'https',
             },
         },
+        'valkey': {
+            'main': {
+                'host': 'localhost',
+                'username': 'default',
+                'password': 'mrsm',
+                'port': 6379,
+            },
+        },
     },
 }
 default_system_config = {
@@ -75,7 +82,7 @@ default_system_config = {
             },
         },
 
-        'api' : {
+        'api': {
         },
     },
     ### not to be confused with system_config['connectors']['api'], this is the configuration
@@ -89,6 +96,10 @@ default_system_config = {
             'proxy_headers': True,
             'forwarded_allow_ips': '*',
         },
+        'cache': {
+            'connector': 'valkey:main',
+            'session_expires_minutes': 43200,
+        },
         'permissions':       {
             'registration': {
                 'users': True,
@@ -98,7 +109,7 @@ default_system_config = {
             'actions': {
                 'non_admin': True,
             },
-            'chaining' : {
+            'chaining': {
                 'insecure_parent_instance': False,
                 'child_apis': False,
             },
@@ -113,6 +124,7 @@ default_system_config = {
         'inplace_sync': True,
         'uv_pip': True,
         'systemd_healthcheck': False,
+        'valkey_session_cache': True,
     },
 }
 default_pipes_config = {

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.3.5.dev0"
+__version__ = "2.4.0"

meerschaum/config/stack/__init__.py

@@ -31,21 +31,29 @@ db_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:db:hostname}'
 api_port = "MRSM{meerschaum:connectors:api:main:port}"
 api_host = "api"
 
+valkey_hostname = "valkey"
+valkey_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:valkey:hostname}'
+valkey_port = "MRSM{meerschaum:connectors:valkey:main:port}"
+valkey_username = 'MRSM{meerschaum:connectors:valkey:main:username}'
+valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
+
 env_dict = {
-    'COMPOSE_PROJECT_NAME' : 'mrsm',
-    'TIMESCALEDB_VERSION' : 'latest-pg16-oss',
-    'POSTGRES_USER' : f'{db_user}',
-    'POSTGRES_PASSWORD' : f'{db_pass}',
-    'POSTGRES_DB' : f'{db_base}',
-    'MEERSCHAUM_API_HOSTNAME' : f'{api_host}',
-    'ALLOW_IP_RANGE' : '0.0.0.0/0',
-    'MEERSCHAUM_API_CONFIG_RESOURCES' : '/meerschaum',
+    'COMPOSE_PROJECT_NAME': 'mrsm',
+    'TIMESCALEDB_VERSION': 'latest-pg16-oss',
+    'POSTGRES_USER': db_user,
+    'POSTGRES_PASSWORD': db_pass,
+    'POSTGRES_DB': db_base,
+    'VALKEY_USERNAME': valkey_username,
+    'VALKEY_PASSWORD': valkey_password,
+    'MEERSCHAUM_API_HOSTNAME': api_host,
+    'ALLOW_IP_RANGE': '0.0.0.0/0',
+    'MEERSCHAUM_API_CONFIG_RESOURCES': '/meerschaum',
 }
 ### apply patch to host config to change hostname to the Docker service name
 env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
     {
-        'meerschaum' : 'MRSM{!meerschaum}',
-        'system' : 'MRSM{!system}',
+        'meerschaum': 'MRSM{!meerschaum}',
+        'system': 'MRSM{!system}',
     },
     indent = 4,
 ).replace(
@@ -56,9 +64,9 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
 
 volumes = {
     'api_root': '/meerschaum',
-    'api_user_local': '/home/meerschaum/.local',
     'meerschaum_db_data': '/var/lib/postgresql/data',
     'grafana_storage': '/var/lib/grafana',
+    'valkey_data': '/bitnami/valkey/data',
 }
 networks = {
     'frontend': None,
@@ -74,13 +82,19 @@ env_dict['MEERSCHAUM_API_PATCH'] = json.dumps(
                         'port': 5432,
                     },
                     'local': {
-                        'database': volumes['api_root'] + '/sqlite/mrsm_local.db'
+                        'database': volumes['api_root'] + '/sqlite/mrsm_local.db',
+                    },
+                },
+                'valkey': {
+                    'main': {
+                        'host': valkey_host,
+                        'port': 6379,
                     },
                 },
             },
         },
     },
-    indent = 4,
+    indent=4,
 )
 
 compose_header = """
@@ -114,19 +128,19 @@ default_docker_compose_config = {
                 ],
                 'interval': '5s',
                 'timeout': '3s',
-                'retries': 3
+                'retries': 5
             },
             'restart': 'always',
-            'image' : 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
-            'ports' : [
+            'image': 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
+            'ports': [
                 f'{db_port}:5432',
             ],
-            'hostname' : f'{db_hostname}',
-            'volumes' : [
+            'hostname': db_hostname,
+            'volumes': [
                 'meerschaum_db_data:' + volumes['meerschaum_db_data'],
             ],
             'shm_size': '1024m',
-            'networks' : [
+            'networks': [
                 'backend',
             ],
         },
@@ -157,10 +171,39 @@ default_docker_compose_config = {
                 'db': {
                     'condition': 'service_healthy',
                 },
+                'valkey': {
+                    'condition': 'service_healthy',
+                },
             },
-            'volumes' : [
+            'volumes': [
                 'api_root:' + volumes['api_root'],
-                'api_user_local:' + volumes['api_user_local'],
+            ],
+        },
+        'valkey': {
+            'image': 'bitnami/valkey:latest',
+            'restart': 'always',
+            'environment': {
+                'VALKEY_PASSWORD': '<DOLLAR>VALKEY_PASSWORD',
+                'VALKEY_RDB_POLICY_DISABLED': 'no',
+                'VALKEY_RDB_POLICY': '900#1 600#5 300#10 120#50 60#1000 30#10000',
+            },
+            'hostname': valkey_hostname,
+            'ports': [
+                f'{valkey_port}:6379',
+            ],
+            'volumes': [
+                'valkey_data:' + volumes['valkey_data'],
+            ],
+            'healthcheck': {
+                'test': [
+                    'CMD', 'valkey-cli', 'ping',
+                ],
+                'interval': '5s',
+                'timeout': '3s',
+                'retries': 5,
+            },
+            'networks': [
+                'backend',
             ],
         },
         'grafana': {

meerschaum/config/static/__init__.py

@@ -46,12 +46,16 @@ STATIC_CONFIG: Dict[str, Any] = {
             'stdin_message': 'MRSM_STDIN',
             'stop_message': 'MRSM_STOP',
             'metadata_cache_seconds': 5,
+            'temp_prefix': '.api-temp-',
         },
     },
     'sql': {
         'internal_schema': '_mrsm_internal',
         'instance_schema': 'mrsm',
     },
+    'valkey': {
+        'colon': '-_',
+    },
     'environment': {
         'config': 'MRSM_CONFIG',
         'config_dir': 'MRSM_CONFIG_DIR',
@@ -68,11 +72,13 @@ STATIC_CONFIG: Dict[str, Any] = {
         'uid': 'MRSM_UID',
         'gid': 'MRSM_GID',
         'noask': 'MRSM_NOASK',
+        'noninteractive': 'MRSM_NONINTERACTIVE',
         'id': 'MRSM_SERVER_ID',
         'daemon_id': 'MRSM_DAEMON_ID',
         'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
         'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
         'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
+        'systemd_delete_job': 'MRSM_SYSTEMD_DELETE_JOB',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },