meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl

meerschaum/api/routes/_actions.py

@@ -7,46 +7,176 @@ Execute Meerschaum Actions via the API
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Union
+import asyncio
+import traceback
+import shlex
+from functools import partial
+from datetime import datetime, timezone
 
+from fastapi import WebSocket, WebSocketDisconnect
+
+from meerschaum.utils.misc import generate_password
+from meerschaum.jobs import Job
+from meerschaum.utils.warnings import warn
+from meerschaum.utils.typing import SuccessTuple, Union, List, Dict, Any
 from meerschaum.api import (
     fastapi, app, endpoints, get_api_connector, debug, manager, private, no_auth
 )
 from meerschaum.actions import actions
 import meerschaum.core
+from meerschaum.config import get_config
+from meerschaum._internal.arguments._parse_arguments import parse_dict_to_sysargs, parse_arguments
+from meerschaum.api.routes._jobs import clean_sysargs
+
 actions_endpoint = endpoints['actions']
 
+def is_user_allowed_to_execute(user) -> SuccessTuple:
+    if user is None:
+        return False, "Could not load user."
+
+    if user.type == 'admin':
+        return True, "Success"
+
+    allow_non_admin = get_config(
+        'system', 'api', 'permissions', 'actions', 'non_admin', patch=True
+    )
+    if not allow_non_admin:
+        return False, (
+            "The administrator for this server has not allowed users to perform actions.\n\n"
+            + "Please contact the system administrator, or if you are running this server, "
+            + "open the configuration file with `edit config system` "
+            + "and search for 'permissions'. "
+            + "\nUnder the keys 'api:permissions:actions', "
+            + "you can allow non-admin users to perform actions."
+        )
+
+    return True, "Success"
+
+
 @app.get(actions_endpoint, tags=['Actions'])
 def get_actions(
-        curr_user = (
-            fastapi.Depends(manager) if private else None
-        ),
-    ) -> list:
+    curr_user = (
+        fastapi.Depends(manager) if private else None
+    ),
+) -> List[str]:
+    """
+    Return a list of the available actions.
+    """
     return list(actions)
 
+
+async def notify_client(client, content: str):
+    """
+    Send a line of text to a client.
+    """
+    try:
+        await client.send_text(content)
+    except WebSocketDisconnect:
+        pass
+
+_temp_jobs = {}
+@app.websocket(actions_endpoint + '/ws')
+async def do_action_websocket(websocket: WebSocket):
+    """
+    Execute an action and stream the output to the client.
+    """
+    await websocket.accept()
+
+    stop_event = asyncio.Event()
+
+    async def monitor_logs(job):
+        success, msg = job.start()
+        await job.monitor_logs_async(
+            partial(notify_client, websocket),
+            stop_event=stop_event,
+            stop_on_exit=True,
+        )
+
+    job = None
+    job_name = '.' + generate_password(12)
+    try:
+        token = await websocket.receive_text()
+        user = await manager.get_current_user(token) if not no_auth else None
+        if user is None and not no_auth:
+            raise fastapi.HTTPException(
+                status_code=401,
+                detail="Invalid credentials.",
+            )
+
+        auth_success, auth_msg = (
+            is_user_allowed_to_execute(user)
+            if not no_auth
+            else (True, "Success")
+        )
+        auth_payload = {
+            'is_authenticated': auth_success,
+            'timestamp': datetime.now(timezone.utc).isoformat(),
+        }
+        await websocket.send_json(auth_payload)
+        if not auth_success:
+            await websocket.close()
+
+        sysargs = clean_sysargs(await websocket.receive_json())
+        #  kwargs = parse_arguments(sysargs)
+        #  _ = kwargs.pop('executor_keys', None)
+        #  _ = kwargs.pop('shell', None)
+        #  sysargs = parse_dict_to_sysargs(kwargs)
+
+        job = Job(
+            job_name,
+            sysargs,
+            executor_keys='local',
+            _properties={
+                'logs': {
+                    'write_timestamps': False,
+                },
+            },
+        )
+        _temp_jobs[job_name] = job
+        monitor_task = asyncio.create_task(monitor_logs(job))
+        await monitor_task
+        try:
+            await websocket.close()
+        except RuntimeError:
+            pass
+    except fastapi.HTTPException:
+        await websocket.send_text("Invalid credentials.")
+        await websocket.close()
+    except WebSocketDisconnect:
+        pass
+    except asyncio.CancelledError:
+        pass
+    except Exception:
+        warn(f"Error in logs websocket:\n{traceback.format_exc()}")
+    finally:
+        if job is not None:
+            job.delete()
+        _ = _temp_jobs.pop(job_name, None)
+        stop_event.set()
+
+
 @app.post(actions_endpoint + "/{action}", tags=['Actions'])
-def do_action(
-        action: str,
-        keywords: dict = fastapi.Body(...),
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> SuccessTuple:
+def do_action_legacy(
+    action: str,
+    keywords: Dict[str, Any] = fastapi.Body(...),
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
     """
-    Perform a Meerschaum action (if permissions allow it).
+    Perform a Meerschaum action (if permissions allow).
 
     Parameters
     ----------
-    action: str :
+    action: str
         The action to perform.
         
-    keywords: dict :
+    keywords: Dict[str, Any]
         The keywords dictionary to pass to the action.
 
     Returns
     -------
-    A `SuccessTuple` of success, message.
-
+    A `SuccessTuple`.
     """
     if curr_user is not None and curr_user.type != 'admin':
         from meerschaum.config import get_config
@@ -65,6 +195,7 @@ def do_action(
 
     if action not in actions:
         return False, f"Invalid action '{action}'."
+
     keywords['mrsm_instance'] = keywords.get('mrsm_instance', str(get_api_connector()))
     _debug = keywords.get('debug', debug)
     keywords.pop('debug', None)

meerschaum/api/routes/_jobs.py

@@ -0,0 +1,407 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Manage jobs via the Meerschaum API.
+"""
+
+from __future__ import annotations
+
+import os
+import select
+import asyncio
+import traceback
+from datetime import datetime
+from collections import defaultdict
+from functools import partial
+
+from fastapi import WebSocket, WebSocketDisconnect
+
+from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
+from meerschaum.jobs import (
+    get_jobs as _get_jobs,
+    Job,
+    StopMonitoringLogs,
+    get_executor_keys_from_context,
+)
+from meerschaum.utils.warnings import warn
+
+from meerschaum.api import (
+    fastapi,
+    app,
+    endpoints,
+    manager,
+    debug,
+    no_auth,
+    private,
+)
+from meerschaum.config.static import STATIC_CONFIG
+
+JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
+JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
+EXECUTOR_KEYS: str = get_executor_keys_from_context()
+
+
+@app.get(endpoints['jobs'], tags=['Jobs'])
+def get_jobs(
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, Dict[str, Any]]:
+    """
+    Return metadata about the current jobs.
+    """
+    jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=False)
+    return {
+        name: {
+            'sysargs': job.sysargs,
+            'result': job.result,
+            'restart': job.restart,
+            'status': job.status,
+            'daemon': {
+                'status': job.daemon.status if job.executor_keys is None else job.status,
+                'pid': job.pid,
+                'properties': (
+                    job.daemon.properties
+                    if job.executor is None
+                    else job.executor.get_job_properties(name)
+                ),
+            },
+        }
+        for name, job in jobs.items()
+    }
+
+
+@app.get(endpoints['jobs'] + '/{name}', tags=['Jobs'])
+def get_job(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, Any]:
+    """
+    Return metadata for a single job.
+    """
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    if not job.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f"{job} doesn't exist."
+        )
+
+    return {
+        'sysargs': job.sysargs,
+        'result': job.result,
+        'restart': job.restart,
+        'status': job.status,
+        'daemon': {
+            'status': job.daemon.status if job.executor_keys is None else job.status,
+            'pid': job.pid,
+            'properties': (
+                job.daemon.properties
+                if job.executor is None
+                else job.executor.get_job_properties(job.name)
+            ),
+        },
+    }
+
+
+def clean_sysargs(sysargs: List[str]) -> List[str]:
+    """
+    Remove the executor flag or leading `api {label}` action.
+    """
+    clean_sysargs = []
+    executor_flag = False
+    for arg in sysargs:
+        if arg in ('-e', '--executor', 'api'):
+            executor_flag = True
+            continue
+        if executor_flag:
+            executor_flag = False
+            continue
+
+        clean_sysargs.append(arg)
+    return clean_sysargs
+
+
+@app.post(endpoints['jobs'] + '/{name}', tags=['Jobs'])
+def create_job(
+    name: str,
+    sysargs: List[str],
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Create and start a new job.
+    """
+    job = Job(name, clean_sysargs(sysargs), executor_keys=EXECUTOR_KEYS)
+    if job.exists():
+        raise fastapi.HTTPException(
+            status_code=409,
+            detail=f"{job} already exists."
+        )
+
+    return job.start()
+
+
+@app.delete(endpoints['jobs'] + '/{name}', tags=['Jobs'])
+def delete_job(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Delete a job.
+    """
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    return job.delete()
+
+
+@app.get(endpoints['jobs'] + '/{name}/exists', tags=['Jobs'])
+def get_job_exists(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> bool:
+    """
+    Return whether a job exists.
+    """
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    return job.exists()
+
+
+@app.get(endpoints['logs'] + '/{name}', tags=['Jobs'])
+def get_logs(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Union[str, None]:
+    """
+    Return a job's log text.
+    To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
+    """
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    if not job.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f"{job} does not exist.",
+        )
+
+    return job.get_logs()
+
+
+@app.post(endpoints['jobs'] + '/{name}/start', tags=['Jobs'])
+def start_job(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Start a job if stopped.
+    """
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    if not job.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f"{job} does not exist."
+        )
+    return job.start()
+
+
+@app.post(endpoints['jobs'] + '/{name}/stop', tags=['Jobs'])
+def stop_job(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Stop a job if running.
+    """
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    if not job.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f"{job} does not exist."
+        )
+    return job.stop()
+
+
+@app.post(endpoints['jobs'] + '/{name}/pause', tags=['Jobs'])
+def pause_job(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Pause a job if running.
+    """
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    if not job.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f"{job} does not exist."
+        )
+    return job.pause()
+
+
+@app.get(endpoints['jobs'] + '/{name}/stop_time', tags=['Jobs'])
+def get_stop_time(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Union[datetime, None]:
+    """
+    Get the timestamp when the job was manually stopped.
+    """
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    return job.stop_time
+
+
+@app.get(endpoints['jobs'] + '/{name}/is_blocking_on_stdin', tags=['Jobs'])
+def get_is_blocking_on_stdin(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> bool:
+    """
+    Return whether a job is blocking on stdin.
+    """
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    return job.is_blocking_on_stdin()
+
+
+_job_clients = defaultdict(lambda: [])
+_job_stop_events = defaultdict(lambda: asyncio.Event())
+async def notify_clients(name: str, websocket: WebSocket, content: str):
+    """
+    Write the given content to all connected clients.
+    """
+    async def _notify_client(client):
+        try:
+            await client.send_text(content)
+        except WebSocketDisconnect:
+            if client in _job_clients[name]:
+                _job_clients[name].remove(client)
+        except Exception:
+            pass
+
+    await _notify_client(websocket)
+
+
+async def get_input_from_clients(name: str, websocket: WebSocket) -> str:
+    """
+    When a job is blocking on input, return input from the first client which provides it.
+    """
+    if not _job_clients[name]:
+        return ''
+
+    async def _read_client(client):
+        try:
+            await client.send_text(JOBS_STDIN_MESSAGE)
+            data = await client.receive_text()
+        except WebSocketDisconnect:
+            if client in _job_clients[name]:
+                _job_clients[name].remove(client)
+        except Exception:
+            pass
+        return data
+
+    read_tasks = [
+        asyncio.create_task(_read_client(client))
+        for client in _job_clients[name]
+    ]
+    done, pending = await asyncio.wait(read_tasks, return_when=asyncio.FIRST_COMPLETED)
+    for task in pending:
+        task.cancel()
+    for task in done:
+        return task.result()
+
+
+async def send_stop_message(name: str, client: WebSocket, result: SuccessTuple):
+    """
+    Send a stop message to clients when the job stops.
+    """
+    try:
+        await client.send_text(JOBS_STOP_MESSAGE)
+        await client.send_json(result)
+    except WebSocketDisconnect:
+        _job_stop_events[name].set()
+        if client in _job_clients[name]:
+            _job_clients[name].remove(client)
+    except RuntimeError:
+        pass
+    except Exception:
+        warn(traceback.format_exc())
+
+
+@app.websocket(endpoints['logs'] + '/{name}/ws')
+async def logs_websocket(name: str, websocket: WebSocket):
+    """
+    Stream logs from a job over a websocket.
+    """
+    await websocket.accept()
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    _job_clients[name].append(websocket)
+
+    async def monitor_logs():
+        try:
+            callback_function = partial(
+                notify_clients,
+                name,
+                websocket,
+            )
+            input_callback_function = partial(
+                get_input_from_clients,
+                name,
+                websocket,
+            )
+            stop_callback_function = partial(
+                send_stop_message,
+                name,
+                websocket,
+            )
+            await job.monitor_logs_async(
+                callback_function=callback_function,
+                input_callback_function=input_callback_function,
+                stop_callback_function=stop_callback_function,
+                stop_event=_job_stop_events[name],
+                stop_on_exit=True,
+                accept_input=True,
+            )
+        except Exception:
+            warn(traceback.format_exc())
+
+    try:
+        token = await websocket.receive_text()
+        user = await manager.get_current_user(token) if not no_auth else None
+        if user is None and not no_auth:
+            raise fastapi.HTTPException(
+                status_code=401,
+                detail="Invalid credentials.",
+            )
+        monitor_task = asyncio.create_task(monitor_logs())
+        await monitor_task
+    except fastapi.HTTPException:
+        await websocket.send_text("Invalid credentials.")
+        await websocket.close()
+    except WebSocketDisconnect:
+        _job_stop_events[name].set()
+        monitor_task.cancel()
+    except asyncio.CancelledError:
+        pass
+    except Exception:
+        warn(f"Error in logs websocket:\n{traceback.format_exc()}")
+    finally:
+        if websocket in _job_clients[name]:
+            _job_clients[name].remove(websocket)
+        _job_stop_events[name].clear()

meerschaum/api/routes/_pipes.py

@@ -44,14 +44,14 @@ pd = attempt_import('pandas')
 
 @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register', tags=['Pipes'])
 def register_pipe(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        parameters: dict,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ):
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    parameters: dict,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
     """
     Register a new pipe.
     """
@@ -201,13 +201,13 @@ async def fetch_pipes_keys(
 
 @app.get(pipes_endpoint, tags=['Pipes'])
 async def get_pipes(
-        connector_keys : str = "",
-        metric_keys : str = "",
-        location_keys : str = "",
-        curr_user = (
+        connector_keys: str = "",
+        metric_keys: str = "",
+        location_keys: str = "",
+        curr_user=(
             fastapi.Depends(manager) if not no_auth else None
         ),
-        debug : bool = False
+        debug: bool = False,
     ) -> Dict[str, Any]:
     """
     Get all registered Pipes with metadata, excluding parameters.
@@ -376,18 +376,18 @@ def sync_pipe(
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
 def get_pipe_data(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        select_columns: Optional[str] = None,
-        omit_columns: Optional[str] = None,
-        begin: Union[str, int, None] = None,
-        end: Union[str, int, None] = None,
-        params: Optional[str] = None,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> str:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    select_columns: Optional[str] = None,
+    omit_columns: Optional[str] = None,
+    begin: Union[str, int, None] = None,
+    end: Union[str, int, None] = None,
+    params: Optional[str] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> str:
     """
     Get a pipe's data, applying any filtering.
 

meerschaum/config/_default.py

@@ -16,6 +16,7 @@ default_meerschaum_config = {
     'api_instance': 'MRSM{meerschaum:instance}',
     'web_instance': 'MRSM{meerschaum:instance}',
     'default_repository': 'api:mrsm',
+    #  'default_executor': 'local',
     'connectors': {
         'sql': {
             'default': {},

meerschaum/config/_formatting.py

@@ -36,6 +36,7 @@ default_formatting_config = {
         'paused'           : '🟡',
         'stopped'          : '🔴',
         'tag'              : '🔖',
+        'announcement'     : '📢',
     },
     'pipes'                : {
         'unicode'          : {

meerschaum/config/_jobs.py

@@ -7,7 +7,7 @@ Default configuration for jobs.
 """
 
 default_jobs_config = {
-    'timeout_seconds': 8,
+    'timeout_seconds': 4,
     'check_timeout_interval_seconds': 0.1,
     'terminal': {
         'lines': 40,