meerschaum 2.2.6__py3-none-any.whl → 2.3.0.dev1__py3-none-any.whl

meerschaum/api/routes/_jobs.py

@@ -0,0 +1,340 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Manage jobs via the Meerschaum API.
+"""
+
+from __future__ import annotations
+
+import os
+import select
+import asyncio
+import traceback
+from datetime import datetime
+from collections import defaultdict
+from functools import partial
+
+from fastapi import WebSocket, WebSocketDisconnect
+
+from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
+from meerschaum.utils.jobs import get_jobs as _get_jobs, Job, StopMonitoringLogs
+from meerschaum.utils.warnings import warn
+
+from meerschaum.api import (
+    fastapi,
+    app,
+    endpoints,
+    manager,
+    debug,
+    no_auth,
+    private,
+)
+from meerschaum.config.static import STATIC_CONFIG
+
+JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
+
+
+@app.get(endpoints['jobs'], tags=['Jobs'])
+def get_jobs(
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, Dict[str, Any]]:
+    """
+    Return metadata about the current jobs.
+    """
+    jobs = _get_jobs()
+    return {
+        name: {
+            'sysargs': job.sysargs,
+            'result': job.result,
+            'daemon': {
+                'status': job.daemon.status,
+                'pid': job.daemon.pid,
+                'properties': job.daemon.properties,
+            },
+        }
+        for name, job in jobs.items()
+    }
+
+
+@app.get(endpoints['jobs'] + '/{name}', tags=['Jobs'])
+def get_job(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, Any]:
+    """
+    Return metadata for a single job.
+    """
+    job = Job(name)
+    if not job.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f"{job} doesn't exist."
+        )
+
+    return {
+        'sysargs': job.sysargs,
+        'result': job.result,
+        'daemon': {
+            'status': job.daemon.status,
+            'pid': job.daemon.pid,
+            'properties': job.daemon.properties,
+        },
+    }
+
+
+@app.post(endpoints['jobs'] + '/{name}', tags=['Jobs'])
+def create_job(
+    name: str,
+    sysargs: List[str],
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Create and start a new job.
+    """
+    job = Job(name, sysargs)
+    if job.exists():
+        raise fastapi.HTTPException(
+            status_code=409,
+            detail=f"{job} already exists."
+        )
+
+    return job.start()
+
+
+@app.delete(endpoints['jobs'] + '/{name}', tags=['Jobs'])
+def delete_job(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Delete a job.
+    """
+    job = Job(name)
+    return job.delete()
+
+
+@app.get(endpoints['jobs'] + '/{name}/exists', tags=['Jobs'])
+def get_job_exists(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> bool:
+    """
+    Return whether a job exists.
+    """
+    job = Job(name)
+    return job.exists()
+
+
+@app.get(endpoints['logs'] + '/{name}', tags=['Jobs'])
+def get_logs(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Union[str, None]:
+    """
+    Return a job's log text.
+    To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
+    """
+    job = Job(name)
+    if not job.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f"{job} does not exist.",
+        )
+
+    return job.get_logs()
+
+
+@app.post(endpoints['jobs'] + '/{name}/start', tags=['Jobs'])
+def start_job(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Start a job if stopped.
+    """
+    job = Job(name)
+    if not job.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f"{job} does not exist."
+        )
+    return job.start()
+
+
+@app.post(endpoints['jobs'] + '/{name}/stop', tags=['Jobs'])
+def stop_job(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Stop a job if running.
+    """
+    job = Job(name)
+    if not job.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f"{job} does not exist."
+        )
+    return job.stop()
+
+
+@app.post(endpoints['jobs'] + '/{name}/pause', tags=['Jobs'])
+def pause_job(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> SuccessTuple:
+    """
+    Pause a job if running.
+    """
+    job = Job(name)
+    if not job.exists():
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail=f"{job} does not exist."
+        )
+    return job.pause()
+
+
+@app.get(endpoints['jobs'] + '/{name}/stop_time', tags=['Jobs'])
+def get_stop_time(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Union[datetime, None]:
+    """
+    Get the timestamp when the job was manually stopped.
+    """
+    job = Job(name)
+    return job.stop_time
+
+
+@app.get(endpoints['jobs'] + '/{name}/is_blocking_on_stdin', tags=['Jobs'])
+def get_is_blocking_on_stdin(
+    name: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> bool:
+    """
+    Return whether a job is blocking on stdin.
+    """
+    job = Job(name)
+    return job.is_blocking_on_stdin()
+
+
+_job_clients = defaultdict(lambda: [])
+_job_stop_events = defaultdict(lambda: asyncio.Event())
+async def notify_clients(name: str, content: str):
+    """
+    Write the given content to all connected clients.
+    """
+    if not _job_clients[name]:
+        _job_stop_events[name].set()
+
+    async def _notify_client(client):
+        try:
+            await client.send_text(content)
+        except WebSocketDisconnect:
+            if client in _job_clients[name]:
+                _job_clients[name].remove(client)
+        except Exception:
+            pass
+
+    notify_tasks = [
+        asyncio.create_task(_notify_client(client))
+        for client in _job_clients[name]
+    ]
+    await asyncio.wait(notify_tasks)
+
+
+async def get_input_from_clients(name):
+    """
+    When a job is blocking on input, return input from the first client which provides it.
+    """
+    print('GET INPUT FROM CLIENTS')
+    if not _job_clients[name]:
+        print('NO CLIENTS')
+        return ''
+
+    async def _read_client(client):
+        try:
+            await client.send_text(JOBS_STDIN_MESSAGE)
+            data = await client.receive_text()
+        except WebSocketDisconnect:
+            if client in _job_clients[name]:
+                _job_clients[name].remove(client)
+        except Exception:
+            pass
+        return data
+
+    read_tasks = [
+        asyncio.create_task(_read_client(client))
+        for client in _job_clients[name]
+    ]
+    done, pending = await asyncio.wait(read_tasks, return_when=asyncio.FIRST_COMPLETED)
+    for task in pending:
+        task.cancel()
+    for task in done:
+        return task.result()
+
+
+@app.websocket(endpoints['logs'] + '/{name}/ws')
+async def logs_websocket(name: str, websocket: WebSocket):
+    """
+    Stream logs from a job over a websocket.
+    """
+    await websocket.accept()
+    job = Job(name)
+    _job_clients[name].append(websocket)
+
+    async def monitor_logs():
+        await job.monitor_logs_async(
+            partial(notify_clients, name),
+            input_callback_function=partial(get_input_from_clients, name),
+            stop_event=_job_stop_events[name],
+        )
+
+    try:
+        token = await websocket.receive_text()
+        user = await manager.get_current_user(token) if not no_auth else None
+        if user is None and not no_auth:
+            raise fastapi.HTTPException(
+                status_code=401,
+                detail="Invalid credentials.",
+            )
+        monitor_task = asyncio.create_task(monitor_logs())
+        await monitor_task
+    except fastapi.HTTPException:
+        await websocket.send_text("Invalid credentials.")
+        await websocket.close()
+    except WebSocketDisconnect:
+        pass
+    except asyncio.CancelledError:
+        pass
+    except Exception:
+        warn(f"Error in logs websocket:\n{traceback.format_exc()}")
+    finally:
+        if websocket in _job_clients[name]:
+            _job_clients[name].remove(websocket)
+        _job_stop_events[name].clear()

meerschaum/api/routes/_pipes.py

@@ -44,14 +44,14 @@ pd = attempt_import('pandas')
 
 @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register', tags=['Pipes'])
 def register_pipe(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        parameters: dict,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ):
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    parameters: dict,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
     """
     Register a new pipe.
     """
@@ -201,13 +201,13 @@ async def fetch_pipes_keys(
 
 @app.get(pipes_endpoint, tags=['Pipes'])
 async def get_pipes(
-        connector_keys : str = "",
-        metric_keys : str = "",
-        location_keys : str = "",
-        curr_user = (
+        connector_keys: str = "",
+        metric_keys: str = "",
+        location_keys: str = "",
+        curr_user=(
             fastapi.Depends(manager) if not no_auth else None
         ),
-        debug : bool = False
+        debug: bool = False,
     ) -> Dict[str, Any]:
     """
     Get all registered Pipes with metadata, excluding parameters.
@@ -376,18 +376,18 @@ def sync_pipe(
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
 def get_pipe_data(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        select_columns: Optional[str] = None,
-        omit_columns: Optional[str] = None,
-        begin: Union[str, int, None] = None,
-        end: Union[str, int, None] = None,
-        params: Optional[str] = None,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> str:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    select_columns: Optional[str] = None,
+    omit_columns: Optional[str] = None,
+    begin: Union[str, int, None] = None,
+    end: Union[str, int, None] = None,
+    params: Optional[str] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> str:
     """
     Get a pipe's data, applying any filtering.
 

meerschaum/config/_default.py

@@ -16,6 +16,7 @@ default_meerschaum_config = {
     'api_instance': 'MRSM{meerschaum:instance}',
     'web_instance': 'MRSM{meerschaum:instance}',
     'default_repository': 'api:mrsm',
+    'default_executor': 'local',
     'connectors': {
         'sql': {
             'default': {},

meerschaum/config/_formatting.py

@@ -36,6 +36,7 @@ default_formatting_config = {
         'paused'           : '🟡',
         'stopped'          : '🔴',
         'tag'              : '🔖',
+        'announcement'     : '📢',
     },
     'pipes'                : {
         'unicode'          : {

meerschaum/config/_paths.py

@@ -123,6 +123,10 @@ paths = {
     'PERMANENT_PATCH_DIR_PATH'       : ('{ROOT_DIR_PATH}', 'permanent_patch_config'),
     'INTERNAL_RESOURCES_PATH'        : ('{ROOT_DIR_PATH}', '.internal'),
 
+    'UPDATES_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'updates'),
+    'UPDATES_CACHE_PATH'             : ('{UPDATES_RESOURCES_PATH}', 'cache.json'),
+    'UPDATES_LOCK_PATH'              : ('{UPDATES_RESOURCES_PATH}', '.updates.lock'),
+
     'STACK_RESOURCES_PATH'           : ('{ROOT_DIR_PATH}', 'stack'),
     'STACK_COMPOSE_FILENAME'         : 'docker-compose.yaml',
     'STACK_COMPOSE_PATH'             : ('{STACK_RESOURCES_PATH}', '{STACK_COMPOSE_FILENAME}'),
@@ -176,6 +180,7 @@ paths = {
     'DAEMON_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'jobs'),
     'LOGS_RESOURCES_PATH'            : ('{ROOT_DIR_PATH}', 'logs'),
     'DAEMON_ERROR_LOG_PATH'          : ('{ROOT_DIR_PATH}', 'daemon_errors.log'),
+    'CHECK_JOBS_LOCK_PATH'           : ('{INTERNAL_RESOURCES_PATH}', 'check-jobs.lock'),
 }
 
 def set_root(root: Union[Path, str]):

meerschaum/config/_shell.py

@@ -6,127 +6,144 @@
 Default configuration for the Meerschaum shell.
 """
 
-#  import platform
-#  default_cmd = 'cmd' if platform.system() != 'Windows' else 'cmd2'
 default_cmd = 'cmd'
 
 default_shell_config = {
-    'ansi'             : {
-        'intro'        : {
-            'rich'     : {
-                'style' : "bold bright_blue",
+    'ansi'               : {
+        'intro'          : {
+            'rich'       : {
+                'style'  : "bold bright_blue",
             },
-            'color'    : [
+            'color'      : [
                 'bold',
                 'bright blue',
             ],
         },
-        'close_message': {
-            'rich' : {
-                'style' : 'bright_blue',
+        'close_message'  : {
+            'rich'       : {
+                'style'  : 'bright_blue',
             },
-            'color'    : [
+            'color'      : [
                 'bright blue',
             ],
         },
-        'doc_header': {
-            'rich' : {
-                'style' : 'bright_blue',
+        'doc_header'     : {
+            'rich'       : {
+                'style'  : 'bright_blue',
             },
-            'color'    : [
+            'color'      : [
                 'bright blue',
             ],
         },
-        'undoc_header': {
-            'rich' : {
-                'style' : 'bright_blue',
+        'undoc_header'   : {
+            'rich'       : {
+                'style'  : 'bright_blue',
             },
-            'color'    : [
+            'color'      : [
                 'bright blue',
             ],
         },
-        'ruler': {
-            'rich' : {
-                'style' : 'bold bright_blue',
+        'ruler'          : {
+            'rich'       : {
+                'style'  : 'bold bright_blue',
             },
-            'color'    : [
+            'color'      : [
                 'bold',
                 'bright blue',
             ],
         },
-        'prompt': {
-            'rich' : {
-                'style' : 'green',
+        'prompt'         : {
+            'rich'       : {
+                'style'  : 'green',
             },
-            'color'    : [
+            'color'      : [
                 'green',
             ],
         },
-        'instance' : {
-            'rich' : {
-                'style' : 'cyan',
+        'instance'       : {
+            'rich'       : {
+                'style'  : 'cyan',
             },
-            'color'    : [
+            'color'      : [
                 'cyan',
             ],
         },
-        'repo' : {
-            'rich': {
-                'style': 'magenta',
+        'repo'           : {
+            'rich'       : {
+                'style'  : 'magenta',
             },
-            'color': [
+            'color'      : [
                 'magenta',
             ],
         },
-        'username' : {
-            'rich' : {
-                'style' : 'white',
+        'executor'       : {
+            'rich'       : {
+                'style'  : 'gold1',
             },
-            'color'    : [
+        },
+        'username'       : {
+            'rich'       : {
+                'style'  : 'white',
+            },
+            'color'      : [
                 'white',
             ],
         },
-        'connected' : {
-            'rich' : {
-                'style' : 'green',
+        'connected'      : {
+            'rich'       : {
+                'style'  : 'green',
             },
-            'color'    : [
+            'color'      : [
                 'green',
             ],
         },
-        'disconnected' : {
-            'rich' : {
-                'style' : 'red',
+        'disconnected'   : {
+            'rich'       : {
+                'style'  : 'red',
             },
-            'color'    : [
+            'color'      : [
+                'red',
+            ],
+        },
+        'update_message' : {
+            'rich'       : {
+                'style'  : 'red',
+            },
+            'color'      : [
                 'red',
             ],
         },
     },
-    'ascii'            : {
-        'intro'        : """       ___  ___  __   __   __
+    'ascii'              : {
+        'intro'          : r"""       ___  ___  __   __   __
  |\/| |__  |__  |__) /__` /  ` |__|  /\  |  |  |\/|
- |  | |___ |___ |  \ .__/ \__, |  | /~~\ \__/  |  |\n""",
-        'prompt'       : '\n [ {username}@{instance} ] > ',
-        'ruler'        : '-',
-        'close_message': 'Thank you for using Meerschaum!',
-        'doc_header'   : 'Meerschaum actions (`help <action>` for usage):',
-        'undoc_header' : 'Unimplemented actions:',
+ |  | |___ |___ |  \ .__/ \__, |  | /~~\ \__/  |  |""" + '\n',
+        'prompt'         : '\n [ {username}@{instance} | {executor_keys} ] > ',
+        'ruler'          : '-',
+        'close_message'  : 'Thank you for using Meerschaum!',
+        'doc_header'     : 'Meerschaum actions (`help <action>` for usage):',
+        'undoc_header'   : 'Unimplemented actions:',
+        'update_message' : "Update available!",
     },
-    'unicode'          : {
-        'intro'        : """
+    'unicode'            : {
+        'intro'          : """
  █▄ ▄█ ██▀ ██▀ █▀▄ ▄▀▀ ▄▀▀ █▄█ ▄▀▄ █ █ █▄ ▄█
  █ ▀ █ █▄▄ █▄▄ █▀▄ ▄██ ▀▄▄ █ █ █▀█ ▀▄█ █ ▀ █\n""",
-        'prompt'       : '\n [ {username}@{instance} ] ➤ ',
-        'ruler'        : '─',
-        'close_message': ' MRSM{formatting:emoji:hand} Thank you for using Meerschaum! ',
-        'doc_header'   : 'Meerschaum actions (`help <action>` for usage):',
-        'undoc_header' : 'Unimplemented actions:',
+        'prompt'         : '\n [ {username}@{instance} | {executor_keys} ] ➤ ',
+        'ruler'          : '─',
+        'close_message'  : ' MRSM{formatting:emoji:hand} Thank you for using Meerschaum! ',
+        'doc_header'     : 'Meerschaum actions (`help <action>` for usage):',
+        'undoc_header'   : 'Unimplemented actions:',
+        'update_message' : "MRSM{formatting:emoji:announcement} Update available!",
+    },
+    'timeout'            : 60,
+    'max_history'        : 1000,
+    'clear_screen'       : True,
+    'bottom_toolbar'     : {
+        'enabled'        : True,
     },
-    'timeout'          : 60,
-    'max_history'      : 1000,
-    'clear_screen'     : True,
-    'bottom_toolbar'   : {
-        'enabled'      : True,
+    'updates'            : {
+        'check_remote'   : True,
+        'refresh_minutes': 180,
     },
 }

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.2.6"
+__version__ = "2.3.0.dev1"