meerschaum 2.3.5.dev0__tar.gz → 2.3.6__tar.gz

{meerschaum-2.3.5.dev0/meerschaum.egg-info → meerschaum-2.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.3.5.dev0
+Version: 2.3.6
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/_internal/arguments/__init__.py

@@ -9,7 +9,8 @@ This package includes argument parsing utilities.
 from meerschaum._internal.arguments._parse_arguments import (
     parse_arguments, parse_line, remove_leading_action,
     parse_dict_to_sysargs, split_chained_sysargs, split_pipeline_sysargs,
-    sysargs_has_api_executor_keys,
+    sysargs_has_api_executor_keys, get_pipeline_sysargs,
+    compress_pipeline_sysargs, remove_api_executor_keys,
 )
 from meerschaum._internal.arguments._parser import parser
 from meerschaum.plugins import add_plugin_argument

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/_internal/arguments/_parse_arguments.py

@@ -424,12 +424,91 @@ def sysargs_has_api_executor_keys(sysargs: List[str]) -> bool:
     if '-e' not in sysargs and '--executor-keys' not in sysargs:
         return False
 
-    executor_keys_flag = '-e' if '-e' in sysargs else '--executor-keys'
-    executor_keys_flag_ix = sysargs.index(executor_keys_flag)
-    executor_keys_ix = executor_keys_flag_ix + 1
+    for i, arg in enumerate(sysargs):
+        if arg not in ('-e', '--executor-keys'):
+            continue
 
-    if len(sysargs) < executor_keys_ix:
-        return False
+        executor_keys_ix = i + 1
+        if len(sysargs) <= executor_keys_ix:
+            return False
+
+        executor_keys = sysargs[executor_keys_ix]
+        if executor_keys.startswith('api:'):
+            return True
+
+    return False
+
+
+def remove_api_executor_keys(sysargs: List[str]) -> List[str]:
+    """
+    Remove any api executor keys from `sysargs`.
+    """
+    from meerschaum.utils.misc import flatten_list
+
+    if not sysargs_has_api_executor_keys(sysargs):
+        return sysargs
+
+    skip_indices = set(flatten_list(
+        [
+            [i, i+1]
+            for i, arg in enumerate(sysargs)
+            if arg in ('-e', '--executor-keys')
+        ]
+    ))
+
+    return [
+        arg
+        for i, arg in enumerate(sysargs)
+        if i not in skip_indices
+    ]
 
-    executor_keys = sysargs[executor_keys_ix]
-    return executor_keys.startswith('api:')
+
+def get_pipeline_sysargs(
+    sysargs: List[str],
+    pipeline_args: List[str],
+    _patch_args: Optional[Dict[str, Any]] = None,
+) -> List[str]:
+    """
+    Parse `sysargs` and `pipeline_args` into a single `start pipeline` sysargs.
+    """
+    import shlex
+    start_pipeline_params = {
+        'sub_args_line': shlex.join(sysargs),
+        'patch_args': _patch_args,
+    }
+    return (
+        ['start', 'pipeline']
+        + [str(arg) for arg in pipeline_args]
+        + ['-P', json.dumps(start_pipeline_params, separators=(',', ':'))]
+    )
+
+
+def compress_pipeline_sysargs(pipeline_sysargs: List[str]) -> List[str]:
+    """
+    Given a `start pipeline` sysargs, return a condensed syntax rendition.
+    """
+    import shlex
+
+    if pipeline_sysargs[:2] != ['start', 'pipeline']:
+        return pipeline_sysargs
+
+    if '-P' not in pipeline_sysargs:
+        return pipeline_sysargs
+
+    params_ix = pipeline_sysargs.index('-P')
+    pipeline_args = pipeline_sysargs[2:params_ix]
+    params_str = pipeline_sysargs[-1]
+    try:
+        start_pipeline_params = json.loads(params_str)
+    except Exception:
+        return pipeline_sysargs
+
+    sub_args_line = start_pipeline_params.get('sub_args_line', None)
+    if not sub_args_line:
+        return pipeline_sysargs
+
+    return (
+        shlex.split(sub_args_line)
+        + [':']
+        + pipeline_args
+    )

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/_internal/entry.py

@@ -11,6 +11,8 @@ from __future__ import annotations
 
 import os
 import sys
+import pathlib
+
 from meerschaum.utils.typing import SuccessTuple, List, Optional, Dict, Callable, Any
 from meerschaum.config.static import STATIC_CONFIG as _STATIC_CONFIG
 
@@ -19,8 +21,17 @@ if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
     from meerschaum.utils.daemon import RotatingFile as _RotatingFile, StdinFile as _StdinFile
     from meerschaum.config import get_config as _get_config
 
-    _systemd_result_path = os.environ[_STATIC_CONFIG['environment']['systemd_result_path']]
-    _systemd_log_path = os.environ[_STATIC_CONFIG['environment']['systemd_log_path']]
+    _systemd_result_path = pathlib.Path(
+        os.environ[_STATIC_CONFIG['environment']['systemd_result_path']]
+    )
+    _systemd_log_path = pathlib.Path(
+        os.environ[_STATIC_CONFIG['environment']['systemd_log_path']]
+    )
+    _systemd_delete_job = (
+        (os.environ.get(_STATIC_CONFIG['environment']['systemd_delete_job'], None) or '0')
+        not in (None, '0', 'false')
+    )
+    _job_name = os.environ[_STATIC_CONFIG['environment']['daemon_id']]
     _systemd_log = _RotatingFile(
         _systemd_log_path,
         write_timestamps=True,
@@ -51,6 +62,7 @@ def entry(
         split_chained_sysargs,
         split_pipeline_sysargs,
         sysargs_has_api_executor_keys,
+        get_pipeline_sysargs,
     )
     from meerschaum.config.static import STATIC_CONFIG
     if sysargs is None:
@@ -72,15 +84,7 @@ def entry(
         else split_chained_sysargs(sysargs)
     )
     if pipeline_args:
-        start_pipeline_params = {
-            'sub_args_line': shlex.join(sysargs),
-            'patch_args': _patch_args,
-        }
-        chained_sysargs = [
-            ['start', 'pipeline']
-            + [str(arg) for arg in pipeline_args]
-            + ['-P', json.dumps(start_pipeline_params, separators=(',', ':'))]
-        ]
+        chained_sysargs = [get_pipeline_sysargs(sysargs, pipeline_args, _patch_args=_patch_args)]
 
     results: List[SuccessTuple] = []
 
@@ -169,8 +173,20 @@ def entry(
 
     if _systemd_result_path:
         import json
-        with open(_systemd_result_path, 'w+', encoding='utf-8') as f:
-            json.dump((success, msg), f)
+        from meerschaum.utils.warnings import warn
+        import meerschaum as mrsm
+
+        job = mrsm.Job(_job_name, executor_keys='systemd')
+        if job.delete_after_completion:
+            delete_success, delete_msg = job.delete()
+            mrsm.pprint((delete_success, delete_msg))
+        else:
+            try:
+                if _systemd_result_path.parent.exists():
+                    with open(_systemd_result_path, 'w+', encoding='utf-8') as f:
+                        json.dump((success, msg), f)
+            except Exception as e:
+                warn(f"Failed to write job result:\n{e}")
 
     return success, msg
 

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/actions/api.py

@@ -89,22 +89,22 @@ def api(
     return success, message
 
 def _api_start(
-        action: Optional[List[str]] = None,
-        host: Optional[str] = None,
-        port: Optional[int] = None,
-        workers: Optional[int] = None,
-        mrsm_instance: Optional[str] = None,
-        no_dash: bool = False,
-        no_auth: bool = False,
-        private: bool = False,
-        secure: bool = False,
-        debug: bool = False,
-        nopretty: bool = False,
-        production: bool = False,
-        keyfile: Optional[str] = None,
-        certfile: Optional[str] = None,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    host: Optional[str] = None,
+    port: Optional[int] = None,
+    workers: Optional[int] = None,
+    mrsm_instance: Optional[str] = None,
+    no_dash: bool = False,
+    no_auth: bool = False,
+    private: bool = False,
+    secure: bool = False,
+    debug: bool = False,
+    nopretty: bool = False,
+    production: bool = False,
+    keyfile: Optional[str] = None,
+    certfile: Optional[str] = None,
+    **kw: Any
+) -> SuccessTuple:
     """Start the API server.
    
     Parameters

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/actions/start.py

@@ -89,6 +89,7 @@ def _start_jobs(
     name: Optional[str] = None,
     sysargs: Optional[List[str]] = None,
     executor_keys: Optional[str] = None,
+    rm: bool = False,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -210,7 +211,7 @@ def _start_jobs(
 
     def _run_new_job(name: Optional[str] = None):
         name = name or get_new_daemon_name()
-        job = Job(name, sysargs, executor_keys=executor_keys)
+        job = Job(name, sysargs, executor_keys=executor_keys, delete_after_completion=rm)
         return job.start(debug=debug), name
 
     def _run_existing_job(name: str):
@@ -568,6 +569,7 @@ def _start_pipeline(
         else 1
     )
 
+    params = params or {}
     sub_args_line = params.get('sub_args_line', None)
     patch_args = params.get('patch_args', None)
 

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/api/_events.py

@@ -20,12 +20,14 @@ from meerschaum.connectors.poll import retry_connect
 from meerschaum.utils.warnings import warn
 from meerschaum._internal.term.tools import is_webterm_running
 from meerschaum.jobs import (
+    get_jobs,
     start_check_jobs_thread,
     stop_check_jobs_thread,
     get_executor_keys_from_context,
 )
+from meerschaum.config.static import STATIC_CONFIG
 
-_check_jobs_thread = None
+TEMP_PREFIX: str = STATIC_CONFIG['api']['jobs']['temp_prefix']
 
 @app.on_event("startup")
 async def startup():
@@ -51,8 +53,7 @@ async def startup():
         await shutdown()
         os._exit(1)
 
-    if get_executor_keys_from_context() == 'local':
-        start_check_jobs_thread()
+    start_check_jobs_thread()
 
 
 @app.on_event("shutdown")
@@ -65,11 +66,14 @@ async def shutdown():
     if get_api_connector().type == 'sql':
         get_api_connector().engine.dispose()
 
-    if get_executor_keys_from_context() == 'local':
-        stop_check_jobs_thread()
+    stop_check_jobs_thread()
 
-    from meerschaum.api.routes._actions import _temp_jobs
-    for name, job in _temp_jobs.items():
+    temp_jobs = {
+        name: job
+        for name, job in get_jobs(include_hidden=True).items()
+        if name.startswith(TEMP_PREFIX)
+    }
+    for job in temp_jobs.values():
         job.delete()
 
     ### Terminate any running jobs left over.

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/api/routes/_actions.py

@@ -67,104 +67,6 @@ def get_actions(
     return list(actions)
 
 
-async def notify_client(client, content: str):
-    """
-    Send a line of text to a client.
-    """
-    try:
-        await client.send_text(content)
-    except (RuntimeError, WebSocketDisconnect, Exception):
-        raise StopMonitoringLogs
-
-_temp_jobs = {}
-@app.websocket(actions_endpoint + '/ws')
-async def do_action_websocket(websocket: WebSocket):
-    """
-    Execute an action and stream the output to the client.
-    """
-    await websocket.accept()
-
-    stop_event = asyncio.Event()
-    job_name = '.' + generate_password(12)
-    job = None
-
-    async def monitor_logs(job):
-        success, msg = job.start()
-        await job.monitor_logs_async(
-            partial(notify_client, websocket),
-            stop_event=stop_event,
-            stop_on_exit=True,
-        )
-
-    try:
-        token = await websocket.receive_text()
-        user = await manager.get_current_user(token) if not no_auth else None
-        if user is None and not no_auth:
-            stop_event.set()
-            if job is not None:
-                job.delete()
-                _ = _temp_jobs.pop(job_name, None)
-            raise fastapi.HTTPException(
-                status_code=401,
-                detail="Invalid credentials.",
-            )
-
-        auth_success, auth_msg = (
-            is_user_allowed_to_execute(user)
-            if not no_auth
-            else (True, "Success")
-        )
-        auth_payload = {
-            'is_authenticated': auth_success,
-            'timestamp': datetime.now(timezone.utc).isoformat(),
-        }
-        await websocket.send_json(auth_payload)
-        if not auth_success:
-            stop_event.set()
-            job.stop()
-            await websocket.close()
-
-        sysargs = clean_sysargs(await websocket.receive_json())
-        job = Job(
-            job_name,
-            sysargs,
-            executor_keys='local',
-            _properties={
-                'logs': {
-                    'write_timestamps': False,
-                },
-            },
-        )
-        _temp_jobs[job_name] = job
-        monitor_task = asyncio.create_task(monitor_logs(job))
-
-        ### NOTE: Await incoming text to trigger `WebSocketDisconnect`.
-        while True:
-            await websocket.receive_text()
-
-    except fastapi.HTTPException:
-        await websocket.send_text("Invalid credentials.")
-        await websocket.close()
-    except (WebSocketDisconnect, asyncio.CancelledError):
-        stop_event.set()
-        job.stop()
-    except Exception:
-        stop_event.set()
-        job.stop()
-        warn(f"Error in logs websocket:\n{traceback.format_exc()}")
-    finally:
-        stop_event.set()
-        job.stop()
-        monitor_task.cancel()
-        if job is not None:
-            job.delete()
-        _ = _temp_jobs.pop(job_name, None)
-        try:
-            await websocket.close()
-        except RuntimeError:
-            pass
-
-
 @app.post(actions_endpoint + "/{action}", tags=['Actions'])
 def do_action_legacy(
     action: str,

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/api/routes/_jobs.py

@@ -40,7 +40,15 @@ from meerschaum.config.static import STATIC_CONFIG
 
 JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
 JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
-EXECUTOR_KEYS: str = get_executor_keys_from_context()
+EXECUTOR_KEYS: str = 'local'
+
+
+def _get_job(name: str):
+    systemd_job = Job(name, executor_keys='systemd')
+    if systemd_job.exists():
+        return systemd_job
+
+    return Job(name, executor_keys=EXECUTOR_KEYS)
 
 
 @app.get(endpoints['jobs'], tags=['Jobs'])
@@ -52,7 +60,7 @@ def get_jobs(
     """
     Return metadata about the current jobs.
     """
-    jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=False)
+    jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=True)
     return {
         name: {
             'sysargs': job.sysargs,
@@ -83,7 +91,7 @@ def get_job(
     """
     Return metadata for a single job.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -163,7 +171,7 @@ def delete_job(
     """
     Delete a job.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.delete()
 
 
@@ -177,7 +185,7 @@ def get_job_exists(
     """
     Return whether a job exists.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.exists()
 
 
@@ -192,7 +200,7 @@ def get_logs(
     Return a job's log text.
     To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -212,7 +220,7 @@ def start_job(
     """
     Start a job if stopped.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -231,7 +239,7 @@ def stop_job(
     """
     Stop a job if running.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -250,7 +258,7 @@ def pause_job(
     """
     Pause a job if running.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -269,7 +277,7 @@ def get_stop_time(
     """
     Get the timestamp when the job was manually stopped.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.stop_time
 
 
@@ -283,12 +291,13 @@ def get_is_blocking_on_stdin(
     """
     Return whether a job is blocking on stdin.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.is_blocking_on_stdin()
 
 
 _job_clients = defaultdict(lambda: [])
 _job_stop_events = defaultdict(lambda: asyncio.Event())
+_job_queues = defaultdict(lambda: asyncio.Queue())
 async def notify_clients(name: str, websocket: WebSocket, content: str):
     """
     Write the given content to all connected clients.
@@ -315,12 +324,16 @@ async def get_input_from_clients(name: str, websocket: WebSocket) -> str:
     async def _read_client(client):
         try:
             await client.send_text(JOBS_STDIN_MESSAGE)
-            data = await client.receive_text()
+            data = await _job_queues[name].get()
         except WebSocketDisconnect:
             if client in _job_clients[name]:
                 _job_clients[name].remove(client)
+            if not _job_clients[name]:
+                _job_stop_events[name].set()
         except Exception:
             pass
+        finally:
+            _job_queues[name].task_done()
         return data
 
     read_tasks = [
@@ -357,10 +370,12 @@ async def logs_websocket(name: str, websocket: WebSocket):
     Stream logs from a job over a websocket.
     """
     await websocket.accept()
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     _job_clients[name].append(websocket)
 
+    _task = None
     async def monitor_logs():
+        nonlocal _task
         try:
             callback_function = partial(
                 notify_clients,
@@ -377,16 +392,17 @@ async def logs_websocket(name: str, websocket: WebSocket):
                 name,
                 websocket,
             )
-            await job.monitor_logs_async(
+            _task = asyncio.create_task(job.monitor_logs_async(
                 callback_function=callback_function,
                 input_callback_function=input_callback_function,
                 stop_callback_function=stop_callback_function,
                 stop_event=_job_stop_events[name],
                 stop_on_exit=True,
                 accept_input=True,
-            )
+            ))
         except Exception:
             warn(traceback.format_exc())
+            _task.cancel()
 
     try:
         token = await websocket.receive_text()
@@ -397,13 +413,17 @@ async def logs_websocket(name: str, websocket: WebSocket):
                 detail="Invalid credentials.",
             )
         monitor_task = asyncio.create_task(monitor_logs())
-        await monitor_task
+        while True:
+            text = await websocket.receive_text()
+            await _job_queues[name].put(text)
+
     except fastapi.HTTPException:
         await websocket.send_text("Invalid credentials.")
         await websocket.close()
     except WebSocketDisconnect:
-        _job_stop_events[name].set()
-        monitor_task.cancel()
+        if not _job_clients[name]:
+            _job_stop_events[name].set()
+            monitor_task.cancel()
     except asyncio.CancelledError:
         pass
     except Exception:

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/config/_default.py

@@ -16,7 +16,6 @@ default_meerschaum_config = {
     'api_instance': 'MRSM{meerschaum:instance}',
     'web_instance': 'MRSM{meerschaum:instance}',
     'default_repository': 'api:mrsm',
-    #  'default_executor': 'local',
     'connectors': {
         'sql': {
             'default': {},

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.3.5.dev0"
+__version__ = "2.3.6"

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/config/stack/__init__.py

@@ -56,7 +56,6 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
 
 volumes = {
     'api_root': '/meerschaum',
-    'api_user_local': '/home/meerschaum/.local',
     'meerschaum_db_data': '/var/lib/postgresql/data',
     'grafana_storage': '/var/lib/grafana',
 }
@@ -160,7 +159,6 @@ default_docker_compose_config = {
             },
             'volumes' : [
                 'api_root:' + volumes['api_root'],
-                'api_user_local:' + volumes['api_user_local'],
             ],
         },
         'grafana': {

{meerschaum-2.3.5.dev0 → meerschaum-2.3.6}/meerschaum/config/static/__init__.py

@@ -46,6 +46,7 @@ STATIC_CONFIG: Dict[str, Any] = {
             'stdin_message': 'MRSM_STDIN',
             'stop_message': 'MRSM_STOP',
             'metadata_cache_seconds': 5,
+            'temp_prefix': '.api-temp-',
         },
     },
     'sql': {
@@ -73,6 +74,7 @@ STATIC_CONFIG: Dict[str, Any] = {
         'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
         'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
         'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
+        'systemd_delete_job': 'MRSM_SYSTEMD_DELETE_JOB',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },