meerschaum 2.3.4__tar.gz → 2.3.5__tar.gz

{meerschaum-2.3.4/meerschaum.egg-info → meerschaum-2.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.3.4
+Version: 2.3.5
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.3.4 → meerschaum-2.3.5}/meerschaum/_internal/arguments/__init__.py

@@ -9,11 +9,8 @@ This package includes argument parsing utilities.
 from meerschaum._internal.arguments._parse_arguments import (
     parse_arguments, parse_line, remove_leading_action,
     parse_dict_to_sysargs, split_chained_sysargs, split_pipeline_sysargs,
+    sysargs_has_api_executor_keys, get_pipeline_sysargs,
+    compress_pipeline_sysargs, remove_api_executor_keys,
 )
 from meerschaum._internal.arguments._parser import parser
 from meerschaum.plugins import add_plugin_argument
-
-__all__ = [
-    'parser', 'parse_arguments', 'parse_line', 'add_plugin_argument', 'parse_dict_to_sysargs',
-    'remove_leading_action',
-]

{meerschaum-2.3.4 → meerschaum-2.3.5}/meerschaum/_internal/arguments/_parse_arguments.py

@@ -283,18 +283,29 @@ def parse_dict_to_sysargs(
             ### Add list flags
             if isinstance(args_dict[a], (list, tuple)):
                 if len(args_dict[a]) > 0:
-                    sysargs.extend(
-                        [t[0]]
-                        + [
-                            str(item)
-                            for item in args_dict[a]
-                        ]
-                    )
+                    if a == 'sub_args' and args_dict[a] != ['']:
+                        print(f"{args_dict[a]=}")
+                        sysargs.extend(
+                            [
+                                '-A',
+                                shlex.join([
+                                    str(item) for item in args_dict[a]
+                                ]),
+                            ]
+                        )
+                    else:
+                        sysargs.extend(
+                            [t[0]]
+                            + [
+                                str(item)
+                                for item in args_dict[a]
+                            ]
+                        )
 
             ### Add dict flags
             elif isinstance(args_dict[a], dict):
                 if len(args_dict[a]) > 0:
-                    sysargs += [t[0], json.dumps(args_dict[a])]
+                    sysargs += [t[0], json.dumps(args_dict[a], separators=(',', ':'))]
 
             ### Account for None and other values
             elif (args_dict[a] is not None) or (args_dict[a] is None and a in allow_none_args):
@@ -403,4 +414,101 @@ def load_plugin_args() -> None:
             to_import.append(plugin.name)
     if not to_import:
         return
-    import_plugins(*to_import) 
+    import_plugins(*to_import)
+
+
+def sysargs_has_api_executor_keys(sysargs: List[str]) -> bool:
+    """
+    Check whether a `sysargs` list contains an `api` executor.
+    """
+    if '-e' not in sysargs and '--executor-keys' not in sysargs:
+        return False
+
+    for i, arg in enumerate(sysargs):
+        if arg not in ('-e', '--executor-keys'):
+            continue
+
+        executor_keys_ix = i + 1
+        if len(sysargs) <= executor_keys_ix:
+            return False
+
+        executor_keys = sysargs[executor_keys_ix]
+        if executor_keys.startswith('api:'):
+            return True
+
+    return False
+
+
+def remove_api_executor_keys(sysargs: List[str]) -> List[str]:
+    """
+    Remove any api executor keys from `sysargs`.
+    """
+    from meerschaum.utils.misc import flatten_list
+
+    if not sysargs_has_api_executor_keys(sysargs):
+        return sysargs
+
+    skip_indices = set(flatten_list(
+        [
+            [i, i+1]
+            for i, arg in enumerate(sysargs)
+            if arg in ('-e', '--executor-keys')
+        ]
+    ))
+
+    return [
+        arg
+        for i, arg in enumerate(sysargs)
+        if i not in skip_indices
+    ]
+
+
+def get_pipeline_sysargs(
+    sysargs: List[str],
+    pipeline_args: List[str],
+    _patch_args: Optional[Dict[str, Any]] = None,
+) -> List[str]:
+    """
+    Parse `sysargs` and `pipeline_args` into a single `start pipeline` sysargs.
+    """
+    import shlex
+    start_pipeline_params = {
+        'sub_args_line': shlex.join(sysargs),
+        'patch_args': _patch_args,
+    }
+    return (
+        ['start', 'pipeline']
+        + [str(arg) for arg in pipeline_args]
+        + ['-P', json.dumps(start_pipeline_params, separators=(',', ':'))]
+    )
+
+
+def compress_pipeline_sysargs(pipeline_sysargs: List[str]) -> List[str]:
+    """
+    Given a `start pipeline` sysargs, return a condensed syntax rendition.
+    """
+    import shlex
+
+    if pipeline_sysargs[:2] != ['start', 'pipeline']:
+        return pipeline_sysargs
+
+    if '-P' not in pipeline_sysargs:
+        return pipeline_sysargs
+
+    params_ix = pipeline_sysargs.index('-P')
+    pipeline_args = pipeline_sysargs[2:params_ix]
+    params_str = pipeline_sysargs[-1]
+    try:
+        start_pipeline_params = json.loads(params_str)
+    except Exception:
+        return pipeline_sysargs
+
+    sub_args_line = start_pipeline_params.get('sub_args_line', None)
+    if not sub_args_line:
+        return pipeline_sysargs
+
+    return (
+        shlex.split(sub_args_line)
+        + [':']
+        + pipeline_args
+    )

{meerschaum-2.3.4 → meerschaum-2.3.5}/meerschaum/_internal/docs/index.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 # vim:fenc=utf-8
 
-"""
+r"""
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <style>
 

{meerschaum-2.3.4 → meerschaum-2.3.5}/meerschaum/_internal/entry.py

@@ -11,6 +11,8 @@ from __future__ import annotations
 
 import os
 import sys
+import pathlib
+
 from meerschaum.utils.typing import SuccessTuple, List, Optional, Dict, Callable, Any
 from meerschaum.config.static import STATIC_CONFIG as _STATIC_CONFIG
 
@@ -19,8 +21,17 @@ if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
     from meerschaum.utils.daemon import RotatingFile as _RotatingFile, StdinFile as _StdinFile
     from meerschaum.config import get_config as _get_config
 
-    _systemd_result_path = os.environ[_STATIC_CONFIG['environment']['systemd_result_path']]
-    _systemd_log_path = os.environ[_STATIC_CONFIG['environment']['systemd_log_path']]
+    _systemd_result_path = pathlib.Path(
+        os.environ[_STATIC_CONFIG['environment']['systemd_result_path']]
+    )
+    _systemd_log_path = pathlib.Path(
+        os.environ[_STATIC_CONFIG['environment']['systemd_log_path']]
+    )
+    _systemd_delete_job = (
+        (os.environ.get(_STATIC_CONFIG['environment']['systemd_delete_job'], None) or '0')
+        not in (None, '0', 'false')
+    )
+    _job_name = os.environ[_STATIC_CONFIG['environment']['daemon_id']]
     _systemd_log = _RotatingFile(
         _systemd_log_path,
         write_timestamps=True,
@@ -50,6 +61,8 @@ def entry(
         parse_arguments,
         split_chained_sysargs,
         split_pipeline_sysargs,
+        sysargs_has_api_executor_keys,
+        get_pipeline_sysargs,
     )
     from meerschaum.config.static import STATIC_CONFIG
     if sysargs is None:
@@ -63,22 +76,15 @@ def entry(
 
     has_daemon = '-d' in sysargs or '--daemon' in sysargs
     has_start_job = sysargs[:2] == ['start', 'job']
+    pipeline_has_api_executor_keys = sysargs_has_api_executor_keys(pipeline_args)
+
     chained_sysargs = (
         [sysargs]
-        if has_daemon or has_start_job
+        if has_daemon or has_start_job or pipeline_has_api_executor_keys
         else split_chained_sysargs(sysargs)
     )
     if pipeline_args:
-        chained_sysargs = [
-            ['start', 'pipeline']
-            + [str(arg) for arg in pipeline_args]
-            + (
-                ['--params', json.dumps(_patch_args)]
-                if _patch_args
-                else []
-            )
-            + ['--sub-args', shlex.join(sysargs)]
-        ]
+        chained_sysargs = [get_pipeline_sysargs(sysargs, pipeline_args, _patch_args=_patch_args)]
 
     results: List[SuccessTuple] = []
 
@@ -167,8 +173,20 @@ def entry(
 
     if _systemd_result_path:
         import json
-        with open(_systemd_result_path, 'w+', encoding='utf-8') as f:
-            json.dump((success, msg), f)
+        from meerschaum.utils.warnings import warn
+        import meerschaum as mrsm
+
+        job = mrsm.Job(_job_name, executor_keys='systemd')
+        if job.delete_after_completion:
+            delete_success, delete_msg = job.delete()
+            mrsm.pprint((delete_success, delete_msg))
+        else:
+            try:
+                if _systemd_result_path.parent.exists():
+                    with open(_systemd_result_path, 'w+', encoding='utf-8') as f:
+                        json.dump((success, msg), f)
+            except Exception as e:
+                warn(f"Failed to write job result:\n{e}")
 
     return success, msg
 

{meerschaum-2.3.4 → meerschaum-2.3.5}/meerschaum/actions/api.py

@@ -89,22 +89,22 @@ def api(
     return success, message
 
 def _api_start(
-        action: Optional[List[str]] = None,
-        host: Optional[str] = None,
-        port: Optional[int] = None,
-        workers: Optional[int] = None,
-        mrsm_instance: Optional[str] = None,
-        no_dash: bool = False,
-        no_auth: bool = False,
-        private: bool = False,
-        secure: bool = False,
-        debug: bool = False,
-        nopretty: bool = False,
-        production: bool = False,
-        keyfile: Optional[str] = None,
-        certfile: Optional[str] = None,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    host: Optional[str] = None,
+    port: Optional[int] = None,
+    workers: Optional[int] = None,
+    mrsm_instance: Optional[str] = None,
+    no_dash: bool = False,
+    no_auth: bool = False,
+    private: bool = False,
+    secure: bool = False,
+    debug: bool = False,
+    nopretty: bool = False,
+    production: bool = False,
+    keyfile: Optional[str] = None,
+    certfile: Optional[str] = None,
+    **kw: Any
+) -> SuccessTuple:
     """Start the API server.
    
     Parameters

{meerschaum-2.3.4 → meerschaum-2.3.5}/meerschaum/actions/python.py

@@ -14,6 +14,7 @@ def python(
     nopretty: bool = False,
     noask: bool = False,
     venv: Optional[str] = None,
+    executor_keys: Optional[str] = None,
     debug: bool = False,
     **kw: Any
 ) -> SuccessTuple:
@@ -53,6 +54,9 @@ def python(
     from meerschaum.utils.packages import run_python_package, attempt_import
     from meerschaum.utils.process import run_process
 
+    if executor_keys and executor_keys.startswith('api:'):
+        warn("Cannot open a Python REPL remotely, falling back to local...", stack=False)
+
     if action is None:
         action = []
 

{meerschaum-2.3.4 → meerschaum-2.3.5}/meerschaum/actions/start.py

@@ -7,7 +7,7 @@ Start subsystems (API server, logging daemon, etc.).
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Optional, List, Any, Union
+from meerschaum.utils.typing import SuccessTuple, Optional, List, Any, Union, Dict
 
 def start(
     action: Optional[List[str]] = None,
@@ -89,6 +89,7 @@ def _start_jobs(
     name: Optional[str] = None,
     sysargs: Optional[List[str]] = None,
     executor_keys: Optional[str] = None,
+    rm: bool = False,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -210,7 +211,7 @@ def _start_jobs(
 
     def _run_new_job(name: Optional[str] = None):
         name = name or get_new_daemon_name()
-        job = Job(name, sysargs, executor_keys=executor_keys)
+        job = Job(name, sysargs, executor_keys=executor_keys, delete_after_completion=rm)
         return job.start(debug=debug), name
 
     def _run_existing_job(name: str):
@@ -539,7 +540,6 @@ def _complete_start_connectors(**kw) -> List[str]:
 
 def _start_pipeline(
     action: Optional[List[str]] = None,
-    sub_args: Optional[List[str]] = None,
     loop: bool = False,
     min_seconds: Union[float, int, None] = 1.0,
     params: Optional[Dict[str, Any]] = None,
@@ -569,7 +569,11 @@ def _start_pipeline(
         else 1
     )
 
-    if not sub_args:
+    params = params or {}
+    sub_args_line = params.get('sub_args_line', None)
+    patch_args = params.get('patch_args', None)
+
+    if not sub_args_line:
         return False, "Nothing to do."
 
     if min_seconds is None:
@@ -580,7 +584,7 @@ def _start_pipeline(
     def run_loop():
         nonlocal ran_n_times, success, msg
         while True:
-            success, msg = entry(sub_args, _patch_args=params)
+            success, msg = entry(sub_args_line, _patch_args=patch_args)
             ran_n_times += 1
 
             if not loop and do_n_times == 1:

{meerschaum-2.3.4 → meerschaum-2.3.5}/meerschaum/api/_events.py

@@ -20,12 +20,14 @@ from meerschaum.connectors.poll import retry_connect
 from meerschaum.utils.warnings import warn
 from meerschaum._internal.term.tools import is_webterm_running
 from meerschaum.jobs import (
+    get_jobs,
     start_check_jobs_thread,
     stop_check_jobs_thread,
     get_executor_keys_from_context,
 )
+from meerschaum.config.static import STATIC_CONFIG
 
-_check_jobs_thread = None
+TEMP_PREFIX: str = STATIC_CONFIG['api']['jobs']['temp_prefix']
 
 @app.on_event("startup")
 async def startup():
@@ -51,8 +53,7 @@ async def startup():
         await shutdown()
         os._exit(1)
 
-    if get_executor_keys_from_context() == 'local':
-        start_check_jobs_thread()
+    start_check_jobs_thread()
 
 
 @app.on_event("shutdown")
@@ -65,11 +66,14 @@ async def shutdown():
     if get_api_connector().type == 'sql':
         get_api_connector().engine.dispose()
 
-    if get_executor_keys_from_context() == 'local':
-        stop_check_jobs_thread()
+    stop_check_jobs_thread()
 
-    from meerschaum.api.routes._actions import _temp_jobs
-    for name, job in _temp_jobs.items():
+    temp_jobs = {
+        name: job
+        for name, job in get_jobs(include_hidden=True).items()
+        if name.startswith(TEMP_PREFIX)
+    }
+    for job in temp_jobs.values():
         job.delete()
 
     ### Terminate any running jobs left over.

{meerschaum-2.3.4 → meerschaum-2.3.5}/meerschaum/api/routes/_actions.py

@@ -14,6 +14,7 @@ from functools import partial
 from datetime import datetime, timezone
 
 from fastapi import WebSocket, WebSocketDisconnect
+from websockets.exceptions import ConnectionClosedError
 
 from meerschaum.utils.misc import generate_password
 from meerschaum.jobs import Job
@@ -27,6 +28,7 @@ import meerschaum.core
 from meerschaum.config import get_config
 from meerschaum._internal.arguments._parse_arguments import parse_dict_to_sysargs, parse_arguments
 from meerschaum.api.routes._jobs import clean_sysargs
+from meerschaum.jobs._Job import StopMonitoringLogs
 
 actions_endpoint = endpoints['actions']
 
@@ -65,96 +67,6 @@ def get_actions(
     return list(actions)
 
 
-async def notify_client(client, content: str):
-    """
-    Send a line of text to a client.
-    """
-    try:
-        await client.send_text(content)
-    except WebSocketDisconnect:
-        pass
-
-_temp_jobs = {}
-@app.websocket(actions_endpoint + '/ws')
-async def do_action_websocket(websocket: WebSocket):
-    """
-    Execute an action and stream the output to the client.
-    """
-    await websocket.accept()
-
-    stop_event = asyncio.Event()
-
-    async def monitor_logs(job):
-        success, msg = job.start()
-        await job.monitor_logs_async(
-            partial(notify_client, websocket),
-            stop_event=stop_event,
-            stop_on_exit=True,
-        )
-
-    job = None
-    job_name = '.' + generate_password(12)
-    try:
-        token = await websocket.receive_text()
-        user = await manager.get_current_user(token) if not no_auth else None
-        if user is None and not no_auth:
-            raise fastapi.HTTPException(
-                status_code=401,
-                detail="Invalid credentials.",
-            )
-
-        auth_success, auth_msg = (
-            is_user_allowed_to_execute(user)
-            if not no_auth
-            else (True, "Success")
-        )
-        auth_payload = {
-            'is_authenticated': auth_success,
-            'timestamp': datetime.now(timezone.utc).isoformat(),
-        }
-        await websocket.send_json(auth_payload)
-        if not auth_success:
-            await websocket.close()
-
-        sysargs = clean_sysargs(await websocket.receive_json())
-        #  kwargs = parse_arguments(sysargs)
-        #  _ = kwargs.pop('executor_keys', None)
-        #  _ = kwargs.pop('shell', None)
-        #  sysargs = parse_dict_to_sysargs(kwargs)
-
-        job = Job(
-            job_name,
-            sysargs,
-            executor_keys='local',
-            _properties={
-                'logs': {
-                    'write_timestamps': False,
-                },
-            },
-        )
-        _temp_jobs[job_name] = job
-        monitor_task = asyncio.create_task(monitor_logs(job))
-        await monitor_task
-        try:
-            await websocket.close()
-        except RuntimeError:
-            pass
-    except fastapi.HTTPException:
-        await websocket.send_text("Invalid credentials.")
-        await websocket.close()
-    except WebSocketDisconnect:
-        pass
-    except asyncio.CancelledError:
-        pass
-    except Exception:
-        warn(f"Error in logs websocket:\n{traceback.format_exc()}")
-    finally:
-        if job is not None:
-            job.delete()
-        _ = _temp_jobs.pop(job_name, None)
-        stop_event.set()
-
-
 @app.post(actions_endpoint + "/{action}", tags=['Actions'])
 def do_action_legacy(
     action: str,