meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0.dev0__py3-none-any.whl

meerschaum/_internal/arguments/__init__.py

@@ -9,7 +9,8 @@ This package includes argument parsing utilities.
 from meerschaum._internal.arguments._parse_arguments import (
     parse_arguments, parse_line, remove_leading_action,
     parse_dict_to_sysargs, split_chained_sysargs, split_pipeline_sysargs,
-    sysargs_has_api_executor_keys,
+    sysargs_has_api_executor_keys, get_pipeline_sysargs,
+    compress_pipeline_sysargs, remove_api_executor_keys,
 )
 from meerschaum._internal.arguments._parser import parser
 from meerschaum.plugins import add_plugin_argument

meerschaum/_internal/arguments/_parse_arguments.py

@@ -424,12 +424,91 @@ def sysargs_has_api_executor_keys(sysargs: List[str]) -> bool:
     if '-e' not in sysargs and '--executor-keys' not in sysargs:
         return False
 
-    executor_keys_flag = '-e' if '-e' in sysargs else '--executor-keys'
-    executor_keys_flag_ix = sysargs.index(executor_keys_flag)
-    executor_keys_ix = executor_keys_flag_ix + 1
+    for i, arg in enumerate(sysargs):
+        if arg not in ('-e', '--executor-keys'):
+            continue
 
-    if len(sysargs) < executor_keys_ix:
-        return False
+        executor_keys_ix = i + 1
+        if len(sysargs) <= executor_keys_ix:
+            return False
+
+        executor_keys = sysargs[executor_keys_ix]
+        if executor_keys.startswith('api:'):
+            return True
+
+    return False
+
+
+def remove_api_executor_keys(sysargs: List[str]) -> List[str]:
+    """
+    Remove any api executor keys from `sysargs`.
+    """
+    from meerschaum.utils.misc import flatten_list
+
+    if not sysargs_has_api_executor_keys(sysargs):
+        return sysargs
+
+    skip_indices = set(flatten_list(
+        [
+            [i, i+1]
+            for i, arg in enumerate(sysargs)
+            if arg in ('-e', '--executor-keys')
+        ]
+    ))
+
+    return [
+        arg
+        for i, arg in enumerate(sysargs)
+        if i not in skip_indices
+    ]
 
-    executor_keys = sysargs[executor_keys_ix]
-    return executor_keys.startswith('api:')
+
+def get_pipeline_sysargs(
+    sysargs: List[str],
+    pipeline_args: List[str],
+    _patch_args: Optional[Dict[str, Any]] = None,
+) -> List[str]:
+    """
+    Parse `sysargs` and `pipeline_args` into a single `start pipeline` sysargs.
+    """
+    import shlex
+    start_pipeline_params = {
+        'sub_args_line': shlex.join(sysargs),
+        'patch_args': _patch_args,
+    }
+    return (
+        ['start', 'pipeline']
+        + [str(arg) for arg in pipeline_args]
+        + ['-P', json.dumps(start_pipeline_params, separators=(',', ':'))]
+    )
+
+
+def compress_pipeline_sysargs(pipeline_sysargs: List[str]) -> List[str]:
+    """
+    Given a `start pipeline` sysargs, return a condensed syntax rendition.
+    """
+    import shlex
+
+    if pipeline_sysargs[:2] != ['start', 'pipeline']:
+        return pipeline_sysargs
+
+    if '-P' not in pipeline_sysargs:
+        return pipeline_sysargs
+
+    params_ix = pipeline_sysargs.index('-P')
+    pipeline_args = pipeline_sysargs[2:params_ix]
+    params_str = pipeline_sysargs[-1]
+    try:
+        start_pipeline_params = json.loads(params_str)
+    except Exception:
+        return pipeline_sysargs
+
+    sub_args_line = start_pipeline_params.get('sub_args_line', None)
+    if not sub_args_line:
+        return pipeline_sysargs
+
+    return (
+        shlex.split(sub_args_line)
+        + [':']
+        + pipeline_args
+    )

meerschaum/_internal/entry.py

@@ -11,6 +11,8 @@ from __future__ import annotations
 
 import os
 import sys
+import pathlib
+
 from meerschaum.utils.typing import SuccessTuple, List, Optional, Dict, Callable, Any
 from meerschaum.config.static import STATIC_CONFIG as _STATIC_CONFIG
 
@@ -19,8 +21,17 @@ if (_STATIC_CONFIG['environment']['systemd_log_path']) in os.environ:
     from meerschaum.utils.daemon import RotatingFile as _RotatingFile, StdinFile as _StdinFile
     from meerschaum.config import get_config as _get_config
 
-    _systemd_result_path = os.environ[_STATIC_CONFIG['environment']['systemd_result_path']]
-    _systemd_log_path = os.environ[_STATIC_CONFIG['environment']['systemd_log_path']]
+    _systemd_result_path = pathlib.Path(
+        os.environ[_STATIC_CONFIG['environment']['systemd_result_path']]
+    )
+    _systemd_log_path = pathlib.Path(
+        os.environ[_STATIC_CONFIG['environment']['systemd_log_path']]
+    )
+    _systemd_delete_job = (
+        (os.environ.get(_STATIC_CONFIG['environment']['systemd_delete_job'], None) or '0')
+        not in (None, '0', 'false')
+    )
+    _job_name = os.environ[_STATIC_CONFIG['environment']['daemon_id']]
     _systemd_log = _RotatingFile(
         _systemd_log_path,
         write_timestamps=True,
@@ -51,6 +62,7 @@ def entry(
         split_chained_sysargs,
         split_pipeline_sysargs,
         sysargs_has_api_executor_keys,
+        get_pipeline_sysargs,
     )
     from meerschaum.config.static import STATIC_CONFIG
     if sysargs is None:
@@ -72,15 +84,7 @@ def entry(
         else split_chained_sysargs(sysargs)
     )
     if pipeline_args:
-        start_pipeline_params = {
-            'sub_args_line': shlex.join(sysargs),
-            'patch_args': _patch_args,
-        }
-        chained_sysargs = [
-            ['start', 'pipeline']
-            + [str(arg) for arg in pipeline_args]
-            + ['-P', json.dumps(start_pipeline_params, separators=(',', ':'))]
-        ]
+        chained_sysargs = [get_pipeline_sysargs(sysargs, pipeline_args, _patch_args=_patch_args)]
 
     results: List[SuccessTuple] = []
 
@@ -169,8 +173,20 @@ def entry(
 
     if _systemd_result_path:
         import json
-        with open(_systemd_result_path, 'w+', encoding='utf-8') as f:
-            json.dump((success, msg), f)
+        from meerschaum.utils.warnings import warn
+        import meerschaum as mrsm
+
+        job = mrsm.Job(_job_name, executor_keys='systemd')
+        if job.delete_after_completion:
+            delete_success, delete_msg = job.delete()
+            mrsm.pprint((delete_success, delete_msg))
+        else:
+            try:
+                if _systemd_result_path.parent.exists():
+                    with open(_systemd_result_path, 'w+', encoding='utf-8') as f:
+                        json.dump((success, msg), f)
+            except Exception as e:
+                warn(f"Failed to write job result:\n{e}")
 
     return success, msg
 

meerschaum/actions/api.py

@@ -89,22 +89,22 @@ def api(
     return success, message
 
 def _api_start(
-        action: Optional[List[str]] = None,
-        host: Optional[str] = None,
-        port: Optional[int] = None,
-        workers: Optional[int] = None,
-        mrsm_instance: Optional[str] = None,
-        no_dash: bool = False,
-        no_auth: bool = False,
-        private: bool = False,
-        secure: bool = False,
-        debug: bool = False,
-        nopretty: bool = False,
-        production: bool = False,
-        keyfile: Optional[str] = None,
-        certfile: Optional[str] = None,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    host: Optional[str] = None,
+    port: Optional[int] = None,
+    workers: Optional[int] = None,
+    mrsm_instance: Optional[str] = None,
+    no_dash: bool = False,
+    no_auth: bool = False,
+    private: bool = False,
+    secure: bool = False,
+    debug: bool = False,
+    nopretty: bool = False,
+    production: bool = False,
+    keyfile: Optional[str] = None,
+    certfile: Optional[str] = None,
+    **kw: Any
+) -> SuccessTuple:
     """Start the API server.
    
     Parameters

meerschaum/actions/bootstrap.py

@@ -234,7 +234,13 @@ def _bootstrap_connectors(
     Prompt the user for the details necessary to create a Connector.
     """
     from meerschaum.connectors.parse import is_valid_connector_keys
-    from meerschaum.connectors import connectors, get_connector, types, custom_types
+    from meerschaum.connectors import (
+        connectors,
+        get_connector,
+        types,
+        custom_types,
+        _load_builtin_custom_connectors,
+    )
     from meerschaum.utils.prompt import prompt, yes_no, choose
     from meerschaum.config import get_config
     from meerschaum.config._edit import write_config
@@ -246,6 +252,7 @@ def _bootstrap_connectors(
 
     abort_tuple = False, "No connectors bootstrapped."
     _clear = get_config('shell', 'clear_screen', patch=True)
+    _load_builtin_custom_connectors()
 
     if action is None:
         action = []
@@ -262,8 +269,8 @@ def _bootstrap_connectors(
                 + '    See https://meerschaum.io/reference/connectors '
                 + 'for documentation on connectors.\n'
             ),
-            sorted(list(connectors)),
-            default = 'sql'
+            sorted([k for k in connectors if k != 'plugin']),
+            default='sql',
         )
     except KeyboardInterrupt:
         return abort_tuple
@@ -288,21 +295,30 @@ def _bootstrap_connectors(
             warn(f"Connector '{_type}:{_label}' already exists.", stack=False)
             overwrite = yes_no(
                 f"Do you want to overwrite connector '{_type}:{_label}'?",
-                default = 'n',
-                yes = yes,
-                noask = noask,
+                default='n',
+                yes=yes,
+                noask=noask,
             )
             if not overwrite and not force:
-                return False, f"No changes made to connector configuration."
+                return False, "No changes made to connector configuration."
                 break
         elif _label == "":
-            warn(f"Please enter a label.", stack=False)
+            warn("Please enter a label.", stack=False)
         else:
             break
 
     cls = types.get(_type)
     cls_required_attrs = getattr(cls, 'REQUIRED_ATTRIBUTES', [])
-    type_attributes = connector_attributes.get(_type, {'required': cls_required_attrs})
+    cls_optional_attrs = getattr(cls, 'OPTIONAL_ATTRIBUTES', [])
+    cls_default_attrs = getattr(cls, 'DEFAULT_ATTRIBUTES', {})
+    type_attributes = connector_attributes.get(
+        _type,
+        {
+            'required': cls_required_attrs,
+            'optional': cls_optional_attrs,
+            'default': cls_default_attrs,
+        }
+    )
 
     new_attributes = {}
     if 'flavors' in type_attributes:
@@ -323,6 +339,7 @@ def _bootstrap_connectors(
         default = type_attributes['flavors'][flavor].get('defaults', {})
     else:
         required = sorted(list(type_attributes.get('required', {})))
+        optional = sorted(list(type_attributes.get('optional', {})))
         default = type_attributes.get('default', {})
     info(
         f"Please answer the following questions to configure the new connector '{_type}:{_label}'."
@@ -330,13 +347,22 @@ def _bootstrap_connectors(
     )
     for r in required:
         try:
-            val = prompt(f"Value for {r}:")
+            default_val = str(default.get(r)) if r in default else None
+            val = prompt(f"Value for {r}:", default=default_val)
         except KeyboardInterrupt:
             continue
         if is_int(val):
             val = int(val)
         new_attributes[r] = val
 
+    for o in optional:
+        try:
+            val = prompt(f"Value for {o} (optional; empty to omit):")
+        except KeyboardInterrupt:
+            continue
+        if val:
+            new_attributes[o] = val
+
     for k, v in default.items():
         ### skip already configured attributes, (e.g. flavor or from required)
         if k in new_attributes:

meerschaum/actions/start.py

@@ -89,6 +89,7 @@ def _start_jobs(
     name: Optional[str] = None,
     sysargs: Optional[List[str]] = None,
     executor_keys: Optional[str] = None,
+    rm: bool = False,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -210,7 +211,7 @@ def _start_jobs(
 
     def _run_new_job(name: Optional[str] = None):
         name = name or get_new_daemon_name()
-        job = Job(name, sysargs, executor_keys=executor_keys)
+        job = Job(name, sysargs, executor_keys=executor_keys, delete_after_completion=rm)
         return job.start(debug=debug), name
 
     def _run_existing_job(name: str):
@@ -454,12 +455,12 @@ def _start_webterm(
 
 
 def _start_connectors(
-        action: Optional[List[str]] = None,
-        connector_keys: Optional[List[str]] = None,
-        min_seconds: int = 3,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    connector_keys: Optional[List[str]] = None,
+    min_seconds: int = 3,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Start polling connectors to verify a connection can be made.
     """
@@ -479,7 +480,7 @@ def _start_connectors(
     for keys in unique_keys:
         try:
             conn = parse_instance_keys(keys)
-        except Exception as e:
+        except Exception:
             warn(f"Invalid connector keys: '{keys}'. Skipping...", stack=False)
             continue
         valid_conns.append(conn)
@@ -487,20 +488,19 @@ def _start_connectors(
     if not valid_conns:
         return False, "No valid connector keys were provided."
 
-
     connected = {}
     def connect(conn):
         success = retry_connect(
             conn,
-            retry_wait = min_seconds,
-            enforce_chaining = False,
-            enforce_login = False,
-            print_on_connect = True,
-            debug = debug,
+            retry_wait=min_seconds,
+            enforce_chaining=False,
+            enforce_login=False,
+            print_on_connect=True,
+            debug=debug,
         )
         connected[conn] = success
         return success
-    
+
     pool = get_pool()
     try:
         pool.map(connect, valid_conns)
@@ -568,6 +568,7 @@ def _start_pipeline(
         else 1
     )
 
+    params = params or {}
     sub_args_line = params.get('sub_args_line', None)
     patch_args = params.get('patch_args', None)
 

meerschaum/api/_events.py

@@ -20,12 +20,14 @@ from meerschaum.connectors.poll import retry_connect
 from meerschaum.utils.warnings import warn
 from meerschaum._internal.term.tools import is_webterm_running
 from meerschaum.jobs import (
+    get_jobs,
     start_check_jobs_thread,
     stop_check_jobs_thread,
     get_executor_keys_from_context,
 )
+from meerschaum.config.static import STATIC_CONFIG
 
-_check_jobs_thread = None
+TEMP_PREFIX: str = STATIC_CONFIG['api']['jobs']['temp_prefix']
 
 @app.on_event("startup")
 async def startup():
@@ -51,8 +53,7 @@ async def startup():
         await shutdown()
         os._exit(1)
 
-    if get_executor_keys_from_context() == 'local':
-        start_check_jobs_thread()
+    start_check_jobs_thread()
 
 
 @app.on_event("shutdown")
@@ -65,11 +66,14 @@ async def shutdown():
     if get_api_connector().type == 'sql':
         get_api_connector().engine.dispose()
 
-    if get_executor_keys_from_context() == 'local':
-        stop_check_jobs_thread()
+    stop_check_jobs_thread()
 
-    from meerschaum.api.routes._actions import _temp_jobs
-    for name, job in _temp_jobs.items():
+    temp_jobs = {
+        name: job
+        for name, job in get_jobs(include_hidden=True).items()
+        if name.startswith(TEMP_PREFIX)
+    }
+    for job in temp_jobs.values():
         job.delete()
 
     ### Terminate any running jobs left over.

meerschaum/api/dash/__init__.py

@@ -50,16 +50,17 @@ stylesheets = [
     '/static/css/bootstrap.min.css',
     '/static/css/dbc_dark.css',
     '/static/css/dash.css',
+    '/static/css/bootstrap-icons/font/bootstrap-icons.min.css',
 ]
 scripts = ['/static/js/node_modules/xterm/lib/xterm.js']
 dash_app = enrich.DashProxy(
     __name__,
-    title = 'Meerschaum Web',
-    requests_pathname_prefix = endpoints['dash'] + '/',
-    external_stylesheets = stylesheets,
-    update_title = None,
-    suppress_callback_exceptions = True,
-    transforms = [
+    title='Meerschaum Web',
+    requests_pathname_prefix=endpoints['dash'] + '/',
+    external_stylesheets=stylesheets,
+    update_title=None,
+    suppress_callback_exceptions=True,
+    transforms=[
         enrich.TriggerTransform(),
         enrich.MultiplexerTransform(),
     ],

meerschaum/api/dash/callbacks/__init__.py

@@ -11,6 +11,7 @@ import meerschaum.api.dash.callbacks.login
 import meerschaum.api.dash.callbacks.plugins
 import meerschaum.api.dash.callbacks.jobs
 import meerschaum.api.dash.callbacks.register
+import meerschaum.api.dash.callbacks.pipes
 from meerschaum.api.dash.callbacks.custom import init_dash_plugins, add_plugin_pages
 
 init_dash_plugins()

meerschaum/api/dash/callbacks/dashboard.py

@@ -101,6 +101,7 @@ _paths = {
     ''        : pages.dashboard.layout,
     'plugins' : pages.plugins.layout,
     'register': pages.register.layout,
+    'pipes'   : pages.pipes.layout,
 }
 _required_login = {''}
 
@@ -113,10 +114,10 @@ _required_login = {''}
     State('location', 'href'),
 )
 def update_page_layout_div(
-        pathname: str, 
-        session_store_data: Dict[str, Any],
-        location_href: str,
-    ) -> Tuple[List[Any], Dict[str, Any]]:
+    pathname: str, 
+    session_store_data: Dict[str, Any],
+    location_href: str,
+) -> Tuple[List[Any], Dict[str, Any]]:
     """
     Route the user to the correct page.
 
@@ -135,7 +136,7 @@ def update_page_layout_div(
     ctx = dash.callback_context
     dash_endpoint = endpoints['dash']
     try:
-        session_id = session_store_data.get('session-id', None) 
+        session_id = session_store_data.get('session-id', None)
     except AttributeError:
         session_id = None
 
@@ -760,6 +761,7 @@ def update_pipe_accordion(item, session_store_data):
         raise PreventUpdate
 
     session_id = session_store_data.get('session-id', None)
+    print(f"{session_id=}")
     authenticated = is_session_authenticated(str(session_id))
     return accordion_items_from_pipe(pipe, active_items=[item], authenticated=authenticated)
 

meerschaum/api/dash/callbacks/pipes.py

@@ -0,0 +1,42 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define callbacks for the `/dash/pipes/` page.
+"""
+
+from dash.exceptions import PreventUpdate
+from dash.dependencies import Input, Output, State
+from dash import no_update
+
+import meerschaum as mrsm
+from meerschaum.api.dash import dash_app, debug, active_sessions
+from meerschaum.api.dash.pipes import build_pipe_card
+from meerschaum.api import get_api_connector, CHECK_UPDATE
+from meerschaum.utils.packages import import_html, import_dcc
+html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
+
+
+@dash_app.callback(
+    Output('pipe-output-div', 'children'),
+    Input('pipes-location', 'pathname'),
+)
+def render_page_from_url(pathname):
+    if not str(pathname).startswith('/dash/pipes'):
+        return no_update
+
+    keys = pathname.replace('/dash/pipes', '').lstrip('/').rstrip('/').split('/')
+    if len(keys) not in (2, 3):
+        return no_update
+
+    ck = keys[0]
+    mk = keys[1]
+    lk = keys[2] if len(keys) == 3 else None
+
+    pipe = mrsm.Pipe(ck, mk, lk)
+    ### TODO Check if logged in
+    return [
+        html.Br(),
+        build_pipe_card(pipe, authenticated=False),
+        html.Br(),
+    ]

meerschaum/api/dash/pages/__init__.py

@@ -10,3 +10,4 @@ import meerschaum.api.dash.pages.login
 import meerschaum.api.dash.pages.dashboard
 import meerschaum.api.dash.pages.plugins
 import meerschaum.api.dash.pages.register
+import meerschaum.api.dash.pages.pipes

meerschaum/api/dash/pages/pipes.py

@@ -0,0 +1,16 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Display pipes via a shareable URL.
+"""
+
+from meerschaum.api import get_api_connector, CHECK_UPDATE
+from meerschaum.utils.packages import import_html, import_dcc
+html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
+import dash_bootstrap_components as dbc
+
+layout = dbc.Container([
+    dcc.Location('pipes-location'),
+    html.Div(id='pipe-output-div'),
+])