meerschaum 2.4.11__py3-none-any.whl → 2.4.13__py3-none-any.whl

meerschaum/actions/start.py

@@ -9,6 +9,7 @@ Start subsystems (API server, logging daemon, etc.).
 from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Optional, List, Any, Union, Dict
 
+
 def start(
     action: Optional[List[str]] = None,
     **kw: Any,
@@ -375,8 +376,8 @@ def _start_gui(
         webview.create_window(
             'Meerschaum Shell', 
             f'http://127.0.0.1:{port}',
-            height = 650,
-            width = 1000
+            height=650,
+            width=1000
         )
         webview.start(debug=debug)
     except Exception as e:
@@ -540,6 +541,7 @@ def _start_pipeline(
     action: Optional[List[str]] = None,
     loop: bool = False,
     min_seconds: Union[float, int, None] = 1.0,
+    timeout_seconds: Optional[int] = None,
     params: Optional[Dict[str, Any]] = None,
     **kwargs
 ) -> SuccessTuple:
@@ -556,10 +558,39 @@ def _start_pipeline(
     `show version + show arguments :: --loop`
 
     """
+    import json
     import time
+    import sys
     from meerschaum._internal.entry import entry
     from meerschaum.utils.warnings import info, warn
     from meerschaum.utils.misc import is_int
+    from meerschaum.utils.venv import venv_exec
+    from meerschaum.utils.process import poll_process
+    fence_begin, fence_end = '<MRSM_RESULT>', '</MRSM_RESULT>'
+
+    success, msg = False, "Did not run pipeline."
+    def write_line(line):
+        nonlocal success, msg
+        decoded = line.decode('utf-8')
+        begin_index, end_index = decoded.find(fence_begin), decoded.find(fence_end)
+
+        ### Found the beginning of the return value.
+        ### Don't write the parsed success tuple message.
+        if begin_index >= 0:
+            success, msg = tuple(json.loads(
+                decoded[begin_index + len(fence_begin):end_index]
+            ))
+            return
+
+        print(decoded)
+
+    def timeout_handler(*args, **kw):
+        nonlocal success, msg
+        success, msg = False, (
+            f"Failed to execute pipeline within {timeout_seconds} second"
+            + ('s' if timeout_seconds != 1 else '') + '.'
+        )
+        write_line((fence_begin + json.dumps((success, msg)) + fence_end).encode('utf-8'))
 
     do_n_times = (
         int(action[0].lstrip('x'))
@@ -577,12 +608,38 @@ def _start_pipeline(
     if min_seconds is None:
         min_seconds = 1.0
 
+    def do_entry() -> None:
+        nonlocal success, msg
+        if timeout_seconds is None:
+            success, msg = entry(sub_args_line, _patch_args=patch_args)
+            return
+
+        sub_args_line_escaped = sub_args_line.replace("'", "<QUOTE>")
+        patch_args_escaped_str = json.dumps(patch_args).replace("'", "<QUOTE>")
+        src = (
+            "import json\n"
+            "from meerschaum._internal.entry import entry\n\n"
+            f"sub_args_line = '{sub_args_line_escaped}'.replace(\"<QUOTE>\", \"'\")\n"
+            f"patch_args = json.loads('{patch_args_escaped_str}'.replace('<QUOTE>', \"'\"))\n"
+            "success, msg = entry(sub_args_line, _patch_args=patch_args)\n"
+            f"print('{fence_begin}' + json.dumps((success, msg)) + '{fence_end}')"
+        )
+        proc = venv_exec(src, venv=None, as_proc=True)
+        poll_process(
+            proc,
+            write_line,
+            timeout_seconds,
+            timeout_handler,
+        )
+
     ran_n_times = 0
-    success, msg = False, "Did not run pipeline."
     def run_loop():
         nonlocal ran_n_times, success, msg
         while True:
-            success, msg = entry(sub_args_line, _patch_args=patch_args)
+            try:
+                do_entry()
+            except Exception as e:
+                warn(e)
             ran_n_times += 1
 
             if not loop and do_n_times == 1:

meerschaum/actions/sync.py

@@ -129,7 +129,6 @@ def _pipes_lap(
             stack=False,
         )
 
-
     def _task_label(count: int):
         return f"[cyan]Syncing {count} pipe{'s' if count != 1 else ''}..."
 
@@ -137,7 +136,6 @@ def _pipes_lap(
         _progress.add_task(_task_label(len(pipes)), start=True, total=len(pipes))
     ) if _progress is not None else None
 
-
     def worker_fn():
         while not stop_event.is_set():
             try:
@@ -188,7 +186,7 @@ def _pipes_lap(
                     decoded[begin_index + len(fence_begin):end_index]
                 ))
                 return
-            sys.stdout.buffer.write(line)
+            print(decoded)
 
         def timeout_handler(p, *args, **kw):
             success, msg = False, (
@@ -227,7 +225,7 @@ def _pipes_lap(
             write_line,
             timeout_seconds,
             timeout_handler,
-            (p,)
+            (p,),
         )
         return _success_tuple
 
@@ -249,18 +247,18 @@ def _pipes_lap(
 
 
 def _sync_pipes(
-        loop: bool = False,
-        min_seconds: int = 1,
-        unblock: bool = False,
-        verify: bool = False,
-        deduplicate: bool = False,
-        bounded: Optional[bool] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        shell: bool = False,
-        nopretty: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    loop: bool = False,
+    min_seconds: int = 1,
+    unblock: bool = False,
+    verify: bool = False,
+    deduplicate: bool = False,
+    bounded: Optional[bool] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    shell: bool = False,
+    nopretty: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Fetch and sync new data for pipes.
 

meerschaum/actions/upgrade.py

@@ -9,16 +9,17 @@ Upgrade your current Meerschaum environment
 from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Union
 
+
 def upgrade(
     action: Optional[List[str]] = None,
     **kw: Any
 ) -> SuccessTuple:
     """
     Upgrade Meerschaum, plugins, or packages.
-    
+
     Command:
         `upgrade {option}`
-    
+
     Example:
         `upgrade meerschaum`
     """
@@ -77,7 +78,7 @@ def _upgrade_meerschaum(
         if force:
             answer = True
         else:
-            answer = yes_no(f"Take down the stack?", default='y', yes=yes, noask=noask)
+            answer = yes_no("Take down the stack?", default='n', yes=yes, noask=noask)
 
         if answer:
             if debug:
@@ -95,7 +96,7 @@ def _upgrade_meerschaum(
     if debug:
         dprint('Upgrade meerschaum with dependencies: \"' + f'{dependencies}' + '\"')
     if not pip_install(install_name, venv=None, debug=debug):
-        return False, f"Failed to upgrade Meerschaum via pip."
+        return False, "Failed to upgrade Meerschaum via pip."
 
     if debug:
         dprint("Pulling new Docker images...")

meerschaum/api/dash/callbacks/dashboard.py

@@ -98,6 +98,7 @@ _paths = {
     'plugins' : pages.plugins.layout,
     'register': pages.register.layout,
     'pipes'   : pages.pipes.layout,
+    'job'     : pages.job.layout,
 }
 _required_login = {''}
 
@@ -121,7 +122,7 @@ def update_page_layout_div(
     ----------
     pathname: str
         The path in the browser.
-        
+
     session_store_data: Dict[str, Any]:
         The stored session data.
 

meerschaum/api/dash/callbacks/jobs.py

@@ -12,6 +12,7 @@ import json
 import time
 import traceback
 from datetime import datetime, timezone
+
 from meerschaum.utils.typing import Optional, Dict, Any
 from meerschaum.api import CHECK_UPDATE
 from meerschaum.api.dash import dash_app
@@ -19,17 +20,19 @@ from meerschaum.api.dash.sessions import get_username_from_session
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html
 from meerschaum.api.dash.components import alert_from_success_tuple
 from meerschaum.api.dash.jobs import (
+    build_job_card,
     build_manage_job_buttons_div_children,
     build_status_children,
     build_process_timestamps_children,
 )
-from meerschaum.jobs import Job
+from meerschaum.api.routes._jobs import _get_job
 from meerschaum.api.dash.sessions import is_session_authenticated
 dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 from dash.exceptions import PreventUpdate
 from dash.dependencies import Input, Output, State, ALL, MATCH
 import dash_bootstrap_components as dbc
+from dash import no_update
 
 
 @dash_app.callback(
@@ -53,6 +56,13 @@ def download_job_logs(n_clicks):
 
     component_dict = json.loads(ctx[0]['prop_id'].split('.' + 'n_clicks')[0])
     job_name = component_dict['index']
+    try:
+        job = _get_job(job_name)
+    except Exception:
+        job = None
+    if job is None or not job.exists():
+        raise PreventUpdate
+
     now = datetime.now(timezone.utc)
     filename = job_name + '_' + str(int(now.timestamp())) + '.log'
     return {
@@ -69,7 +79,7 @@ def download_job_logs(n_clicks):
     Input({'type': 'manage-job-button', 'action': ALL, 'index': MATCH}, 'n_clicks'),
     State('session-store', 'data'),
     State({'type': 'job-label-p', 'index': MATCH}, 'children'),
-    prevent_initial_call = True,
+    prevent_initial_call=True,
 )
 def manage_job_button_click(
     n_clicks: Optional[int] = None,
@@ -102,10 +112,10 @@ def manage_job_button_click(
     job_name = component_dict['index']
     manage_job_action = component_dict['action']
     try:
-        job = Job(job_name, job_label.replace('\n', ' ') if job_label else None)
-    except Exception as e:
+        job = _get_job(job_name, job_label.replace('\n', ' ') if job_label else None)
+    except Exception:
         job = None
-    if job is None:
+    if job is None or not job.exists():
         raise PreventUpdate
 
     manage_functions = {
@@ -191,7 +201,7 @@ dash_app.clientside_callback(
     Output({'type': 'process-timestamps-div', 'index': ALL}, 'children'),
     Input('refresh-jobs-interval', 'n_intervals'),
     State('session-store', 'data'),
-    prevent_initial_call = True,
+    prevent_initial_call=True,
 )
 def refresh_jobs_on_interval(
     n_intervals: Optional[int] = None,
@@ -209,7 +219,7 @@ def refresh_jobs_on_interval(
     ]
 
     ### NOTE: The job may have been deleted, but the card may still exist.
-    jobs = [Job(name) for name in job_names]
+    jobs = [_get_job(name) for name in job_names]
 
     return (
         [
@@ -229,3 +239,39 @@ def refresh_jobs_on_interval(
             for job in jobs
         ],
     )
+
+
+@dash_app.callback(
+    Output('job-output-div', 'children'),
+    Input('job-location', 'pathname'),
+    State('session-store', 'data'),
+)
+def render_job_page_from_url(
+    pathname: str,
+    session_data: Optional[Dict[str, Any]],
+):
+    """
+    Load the `/job/{name}` page.
+    """
+    if not str(pathname).startswith('/dash/job'):
+        return no_update
+
+    session_id = (session_data or {}).get('session-id', None)
+    authenticated = is_session_authenticated(str(session_id))
+
+    job_name = pathname.replace('/dash/job', '').lstrip('/').rstrip('/')
+    if not job_name:
+        return no_update
+
+    job = _get_job(job_name)
+    if not job.exists():
+        return [
+            html.Br(),
+            html.H2("404: Job does not exist."),
+        ]
+
+    return [
+        html.Br(),
+        build_job_card(job, authenticated=authenticated, include_follow=False),
+        html.Br(),
+    ]

meerschaum/api/dash/callbacks/pipes.py

@@ -26,7 +26,7 @@ html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHEC
     State('pipes-location', 'search'),
     State('session-store', 'data'),
 )
-def render_page_from_url(
+def render_pipe_page_from_url(
     pathname: str,
     pipe_search: str,
     session_data: Optional[Dict[str, Any]],

meerschaum/api/dash/jobs.py

@@ -31,6 +31,7 @@ STATUS_EMOJI: Dict[str, str] = {
 
 EXECUTOR_KEYS: str = get_executor_keys_from_context()
 
+
 def get_jobs_cards(state: WebState):
     """
     Build cards and alerts lists for jobs.
@@ -42,71 +43,96 @@ def get_jobs_cards(state: WebState):
     cards = []
 
     for name, job in jobs.items():
-        footer_children = html.Div(
-            build_process_timestamps_children(job),
-            id = {'type': 'process-timestamps-div', 'index': name},
-        )
-        follow_logs_button = dbc.DropdownMenuItem(
-            "Follow logs",
-            id = {'type': 'follow-logs-button', 'index': name},
-        )
-        download_logs_button = dbc.DropdownMenuItem(
-            "Download logs",
-            id = {'type': 'job-download-logs-button', 'index': name},
-        )
-        logs_menu_children = (
-            ([follow_logs_button] if is_authenticated else []) + [download_logs_button]
-        )
-        header_children = [
-            html.Div(
-                build_status_children(job),
-                id={'type': 'manage-job-status-div', 'index': name},
-                style={'float': 'left'},
+        cards.append(build_job_card(job, authenticated=is_authenticated))
+
+    return cards, []
+
+
+def build_job_card(
+    job: Job,
+    authenticated: bool = False,
+    include_follow: bool = True,
+):
+    """
+    Return a card for a given job.
+    """
+    footer_children = html.Div(
+        build_process_timestamps_children(job),
+        id={'type': 'process-timestamps-div', 'index': job.name},
+    )
+    follow_logs_button = dbc.DropdownMenuItem(
+        "Follow logs",
+        id={'type': 'follow-logs-button', 'index': job.name},
+    )
+    download_logs_button = dbc.DropdownMenuItem(
+        "Download logs",
+        id={'type': 'job-download-logs-button', 'index': job.name},
+    )
+    logs_menu_children = (
+        ([follow_logs_button] if include_follow else []) + [download_logs_button]
+        if authenticated
+        else []
+    )
+    header_children = [
+        html.Div(
+            build_status_children(job),
+            id={'type': 'manage-job-status-div', 'index': job.name},
+            style={'float': 'left'},
+        ),
+    ] + ([
+        html.Div(
+            dbc.DropdownMenu(
+                logs_menu_children,
+                label="Logs",
+                size="sm",
+                align_end=True,
+                color="secondary",
+                menu_variant='dark',
             ),
-            html.Div(
-                dbc.DropdownMenu(
-                    logs_menu_children,
-                    label="Logs",
-                    size="sm",
-                    align_end=True,
-                    color="secondary",
-                    menu_variant='dark',
-                ),
-                style={'float': 'right'},
+            style={'float': 'right'},
+        ),
+    ] if authenticated else [])
+
+    body_children = [
+        html.H4(
+            html.B(
+                html.A(
+                    "🔗 " + job.name,
+                    href=f"/dash/job/{job.name}",
+                    target="_blank",
+                    style={
+                        'color': 'white',
+                        'text-decoration': 'none',
+                    },
+                )
             ),
-        ]
-
-        body_children = [
-            html.H4(html.B(name), className="card-title"),
-            html.Div(
-                html.P(
-                    job.label,
-                    className="card-text job-card-text",
-                    style={"word-wrap": "break-word"},
-                    id={'type': 'job-label-p', 'index': name},
-                ),
-                style={"white-space": "pre-wrap"},
+            className="card-title",
+        ),
+        html.Div(
+            html.P(
+                job.label,
+                className="card-text job-card-text",
+                style={"word-wrap": "break-word"},
+                id={'type': 'job-label-p', 'index': job.name},
             ),
-            html.Div(
-                (
-                    build_manage_job_buttons_div_children(job)
-                    if is_authenticated
-                    else []
-                ),
-                id={'type': 'manage-job-buttons-div', 'index': name},
+            style={"white-space": "pre-wrap"},
+        ),
+        html.Div(
+            (
+                build_manage_job_buttons_div_children(job)
+                if authenticated
+                else []
             ),
-            html.Div(id={'type': 'manage-job-alert-div', 'index': name}),
-        ]
-
-        cards.append(
-            dbc.Card([
-                dbc.CardHeader(header_children),
-                dbc.CardBody(body_children),
-                dbc.CardFooter(footer_children),
-            ])
-        )
+            id={'type': 'manage-job-buttons-div', 'index': job.name},
+        ),
+        html.Div(id={'type': 'manage-job-alert-div', 'index': job.name}),
+    ]
 
-    return cards, []
+    return dbc.Card([
+        dbc.CardHeader(header_children),
+        dbc.CardBody(body_children),
+        dbc.CardFooter(footer_children),
+    ])
 
 
 def build_manage_job_buttons_div_children(job: Job):

meerschaum/api/dash/pages/__init__.py

@@ -11,3 +11,4 @@ import meerschaum.api.dash.pages.dashboard
 import meerschaum.api.dash.pages.plugins
 import meerschaum.api.dash.pages.register
 import meerschaum.api.dash.pages.pipes
+import meerschaum.api.dash.pages.job

meerschaum/api/dash/pages/job.py

@@ -0,0 +1,21 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Display pipes via a shareable URL.
+"""
+
+from meerschaum.api import CHECK_UPDATE
+from meerschaum.utils.packages import import_html, import_dcc
+
+html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
+import dash_bootstrap_components as dbc
+
+from meerschaum.api.dash.components import download_logs, refresh_jobs_interval
+
+layout = dbc.Container([
+    dcc.Location('job-location'),
+    html.Div(id='job-output-div'),
+    download_logs,
+    refresh_jobs_interval,
+])

meerschaum/api/routes/_jobs.py

@@ -39,12 +39,12 @@ NONINTERACTIVE_ENV: str = STATIC_CONFIG['environment']['noninteractive']
 EXECUTOR_KEYS: str = 'local'
 
 
-def _get_job(name: str):
-    systemd_job = Job(name, executor_keys='systemd')
+def _get_job(name: str, sysargs: Union[str, List[str], None] = None):
+    systemd_job = Job(name, sysargs, executor_keys='systemd')
     if systemd_job.exists():
         return systemd_job
 
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = Job(name, sysargs, executor_keys=EXECUTOR_KEYS)
     return job
 
 

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.11"
+__version__ = "2.4.13"