meerschaum 2.4.10__py3-none-any.whl → 2.4.12__py3-none-any.whl

meerschaum/api/dash/callbacks/jobs.py

@@ -12,6 +12,7 @@ import json
 import time
 import traceback
 from datetime import datetime, timezone
+
 from meerschaum.utils.typing import Optional, Dict, Any
 from meerschaum.api import CHECK_UPDATE
 from meerschaum.api.dash import dash_app
@@ -19,17 +20,19 @@ from meerschaum.api.dash.sessions import get_username_from_session
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html
 from meerschaum.api.dash.components import alert_from_success_tuple
 from meerschaum.api.dash.jobs import (
+    build_job_card,
     build_manage_job_buttons_div_children,
     build_status_children,
     build_process_timestamps_children,
 )
-from meerschaum.jobs import Job
+from meerschaum.api.routes._jobs import _get_job
 from meerschaum.api.dash.sessions import is_session_authenticated
 dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 from dash.exceptions import PreventUpdate
 from dash.dependencies import Input, Output, State, ALL, MATCH
 import dash_bootstrap_components as dbc
+from dash import no_update
 
 
 @dash_app.callback(
@@ -53,6 +56,13 @@ def download_job_logs(n_clicks):
 
     component_dict = json.loads(ctx[0]['prop_id'].split('.' + 'n_clicks')[0])
     job_name = component_dict['index']
+    try:
+        job = _get_job(job_name)
+    except Exception:
+        job = None
+    if job is None or not job.exists():
+        raise PreventUpdate
+
     now = datetime.now(timezone.utc)
     filename = job_name + '_' + str(int(now.timestamp())) + '.log'
     return {
@@ -69,7 +79,7 @@ def download_job_logs(n_clicks):
     Input({'type': 'manage-job-button', 'action': ALL, 'index': MATCH}, 'n_clicks'),
     State('session-store', 'data'),
     State({'type': 'job-label-p', 'index': MATCH}, 'children'),
-    prevent_initial_call = True,
+    prevent_initial_call=True,
 )
 def manage_job_button_click(
     n_clicks: Optional[int] = None,
@@ -102,10 +112,10 @@ def manage_job_button_click(
     job_name = component_dict['index']
     manage_job_action = component_dict['action']
     try:
-        job = Job(job_name, job_label.replace('\n', ' ') if job_label else None)
-    except Exception as e:
+        job = _get_job(job_name, job_label.replace('\n', ' ') if job_label else None)
+    except Exception:
         job = None
-    if job is None:
+    if job is None or not job.exists():
         raise PreventUpdate
 
     manage_functions = {
@@ -191,7 +201,7 @@ dash_app.clientside_callback(
     Output({'type': 'process-timestamps-div', 'index': ALL}, 'children'),
     Input('refresh-jobs-interval', 'n_intervals'),
     State('session-store', 'data'),
-    prevent_initial_call = True,
+    prevent_initial_call=True,
 )
 def refresh_jobs_on_interval(
     n_intervals: Optional[int] = None,
@@ -209,7 +219,7 @@ def refresh_jobs_on_interval(
     ]
 
     ### NOTE: The job may have been deleted, but the card may still exist.
-    jobs = [Job(name) for name in job_names]
+    jobs = [_get_job(name) for name in job_names]
 
     return (
         [
@@ -229,3 +239,39 @@ def refresh_jobs_on_interval(
             for job in jobs
         ],
     )
+
+
+@dash_app.callback(
+    Output('job-output-div', 'children'),
+    Input('job-location', 'pathname'),
+    State('session-store', 'data'),
+)
+def render_job_page_from_url(
+    pathname: str,
+    session_data: Optional[Dict[str, Any]],
+):
+    """
+    Load the `/job/{name}` page.
+    """
+    if not str(pathname).startswith('/dash/job'):
+        return no_update
+
+    session_id = (session_data or {}).get('session-id', None)
+    authenticated = is_session_authenticated(str(session_id))
+
+    job_name = pathname.replace('/dash/job', '').lstrip('/').rstrip('/')
+    if not job_name:
+        return no_update
+
+    job = _get_job(job_name)
+    if not job.exists():
+        return [
+            html.Br(),
+            html.H2("404: Job does not exist."),
+        ]
+
+    return [
+        html.Br(),
+        build_job_card(job, authenticated=authenticated, include_follow=False),
+        html.Br(),
+    ]

meerschaum/api/dash/callbacks/pipes.py

@@ -26,7 +26,7 @@ html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHEC
     State('pipes-location', 'search'),
     State('session-store', 'data'),
 )
-def render_page_from_url(
+def render_pipe_page_from_url(
     pathname: str,
     pipe_search: str,
     session_data: Optional[Dict[str, Any]],

meerschaum/api/dash/jobs.py

@@ -31,6 +31,7 @@ STATUS_EMOJI: Dict[str, str] = {
 
 EXECUTOR_KEYS: str = get_executor_keys_from_context()
 
+
 def get_jobs_cards(state: WebState):
     """
     Build cards and alerts lists for jobs.
@@ -42,71 +43,96 @@ def get_jobs_cards(state: WebState):
     cards = []
 
     for name, job in jobs.items():
-        footer_children = html.Div(
-            build_process_timestamps_children(job),
-            id = {'type': 'process-timestamps-div', 'index': name},
-        )
-        follow_logs_button = dbc.DropdownMenuItem(
-            "Follow logs",
-            id = {'type': 'follow-logs-button', 'index': name},
-        )
-        download_logs_button = dbc.DropdownMenuItem(
-            "Download logs",
-            id = {'type': 'job-download-logs-button', 'index': name},
-        )
-        logs_menu_children = (
-            ([follow_logs_button] if is_authenticated else []) + [download_logs_button]
-        )
-        header_children = [
-            html.Div(
-                build_status_children(job),
-                id={'type': 'manage-job-status-div', 'index': name},
-                style={'float': 'left'},
+        cards.append(build_job_card(job, authenticated=is_authenticated))
+
+    return cards, []
+
+
+def build_job_card(
+    job: Job,
+    authenticated: bool = False,
+    include_follow: bool = True,
+):
+    """
+    Return a card for a given job.
+    """
+    footer_children = html.Div(
+        build_process_timestamps_children(job),
+        id={'type': 'process-timestamps-div', 'index': job.name},
+    )
+    follow_logs_button = dbc.DropdownMenuItem(
+        "Follow logs",
+        id={'type': 'follow-logs-button', 'index': job.name},
+    )
+    download_logs_button = dbc.DropdownMenuItem(
+        "Download logs",
+        id={'type': 'job-download-logs-button', 'index': job.name},
+    )
+    logs_menu_children = (
+        ([follow_logs_button] if include_follow else []) + [download_logs_button]
+        if authenticated
+        else []
+    )
+    header_children = [
+        html.Div(
+            build_status_children(job),
+            id={'type': 'manage-job-status-div', 'index': job.name},
+            style={'float': 'left'},
+        ),
+    ] + ([
+        html.Div(
+            dbc.DropdownMenu(
+                logs_menu_children,
+                label="Logs",
+                size="sm",
+                align_end=True,
+                color="secondary",
+                menu_variant='dark',
             ),
-            html.Div(
-                dbc.DropdownMenu(
-                    logs_menu_children,
-                    label="Logs",
-                    size="sm",
-                    align_end=True,
-                    color="secondary",
-                    menu_variant='dark',
-                ),
-                style={'float': 'right'},
+            style={'float': 'right'},
+        ),
+    ] if authenticated else [])
+
+    body_children = [
+        html.H4(
+            html.B(
+                html.A(
+                    "🔗 " + job.name,
+                    href=f"/dash/job/{job.name}",
+                    target="_blank",
+                    style={
+                        'color': 'white',
+                        'text-decoration': 'none',
+                    },
+                )
             ),
-        ]
-
-        body_children = [
-            html.H4(html.B(name), className="card-title"),
-            html.Div(
-                html.P(
-                    job.label,
-                    className="card-text job-card-text",
-                    style={"word-wrap": "break-word"},
-                    id={'type': 'job-label-p', 'index': name},
-                ),
-                style={"white-space": "pre-wrap"},
+            className="card-title",
+        ),
+        html.Div(
+            html.P(
+                job.label,
+                className="card-text job-card-text",
+                style={"word-wrap": "break-word"},
+                id={'type': 'job-label-p', 'index': job.name},
             ),
-            html.Div(
-                (
-                    build_manage_job_buttons_div_children(job)
-                    if is_authenticated
-                    else []
-                ),
-                id={'type': 'manage-job-buttons-div', 'index': name},
+            style={"white-space": "pre-wrap"},
+        ),
+        html.Div(
+            (
+                build_manage_job_buttons_div_children(job)
+                if authenticated
+                else []
             ),
-            html.Div(id={'type': 'manage-job-alert-div', 'index': name}),
-        ]
-
-        cards.append(
-            dbc.Card([
-                dbc.CardHeader(header_children),
-                dbc.CardBody(body_children),
-                dbc.CardFooter(footer_children),
-            ])
-        )
+            id={'type': 'manage-job-buttons-div', 'index': job.name},
+        ),
+        html.Div(id={'type': 'manage-job-alert-div', 'index': job.name}),
+    ]
 
-    return cards, []
+    return dbc.Card([
+        dbc.CardHeader(header_children),
+        dbc.CardBody(body_children),
+        dbc.CardFooter(footer_children),
+    ])
 
 
 def build_manage_job_buttons_div_children(job: Job):

meerschaum/api/dash/pages/__init__.py

@@ -11,3 +11,4 @@ import meerschaum.api.dash.pages.dashboard
 import meerschaum.api.dash.pages.plugins
 import meerschaum.api.dash.pages.register
 import meerschaum.api.dash.pages.pipes
+import meerschaum.api.dash.pages.job

meerschaum/api/dash/pages/job.py

@@ -0,0 +1,21 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Display pipes via a shareable URL.
+"""
+
+from meerschaum.api import CHECK_UPDATE
+from meerschaum.utils.packages import import_html, import_dcc
+
+html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
+import dash_bootstrap_components as dbc
+
+from meerschaum.api.dash.components import download_logs, refresh_jobs_interval
+
+layout = dbc.Container([
+    dcc.Location('job-location'),
+    html.Div(id='job-output-div'),
+    download_logs,
+    refresh_jobs_interval,
+])

meerschaum/api/routes/_jobs.py

@@ -39,12 +39,12 @@ NONINTERACTIVE_ENV: str = STATIC_CONFIG['environment']['noninteractive']
 EXECUTOR_KEYS: str = 'local'
 
 
-def _get_job(name: str):
-    systemd_job = Job(name, executor_keys='systemd')
+def _get_job(name: str, sysargs: Union[str, List[str], None] = None):
+    systemd_job = Job(name, sysargs, executor_keys='systemd')
     if systemd_job.exists():
         return systemd_job
 
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = Job(name, sysargs, executor_keys=EXECUTOR_KEYS)
     return job
 
 

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.10"
+__version__ = "2.4.12"

meerschaum/connectors/sql/_fetch.py

@@ -7,27 +7,29 @@ Implement the Connector fetch() method
 """
 
 from __future__ import annotations
+
 from datetime import datetime, timedelta
 import meerschaum as mrsm
-from meerschaum.utils.typing import Optional, Union, Callable, Any
+from meerschaum.utils.typing import Optional, Union, Callable, Any, List, Dict
+
 
 def fetch(
-        self,
-        pipe: meerschaum.Pipe,
-        begin: Union[datetime, int, str, None] = '',
-        end: Union[datetime, int, str, None] = None,
-        check_existing: bool = True,
-        chunk_hook: Optional[Callable[[pd.DataFrame], Any]] = None,
-        chunksize: Optional[int] = -1,
-        workers: Optional[int] = None,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union['pd.DataFrame', List[Any], None]:
+    self,
+    pipe: mrsm.Pipe,
+    begin: Union[datetime, int, str, None] = '',
+    end: Union[datetime, int, str, None] = None,
+    check_existing: bool = True,
+    chunk_hook: Optional[Callable[['pd.DataFrame'], Any]] = None,
+    chunksize: Optional[int] = -1,
+    workers: Optional[int] = None,
+    debug: bool = False,
+    **kw: Any
+) -> Union['pd.DataFrame', List[Any], None]:
     """Execute the SQL definition and return a Pandas DataFrame.
 
     Parameters
     ----------
-    pipe: meerschaum.Pipe
+    pipe: mrsm.Pipe
         The pipe object which contains the `fetch` metadata.
         
         - pipe.columns['datetime']: str
@@ -63,7 +65,7 @@ def fetch(
 
     debug: bool, default False
         Verbosity toggle.
-       
+
     Returns
     -------
     A pandas DataFrame or `None`.
@@ -71,20 +73,20 @@ def fetch(
     """
     meta_def = self.get_pipe_metadef(
         pipe,
-        begin = begin,
-        end = end,
-        check_existing = check_existing,
-        debug = debug,
+        begin=begin,
+        end=end,
+        check_existing=check_existing,
+        debug=debug,
         **kw
     )
     as_hook_results = chunk_hook is not None
     chunks = self.read(
         meta_def,
-        chunk_hook = chunk_hook,
-        as_hook_results = as_hook_results,
-        chunksize = chunksize,
-        workers = workers,
-        debug = debug,
+        chunk_hook=chunk_hook,
+        as_hook_results=as_hook_results,
+        chunksize=chunksize,
+        workers=workers,
+        debug=debug,
     )
     ### if sqlite, parse for datetimes
     if not as_hook_results and self.flavor == 'sqlite':
@@ -97,8 +99,8 @@ def fetch(
         return (
             parse_df_datetimes(
                 chunk,
-                ignore_cols = ignore_cols,
-                debug = debug,
+                ignore_cols=ignore_cols,
+                debug=debug,
             )
             for chunk in chunks
         )
@@ -106,15 +108,15 @@ def fetch(
 
 
 def get_pipe_metadef(
-        self,
-        pipe: meerschaum.Pipe,
-        params: Optional[Dict[str, Any]] = None,
-        begin: Union[datetime, int, str, None] = '',
-        end: Union[datetime, int, str, None] = None,
-        check_existing: bool = True,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union[str, None]:
+    self,
+    pipe: mrsm.Pipe,
+    params: Optional[Dict[str, Any]] = None,
+    begin: Union[datetime, int, str, None] = '',
+    end: Union[datetime, int, str, None] = None,
+    check_existing: bool = True,
+    debug: bool = False,
+    **kw: Any
+) -> Union[str, None]:
     """
     Return a pipe's meta definition fetch query.
 
@@ -173,7 +175,6 @@ def get_pipe_metadef(
                     stack = False
                 )
 
-
     apply_backtrack = begin == '' and check_existing
     backtrack_interval = pipe.get_backtrack_interval(check_existing=check_existing, debug=debug)
     btm = (
@@ -189,35 +190,34 @@ def get_pipe_metadef(
 
     if begin and end and begin >= end:
         begin = None
-    
-    da = None
+
     if dt_name:
         begin_da = (
             dateadd_str(
-                flavor = self.flavor,
-                datepart = 'minute',
-                number = ((-1 * btm) if apply_backtrack else 0), 
-                begin = begin,
+                flavor=self.flavor,
+                datepart='minute',
+                number=((-1 * btm) if apply_backtrack else 0), 
+                begin=begin,
             )
             if begin
             else None
         )
         end_da = (
             dateadd_str(
-                flavor = self.flavor,
-                datepart = 'minute',
-                number = 0,
-                begin = end,
+                flavor=self.flavor,
+                datepart='minute',
+                number=0,
+                begin=end,
             )
             if end
             else None
         )
 
     meta_def = (
-        _simple_fetch_query(pipe) if (
+        _simple_fetch_query(pipe, self.flavor) if (
             (not (pipe.columns or {}).get('id', None))
             or (not get_config('system', 'experimental', 'join_fetch'))
-        ) else _join_fetch_query(pipe, debug=debug, **kw)
+        ) else _join_fetch_query(pipe, self.flavor, debug=debug, **kw)
     )
 
     has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
@@ -300,25 +300,28 @@ def set_pipe_query(pipe: mrsm.Pipe, query: str) -> None:
     dict_to_set[key_to_set] = query
 
 
-def _simple_fetch_query(pipe, debug: bool=False, **kw) -> str:
+def _simple_fetch_query(
+    pipe: mrsm.Pipe,
+    flavor: str,
+    debug: bool = False,
+    **kw
+) -> str:
     """Build a fetch query from a pipe's definition."""
-    def_name = 'definition'
+    from meerschaum.utils.sql import format_cte_subquery
     definition = get_pipe_query(pipe)
-    return (
-        f"WITH {def_name} AS (\n{definition}\n) SELECT * FROM {def_name}"
-        if pipe.connector.flavor not in ('mysql', 'mariadb')
-        else f"SELECT * FROM (\n{definition}\n) AS {def_name}"
-    )
+    return format_cte_subquery(definition, flavor, 'definition')
+
 
 def _join_fetch_query(
-        pipe,
-        debug: bool = False,
-        new_ids: bool = True,
-        **kw
-    ) -> str:
+    pipe: mrsm.Pipe,
+    flavor: str,
+    debug: bool = False,
+    new_ids: bool = True,
+    **kw
+) -> str:
     """Build a fetch query based on the datetime and ID indices."""
     if not pipe.exists(debug=debug):
-        return _simple_fetch_query(pipe, debug=debug, **kw)
+        return _simple_fetch_query(pipe, flavor, debug=debug, **kw)
 
     from meerschaum.utils.sql import sql_item_name, dateadd_str
     pipe_instance_name = sql_item_name(
@@ -350,7 +353,8 @@ def _join_fetch_query(
     """
     sync_times = pipe.instance_connector.read(sync_times_query, debug=debug, silent=False)
     if sync_times is None:
-        return _simple_fetch_query(pipe, debug=debug, **kw)
+        return _simple_fetch_query(pipe, flavor, debug=debug, **kw)
+
     _sync_times_q = f",\n{sync_times_remote_name} AS ("
     for _id, _st in sync_times.itertuples(index=False):
         _sync_times_q += (