meerschaum 2.4.11__py3-none-any.whl → 2.4.12__py3-none-any.whl

meerschaum/api/dash/callbacks/pipes.py

@@ -26,7 +26,7 @@ html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHEC
     State('pipes-location', 'search'),
     State('session-store', 'data'),
 )
-def render_page_from_url(
+def render_pipe_page_from_url(
     pathname: str,
     pipe_search: str,
     session_data: Optional[Dict[str, Any]],

meerschaum/api/dash/jobs.py

@@ -31,6 +31,7 @@ STATUS_EMOJI: Dict[str, str] = {
 
 EXECUTOR_KEYS: str = get_executor_keys_from_context()
 
+
 def get_jobs_cards(state: WebState):
     """
     Build cards and alerts lists for jobs.
@@ -42,71 +43,96 @@ def get_jobs_cards(state: WebState):
     cards = []
 
     for name, job in jobs.items():
-        footer_children = html.Div(
-            build_process_timestamps_children(job),
-            id = {'type': 'process-timestamps-div', 'index': name},
-        )
-        follow_logs_button = dbc.DropdownMenuItem(
-            "Follow logs",
-            id = {'type': 'follow-logs-button', 'index': name},
-        )
-        download_logs_button = dbc.DropdownMenuItem(
-            "Download logs",
-            id = {'type': 'job-download-logs-button', 'index': name},
-        )
-        logs_menu_children = (
-            ([follow_logs_button] if is_authenticated else []) + [download_logs_button]
-        )
-        header_children = [
-            html.Div(
-                build_status_children(job),
-                id={'type': 'manage-job-status-div', 'index': name},
-                style={'float': 'left'},
+        cards.append(build_job_card(job, authenticated=is_authenticated))
+
+    return cards, []
+
+
+def build_job_card(
+    job: Job,
+    authenticated: bool = False,
+    include_follow: bool = True,
+):
+    """
+    Return a card for a given job.
+    """
+    footer_children = html.Div(
+        build_process_timestamps_children(job),
+        id={'type': 'process-timestamps-div', 'index': job.name},
+    )
+    follow_logs_button = dbc.DropdownMenuItem(
+        "Follow logs",
+        id={'type': 'follow-logs-button', 'index': job.name},
+    )
+    download_logs_button = dbc.DropdownMenuItem(
+        "Download logs",
+        id={'type': 'job-download-logs-button', 'index': job.name},
+    )
+    logs_menu_children = (
+        ([follow_logs_button] if include_follow else []) + [download_logs_button]
+        if authenticated
+        else []
+    )
+    header_children = [
+        html.Div(
+            build_status_children(job),
+            id={'type': 'manage-job-status-div', 'index': job.name},
+            style={'float': 'left'},
+        ),
+    ] + ([
+        html.Div(
+            dbc.DropdownMenu(
+                logs_menu_children,
+                label="Logs",
+                size="sm",
+                align_end=True,
+                color="secondary",
+                menu_variant='dark',
             ),
-            html.Div(
-                dbc.DropdownMenu(
-                    logs_menu_children,
-                    label="Logs",
-                    size="sm",
-                    align_end=True,
-                    color="secondary",
-                    menu_variant='dark',
-                ),
-                style={'float': 'right'},
+            style={'float': 'right'},
+        ),
+    ] if authenticated else [])
+
+    body_children = [
+        html.H4(
+            html.B(
+                html.A(
+                    "🔗 " + job.name,
+                    href=f"/dash/job/{job.name}",
+                    target="_blank",
+                    style={
+                        'color': 'white',
+                        'text-decoration': 'none',
+                    },
+                )
             ),
-        ]
-
-        body_children = [
-            html.H4(html.B(name), className="card-title"),
-            html.Div(
-                html.P(
-                    job.label,
-                    className="card-text job-card-text",
-                    style={"word-wrap": "break-word"},
-                    id={'type': 'job-label-p', 'index': name},
-                ),
-                style={"white-space": "pre-wrap"},
+            className="card-title",
+        ),
+        html.Div(
+            html.P(
+                job.label,
+                className="card-text job-card-text",
+                style={"word-wrap": "break-word"},
+                id={'type': 'job-label-p', 'index': job.name},
             ),
-            html.Div(
-                (
-                    build_manage_job_buttons_div_children(job)
-                    if is_authenticated
-                    else []
-                ),
-                id={'type': 'manage-job-buttons-div', 'index': name},
+            style={"white-space": "pre-wrap"},
+        ),
+        html.Div(
+            (
+                build_manage_job_buttons_div_children(job)
+                if authenticated
+                else []
             ),
-            html.Div(id={'type': 'manage-job-alert-div', 'index': name}),
-        ]
-
-        cards.append(
-            dbc.Card([
-                dbc.CardHeader(header_children),
-                dbc.CardBody(body_children),
-                dbc.CardFooter(footer_children),
-            ])
-        )
+            id={'type': 'manage-job-buttons-div', 'index': job.name},
+        ),
+        html.Div(id={'type': 'manage-job-alert-div', 'index': job.name}),
+    ]
 
-    return cards, []
+    return dbc.Card([
+        dbc.CardHeader(header_children),
+        dbc.CardBody(body_children),
+        dbc.CardFooter(footer_children),
+    ])
 
 
 def build_manage_job_buttons_div_children(job: Job):

meerschaum/api/dash/pages/__init__.py

@@ -11,3 +11,4 @@ import meerschaum.api.dash.pages.dashboard
 import meerschaum.api.dash.pages.plugins
 import meerschaum.api.dash.pages.register
 import meerschaum.api.dash.pages.pipes
+import meerschaum.api.dash.pages.job

meerschaum/api/dash/pages/job.py

@@ -0,0 +1,21 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Display pipes via a shareable URL.
+"""
+
+from meerschaum.api import CHECK_UPDATE
+from meerschaum.utils.packages import import_html, import_dcc
+
+html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
+import dash_bootstrap_components as dbc
+
+from meerschaum.api.dash.components import download_logs, refresh_jobs_interval
+
+layout = dbc.Container([
+    dcc.Location('job-location'),
+    html.Div(id='job-output-div'),
+    download_logs,
+    refresh_jobs_interval,
+])

meerschaum/api/routes/_jobs.py

@@ -39,12 +39,12 @@ NONINTERACTIVE_ENV: str = STATIC_CONFIG['environment']['noninteractive']
 EXECUTOR_KEYS: str = 'local'
 
 
-def _get_job(name: str):
-    systemd_job = Job(name, executor_keys='systemd')
+def _get_job(name: str, sysargs: Union[str, List[str], None] = None):
+    systemd_job = Job(name, sysargs, executor_keys='systemd')
     if systemd_job.exists():
         return systemd_job
 
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = Job(name, sysargs, executor_keys=EXECUTOR_KEYS)
     return job
 
 

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.11"
+__version__ = "2.4.12"

meerschaum/connectors/sql/_fetch.py

@@ -7,27 +7,29 @@ Implement the Connector fetch() method
 """
 
 from __future__ import annotations
+
 from datetime import datetime, timedelta
 import meerschaum as mrsm
-from meerschaum.utils.typing import Optional, Union, Callable, Any
+from meerschaum.utils.typing import Optional, Union, Callable, Any, List, Dict
+
 
 def fetch(
-        self,
-        pipe: meerschaum.Pipe,
-        begin: Union[datetime, int, str, None] = '',
-        end: Union[datetime, int, str, None] = None,
-        check_existing: bool = True,
-        chunk_hook: Optional[Callable[[pd.DataFrame], Any]] = None,
-        chunksize: Optional[int] = -1,
-        workers: Optional[int] = None,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union['pd.DataFrame', List[Any], None]:
+    self,
+    pipe: mrsm.Pipe,
+    begin: Union[datetime, int, str, None] = '',
+    end: Union[datetime, int, str, None] = None,
+    check_existing: bool = True,
+    chunk_hook: Optional[Callable[['pd.DataFrame'], Any]] = None,
+    chunksize: Optional[int] = -1,
+    workers: Optional[int] = None,
+    debug: bool = False,
+    **kw: Any
+) -> Union['pd.DataFrame', List[Any], None]:
     """Execute the SQL definition and return a Pandas DataFrame.
 
     Parameters
     ----------
-    pipe: meerschaum.Pipe
+    pipe: mrsm.Pipe
         The pipe object which contains the `fetch` metadata.
         
         - pipe.columns['datetime']: str
@@ -63,7 +65,7 @@ def fetch(
 
     debug: bool, default False
         Verbosity toggle.
-       
+
     Returns
     -------
     A pandas DataFrame or `None`.
@@ -71,20 +73,20 @@ def fetch(
     """
     meta_def = self.get_pipe_metadef(
         pipe,
-        begin = begin,
-        end = end,
-        check_existing = check_existing,
-        debug = debug,
+        begin=begin,
+        end=end,
+        check_existing=check_existing,
+        debug=debug,
         **kw
     )
     as_hook_results = chunk_hook is not None
     chunks = self.read(
         meta_def,
-        chunk_hook = chunk_hook,
-        as_hook_results = as_hook_results,
-        chunksize = chunksize,
-        workers = workers,
-        debug = debug,
+        chunk_hook=chunk_hook,
+        as_hook_results=as_hook_results,
+        chunksize=chunksize,
+        workers=workers,
+        debug=debug,
     )
     ### if sqlite, parse for datetimes
     if not as_hook_results and self.flavor == 'sqlite':
@@ -97,8 +99,8 @@ def fetch(
         return (
             parse_df_datetimes(
                 chunk,
-                ignore_cols = ignore_cols,
-                debug = debug,
+                ignore_cols=ignore_cols,
+                debug=debug,
             )
             for chunk in chunks
         )
@@ -106,15 +108,15 @@ def fetch(
 
 
 def get_pipe_metadef(
-        self,
-        pipe: meerschaum.Pipe,
-        params: Optional[Dict[str, Any]] = None,
-        begin: Union[datetime, int, str, None] = '',
-        end: Union[datetime, int, str, None] = None,
-        check_existing: bool = True,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union[str, None]:
+    self,
+    pipe: mrsm.Pipe,
+    params: Optional[Dict[str, Any]] = None,
+    begin: Union[datetime, int, str, None] = '',
+    end: Union[datetime, int, str, None] = None,
+    check_existing: bool = True,
+    debug: bool = False,
+    **kw: Any
+) -> Union[str, None]:
     """
     Return a pipe's meta definition fetch query.
 
@@ -173,7 +175,6 @@ def get_pipe_metadef(
                     stack = False
                 )
 
-
     apply_backtrack = begin == '' and check_existing
     backtrack_interval = pipe.get_backtrack_interval(check_existing=check_existing, debug=debug)
     btm = (
@@ -189,35 +190,34 @@ def get_pipe_metadef(
 
     if begin and end and begin >= end:
         begin = None
-    
-    da = None
+
     if dt_name:
         begin_da = (
             dateadd_str(
-                flavor = self.flavor,
-                datepart = 'minute',
-                number = ((-1 * btm) if apply_backtrack else 0), 
-                begin = begin,
+                flavor=self.flavor,
+                datepart='minute',
+                number=((-1 * btm) if apply_backtrack else 0), 
+                begin=begin,
             )
             if begin
             else None
         )
         end_da = (
             dateadd_str(
-                flavor = self.flavor,
-                datepart = 'minute',
-                number = 0,
-                begin = end,
+                flavor=self.flavor,
+                datepart='minute',
+                number=0,
+                begin=end,
             )
             if end
             else None
         )
 
     meta_def = (
-        _simple_fetch_query(pipe) if (
+        _simple_fetch_query(pipe, self.flavor) if (
             (not (pipe.columns or {}).get('id', None))
             or (not get_config('system', 'experimental', 'join_fetch'))
-        ) else _join_fetch_query(pipe, debug=debug, **kw)
+        ) else _join_fetch_query(pipe, self.flavor, debug=debug, **kw)
     )
 
     has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
@@ -300,25 +300,28 @@ def set_pipe_query(pipe: mrsm.Pipe, query: str) -> None:
     dict_to_set[key_to_set] = query
 
 
-def _simple_fetch_query(pipe, debug: bool=False, **kw) -> str:
+def _simple_fetch_query(
+    pipe: mrsm.Pipe,
+    flavor: str,
+    debug: bool = False,
+    **kw
+) -> str:
     """Build a fetch query from a pipe's definition."""
-    def_name = 'definition'
+    from meerschaum.utils.sql import format_cte_subquery
     definition = get_pipe_query(pipe)
-    return (
-        f"WITH {def_name} AS (\n{definition}\n) SELECT * FROM {def_name}"
-        if pipe.connector.flavor not in ('mysql', 'mariadb')
-        else f"SELECT * FROM (\n{definition}\n) AS {def_name}"
-    )
+    return format_cte_subquery(definition, flavor, 'definition')
+
 
 def _join_fetch_query(
-        pipe,
-        debug: bool = False,
-        new_ids: bool = True,
-        **kw
-    ) -> str:
+    pipe: mrsm.Pipe,
+    flavor: str,
+    debug: bool = False,
+    new_ids: bool = True,
+    **kw
+) -> str:
     """Build a fetch query based on the datetime and ID indices."""
     if not pipe.exists(debug=debug):
-        return _simple_fetch_query(pipe, debug=debug, **kw)
+        return _simple_fetch_query(pipe, flavor, debug=debug, **kw)
 
     from meerschaum.utils.sql import sql_item_name, dateadd_str
     pipe_instance_name = sql_item_name(
@@ -350,7 +353,8 @@ def _join_fetch_query(
     """
     sync_times = pipe.instance_connector.read(sync_times_query, debug=debug, silent=False)
     if sync_times is None:
-        return _simple_fetch_query(pipe, debug=debug, **kw)
+        return _simple_fetch_query(pipe, flavor, debug=debug, **kw)
+
     _sync_times_q = f",\n{sync_times_remote_name} AS ("
     for _id, _st in sync_times.itertuples(index=False):
         _sync_times_q += (

meerschaum/connectors/sql/_pipes.py

@@ -2096,7 +2096,7 @@ def get_pipe_rowcount(
     An `int` for the number of rows if the `pipe` exists, otherwise `None`.
 
     """
-    from meerschaum.utils.sql import dateadd_str, sql_item_name, NO_CTE_FLAVORS
+    from meerschaum.utils.sql import dateadd_str, sql_item_name, wrap_query_with_cte
     from meerschaum.connectors.sql._fetch import get_pipe_query
     if remote:
         msg = f"'fetch:definition' must be an attribute of {pipe} to get a remote rowcount."
@@ -2175,20 +2175,10 @@ def get_pipe_rowcount(
         if not remote
         else get_pipe_query(pipe)
     )
-    query = (
-        f"""
-        WITH src AS ({src})
-        SELECT COUNT(*)
-        FROM src
-        """
-    ) if self.flavor not in ('mysql', 'mariadb') else (
-        f"""
-        SELECT COUNT(*)
-        FROM ({src}) AS src
-        """
-    )
+    parent_query = f"SELECT COUNT(*)\nFROM {sql_item_name('src', self.flavor)}"
+    query = wrap_query_with_cte(src, parent_query, self.flavor)
     if begin is not None or end is not None:
-        query += "WHERE"
+        query += "\nWHERE"
     if begin is not None:
         query += f"""
         {dt} >= {dateadd_str(self.flavor, datepart='minute', number=0, begin=begin)}
@@ -2330,10 +2320,10 @@ def clear_pipe(
 
 
 def get_pipe_table(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> sqlalchemy.Table:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Union['sqlalchemy.Table', None]:
     """
     Return the `sqlalchemy.Table` object for a `mrsm.Pipe`.
 
@@ -2352,18 +2342,18 @@ def get_pipe_table(
         return None
     return get_sqlalchemy_table(
         pipe.target,
-        connector = self,
-        schema = self.get_pipe_schema(pipe),
-        debug = debug,
-        refresh = True,
+        connector=self,
+        schema=self.get_pipe_schema(pipe),
+        debug=debug,
+        refresh=True,
     )
 
 
 def get_pipe_columns_types(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> Dict[str, str]:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Dict[str, str]:
     """
     Get the pipe's columns and types.
 
@@ -2394,8 +2384,8 @@ def get_pipe_columns_types(
         return get_table_cols_types(
             pipe.target,
             self,
-            flavor = self.flavor,
-            schema = self.schema,
+            flavor=self.flavor,
+            schema=self.get_pipe_schema(pipe),
         )
 
     table_columns = {}
@@ -2448,6 +2438,7 @@ def get_add_columns_queries(
     from meerschaum.utils.sql import (
         sql_item_name,
         SINGLE_ALTER_TABLE_FLAVORS,
+        get_table_cols_types,
     )
     from meerschaum.utils.dtypes.sql import (
         get_pd_type_from_db_type,
@@ -2480,6 +2471,14 @@ def get_add_columns_queries(
     db_cols_types = {
         col: get_pd_type_from_db_type(str(typ.type))
         for col, typ in table_obj.columns.items()
+    } if table_obj is not None else {
+        col: get_pd_type_from_db_type(typ)
+        for col, typ in get_table_cols_types(
+            pipe.target,
+            self,
+            schema=self.get_pipe_schema(pipe),
+            debug=debug,
+        ).items()
     }
     new_cols = set(df_cols_types) - set(db_cols_types)
     if not new_cols:
@@ -2552,7 +2551,7 @@ def get_alter_columns_queries(
     """
     if not pipe.exists(debug=debug):
         return []
-    from meerschaum.utils.sql import sql_item_name, DROP_IF_EXISTS_FLAVORS
+    from meerschaum.utils.sql import sql_item_name, DROP_IF_EXISTS_FLAVORS, get_table_cols_types
     from meerschaum.utils.dataframe import get_numeric_cols
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import (
@@ -2583,6 +2582,14 @@ def get_alter_columns_queries(
     db_cols_types = {
         col: get_pd_type_from_db_type(str(typ.type))
         for col, typ in table_obj.columns.items()
+    } if table_obj is not None else {
+        col: get_pd_type_from_db_type(typ)
+        for col, typ in get_table_cols_types(
+            pipe.target,
+            self,
+            schema=self.get_pipe_schema(pipe),
+            debug=debug,
+        ).items()
     }
     pipe_bool_cols = [col for col, typ in pipe.dtypes.items() if are_dtypes_equal(str(typ), 'bool')]
     pd_db_df_aliases = {