meerschaum 2.8.3__py3-none-any.whl → 2.9.0.dev1__py3-none-any.whl

meerschaum/_internal/arguments/_parser.py

@@ -367,6 +367,11 @@ groups['sync'].add_argument(
         "Only compare row-counts when verifying pipes."
     ),
 )
+groups['sync'].add_argument(
+    '--skip-hooks', action='store_true', help=(
+        "Skip calling the sync hooks."
+    )
+)
 groups['sync'].add_argument(
     '--cache', action='store_true',
     help=(

meerschaum/actions/drop.py

@@ -86,7 +86,7 @@ def _drop_pipes(
         pprint(success_dict)
 
     msg = (
-        f"Finished dropping {len(pipes)} pipes"
+        f"Finished dropping {len(pipes)} pipe"
         + ('s' if len(pipes) != 1 else '')
         + f"\n    ({successes} succeeded, {fails} failed)."
     )

meerschaum/actions/start.py

@@ -568,9 +568,9 @@ def _start_pipeline(
     Examples
     --------
 
-    `sync pipes -i sql:local + sync pipes -i sql:main :: -s 'daily'`
+    `sync pipes -i sql:local + sync pipes -i sql:main : -s 'daily'`
 
-    `show version + show arguments :: --loop`
+    `show version + show arguments : --loop`
 
     """
     import json
@@ -580,10 +580,11 @@ def _start_pipeline(
     from meerschaum.utils.warnings import info, warn
     from meerschaum.utils.misc import is_int
     from meerschaum.utils.venv import venv_exec
-    from meerschaum.utils.process import poll_process
+    from meerschaum.utils.process import poll_process, _stop_process
     fence_begin, fence_end = '<MRSM_RESULT>', '</MRSM_RESULT>'
 
-    success, msg = False, "Did not run pipeline."
+    default_msg = "Did not pipeline."
+    success, msg = False, default_msg
     def write_line(line):
         nonlocal success, msg
         decoded = line.decode('utf-8')
@@ -610,7 +611,9 @@ def _start_pipeline(
     do_n_times = (
         int(action[0].lstrip('x'))
         if action and is_int(action[0].lstrip('x'))
-        else 1
+        else (
+            1 if not loop else -1
+        )
     )
 
     params = params or {}
@@ -623,8 +626,9 @@ def _start_pipeline(
     if min_seconds is None:
         min_seconds = 1.0
 
+    proc = None
     def do_entry() -> None:
-        nonlocal success, msg
+        nonlocal success, msg, proc
         if timeout_seconds is None:
             success, msg = entry(sub_args_line, _patch_args=patch_args)
             return
@@ -674,6 +678,10 @@ def _start_pipeline(
         run_loop()
     except KeyboardInterrupt:
         warn("Cancelled pipeline.", stack=False)
+        if proc is not None:
+            _stop_process(proc)
+        if msg == default_msg:
+            msg = "Pipeline was cancelled."
 
     if do_n_times != 1:
         info(f"Ran pipeline {ran_n_times} time" + ('s' if ran_n_times != 1 else '') + '.')

meerschaum/actions/sync.py

@@ -50,6 +50,7 @@ def _pipes_lap(
     bounded: Optional[bool] = None,
     chunk_interval: Union[timedelta, int, None] = None,
     check_rowcounts_only: bool = False,
+    skip_hooks: bool = False,
     mrsm_instance: Optional[str] = None,
     timeout_seconds: Optional[int] = None,
     nopretty: bool = False,
@@ -95,6 +96,7 @@ def _pipes_lap(
         'bounded': bounded,
         'chunk_interval': chunk_interval,
         'check_rowcounts_only': check_rowcounts_only,
+        'skip_hooks': skip_hooks,
     })
     locks = {'remaining_count': Lock(), 'results_dict': Lock(), 'pipes_threads': Lock(),}
     pipes = get_pipes(
@@ -257,6 +259,7 @@ def _sync_pipes(
     bounded: Optional[bool] = None,
     chunk_interval: Union[timedelta, int, None] = None,
     check_rowcounts_only: bool = False,
+    skip_hooks: bool = False,
     shell: bool = False,
     nopretty: bool = False,
     debug: bool = False,
@@ -289,6 +292,8 @@ def _sync_pipes(
 
     noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
     noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
+    if check_rowcounts_only:
+        skip_hooks = True
 
     run = True
     msg = ""
@@ -312,6 +317,7 @@ def _sync_pipes(
                     bounded=bounded,
                     chunk_interval=chunk_interval,
                     check_rowcounts_only=check_rowcounts_only,
+                    skip_hooks=skip_hooks,
                     unblock=unblock,
                     debug=debug,
                     nopretty=nopretty,
@@ -403,6 +409,7 @@ def _wrap_pipe(
     deduplicate: bool = False,
     bounded: Optional[bool] = None,
     chunk_interval: Union[timedelta, int, None] = None,
+    skip_hooks: bool = False,
     **kw
 ):
     """
@@ -462,6 +469,8 @@ def _wrap_pipe(
 
     pre_hook_results, post_hook_results = [], []
     def apply_hooks(is_pre_sync: bool):
+        if skip_hooks:
+            return
         _sync_hooks = (_pre_sync_hooks if is_pre_sync else _post_sync_hooks)
         _hook_results = (pre_hook_results if is_pre_sync else post_hook_results)
         for module_name, sync_hooks in _sync_hooks.items():

meerschaum/api/__init__.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import os
 from collections import defaultdict
+from fnmatch import fnmatch
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Any, Optional, PipesDict
@@ -119,14 +120,19 @@ def get_api_connector(instance_keys: Optional[str] = None):
         )
 
     allowed_instance_keys = permissions_config.get(
-        'instance', {}
+        'instances', {}
     ).get(
         'allowed_instance_keys',
         ['*']
     )
-    if allowed_instance_keys != ['*'] and instance_keys not in allowed_instance_keys:
+    found_match: bool = False
+    for allowed_keys_pattern in allowed_instance_keys:
+        if fnmatch(instance_keys, allowed_keys_pattern):
+            found_match = True
+            break
+    if not found_match:
         raise APIPermissionError(
-            f"Instance keys '{instance_keys}' not in list of allowed instances."
+            f"Instance keys '{instance_keys}' does not match the allowed instances patterns."
         )
 
     with _locks[f'instance-{instance_keys}']:

meerschaum/api/_chunks.py

@@ -0,0 +1,67 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Utility functions for the retrieval, caching, and response of chunk data.
+"""
+
+import random
+from datetime import datetime, timedelta, timezone
+from typing import Dict, Generator, Any, Union, Optional, List
+
+import meerschaum as mrsm
+from meerschaum.api import get_cache_connector
+from meerschaum.utils.misc import generate_password
+
+CHUNKS_TOKENS_GENERATORS: Dict[str, Dict[str, Union[Generator[Any, None, None], datetime, int]]]
+DEFAULT_TTL_SECONDS = mrsm.get_config('system', 'api', 'data', 'chunks', 'ttl_seconds')
+
+
+def generate_chunks_cursor_token(
+    pipe: mrsm.Pipe,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    limit: Optional[int] = None,
+    order: Optional[str] = 'asc',
+    ttl_seconds: Optional[int] = None,
+    debug: bool = False,
+) -> str:
+    """
+    Store a generator in the `CHUNKS_TOKENS_GENERATORS`
+    """
+    now = datetime.now(timezone.utc)
+    cache_connector = get_cache_connector()
+    if cache_connector is None:
+        pass
+
+    ttl_seconds = ttl_seconds or DEFAULT_TTL_SECONDS
+    chunk_bounds = pipe.get_chunk_bounds(
+        begin=begin,
+        end=end,
+        bounded=True,
+    )
+
+    while True:
+        chunk_token = prefix + generate_password(random.randint(6, 12))
+        if chunk_token in CHUNKS_TOKENS_GENERATORS:
+            continue
+        break
+
+    CHUNKS_TOKENS_GENERATORS[chunk_token] = {
+        'generator': chunk_generator,
+        'created': now,
+        'ttl': ttl_seconds,
+        'last_accessed': now,
+    }
+
+    return chunk_token
+
+
+def deallocate_expired_generators():
+    """
+    Periodically delete chunk tokens with an expired ttl timestamp.
+    """
+    chunk_tokens = list(CHUNKS_TOKENS_GENERATORS)

meerschaum/api/dash/callbacks/custom.py

@@ -7,10 +7,13 @@ Import custom callbacks created by plugins.
 """
 
 import traceback
+from typing import Any, Dict
+
 from meerschaum.api.dash import dash_app
 from meerschaum.plugins import _dash_plugins, _plugin_endpoints_to_pages
 from meerschaum.utils.warnings import warn
-from meerschaum.api.dash.callbacks.dashboard import _paths, _required_login
+from meerschaum.api.dash.callbacks.dashboard import _paths, _required_login, _pages
+from meerschaum.api.dash.components import pages_navbar
 
 
 def init_dash_plugins():
@@ -34,6 +37,24 @@ def add_plugin_pages():
     Allow users to add pages via the `@web_page` decorator.
     """
     for _endpoint, _page_dict in _plugin_endpoints_to_pages.items():
-        _paths[_endpoint] = _page_dict['function']()
+        page_layout = _page_dict['function']()
+        if not _page_dict['skip_navbar']:
+            if isinstance(page_layout, list):
+                page_layout = [pages_navbar] + page_layout
+            else:
+                page_layout = [pages_navbar, page_layout]
+        page_key = (
+            ' '.join(
+                [
+                    word.capitalize()
+                    for word in (
+                        _endpoint.replace('/dash', '').lstrip('/').rstrip('/').strip()
+                        .replace('-', ' ').replace('_', ' ').split(' ')
+                    )
+                ]
+            )
+        )
+        _pages[page_key] = _endpoint
+        _paths[_endpoint] = page_layout
         if _page_dict['login_required']:
             _required_login.add(_endpoint)

meerschaum/api/dash/callbacks/dashboard.py

@@ -103,6 +103,10 @@ _paths = {
     'job'     : pages.job.layout,
 }
 _required_login = {''}
+_pages = {
+    'Web Console': '/dash/',
+    'Plugins': '/dash/plugins',
+}
 
 
 @dash_app.callback(
@@ -696,7 +700,6 @@ dash_app.clientside_callback(
 dash_app.clientside_callback(
     """
     function(n_clicks){
-        console.log('fullscreen');
         if (!n_clicks) { return dash_clientside.no_update; }
         iframe = document.getElementById('webterm-iframe');
         if (!iframe){ return dash_clientside.no_update; }
@@ -707,11 +710,11 @@ dash_app.clientside_callback(
         if (leftCol.style.display === 'none') {
             leftCol.style.display = '';
             rightCol.className = 'col-6';
-            button.innerHTML = "Full View";
+            button.innerHTML = "⛶";
         } else {
             leftCol.style.display = 'none';
             rightCol.className = 'col-12';
-            button.innerHTML = "Side-by-side View";
+            button.innerHTML = "🀲";
         }
 
         return dash_clientside.no_update;
@@ -1078,3 +1081,38 @@ def parameters_as_yaml_or_json_click(
     if as_yaml:
         return yaml.dump(pipe.parameters)
     return json.dumps(pipe.parameters, indent=4, separators=(',', ': '), sort_keys=True)
+
+
+@dash_app.callback(
+    Output('pages-offcanvas', 'is_open'),
+    Output('pages-offcanvas', 'children'),
+    Input('logo-img', 'n_clicks'),
+    State('pages-offcanvas', 'is_open'),
+)
+def toggle_pages_offcanvas(n_clicks: Optional[int], is_open: bool):
+    """
+    Toggle the pages sidebar.
+    """
+    pages_children = dbc.Card(
+        dbc.ListGroup(
+            [
+                dbc.ListGroupItem(
+                    dbc.Button(
+                        html.P(
+                            ' '.join([word.capitalize() for word in page_key.split(' ')]),
+                            style={'text-decoration': 'none', 'fontSize': '18px'},
+                        ),
+                        style={'width': '100%', 'text-align': 'left'},
+                        href=page_href,
+                        color='dark',
+                    )
+                )
+                for page_key, page_href in _pages.items()
+            ],
+            flush=True,
+        ),
+        outline=True,
+    )
+    if n_clicks:
+        return not is_open, pages_children
+    return is_open, pages_children

meerschaum/api/dash/components.py

@@ -92,9 +92,9 @@ search_parameters_editor = dash_ace.DashAceEditor(
     style={'height': 100},
 )
 
-sidebar = dbc.Offcanvas(
-    children=[],
-    title='Pages',
+pages_offcanvas = dbc.Offcanvas(
+    title='',
+    id='pages-offcanvas',
 )
 
 download_dataframe = dcc.Download(id='download-dataframe-csv')
@@ -110,26 +110,33 @@ instance_select = dbc.Select(
     class_name='dbc_dark custom-select custom-select-sm',
 )
 
+logo_row = dbc.Row(
+    [
+        dbc.Col(
+            html.Img(
+                src=endpoints['dash'] + "/assets/logo_48x48.png",
+                title=doc,
+                id="logo-img",
+                style={'cursor': 'pointer'},
+            ),
+        ),
+    ],
+    align='center',
+    className='g-0 navbar-logo-row',
+)
+
+pages_navbar = html.Div(
+    [
+        pages_offcanvas,
+        dbc.Navbar(dbc.Container(logo_row), dark=True, color='dark'),
+    ],
+    id='pages-navbar-div',
+)
 
 navbar = dbc.Navbar(
     dbc.Container(
         [
-            html.A(
-                dbc.Row(
-                    [
-                        dbc.Col(
-                            html.Img(
-                                src=endpoints['dash'] + "/assets/logo_48x48.png",
-                                title=doc,
-                            ),
-                        ),
-                    ],
-                    align='center',
-                    className='g-0 navbar-logo-row',
-                ),
-                href=('/docs' if docs_enabled else '#'),
-                style={"textDecoration": "none"},
-            ),
+            logo_row,
             dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
             dbc.Collapse(
                 dbc.Row(
@@ -142,6 +149,7 @@ navbar = dbc.Navbar(
                                 style={'margin-left': '30px'},
                                 id='sign-out-button',
                             ),
+                            className="ms-auto",
                         ),
                     ],
                     className="g-0 ms-auto flex-nowrap mt-3 mt-md-0",

meerschaum/api/dash/pages/dashboard.py

@@ -37,6 +37,7 @@ from meerschaum.api.dash.components import (
     console_div,
     download_dataframe,
     navbar,
+    pages_offcanvas,
     download_logs,
     refresh_jobs_interval,
 )
@@ -47,22 +48,23 @@ from meerschaum.api.dash.keys import (
 )
 
 layout = html.Div(
-    id = 'main-div',
-    children = [
+    id='main-div',
+    children=[
         keys_lists_content,
         download_dataframe,
         download_logs,
         refresh_jobs_interval,
         navbar,
+        pages_offcanvas,
         html.Div(
             dbc.Row(
-                id = 'content-row',
-                children = [
+                id='content-row',
+                children=[
                     dbc.Col(
-                        children = [
+                        children=[
                             dbc.Tabs(
-                                id = 'pipes-filter-tabs',
-                                children = [
+                                id='pipes-filter-tabs',
+                                children=[
                                     dbc.Tab(
                                         dropdown_tab_content,
                                         label='Filters',
@@ -107,9 +109,9 @@ layout = html.Div(
                         id='content-col-right',
                     ),
                 ],
-                style = {'max-width': '100%', 'padding': '15px'},
+                style={'max-width': '100%', 'padding': '15px'},
             ), ### end of Row
-            className = 'container-fluid',
+            className='container-fluid',
         ), ### end of Div
         html.P('', id='line-buffer', style={'display': 'none'}),
     ],

meerschaum/api/dash/pages/plugins.py

@@ -13,6 +13,7 @@ html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHEC
 import dash_bootstrap_components as dbc
 from meerschaum.core import Plugin
 from meerschaum.utils.typing import Optional
+from meerschaum.api.dash.components import pages_navbar
 
 search_box = dbc.Input(
     id = "search-plugins-input",
@@ -20,32 +21,35 @@ search_box = dbc.Input(
     type = "text",
 )
 
-layout = dbc.Container([
-    html.Div([
-        html.Br(),
-        html.Div(
-            dbc.Container([
-                html.H3('Plugins'),
-                html.P([
-                    (
-                        'Plugins extend the functionality of Meerschaum.'
-                        + ' To find out more, check out the '
-                    ),
-                    html.A(
-                        'plugins documentation',
-                        href = 'https://meerschaum.io/reference/plugins/',
-                        rel = "noreferrer noopener",
-                        target = "_blank",
-                    ),
-                    '.',
+layout = [
+    pages_navbar,
+    dbc.Container([
+        html.Div([
+            html.Br(),
+            html.Div(
+                dbc.Container([
+                    html.H3('Plugins'),
+                    html.P([
+                        (
+                            'Plugins extend the functionality of Meerschaum.'
+                            ' To find out more, check out the '
+                        ),
+                        html.A(
+                            'plugins documentation',
+                            href='https://meerschaum.io/reference/plugins/',
+                            rel="noreferrer noopener",
+                            target="_blank",
+                        ),
+                        '.',
+                    ]),
                 ]),
-            ]),
-            className = 'page-header',
-            style = {'background-color': 'var(--dark)', 'padding': '1em'},
-        ),
-        html.Br(),
-        search_box,
-        html.Br(),
-        html.Div([], id='plugins-cards-div'),
+                className='page-header',
+                style={'background-color': 'var(--dark)', 'padding': '1em'},
+            ),
+            html.Br(),
+            search_box,
+            html.Br(),
+            html.Div([], id='plugins-cards-div'),
+        ])
     ])
-])
+]

meerschaum/api/dash/webterm.py

@@ -53,19 +53,22 @@ def get_webterm(state: WebState) -> Tuple[Any, Any]:
                     html.Div(
                         [
                             dbc.Button(
-                                'Refresh',
+                                "⟳",
                                 color='black',
+                                size='sm',
                                 id='webterm-refresh-button',
                             ),
                             dbc.Button(
-                                'Full View',
+                                '⛶',
                                 color='black',
+                                size='sm',
                                 id='webterm-fullscreen-button',
                             ),
                         ] + [
                             dbc.Button(
-                                'New Tab',
+                                html.B('+'),
                                 color='black',
+                                size='sm',
                                 id='webterm-new-tab-button',
                             ),
                         ] if TMUX_IS_ENABLED else [],

meerschaum/api/resources/static/css/dash.css

@@ -23,7 +23,7 @@ a {
   padding: 0.5em;
 }
 #webterm-iframe {
-  width: 102%;
+  width: 100%;
   height: 83vh !important;
   max-height: 83vh;
 }

meerschaum/api/resources/templates/termpage.html

@@ -111,6 +111,10 @@ window.addEventListener(
   false,
 );
 
+document.addEventListener('contextmenu', function(e) {
+    e.preventDefault();
+});
+
 let reconnectTimeout;
 
 function cleanUpWebSocket(socket) {