meerschaum 2.8.3__py3-none-any.whl → 2.9.0__py3-none-any.whl

meerschaum/_internal/arguments/_parser.py

@@ -367,6 +367,11 @@ groups['sync'].add_argument(
         "Only compare row-counts when verifying pipes."
     ),
 )
+groups['sync'].add_argument(
+    '--skip-hooks', action='store_true', help=(
+        "Skip calling the sync hooks."
+    )
+)
 groups['sync'].add_argument(
     '--cache', action='store_true',
     help=(

meerschaum/actions/drop.py

@@ -86,7 +86,7 @@ def _drop_pipes(
         pprint(success_dict)
 
     msg = (
-        f"Finished dropping {len(pipes)} pipes"
+        f"Finished dropping {len(pipes)} pipe"
         + ('s' if len(pipes) != 1 else '')
         + f"\n    ({successes} succeeded, {fails} failed)."
     )

meerschaum/actions/start.py

@@ -568,9 +568,9 @@ def _start_pipeline(
     Examples
     --------
 
-    `sync pipes -i sql:local + sync pipes -i sql:main :: -s 'daily'`
+    `sync pipes -i sql:local + sync pipes -i sql:main : -s 'daily'`
 
-    `show version + show arguments :: --loop`
+    `show version + show arguments : --loop`
 
     """
     import json
@@ -580,10 +580,11 @@ def _start_pipeline(
     from meerschaum.utils.warnings import info, warn
     from meerschaum.utils.misc import is_int
     from meerschaum.utils.venv import venv_exec
-    from meerschaum.utils.process import poll_process
+    from meerschaum.utils.process import poll_process, _stop_process
     fence_begin, fence_end = '<MRSM_RESULT>', '</MRSM_RESULT>'
 
-    success, msg = False, "Did not run pipeline."
+    default_msg = "Did not pipeline."
+    success, msg = False, default_msg
     def write_line(line):
         nonlocal success, msg
         decoded = line.decode('utf-8')
@@ -610,7 +611,9 @@ def _start_pipeline(
     do_n_times = (
         int(action[0].lstrip('x'))
         if action and is_int(action[0].lstrip('x'))
-        else 1
+        else (
+            1 if not loop else -1
+        )
     )
 
     params = params or {}
@@ -623,8 +626,9 @@ def _start_pipeline(
     if min_seconds is None:
         min_seconds = 1.0
 
+    proc = None
     def do_entry() -> None:
-        nonlocal success, msg
+        nonlocal success, msg, proc
         if timeout_seconds is None:
             success, msg = entry(sub_args_line, _patch_args=patch_args)
             return
@@ -674,6 +678,10 @@ def _start_pipeline(
         run_loop()
     except KeyboardInterrupt:
         warn("Cancelled pipeline.", stack=False)
+        if proc is not None:
+            _stop_process(proc)
+        if msg == default_msg:
+            msg = "Pipeline was cancelled."
 
     if do_n_times != 1:
         info(f"Ran pipeline {ran_n_times} time" + ('s' if ran_n_times != 1 else '') + '.')

meerschaum/actions/sync.py

@@ -50,6 +50,7 @@ def _pipes_lap(
     bounded: Optional[bool] = None,
     chunk_interval: Union[timedelta, int, None] = None,
     check_rowcounts_only: bool = False,
+    skip_hooks: bool = False,
     mrsm_instance: Optional[str] = None,
     timeout_seconds: Optional[int] = None,
     nopretty: bool = False,
@@ -95,6 +96,7 @@ def _pipes_lap(
         'bounded': bounded,
         'chunk_interval': chunk_interval,
         'check_rowcounts_only': check_rowcounts_only,
+        'skip_hooks': skip_hooks,
     })
     locks = {'remaining_count': Lock(), 'results_dict': Lock(), 'pipes_threads': Lock(),}
     pipes = get_pipes(
@@ -257,6 +259,7 @@ def _sync_pipes(
     bounded: Optional[bool] = None,
     chunk_interval: Union[timedelta, int, None] = None,
     check_rowcounts_only: bool = False,
+    skip_hooks: bool = False,
     shell: bool = False,
     nopretty: bool = False,
     debug: bool = False,
@@ -289,6 +292,8 @@ def _sync_pipes(
 
     noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
     noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
+    if check_rowcounts_only:
+        skip_hooks = True
 
     run = True
     msg = ""
@@ -312,6 +317,7 @@ def _sync_pipes(
                     bounded=bounded,
                     chunk_interval=chunk_interval,
                     check_rowcounts_only=check_rowcounts_only,
+                    skip_hooks=skip_hooks,
                     unblock=unblock,
                     debug=debug,
                     nopretty=nopretty,
@@ -403,6 +409,7 @@ def _wrap_pipe(
     deduplicate: bool = False,
     bounded: Optional[bool] = None,
     chunk_interval: Union[timedelta, int, None] = None,
+    skip_hooks: bool = False,
     **kw
 ):
     """
@@ -462,6 +469,8 @@ def _wrap_pipe(
 
     pre_hook_results, post_hook_results = [], []
     def apply_hooks(is_pre_sync: bool):
+        if skip_hooks:
+            return
         _sync_hooks = (_pre_sync_hooks if is_pre_sync else _post_sync_hooks)
         _hook_results = (pre_hook_results if is_pre_sync else post_hook_results)
         for module_name, sync_hooks in _sync_hooks.items():

meerschaum/api/__init__.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import os
 from collections import defaultdict
+from fnmatch import fnmatch
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Any, Optional, PipesDict
@@ -119,14 +120,19 @@ def get_api_connector(instance_keys: Optional[str] = None):
         )
 
     allowed_instance_keys = permissions_config.get(
-        'instance', {}
+        'instances', {}
     ).get(
         'allowed_instance_keys',
         ['*']
     )
-    if allowed_instance_keys != ['*'] and instance_keys not in allowed_instance_keys:
+    found_match: bool = False
+    for allowed_keys_pattern in allowed_instance_keys:
+        if fnmatch(instance_keys, allowed_keys_pattern):
+            found_match = True
+            break
+    if not found_match:
         raise APIPermissionError(
-            f"Instance keys '{instance_keys}' not in list of allowed instances."
+            f"Instance keys '{instance_keys}' does not match the allowed instances patterns."
         )
 
     with _locks[f'instance-{instance_keys}']:

meerschaum/api/_chunks.py

@@ -0,0 +1,67 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Utility functions for the retrieval, caching, and response of chunk data.
+"""
+
+import random
+from datetime import datetime, timedelta, timezone
+from typing import Dict, Generator, Any, Union, Optional, List
+
+import meerschaum as mrsm
+from meerschaum.api import get_cache_connector
+from meerschaum.utils.misc import generate_password
+
+CHUNKS_TOKENS_GENERATORS: Dict[str, Dict[str, Union[Generator[Any, None, None], datetime, int]]]
+DEFAULT_TTL_SECONDS = mrsm.get_config('system', 'api', 'data', 'chunks', 'ttl_seconds')
+
+
+def generate_chunks_cursor_token(
+    pipe: mrsm.Pipe,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    limit: Optional[int] = None,
+    order: Optional[str] = 'asc',
+    ttl_seconds: Optional[int] = None,
+    debug: bool = False,
+) -> str:
+    """
+    Store a generator in the `CHUNKS_TOKENS_GENERATORS`
+    """
+    now = datetime.now(timezone.utc)
+    cache_connector = get_cache_connector()
+    if cache_connector is None:
+        pass
+
+    ttl_seconds = ttl_seconds or DEFAULT_TTL_SECONDS
+    chunk_bounds = pipe.get_chunk_bounds(
+        begin=begin,
+        end=end,
+        bounded=True,
+    )
+
+    while True:
+        chunk_token = prefix + generate_password(random.randint(6, 12))
+        if chunk_token in CHUNKS_TOKENS_GENERATORS:
+            continue
+        break
+
+    CHUNKS_TOKENS_GENERATORS[chunk_token] = {
+        'generator': chunk_generator,
+        'created': now,
+        'ttl': ttl_seconds,
+        'last_accessed': now,
+    }
+
+    return chunk_token
+
+
+def deallocate_expired_generators():
+    """
+    Periodically delete chunk tokens with an expired ttl timestamp.
+    """
+    chunk_tokens = list(CHUNKS_TOKENS_GENERATORS)

meerschaum/api/dash/callbacks/__init__.py

@@ -6,13 +6,16 @@
 Define callbacks for pages.
 """
 
+from meerschaum.api import debug as _debug
 import meerschaum.api.dash.callbacks.dashboard
 import meerschaum.api.dash.callbacks.login
 import meerschaum.api.dash.callbacks.plugins
 import meerschaum.api.dash.callbacks.jobs
 import meerschaum.api.dash.callbacks.register
 import meerschaum.api.dash.callbacks.pipes
+import meerschaum.api.dash.callbacks.settings
 from meerschaum.api.dash.callbacks.custom import init_dash_plugins, add_plugin_pages
 
-init_dash_plugins()
-add_plugin_pages()
+init_dash_plugins(debug=_debug)
+add_plugin_pages(debug=_debug)
+

meerschaum/api/dash/callbacks/custom.py

@@ -7,13 +7,16 @@ Import custom callbacks created by plugins.
 """
 
 import traceback
+from typing import Any, Dict
+
 from meerschaum.api.dash import dash_app
 from meerschaum.plugins import _dash_plugins, _plugin_endpoints_to_pages
-from meerschaum.utils.warnings import warn
-from meerschaum.api.dash.callbacks.dashboard import _paths, _required_login
+from meerschaum.utils.warnings import warn, dprint
+from meerschaum.api.dash.callbacks.dashboard import _paths, _required_login, _pages
+from meerschaum.api.dash.components import pages_navbar
 
 
-def init_dash_plugins():
+def init_dash_plugins(debug: bool = False):
     """
     Fire the initial callbacks for Dash plugins.
     """
@@ -29,11 +32,21 @@ def init_dash_plugins():
                 )
 
 
-def add_plugin_pages():
+def add_plugin_pages(debug: bool = False):
     """
     Allow users to add pages via the `@web_page` decorator.
     """
-    for _endpoint, _page_dict in _plugin_endpoints_to_pages.items():
-        _paths[_endpoint] = _page_dict['function']()
-        if _page_dict['login_required']:
-            _required_login.add(_endpoint)
+    for plugin_name, pages_dicts in _plugin_endpoints_to_pages.items():
+        if debug:
+            dprint(f"Adding pages from plugin '{plugin_name}'...")
+        for _endpoint, _page_dict in pages_dicts.items():
+            page_layout = _page_dict['function']()
+            if not _page_dict['skip_navbar']:
+                if isinstance(page_layout, list):
+                    page_layout = [pages_navbar] + page_layout
+                else:
+                    page_layout = [pages_navbar, page_layout]
+            _pages[_page_dict['page_key']] = _endpoint
+            _paths[_endpoint] = page_layout
+            if _page_dict['login_required']:
+                _required_login.add(_endpoint)

meerschaum/api/dash/callbacks/dashboard.py

@@ -33,7 +33,10 @@ from meerschaum.api.dash.users import get_users_cards
 from meerschaum.api.dash.graphs import get_graphs_cards
 from meerschaum.api.dash.webterm import get_webterm
 from meerschaum.api.dash.components import (
-    alert_from_success_tuple, console_div, build_cards_grid,
+    alert_from_success_tuple,
+    console_div,
+    build_cards_grid,
+    build_pages_offcanvas_children,
 )
 from meerschaum.api.dash import pages
 from meerschaum.utils.typing import Dict
@@ -103,6 +106,10 @@ _paths = {
     'job'     : pages.job.layout,
 }
 _required_login = {''}
+_pages = {
+    'Web Console': '/dash/',
+    'Plugins': '/dash/plugins',
+}
 
 
 @dash_app.callback(
@@ -696,7 +703,6 @@ dash_app.clientside_callback(
 dash_app.clientside_callback(
     """
     function(n_clicks){
-        console.log('fullscreen');
         if (!n_clicks) { return dash_clientside.no_update; }
         iframe = document.getElementById('webterm-iframe');
         if (!iframe){ return dash_clientside.no_update; }
@@ -707,11 +713,11 @@ dash_app.clientside_callback(
         if (leftCol.style.display === 'none') {
             leftCol.style.display = '';
             rightCol.className = 'col-6';
-            button.innerHTML = "Full View";
+            button.innerHTML = "⛶";
         } else {
             leftCol.style.display = 'none';
             rightCol.className = 'col-12';
-            button.innerHTML = "Side-by-side View";
+            button.innerHTML = "🀲";
         }
 
         return dash_clientside.no_update;
@@ -1078,3 +1084,19 @@ def parameters_as_yaml_or_json_click(
     if as_yaml:
         return yaml.dump(pipe.parameters)
     return json.dumps(pipe.parameters, indent=4, separators=(',', ': '), sort_keys=True)
+
+
+@dash_app.callback(
+    Output('pages-offcanvas', 'is_open'),
+    Output('pages-offcanvas', 'children'),
+    Input('logo-img', 'n_clicks'),
+    State('pages-offcanvas', 'is_open'),
+)
+def toggle_pages_offcanvas(n_clicks: Optional[int], is_open: bool):
+    """
+    Toggle the pages sidebar.
+    """
+    pages_children = build_pages_offcanvas_children()
+    if n_clicks:
+        return not is_open, pages_children
+    return is_open, pages_children

meerschaum/api/dash/callbacks/settings/__init__.py

@@ -0,0 +1,8 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define the callbacks for the settings pages.
+"""
+
+import meerschaum.api.dash.callbacks.settings.password_reset

meerschaum/api/dash/callbacks/settings/password_reset.py

@@ -0,0 +1,76 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define the callbacks for the password reset page.
+"""
+
+import dash
+from dash.exceptions import PreventUpdate
+from dash.dependencies import Input, Output, State
+import dash_bootstrap_components as dbc
+
+from meerschaum.core.User import User
+from meerschaum.config.static import STATIC_CONFIG
+from meerschaum.api import get_api_connector, debug
+from meerschaum.api.dash import dash_app
+from meerschaum.api.dash.components import alert_from_success_tuple
+from meerschaum.api.dash.sessions import get_username_from_session
+
+
+@dash_app.callback(
+    Output('password-reset-alert-div', 'children'),
+    Input('password-reset-button', 'n_clicks'),
+    State('password-reset-input', 'value',),
+    State('session-store', 'data'),
+)
+def password_reset_button_click(n_clicks, new_password_value, session_store_data):
+    """
+    Attempt the password reset with the form data.
+    """
+    if not n_clicks:
+        raise PreventUpdate
+
+    session_id = session_store_data.get('session-id', None)
+    username = get_username_from_session(session_id)
+    if not username:
+        return alert_from_success_tuple(
+            (False, "Invalid session. Are you logged in correctly?")
+        )
+
+    instance_connector = get_api_connector()
+    user = User(username, new_password_value)
+    success, msg = instance_connector.edit_user(user, debug=debug)
+    return alert_from_success_tuple((success, msg))
+
+
+@dash_app.callback(
+    Output('password-reset-input', 'valid'),
+    Output('password-reset-input', 'invalid'),
+    Input('password-reset-input', 'value'),
+)
+def validate_new_password(new_password_value):
+    if not new_password_value:
+        raise PreventUpdate
+
+    valid = (len(new_password_value) >= STATIC_CONFIG['users']['min_password_length'])
+    return valid, not valid
+ 
+
+@dash_app.callback(
+    Output("password-reset-confirm-input", "valid"),
+    Output("password-reset-confirm-input", "invalid"),
+    Output("password-reset-button", 'disabled'),
+    Input("password-reset-confirm-input", "value"),
+    State("password-reset-input", "value"),
+)
+def validate_new_passwords_match(confirm_password_value, new_password_value):
+    if not confirm_password_value:
+        raise PreventUpdate
+
+    new_password_is_valid, _ = validate_new_password(new_password_value)
+    if not new_password_is_valid:
+        raise PreventUpdate
+
+    valid = confirm_password_value == new_password_value
+    return valid, not valid, not valid

meerschaum/api/dash/components.py

@@ -92,9 +92,9 @@ search_parameters_editor = dash_ace.DashAceEditor(
     style={'height': 100},
 )
 
-sidebar = dbc.Offcanvas(
-    children=[],
-    title='Pages',
+pages_offcanvas = dbc.Offcanvas(
+    title='',
+    id='pages-offcanvas',
 )
 
 download_dataframe = dcc.Download(id='download-dataframe-csv')
@@ -110,38 +110,71 @@ instance_select = dbc.Select(
     class_name='dbc_dark custom-select custom-select-sm',
 )
 
+sign_out_button = dbc.Button(
+    "Sign out",
+    color='link',
+    style={'margin-left': '30px'},
+    id='sign-out-button',
+)
+
+logo_row = dbc.Row(
+    [
+        dbc.Col(
+            html.Img(
+                src=endpoints['dash'] + "/assets/logo_48x48.png",
+                title=doc,
+                id="logo-img",
+                style={'cursor': 'pointer'},
+            ),
+        ),
+    ],
+    align='center',
+    className='g-0 navbar-logo-row',
+)
+
+pages_navbar = html.Div(
+    [
+        pages_offcanvas,
+        dbc.Navbar(
+            dbc.Container(
+                [
+                    logo_row,
+                    dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
+                    dbc.Collapse(
+                        dbc.Row(
+                            [
+                                dbc.Col(
+                                    sign_out_button,
+                                    className="ms-auto",
+                                ),
+                            ],
+                            className="g-0 ms-auto flex-nowrap mt-3 mt-md-0",
+                        ),
+                        id='navbar-collapse',
+                        is_open=False,
+                        navbar=True,
+                    ),
+                ]
+            ),
+            dark=True,
+            color='dark'
+        ),
+    ],
+    id='pages-navbar-div',
+)
 
 navbar = dbc.Navbar(
     dbc.Container(
         [
-            html.A(
-                dbc.Row(
-                    [
-                        dbc.Col(
-                            html.Img(
-                                src=endpoints['dash'] + "/assets/logo_48x48.png",
-                                title=doc,
-                            ),
-                        ),
-                    ],
-                    align='center',
-                    className='g-0 navbar-logo-row',
-                ),
-                href=('/docs' if docs_enabled else '#'),
-                style={"textDecoration": "none"},
-            ),
+            logo_row,
             dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
             dbc.Collapse(
                 dbc.Row(
                     [
                         dbc.Col(instance_select),
                         dbc.Col(
-                            dbc.Button(
-                                "Sign out",
-                                color='link',
-                                style={'margin-left': '30px'},
-                                id='sign-out-button',
-                            ),
+                            sign_out_button,
+                            className="ms-auto",
                         ),
                     ],
                     className="g-0 ms-auto flex-nowrap mt-3 mt-md-0",
@@ -199,3 +232,81 @@ def build_cards_grid(cards: List[dbc.Card], num_columns: int = 3) -> html.Div:
         rows.append(r)
         rows.append(html.Br())
     return html.Div(rows)
+
+
+def build_pages_offcanvas_children():
+    """
+    Return the contents of the pages offcanvas.
+    """
+    from meerschaum.api.dash.callbacks.dashboard import _pages
+    from meerschaum.api.dash.callbacks.custom import _plugin_endpoints_to_pages
+    pages_listgroup_items = []
+    custom_pages = []
+    for pages_dicts in _plugin_endpoints_to_pages.values():
+        for page_dict in pages_dicts.values():
+            custom_pages.append(page_dict['page_key'])
+
+    for page_key, page_href in _pages.items():
+        if page_key in custom_pages:
+            continue
+        pages_listgroup_items.append(
+            dbc.ListGroupItem(
+                dbc.Button(
+                    ' '.join([word.capitalize() for word in page_key.split(' ')]),
+                    style={
+                        'width': '100%',
+                        'text-align': 'left',
+                        'text-decoration': 'none',
+                        'padding-left': '0',
+                    },
+                    href=page_href,
+                    color='dark',
+                )
+            )
+        )
+
+    plugins_accordion_items = []
+    for page_group, pages_dicts in _plugin_endpoints_to_pages.items():
+        plugin_listgroup_items = [
+            dbc.ListGroupItem(
+                dbc.Button(
+                    ' '.join([word.capitalize() for word in page_dict['page_key'].split(' ')]),
+                    style={
+                        'width': '100%',
+                        'text-align': 'left',
+                        'text-decoration': 'none',
+                        'padding-left': '0',
+                    },
+                    href=page_href,
+                    color='dark',
+                )
+            )
+            for page_href, page_dict in pages_dicts.items()
+        ]
+        plugin_listgroup = dbc.ListGroup(plugin_listgroup_items, flush=True)
+        plugin_accordion_item = dbc.AccordionItem(
+            plugin_listgroup,
+            title=(
+                page_group.capitalize()
+                if page_group and not page_group[0].isupper()
+                else page_group
+            ),
+            class_name='pages-offcanvas-accordion',
+        )
+        plugins_accordion_items.append(plugin_accordion_item)
+    plugins_accordion = dbc.Accordion(
+        plugins_accordion_items,
+        start_collapsed=True,
+        flush=True,
+        always_open=True,
+    )
+    pages_listgroup_items.append(plugins_accordion)
+
+    pages_children = dbc.Card(
+        dbc.ListGroup(
+            pages_listgroup_items,
+            flush=True,
+        ),
+        outline=True,
+    )
+    return pages_children

meerschaum/api/dash/pages/__init__.py

@@ -12,3 +12,4 @@ import meerschaum.api.dash.pages.plugins
 import meerschaum.api.dash.pages.register
 import meerschaum.api.dash.pages.pipes
 import meerschaum.api.dash.pages.job
+import meerschaum.api.dash.pages.settings