meerschaum 3.0.0rc1__py3-none-any.whl → 3.0.0rc3__py3-none-any.whl

meerschaum/api/dash/pages/{job.py → jobs.py}

@@ -11,11 +11,14 @@ from meerschaum.utils.packages import import_html, import_dcc
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 import dash_bootstrap_components as dbc
 
-from meerschaum.api.dash.components import download_logs, refresh_jobs_interval
+from meerschaum.api.dash.components import download_logs, refresh_jobs_interval, pages_navbar
 
-layout = dbc.Container([
-    dcc.Location('job-location'),
-    html.Div(id='job-output-div'),
-    download_logs,
-    refresh_jobs_interval,
-])
+layout = [
+    pages_navbar,
+    dbc.Container([
+        dcc.Location('job-location'),
+        html.Div(id='job-output-div'),
+        download_logs,
+        refresh_jobs_interval,
+    ]),
+]

meerschaum/api/dash/pages/pipes.py

@@ -7,16 +7,17 @@ Display pipes via a shareable URL.
 
 from meerschaum.api import CHECK_UPDATE
 from meerschaum.utils.packages import import_html, import_dcc
-from meerschaum.api.dash.components import download_dataframe, pages_navbar
+from meerschaum.api.dash.components import download_dataframe, pages_navbar, navbar
 
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
 import dash_bootstrap_components as dbc
 
+
 layout = [
     pages_navbar,
+    dcc.Location('pipes-location'),
+    download_dataframe,
     dbc.Container([
-        dcc.Location('pipes-location'),
-        download_dataframe,
         html.Div(id='pipe-output-div'),
     ])
 ]

meerschaum/api/dash/pages/settings/__init__.py

@@ -6,4 +6,3 @@ Define the layouts for the `settings` pages.
 """
 
 import meerschaum.api.dash.pages.settings.password_reset
-import meerschaum.api.dash.pages.settings.tokens

meerschaum/api/dash/pages/{settings/tokens.py → tokens.py}

@@ -8,16 +8,13 @@ Define the tokens page layout.
 import dash_bootstrap_components as dbc
 import dash.html as html
 import dash.dcc as dcc
-from meerschaum.plugins import web_page
 from meerschaum._internal.static import STATIC_CONFIG
+from meerschaum.api.dash.components import pages_navbar
 
 
-@web_page('tokens', login_required=True, page_group='Settings')
-def page_layout():
-    """
-    Return the layout for the tokens page.
-    """
-    return dbc.Container([
+layout = [
+    pages_navbar,
+    dbc.Container([
         html.Br(), 
         html.H3('Tokens'),
         html.Div(id="tokens-alert-div"),
@@ -52,4 +49,5 @@ def page_layout():
             style={'text-align': 'right'},
         ),
         html.Div(id='tokens-output-div'),
-    ])
+    ]),
+]

meerschaum/api/dash/pipes.py

@@ -12,6 +12,7 @@ import shlex
 from textwrap import dedent
 from urllib.parse import urlencode
 
+from meerschaum.utils import fetch_pipes_keys
 from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
 from meerschaum.utils.misc import string_to_dict
 from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
@@ -817,3 +818,133 @@ def get_backtrack_text(
         json_text = '[]'
 
     return json.dumps(json.loads(json_text), indent=4, separators=(',', ': '))
+
+
+def build_pipes_dropdown_keys_row(
+    connector_keys: List[str],
+    metric_keys: List[str],
+    location_keys: List[str],
+    tags: List[str],
+    pipes: List[mrsm.Pipe],
+    instance_connector: mrsm.connectors.InstanceConnector,
+) -> dbc.Row:
+    """
+    Return the dropdown keys row for the dedicated pipes page.
+    """
+    ck_alone = connector_keys and not any([str(x) for x in (tags + metric_keys + location_keys)])
+    mk_alone = metric_keys and not any([str(x) for x in (connector_keys + tags + location_keys)])
+    lk_alone = location_keys and not any([str(x) for x in (connector_keys + metric_keys + tags)])
+    all_keys = fetch_pipes_keys('registered', instance_connector)
+    return dbc.Row(
+        [
+            dbc.Col(
+                html.Div(
+                    [
+                        dcc.Dropdown(
+                            id='pipes-connector-keys-dropdown',
+                            options=(
+                                sorted(list({pipe.connector_keys for pipe in pipes}))
+                                if not ck_alone
+                                else sorted(list({ck for ck, _, _ in all_keys}))
+                            ),
+                            value=[str(ck) for ck in connector_keys],
+                            placeholder='Connectors',
+                            multi=True,
+                        ),
+                    ],
+                    className='dbc_dark',
+                ),
+                lg=4,
+                md=12,
+                sm=12,
+            ),
+            dbc.Col(
+                html.Div(
+                    [
+                        dcc.Dropdown(
+                            id='pipes-metric-keys-dropdown',
+                            options=(
+                                sorted(list({pipe.metric_key for pipe in pipes}))
+                                if not mk_alone
+                                else sorted(list({mk for _, mk, _ in all_keys}))
+                            ),
+                            value=[str(mk) for mk in metric_keys],
+                            placeholder='Metrics',
+                            multi=True,
+                        ),
+                    ],
+                    className='dbc_dark'
+                ),
+                lg=4,
+                md=12,
+                sm=12,
+            ),
+            dbc.Col(
+                html.Div(
+                    [
+                        dcc.Dropdown(
+                            id='pipes-location-keys-dropdown',
+                            options=(
+                                sorted(list({str(pipe.location_key) for pipe in pipes}))
+                                if not lk_alone
+                                else sorted(list({str(lk) for _, _, lk in all_keys}))
+                            ),
+                            value=[str(lk) for lk in location_keys],
+                            placeholder='Locations',
+                            multi=True,
+                        ),
+                    ],
+                    className='dbc_dark'
+                ),
+                lg=4,
+                md=12,
+                sm=12,
+            ),
+        ] ### end of filters row children
+    )
+
+
+def build_pipes_tags_dropdown(
+    connector_keys: List[str],
+    metric_keys: List[str],
+    location_keys: List[str],
+    tags: List[str],
+    instance: str,
+) -> html.Div:
+    """
+    Build the tags dropdown for the dedicated pipes page.
+    """
+    _tags_alone = tags and not any([str(x) for x in (connector_keys + metric_keys + location_keys)])
+    _tags_pipes = mrsm.get_pipes(
+        connector_keys=connector_keys,
+        metric_keys=metric_keys,
+        location_keys=location_keys,
+        tags=tags,
+        instance=instance,
+        as_tags_dict=True,
+    )
+
+    _all_tags = list(
+        mrsm.get_pipes(
+            instance=instance,
+            as_tags_dict=True,
+        )
+    ) if _tags_alone else []
+
+    tags_options = [
+        str(tag)
+        for tag in (_all_tags if _tags_alone else _tags_pipes)
+    ]
+
+    return html.Div(
+        dcc.Dropdown(
+            id='pipes-tags-dropdown',
+            options=tags_options,
+            value=tags,
+            placeholder='Tags',
+            multi=True,
+            searchable=True,
+        ),
+        className="dbc_dark",
+        id="pipes-tags-dropdown-div",
+    )

meerschaum/api/dash/tokens.py

@@ -9,7 +9,7 @@ Dash utility functions for constructing tokens components.
 from __future__ import annotations
 
 from datetime import datetime, timezone, timedelta
-from typing import Any, List
+from typing import Any, List, Optional
 
 import meerschaum as mrsm
 from meerschaum.api import debug, CHECK_UPDATE, get_api_connector
@@ -18,8 +18,8 @@ from meerschaum.api.dash.components import alert_from_success_tuple
 from meerschaum.api.dash.sessions import get_user_from_session
 from meerschaum.utils.typing import WebState, SuccessTuple, List, Tuple
 from meerschaum.utils.packages import attempt_import, import_html, import_dcc
-from meerschaum.utils.misc import interval_str, round_time
-from meerschaum.utils.dtypes import value_is_null
+from meerschaum.utils.misc import interval_str
+from meerschaum.utils.dtypes import value_is_null, round_time
 from meerschaum._internal.static import STATIC_CONFIG
 from meerschaum.core import Token
 from meerschaum.utils.daemon import get_new_daemon_name
@@ -91,39 +91,36 @@ def build_manage_token_popover(token: Token) -> dbc.Popover:
     """
     return dbc.Popover(
         [
-            dbc.PopoverHeader(["Manage token"]),
-            dbc.PopoverBody([
-                dbc.ButtonGroup(
-                    ([
-                        dbc.Button(
-                            "Edit",
-                            outline=True,
-                            color='light',
-                            id={
-                                'type': 'tokens-edit-button',
-                                'index': str(token.id),
-                            },
-                        ),
-                        dbc.Button(
-                            "Invalidate",
-                            outline=True,
-                            color='warning',
-                            id={
-                                'type': 'tokens-invalidate-button',
-                                'index': str(token.id),
-                            },
-                        ),
-                    ] if token.is_valid else []) + [
+            dbc.ButtonGroup(
+                ([
                     dbc.Button(
-                        "Delete",
-                        color='danger',
+                        "Edit",
                         outline=True,
+                        color='light',
                         id={
-                            'type': 'tokens-delete-button',
+                            'type': 'tokens-edit-button',
                             'index': str(token.id),
                         },
                     ),
-                ]),
+                    dbc.Button(
+                        "Invalidate",
+                        outline=True,
+                        color='warning',
+                        id={
+                            'type': 'tokens-invalidate-button',
+                            'index': str(token.id),
+                        },
+                    ),
+                ] if token.is_valid else []) + [
+                dbc.Button(
+                    "Delete",
+                    color='danger',
+                    outline=True,
+                    id={
+                        'type': 'tokens-delete-button',
+                        'index': str(token.id),
+                    },
+                ),
             ]),
         ],
         body=True,
@@ -600,7 +597,7 @@ def build_tokens_register_output_modal(token: Token) -> List[Any]:
             dbc.Button(
                 "Close",
                 id='tokens-close-register-output-modal-button',
-                disabled=True,
+                disabled=success,
             ),
         ]),
     ]

meerschaum/api/routes/_pipes.py

@@ -23,9 +23,7 @@ from meerschaum.api import (
     pipes,
     get_pipe,
     _get_pipes,
-    manager,
     debug,
-    no_auth,
     ScopedAuth,
 )
 from meerschaum.models import (
@@ -74,7 +72,7 @@ def register_pipe(
     instance_keys: Optional[str] = None,
     parameters: Optional[Dict[str, Any]] = None,
     curr_user = fastapi.Security(ScopedAuth(['pipes:write']), scopes=['pipes:write']),
-) -> SuccessTupleResponseModel:
+) -> mrsm.SuccessTuple:
     """
     Register a new pipe.
     """
@@ -94,8 +92,9 @@ def register_pipe(
         )
     if parameters:
         pipe.parameters = parameters
+
     success, msg = get_api_connector(instance_keys).register_pipe(pipe, debug=debug)
-    pipes(instance_keys, refresh=True)
+    _ = pipes(instance_keys, refresh=True)
     return success, msg
 
 
@@ -112,7 +111,7 @@ def edit_pipe(
     instance_keys: Optional[str] = None,
     patch: bool = False,
     curr_user = fastapi.Security(ScopedAuth(['pipes:write'])),
-) -> SuccessTupleResponseModel:
+) -> mrsm.SuccessTuple:
     """
     Edit an existing pipe's parameters.
     """
@@ -125,14 +124,16 @@ def edit_pipe(
             " Under the keys `api:permissions:actions`, "
             "you can toggle non-admin actions."
         )
+
     pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
     if not is_pipe_registered(pipe, pipes(instance_keys, refresh=True)):
         raise fastapi.HTTPException(
             status_code=409, detail=f"{pipe} is not registered."
         )
+
     pipe.parameters = parameters
     success, msg = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
-    pipes(instance_keys, refresh=True)
+    _ = pipes(instance_keys, refresh=True)
     return success, msg
 
 
@@ -147,7 +148,7 @@ def delete_pipe(
     location_key: str,
     instance_keys: Optional[str] = None,
     curr_user = fastapi.Security(ScopedAuth(['pipes:delete'])),
-) -> SuccessTupleResponseModel:
+) -> SuccessTuple:
     """
     Delete a Pipe (without dropping its table).
     """
@@ -156,9 +157,10 @@ def delete_pipe(
         raise fastapi.HTTPException(
             status_code=409, detail=f"{pipe} is not registered."
         )
-    results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
-    pipes(instance_keys, refresh=True)
-    return results
+    
+    success, msg = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
+    _ = pipes(instance_keys, refresh=True)
+    return success, msg
 
 
 @app.get(
@@ -207,6 +209,7 @@ async def get_pipes(
         kw['metric_keys'] = metric_keys
     if location_keys != "":
         kw['location_keys'] = location_keys
+
     pipes_dict = replace_pipes_in_dict(_get_pipes(**kw), lambda p: p.attributes)
     for metrics in pipes_dict.values():
         for locations in metrics.values():
@@ -224,14 +227,19 @@ async def get_pipes_by_connector(
     """
     Get all registered Pipes by connector_keys with metadata, excluding parameters.
     """
-    if connector_keys not in pipes(instance_keys):
+    if connector_keys not in pipes(instance_keys, refresh=True):
         raise fastapi.HTTPException(
             status_code=404, detail=f"Connector '{connector_keys}' not found."
         )
-    metrics = replace_pipes_in_dict(pipes(instance_keys)[connector_keys], lambda p: p.attributes)
+
+    metrics = replace_pipes_in_dict(
+        pipes(instance_keys, refresh=False)[connector_keys],
+        lambda p: p.attributes
+    )
     for locations in metrics.values():
         if None in locations:
             locations['None'] = locations.pop(None)
+
     return metrics
 
 
@@ -245,22 +253,26 @@ async def get_pipes_by_connector_and_metric(
     """
     Get all registered Pipes by `connector_keys` and `metric_key` with metadata, excluding parameters.
     """
-    if connector_keys not in pipes(instance_keys):
+    if connector_keys not in pipes(instance_keys, refresh=True):
         raise fastapi.HTTPException(
             status_code=404,
             detail=f"Connector '{connector_keys}' not found.",
         )
-    if metric_key not in pipes(instance_keys)[connector_keys]:
+
+    if metric_key not in pipes(instance_keys, refresh=False)[connector_keys]:
         raise fastapi.HTTPException(
             status_code=404,
             detail=f"Metric '{metric_key}' not found.",
         )
+
     locations = replace_pipes_in_dict(
-        pipes(instance_keys)[connector_keys][metric_key],
+        pipes(instance_keys, refresh=False)[connector_keys][metric_key],
         lambda p: p.attributes
     )
+
     if None in locations:
         locations['None'] = locations.pop(None)
+
     return locations
 
 
@@ -278,22 +290,25 @@ async def get_pipe_by_connector_and_metric_and_location(
     """
     Get a specific Pipe with metadata, excluding parameters.
     """
-    if connector_keys not in pipes(instance_keys):
+    if connector_keys not in pipes(instance_keys, refresh=True):
         raise fastapi.HTTPException(
             status_code=404,
             detail=f"Connector '{connector_keys}' not found.",
         )
-    if metric_key not in pipes(instance_keys)[connector_keys]:
+
+    if metric_key not in pipes(instance_keys, refresh=False)[connector_keys]:
         raise fastapi.HTTPException(status_code=404, detail=f"Metric '{metric_key}' not found.")
+
     if location_key in ('[None]', 'None', 'null'):
         location_key = None
-    if location_key not in pipes(instance_keys)[connector_keys][metric_key]:
+
+    if location_key not in pipes(instance_keys, refresh=False)[connector_keys][metric_key]:
         raise fastapi.HTTPException(
             status_code=404,
             detail=f"location_key '{location_key}' not found."
         )
 
-    return pipes(instance_keys)[connector_keys][metric_key][location_key].attributes
+    return pipes(instance_keys, refresh=False)[connector_keys][metric_key][location_key].attributes
 
 
 @app.get(
@@ -397,7 +412,7 @@ async def sync_pipe(
             detail="Cannot sync given data.",
         )
 
-    pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys, refresh=True)
     if pipe.target in ('mrsm_users', 'mrsm_plugins', 'mrsm_pipes', 'mrsm_tokens'):
         raise fastapi.HTTPException(
             status_code=409,
@@ -616,23 +631,14 @@ def drop_pipe(
     location_key: str,
     instance_keys: Optional[str] = None,
     curr_user = fastapi.Security(ScopedAuth(['pipes:drop'])),
-) -> SuccessTupleResponseModel:
+) -> mrsm.SuccessTuple:
     """
     Drop a pipe's target table.
     """
-    allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
-    if not allow_actions:
-        return False, (
-            "The administrator for this server has not allowed actions.\n\n"
-            "Please contact the system administrator, or if you are running this server, "
-            "open the configuration file with `edit config system` and search for 'permissions'."
-            " Under the keys `api:permissions:actions`, " + 
-            "you can toggle non-admin actions."
-        )
-    pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
-    results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
-    pipes(instance_keys, refresh=True)
-    return results
+    pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    success, msg = pipe.drop(debug=debug)
+    _ = pipes(instance_keys, refresh=True)
+    return success, msg
 
 
 @app.delete(
@@ -661,6 +667,7 @@ def clear_pipe(
             _params = json.loads(params)
         except Exception:
             _params = None
+
     if not isinstance(_params, dict):
         raise fastapi.HTTPException(
             status_code=409,
@@ -676,7 +683,7 @@ def clear_pipe(
         params=_params,
         debug=debug,
     )
-    pipes(instance_keys, refresh=True)
+    _ = pipes(instance_keys, refresh=True)
     return results
 
 
@@ -894,8 +901,11 @@ def get_pipe_columns_types(
     ```
     """
     pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
-    columns_types = pipe.get_columns_types(debug=debug)
-    return pipe.dtypes
+    columns_types = (
+        pipe.get_columns_types(debug=debug)
+        or pipe.get_dtypes(refresh=True, debug=debug)
+    )
+    return columns_types
 
 
 @app.get(

meerschaum/config/_default.py

@@ -99,6 +99,7 @@ default_system_config = {
                 'mssql': True,
             },
             'instance': {
+                'create_metadata_cache_minutes': 14400,
                 'stale_temporary_tables_minutes': 1440,
                 'temporary_target': {
                     'prefix': '_',
@@ -199,14 +200,25 @@ default_pipes_config = {
         },
     },
     'attributes': {
-        'local_cache_timeout_seconds': 60,
+        'local_cache_timeout_seconds': 600.0,
     },
     'sync': {
         'filter_params_index_limit': 250,
+        'exists_cache_seconds': 60.0,
     },
     'verify': {
         'max_chunks_syncs': 3,
     },
+    'autotime': {
+        'column_name_if_datetime_missing': 'ts',
+    },
+    'dtypes': {
+        'min_ratio_columns_changed_for_full_astype': 0.5,
+        'columns_types_cache_seconds': 60.0,
+    },
+    'static': {
+        'static_schema_cache_seconds': 3600.0,
+    },
 }
 default_plugins_config = {}
 default_experimental_config = {
@@ -214,7 +226,6 @@ default_experimental_config = {
 }
 
 
-
 ### build default config dictionary
 default_config = {}
 default_config['meerschaum'] = default_meerschaum_config

meerschaum/config/_paths.py

@@ -149,6 +149,7 @@ paths = {
     'PIPES_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'pipes'),
     'USERS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'users'),
     'VENVS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'venvs'),
+    'SQL_CONN_CACHE_RESOURCES_PATH'  : ('{CACHE_RESOURCES_PATH}', 'sql'),
 
     'PLUGINS_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
     'PLUGINS_INTERNAL_LOCK_PATH'     : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "3.0.0rc1"
+__version__ = "3.0.0rc3"

meerschaum/config/stack/__init__.py

@@ -122,9 +122,9 @@ default_docker_compose_config = {
                 'POSTGRES_DB': '<DOLLAR>POSTGRES_DB',
                 'POSTGRES_PASSWORD': '<DOLLAR>POSTGRES_PASSWORD',
                 'ALLOW_IP_RANGE': env_dict['ALLOW_IP_RANGE'],
-                'POSTGRES_INITDB_ARGS': '-c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100'
+                #  'POSTGRES_INITDB_ARGS': '-c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100'
             },
-            #  'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB',
+            'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100',
             'healthcheck': {
                 'test': [
                     'CMD-SHELL', 'pg_isready -d <DOLLAR>POSTGRES_DB -U <DOLLAR>POSTGRES_USER',
@@ -305,19 +305,20 @@ def get_necessary_files():
 
 
 def write_stack(
-        debug: bool = False 
-    ):
+    debug: bool = False 
+):
     """Write Docker Compose configuration files."""
     from meerschaum.config._edit import general_write_yaml_config
     from meerschaum.config._sync import sync_files
     general_write_yaml_config(get_necessary_files(), debug=debug)
     return sync_files(['stack'])
    
+
 def edit_stack(
-        action: Optional[List[str]] = None,
-        debug: bool = False,
-        **kw
-    ):
+    action: Optional[List[str]] = None,
+    debug: bool = False,
+    **kw
+):
     """Open docker-compose.yaml or .env for editing."""
     from meerschaum.config._edit import general_edit_config
     if action is None: