meerschaum 2.6.0.dev1__py3-none-any.whl → 2.6.1__py3-none-any.whl

meerschaum/api/dash/pages/login.py

@@ -19,13 +19,13 @@ allow_user_registration = permissions_config['registration']['users']
 
 registration_div = html.Div(
     id = 'registration-div',
-    style = {'height' : '100%'},
-    children = (
+    style={'height' : '100%'},
+    children=(
         [
             dcc.Link(
                 'No account? Create one here.',
-                href = (dash_endpoint + '/register'),
-                refresh = False
+                href=(dash_endpoint + '/register'),
+                refresh=False
             ),
         ] if allow_user_registration
         else [
@@ -36,12 +36,12 @@ registration_div = html.Div(
             dbc.Button(
                 'More information.',
                 id = 'show-user-registration-disabled-button',
-                color = 'link',
-                size = 'sm',
+                color='link',
+                size='sm',
             ),
             dbc.Collapse(
-                id = 'user-registration-disabled-collapse',
-                children = [
+                id='user-registration-disabled-collapse',
+                children=[
                     dcc.Markdown(
                         "For example, to register user `newuser` on instance `sql:main`:"
                     ),
@@ -60,10 +60,10 @@ registration_div = html.Div(
                     html.Pre(
                         html.Code(
                             json.dumps({
-                                'api' : {
-                                    'permissions' : {
-                                        'registration' : {
-                                            'users' : True,
+                                'api': {
+                                    'permissions': {
+                                        'registration': {
+                                            'users': True,
                                         },
                                     }
                                 }
@@ -112,11 +112,11 @@ layout = dbc.Container([
                 dbc.Row([
                     dbc.Col([
                         html.Button(
-                            children = 'Login',
-                            n_clicks = 0,
-                            type = 'submit',
-                            id = 'login-button',
-                            className = 'btn btn-primary btn-lg'
+                            children='Login',
+                            n_clicks=0,
+                            type='submit',
+                            id='login-button',
+                            className='btn btn-primary btn-lg'
                         ),
                     ]),
                 ]),

meerschaum/api/dash/pipes.py

@@ -33,6 +33,11 @@ html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHEC
 humanfriendly = attempt_import('humanfriendly', check_update=CHECK_UPDATE)
 pd = import_pandas()
 
+INDEX_PREFIX_EMOJI: Dict[str, str] = {
+    'datetime': '🕓',
+    'primary': '🔑',
+}
+
 
 def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, None]:
     """
@@ -420,9 +425,9 @@ def accordion_items_from_pipe(
         columns = pipe.columns
         index_column_names = pipe.get_indices()
         indices_rows = []
-        for ix_key, ix_name in index_column_names.items():
+        for ix_key, ix_cols in indices.items():
             col = columns.get(ix_key, None)
-            ix_cols = indices.get(ix_key, None)
+            ix_name = index_column_names.get(ix_key, None)
             if not col and not ix_cols:
                 continue
             if not isinstance(ix_cols, (list, tuple)):
@@ -434,12 +439,16 @@ def accordion_items_from_pipe(
             else:
                 col_item = html.Pre(html.Ul([html.Li(_c) for _c in ix_cols]))
             is_composite_item = "✅" if col else ""
-            ix_key_item = html.Pre(ix_key) if ix_key != 'datetime' else html.Pre(f"🕓 {ix_key}")
+            ix_key_item = html.Pre(
+                INDEX_PREFIX_EMOJI.get(ix_key, '')
+                + (' ' if ix_key in INDEX_PREFIX_EMOJI else '')
+                + ix_key
+            )
             indices_rows.append(
                 html.Tr([
                     html.Td(ix_key_item),
                     html.Td(col_item),
-                    html.Td(html.Pre(ix_name)),
+                    html.Td(html.Pre(ix_name or '')),
                     html.Td(is_composite_item),
                 ])
             )

meerschaum/api/routes/_pipes.py

@@ -7,8 +7,16 @@ Register Pipes via the Meerschaum API.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Any, Optional, Dict, Union
 
+
+import io
+import json
+import fastapi
+from decimal import Decimal
+import datetime
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import Any, Optional, Dict, Union, List
 from meerschaum.api import (
     fastapi,
     app,
@@ -19,27 +27,27 @@ from meerschaum.api import (
     _get_pipes,
     manager,
     debug,
-    no_auth, private,
+    no_auth,
+    private,
 )
-import json
-import fastapi
-from decimal import Decimal
 from meerschaum import Pipe
 from meerschaum.api.models import MetaPipe
 from meerschaum.utils.packages import attempt_import, import_pandas
-from meerschaum.utils.dataframe import get_numeric_cols, to_json
+from meerschaum.utils.dataframe import get_numeric_cols, to_json, parse_df_datetimes
+from meerschaum.utils.dtypes import are_dtypes_equal
 from meerschaum.utils.misc import (
-    is_pipe_registered, round_time, is_int, parse_df_datetimes,
+    is_pipe_registered,
+    round_time,
+    is_int,
     replace_pipes_in_dict,
 )
-from meerschaum.utils.typing import List, Dict, Any, Union
 import meerschaum.core.User
-import datetime
-pipes_endpoint = endpoints['pipes']
-from fastapi.responses import StreamingResponse
-import io
+
+fastapi_responses = attempt_import('fastapi.responses', lazy=False)
+StreamingResponse = fastapi_responses.StreamingResponse
 dateutil_parser = attempt_import('dateutil.parser', lazy=False)
-pd = attempt_import('pandas')
+pipes_endpoint = endpoints['pipes']
+pd = attempt_import('pandas', lazy=False)
 
 
 @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register', tags=['Pipes'])
@@ -79,13 +87,13 @@ def register_pipe(
 
 @app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete', tags=['Pipes'])
 def delete_pipe(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ):
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
     """
     Delete a Pipe (without dropping its table).
     """
@@ -141,15 +149,15 @@ def drop_pipe(
 
 @app.patch(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit', tags=['Pipes'])
 def edit_pipe(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        parameters: dict,
-        patch: bool = False,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ):
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    parameters: dict,
+    patch: bool = False,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
     """
     Edit an existing pipe.
     """
@@ -177,15 +185,15 @@ def edit_pipe(
 
 @app.get(pipes_endpoint + '/keys', tags=['Pipes'])
 async def fetch_pipes_keys(
-        connector_keys: str = "[]",
-        metric_keys: str = "[]",
-        location_keys: str = "[]",
-        tags: str = "[]",
-        params: str = "{}",
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ):
+    connector_keys: str = "[]",
+    metric_keys: str = "[]",
+    location_keys: str = "[]",
+    tags: str = "[]",
+    params: str = "{}",
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
     """
     Get a list of tuples of all registered Pipes' keys.
     """
@@ -201,14 +209,14 @@ async def fetch_pipes_keys(
 
 @app.get(pipes_endpoint, tags=['Pipes'])
 async def get_pipes(
-        connector_keys: str = "",
-        metric_keys: str = "",
-        location_keys: str = "",
-        curr_user=(
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-        debug: bool = False,
-    ) -> Dict[str, Any]:
+    connector_keys: str = "",
+    metric_keys: str = "",
+    location_keys: str = "",
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+    debug: bool = False,
+) -> Dict[str, Any]:
     """
     Get all registered Pipes with metadata, excluding parameters.
     """
@@ -223,11 +231,11 @@ async def get_pipes(
 
 @app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes'])
 async def get_pipes_by_connector(
-        connector_keys: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Dict[str, Any]:
+    connector_keys: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, Any]:
     """
     Get all registered Pipes by connector_keys with metadata, excluding parameters.
     """
@@ -240,13 +248,13 @@ async def get_pipes_by_connector(
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes'])
 async def get_pipes_by_connector_and_metric(
-        connector_keys: str,
-        metric_key: str,
-        parent: bool = False,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ):
+    connector_keys: str,
+    metric_key: str,
+    parent: bool = False,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
     """
     Get all registered Pipes by connector_keys and metric_key with metadata, excluding parameters.
 
@@ -269,13 +277,13 @@ async def get_pipes_by_connector_and_metric(
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}', tags=['Pipes'])
 async def get_pipes_by_connector_and_metric_and_location(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ):
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
     """
     Get a specific Pipe with metadata, excluding parameters.
     """
@@ -297,17 +305,17 @@ async def get_pipes_by_connector_and_metric_and_location(
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time', tags=['Pipes'])
 def get_sync_time(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        params: Optional[Dict[str, Any]] = None,
-        newest: bool = True,
-        round_down: bool = True,
-        debug: bool = False,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Union[str, int, None]:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    params: Optional[Dict[str, Any]] = None,
+    newest: bool = True,
+    round_down: bool = True,
+    debug: bool = False,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Union[str, int, None]:
     """
     Get a Pipe's latest datetime value.
     See `meerschaum.Pipe.get_sync_time`.
@@ -328,20 +336,20 @@ def get_sync_time(
 
 @app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
 def sync_pipe(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        data: dict = None,
-        check_existing: bool = True,
-        blocking: bool = True,
-        force: bool = False,
-        workers: Optional[int] = None,
-        columns: Optional[str] = None,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-        debug: bool = False,
-    ) -> List[Union[bool, str]]:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    data: dict = None,
+    check_existing: bool = True,
+    blocking: bool = True,
+    force: bool = False,
+    workers: Optional[int] = None,
+    columns: Optional[str] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+    debug: bool = False,
+) -> List[Union[bool, str]]:
     """
     Add data to an existing Pipe.
     See `meerschaum.Pipe.sync`.
@@ -365,11 +373,11 @@ def sync_pipe(
 
     result = list(p.sync(
         data,
-        debug = debug,
-        check_existing = check_existing,
-        blocking = blocking,
-        force = force,
-        workers = workers
+        debug=debug,
+        check_existing=check_existing,
+        blocking=blocking,
+        force=force,
+        workers=workers,
     ))
     return result
 
@@ -558,7 +566,7 @@ def get_pipe_id(
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
-) -> int:
+) -> Union[int, str]:
     """
     Get a Pipe's ID.
     """
@@ -573,36 +581,36 @@ def get_pipe_id(
     tags = ['Pipes']
 )
 def get_pipe_attributes(
-        connector_keys : str,
-        metric_key : str,
-        location_key : str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Dict[str, Any]:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, Any]:
     """Get a Pipe's attributes."""
-    return get_pipe(connector_keys, metric_key, location_key).attributes
+    return get_pipe(connector_keys, metric_key, location_key, refresh=True).attributes
 
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists', tags=['Pipes'])
 def get_pipe_exists(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> bool:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> bool:
     """Determine whether a Pipe exists."""
     return get_pipe(connector_keys, metric_key, location_key).exists()
 
 
 @app.post(endpoints['metadata'], tags=['Pipes'])
 def create_metadata(
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> bool:
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> bool:
     """Create Pipe metadata tables"""
     from meerschaum.connectors.sql.tables import get_tables
     try:
@@ -614,16 +622,16 @@ def create_metadata(
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount', tags=['Pipes'])
 def get_pipe_rowcount(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        begin: Union[str, int, None] = None,
-        end: Union[str, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> int:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    begin: Union[str, int, None] = None,
+    end: Union[str, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> int:
     """
     Return a pipe's rowcount.
     """
@@ -632,10 +640,10 @@ def get_pipe_rowcount(
     if is_int(end):
         end = int(end)
     return get_pipe(connector_keys, metric_key, location_key).get_rowcount(
-        begin = begin,
-        end = end,
-        params = params,
-        debug = debug
+        begin=begin,
+        end=end,
+        params=params,
+        debug=debug,
     )
 
 
@@ -644,13 +652,13 @@ def get_pipe_rowcount(
     tags=['Pipes']
 )
 def get_pipe_columns_types(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> Dict[str, str]:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, str]:
     """
     Return a dictionary of column names and types.
 
@@ -663,3 +671,21 @@ def get_pipe_columns_types(
     ```
     """
     return get_pipe(connector_keys, metric_key, location_key).dtypes
+
+
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/indices',
+    tags=['Pipes']
+)
+def get_pipe_columns_indices(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user=(
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> Dict[str, List[Dict[str, str]]]:
+    """
+    Return a dictionary of column names and related indices.
+    """
+    return get_pipe(connector_keys, metric_key, location_key).get_columns_indices()

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.6.0.dev1"
+__version__ = "2.6.1"

meerschaum/config/static/__init__.py

@@ -148,6 +148,7 @@ STATIC_CONFIG: Dict[str, Any] = {
             'min_ratio_columns_changed_for_full_astype': 0.5,
         },
         'exists_timeout_seconds': 5.0,
+        'static_schema_cache_seconds': 60.0,
     },
     'jobs': {
         'check_restart_seconds': 1.0,

meerschaum/connectors/api/_APIConnector.py

@@ -58,6 +58,7 @@ class APIConnector(Connector):
         drop_pipe,
         clear_pipe,
         get_pipe_columns_types,
+        get_pipe_columns_indices,
     )
     from ._fetch import fetch
     from ._plugins import (