meerschaum 3.0.0rc2__py3-none-any.whl → 3.0.0rc4__py3-none-any.whl

meerschaum/api/dash/tokens.py

@@ -9,7 +9,7 @@ Dash utility functions for constructing tokens components.
 from __future__ import annotations
 
 from datetime import datetime, timezone, timedelta
-from typing import Any, List
+from typing import Any, List, Optional
 
 import meerschaum as mrsm
 from meerschaum.api import debug, CHECK_UPDATE, get_api_connector
@@ -65,7 +65,7 @@ def get_tokens_table(session_id: Optional[str] = None) -> Tuple[dbc.Table, List[
             html.Td("✅" if token.is_valid else "❌"),
             html.Td([
                 dbc.Button(
-                    html.B("⠇"),
+                    html.B("⋮"),
                     color='link',
                     size='sm',
                     id={'type': 'tokens-context-button', 'index': str(token.id)},
@@ -91,39 +91,36 @@ def build_manage_token_popover(token: Token) -> dbc.Popover:
     """
     return dbc.Popover(
         [
-            dbc.PopoverHeader(["Manage token"]),
-            dbc.PopoverBody([
-                dbc.ButtonGroup(
-                    ([
-                        dbc.Button(
-                            "Edit",
-                            outline=True,
-                            color='light',
-                            id={
-                                'type': 'tokens-edit-button',
-                                'index': str(token.id),
-                            },
-                        ),
-                        dbc.Button(
-                            "Invalidate",
-                            outline=True,
-                            color='warning',
-                            id={
-                                'type': 'tokens-invalidate-button',
-                                'index': str(token.id),
-                            },
-                        ),
-                    ] if token.is_valid else []) + [
+            dbc.ButtonGroup(
+                ([
                     dbc.Button(
-                        "Delete",
-                        color='danger',
+                        "Edit",
                         outline=True,
+                        color='light',
                         id={
-                            'type': 'tokens-delete-button',
+                            'type': 'tokens-edit-button',
                             'index': str(token.id),
                         },
                     ),
-                ]),
+                    dbc.Button(
+                        "Invalidate",
+                        outline=True,
+                        color='warning',
+                        id={
+                            'type': 'tokens-invalidate-button',
+                            'index': str(token.id),
+                        },
+                    ),
+                ] if token.is_valid else []) + [
+                dbc.Button(
+                    "Delete",
+                    color='danger',
+                    outline=True,
+                    id={
+                        'type': 'tokens-delete-button',
+                        'index': str(token.id),
+                    },
+                ),
             ]),
         ],
         body=True,
@@ -600,7 +597,7 @@ def build_tokens_register_output_modal(token: Token) -> List[Any]:
             dbc.Button(
                 "Close",
                 id='tokens-close-register-output-modal-button',
-                disabled=True,
+                disabled=success,
             ),
         ]),
     ]

meerschaum/config/_default.py

@@ -99,6 +99,7 @@ default_system_config = {
                 'mssql': True,
             },
             'instance': {
+                'create_metadata_cache_minutes': 14400,
                 'stale_temporary_tables_minutes': 1440,
                 'temporary_target': {
                     'prefix': '_',
@@ -199,11 +200,11 @@ default_pipes_config = {
         },
     },
     'attributes': {
-        'local_cache_timeout_seconds': 60,
+        'local_cache_timeout_seconds': 600.0,
     },
     'sync': {
         'filter_params_index_limit': 250,
-        'exists_cache_seconds': 5.0,
+        'exists_cache_seconds': 60.0,
     },
     'verify': {
         'max_chunks_syncs': 3,
@@ -213,10 +214,10 @@ default_pipes_config = {
     },
     'dtypes': {
         'min_ratio_columns_changed_for_full_astype': 0.5,
-        'columns_types_cache_seconds': 5.0,
+        'columns_types_cache_seconds': 60.0,
     },
     'static': {
-        'static_schema_cache_seconds': 60.0,
+        'static_schema_cache_seconds': 3600.0,
     },
 }
 default_plugins_config = {}

meerschaum/config/_paths.py

@@ -149,6 +149,7 @@ paths = {
     'PIPES_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'pipes'),
     'USERS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'users'),
     'VENVS_CACHE_RESOURCES_PATH'     : ('{CACHE_RESOURCES_PATH}', 'venvs'),
+    'SQL_CONN_CACHE_RESOURCES_PATH'  : ('{CACHE_RESOURCES_PATH}', 'sql'),
 
     'PLUGINS_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
     'PLUGINS_INTERNAL_LOCK_PATH'     : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "3.0.0rc2"
+__version__ = "3.0.0rc4"

meerschaum/connectors/instance/_tokens.py

@@ -53,7 +53,11 @@ def get_tokens_pipe(self) -> mrsm.Pipe:
     )
 
 
-def register_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple:
+def register_token(
+    self,
+    token: Token,
+    debug: bool = False,
+) -> mrsm.SuccessTuple:
     """
     Register the new token to the tokens table.
     """
@@ -61,7 +65,7 @@ def register_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple
     tokens_pipe = self.get_tokens_pipe()
     user_id = self.get_user_id(token.user) if token.user is not None else None
     if user_id is None:
-        raise ValueError("Cannot register a token without a user.")
+        return False, "Cannot register a token without a user."
 
     doc = {
         'id': token_id,

meerschaum/connectors/sql/_SQLConnector.py

@@ -7,6 +7,8 @@ Interface with SQL servers using sqlalchemy.
 """
 
 from __future__ import annotations
+
+import pathlib
 import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Any, Union
 
@@ -376,6 +378,18 @@ class SQLConnector(InstanceConnector):
         self.__dict__['schema'] = _schema
         return _schema
 
+    def get_metadata_cache_path(self, kind: str = 'json') -> pathlib.Path:
+        """
+        Return the path to the file to which to write metadata cache.
+        """
+        from meerschaum.config.paths import SQL_CONN_CACHE_RESOURCES_PATH
+        filename = (
+            f'{self.label}-metadata.pkl'
+            if kind == 'pkl'
+            else f'{self.label}.json'
+        )
+        return SQL_CONN_CACHE_RESOURCES_PATH / filename
+
     def __getstate__(self):
         return self.__dict__
 

meerschaum/connectors/sql/_pipes.py

@@ -147,7 +147,9 @@ def fetch_pipes_keys(
     tags: Optional[List[str]] = None,
     params: Optional[Dict[str, Any]] = None,
     debug: bool = False
-) -> Optional[List[Tuple[str, str, Optional[str]]]]:
+) -> List[
+        Tuple[str, str, Union[str, None], Dict[str, Any]]
+    ]:
     """
     Return a list of tuples corresponding to the parameters provided.
 
@@ -162,16 +164,27 @@ def fetch_pipes_keys(
     location_keys: Optional[List[str]], default None
         List of location_keys to search by.
 
+    tags: Optional[List[str]], default None
+        List of pipes to search by.
+
     params: Optional[Dict[str, Any]], default None
         Dictionary of additional parameters to search by.
         E.g. `--params pipe_id:1`
 
     debug: bool, default False
         Verbosity toggle.
+
+    Returns
+    -------
+    A list of tuples of pipes' keys and parameters (connector_keys, metric_key, location_key, parameters).
     """
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.misc import separate_negation_values
-    from meerschaum.utils.sql import OMIT_NULLSFIRST_FLAVORS, table_exists
+    from meerschaum.utils.sql import (
+        OMIT_NULLSFIRST_FLAVORS,
+        table_exists,
+        json_flavors,
+    )
     from meerschaum._internal.static import STATIC_CONFIG
     import json
     from copy import deepcopy
@@ -243,6 +256,11 @@ def fetch_pipes_keys(
             pipes_tbl.c.connector_keys,
             pipes_tbl.c.metric_key,
             pipes_tbl.c.location_key,
+            (
+                pipes_tbl.c.parameters['tags']
+                if self.flavor in json_flavors
+                else pipes_tbl.c.parameters
+            ),
         ]
     )
 
@@ -259,25 +277,47 @@ def fetch_pipes_keys(
     in_ex_tag_groups = [separate_negation_values(tag_group) for tag_group in tag_groups]
 
     ors, nands = [], []
-    for _in_tags, _ex_tags in in_ex_tag_groups:
-        sub_ands = []
-        for nt in _in_tags:
-            sub_ands.append(
-                sqlalchemy.cast(
-                    pipes_tbl.c['parameters'],
-                    sqlalchemy.String,
-                ).like(f'%"tags":%"{nt}"%')
-            )
-        if sub_ands:
-            ors.append(sqlalchemy.and_(*sub_ands))
-
-        for xt in _ex_tags:
-            nands.append(
-                sqlalchemy.cast(
-                    pipes_tbl.c['parameters'],
-                    sqlalchemy.String,
-                ).not_like(f'%"tags":%"{xt}"%')
-            )
+    if self.flavor in json_flavors:
+        from sqlalchemy.dialects import postgresql
+        for _in_tags, _ex_tags in in_ex_tag_groups:
+            if _in_tags:
+                ors.append(
+                    sqlalchemy.and_(
+                        pipes_tbl.c['parameters']['tags'].cast(
+                            postgresql.JSONB
+                        ).contains(_in_tags)
+                    )
+                )
+            for xt in _ex_tags:
+                nands.append(
+                    sqlalchemy.not_(
+                        sqlalchemy.and_(
+                            pipes_tbl.c['parameters']['tags'].cast(
+                                postgresql.JSONB
+                            ).contains([xt])
+                        )
+                    )
+                )
+    else:
+        for _in_tags, _ex_tags in in_ex_tag_groups:
+            sub_ands = []
+            for nt in _in_tags:
+                sub_ands.append(
+                    sqlalchemy.cast(
+                        pipes_tbl.c['parameters'],
+                        sqlalchemy.String,
+                    ).like(f'%"tags":%"{nt}"%')
+                )
+            if sub_ands:
+                ors.append(sqlalchemy.and_(*sub_ands))
+
+            for xt in _ex_tags:
+                nands.append(
+                    sqlalchemy.cast(
+                        pipes_tbl.c['parameters'],
+                        sqlalchemy.String,
+                    ).not_like(f'%"tags":%"{xt}"%')
+                )
 
     q = q.where(sqlalchemy.and_(*nands)) if nands else q
     q = q.where(sqlalchemy.or_(*ors)) if ors else q
@@ -292,7 +332,7 @@ def fetch_pipes_keys(
 
     ### execute the query and return a list of tuples
     if debug:
-        dprint(q.compile(compile_kwargs={'literal_binds': True}))
+        dprint(q)
     try:
         rows = (
             self.execute(q).fetchall()
@@ -305,7 +345,7 @@ def fetch_pipes_keys(
     except Exception as e:
         error(str(e))
 
-    return [(row[0], row[1], row[2]) for row in rows]
+    return rows
 
 
 def create_pipe_indices(