meerschaum 3.0.0rc3__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

meerschaum/connectors/sql/_pipes.py

@@ -146,8 +146,10 @@ def fetch_pipes_keys(
     location_keys: Optional[List[str]] = None,
     tags: Optional[List[str]] = None,
     params: Optional[Dict[str, Any]] = None,
-    debug: bool = False
-) -> List[Tuple[str, str, Optional[str]]]:
+    debug: bool = False,
+) -> List[
+        Tuple[str, str, Union[str, None], Dict[str, Any]]
+    ]:
     """
     Return a list of tuples corresponding to the parameters provided.
 
@@ -174,7 +176,7 @@ def fetch_pipes_keys(
 
     Returns
     -------
-    A list of tuples of pipes' keys (connector_keys, metric_key, location_key).
+    A list of tuples of pipes' keys and parameters (connector_keys, metric_key, location_key, parameters).
     """
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.misc import separate_negation_values
@@ -249,11 +251,18 @@ def fetch_pipes_keys(
         ) for key, val in _params.items()
         if not isinstance(val, (list, tuple)) and key in pipes_tbl.c
     ]
+    if self.flavor in json_flavors:
+        sqlalchemy_dialects = mrsm.attempt_import('sqlalchemy.dialects', lazy=False)
+        JSONB = sqlalchemy_dialects.postgresql.JSONB
+    else:
+        JSONB = sqlalchemy.String
+
     select_cols = (
         [
             pipes_tbl.c.connector_keys,
             pipes_tbl.c.metric_key,
             pipes_tbl.c.location_key,
+            pipes_tbl.c.parameters,
         ]
     )
 
@@ -271,14 +280,12 @@ def fetch_pipes_keys(
 
     ors, nands = [], []
     if self.flavor in json_flavors:
-        from sqlalchemy.dialects import postgresql
         for _in_tags, _ex_tags in in_ex_tag_groups:
             if _in_tags:
                 ors.append(
                     sqlalchemy.and_(
-                        pipes_tbl.c['parameters'].cast(postgresql.JSONB).has_key('tags'),
                         pipes_tbl.c['parameters']['tags'].cast(
-                            postgresql.JSONB
+                            JSONB
                         ).contains(_in_tags)
                     )
                 )
@@ -286,9 +293,8 @@ def fetch_pipes_keys(
                 nands.append(
                     sqlalchemy.not_(
                         sqlalchemy.and_(
-                            pipes_tbl.c['parameters'].cast(postgresql.JSONB).has_key('tags'),
                             pipes_tbl.c['parameters']['tags'].cast(
-                                postgresql.JSONB
+                                JSONB
                             ).contains([xt])
                         )
                     )
@@ -340,7 +346,7 @@ def fetch_pipes_keys(
     except Exception as e:
         error(str(e))
 
-    return [(row[0], row[1], row[2]) for row in rows]
+    return rows
 
 
 def create_pipe_indices(
@@ -371,6 +377,9 @@ def create_indices(
     """
     Create a pipe's indices.
     """
+    if pipe.__dict__.get('_skip_check_indices', False):
+        return True
+
     if debug:
         dprint(f"Creating indices for {pipe}...")
 
@@ -380,7 +389,7 @@ def create_indices(
 
     cols_to_include = set((columns or []) + (indices or [])) or None
 
-    _ = pipe.__dict__.pop('_columns_indices', None)
+    pipe._clear_cache_key('_columns_indices', debug=debug)
     ix_queries = {
         col: queries
         for col, queries in self.get_create_index_queries(pipe, debug=debug).items()
@@ -456,7 +465,7 @@ def get_pipe_index_names(self, pipe: mrsm.Pipe) -> Dict[str, str]:
     -------
     A dictionary of index keys to column names.
     """
-    from meerschaum.utils.sql import DEFAULT_SCHEMA_FLAVORS
+    from meerschaum.utils.sql import DEFAULT_SCHEMA_FLAVORS, truncate_item_name
     _parameters = pipe.parameters
     _index_template = _parameters.get('index_template', "IX_{schema_str}{target}_{column_names}")
     _schema = self.get_pipe_schema(pipe)
@@ -497,7 +506,7 @@ def get_pipe_index_names(self, pipe: mrsm.Pipe) -> Dict[str, str]:
             continue
         seen_index_names[index_name] = ix
     return {
-        ix: index_name
+        ix: truncate_item_name(index_name, flavor=self.flavor)
         for index_name, ix in seen_index_names.items()
     }
 
@@ -1498,7 +1507,7 @@ def get_pipe_attributes(
     """
     from meerschaum.connectors.sql.tables import get_tables
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy = attempt_import('sqlalchemy', lazy=False)
 
     if pipe.get_id(debug=debug) is None:
         return {}
@@ -1509,16 +1518,16 @@ def get_pipe_attributes(
         q = sqlalchemy.select(pipes_tbl).where(pipes_tbl.c.pipe_id == pipe.id)
         if debug:
             dprint(q)
-        attributes = (
-            dict(self.exec(q, silent=True, debug=debug).first()._mapping)
+        rows = (
+            self.exec(q, silent=True, debug=debug).mappings().all()
             if self.flavor != 'duckdb'
-            else self.read(q, debug=debug).to_dict(orient='records')[0]
+            else self.read(q, debug=debug).to_dict(orient='records')
         )
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        warn(e)
-        print(pipe)
+        if not rows:
+            return {}
+        attributes = dict(rows[0])
+    except Exception:
+        warn(traceback.format_exc())
         return {}
 
     ### handle non-PostgreSQL databases (text vs JSON)
@@ -1673,8 +1682,9 @@ def sync_pipe(
         UPDATE_QUERIES,
         get_reset_autoincrement_queries,
     )
-    from meerschaum.utils.dtypes import are_dtypes_equal
+    from meerschaum.utils.dtypes import get_current_timestamp
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
+    from meerschaum.utils.dataframe import get_special_cols
     from meerschaum import Pipe
     import time
     import copy
@@ -1686,6 +1696,7 @@ def sync_pipe(
 
     start = time.perf_counter()
     pipe_name = sql_item_name(pipe.target, self.flavor, schema=self.get_pipe_schema(pipe))
+    dtypes = pipe.get_dtypes(debug=debug)
 
     if not pipe.temporary and not pipe.get_id(debug=debug):
         register_tuple = pipe.register(debug=debug)
@@ -1702,6 +1713,7 @@ def sync_pipe(
             df,
             chunksize=chunksize,
             safe_copy=kw.get('safe_copy', False),
+            dtypes=dtypes,
             debug=debug,
         )
 
@@ -1714,35 +1726,18 @@ def sync_pipe(
         ### Check for new columns.
         add_cols_queries = self.get_add_columns_queries(pipe, df, debug=debug)
         if add_cols_queries:
-            _ = pipe.__dict__.pop('_columns_indices', None)
-            _ = pipe.__dict__.pop('_columns_types', None)
+            pipe._clear_cache_key('_columns_types', debug=debug)
+            pipe._clear_cache_key('_columns_indices', debug=debug)
             if not self.exec_queries(add_cols_queries, debug=debug):
                 warn(f"Failed to add new columns to {pipe}.")
 
         alter_cols_queries = self.get_alter_columns_queries(pipe, df, debug=debug)
         if alter_cols_queries:
-            _ = pipe.__dict__.pop('_columns_indices', None)
-            _ = pipe.__dict__.pop('_columns_types', None)
+            pipe._clear_cache_key('_columns_types', debug=debug)
+            pipe._clear_cache_key('_columns_types', debug=debug)
             if not self.exec_queries(alter_cols_queries, debug=debug):
                 warn(f"Failed to alter columns for {pipe}.")
 
-    ### NOTE: Oracle SQL < 23c (2023) and SQLite does not support booleans,
-    ### so infer bools and persist them to `dtypes`.
-    if self.flavor in ('oracle', 'sqlite', 'mysql', 'mariadb'):
-        pipe_dtypes = pipe.get_dtypes(infer=False, debug=debug)
-        new_bool_cols = {
-            col: 'bool[pyarrow]'
-            for col, typ in df.dtypes.items()
-            if col not in pipe_dtypes
-                and are_dtypes_equal(str(typ), 'bool')
-        }
-        pipe_dtypes.update(new_bool_cols)
-        pipe.dtypes = pipe_dtypes
-        if new_bool_cols and not pipe.temporary:
-            infer_bool_success, infer_bool_msg = pipe.edit(debug=debug)
-            if not infer_bool_success:
-                return infer_bool_success, infer_bool_msg
-
     upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in UPDATE_QUERIES
     if upsert:
         check_existing = False
@@ -1771,7 +1766,7 @@ def sync_pipe(
     if 'name' in kw:
         kw.pop('name')
 
-    ### Insert new data into Pipe's table.
+    ### Insert new data into the target table.
     unseen_kw = copy.deepcopy(kw)
     unseen_kw.update({
         'name': pipe.target,
@@ -1792,7 +1787,7 @@ def sync_pipe(
             is_new
             and primary_key
             and primary_key
-            not in pipe.dtypes
+            not in dtypes
             and primary_key not in unseen_df.columns
         )
     )
@@ -1892,6 +1887,14 @@ def sync_pipe(
             label=('update' if not upsert else 'upsert'),
         )
         self._log_temporary_tables_creation(temp_target, create=(not pipe.temporary), debug=debug)
+        update_dtypes = {
+            **{
+                col: str(typ)
+                for col, typ in update_df.dtypes.items()
+            },
+            **get_special_cols(update_df)
+        }
+
         temp_pipe = Pipe(
             pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
             instance=pipe.instance_keys,
@@ -1900,34 +1903,30 @@ def sync_pipe(
                 for ix_key, ix in pipe.columns.items()
                 if ix and ix in update_df.columns
             },
-            dtypes={
-                col: typ
-                for col, typ in pipe.dtypes.items()
-                if col in update_df.columns
-            },
+            dtypes=update_dtypes,
             target=temp_target,
             temporary=True,
             enforce=False,
             static=True,
             autoincrement=False,
+            cache=False,
             parameters={
                 'schema': self.internal_schema,
                 'hypertable': False,
             },
         )
-        temp_pipe.__dict__['_columns_types'] = {
-            col: get_db_type_from_pd_type(
-                pipe.dtypes.get(col, str(typ)),
-                self.flavor,
-            )
-            for col, typ in update_df.dtypes.items()
+        _temp_columns_types = {
+            col: get_db_type_from_pd_type(typ, self.flavor)
+            for col, typ in update_dtypes.items()
         }
-        now_ts = time.perf_counter()
-        temp_pipe.__dict__['_columns_types_timestamp'] = now_ts
-        temp_pipe.__dict__['_skip_check_indices'] = True
+        temp_pipe._cache_value('_columns_types', _temp_columns_types, memory_only=True, debug=debug)
+        temp_pipe._cache_value('_skip_check_indices', True, memory_only=True, debug=debug)
+        now_ts = get_current_timestamp('ms', as_int=True) / 1000
+        temp_pipe._cache_value('_columns_types_timestamp', now_ts, memory_only=True, debug=debug)
         temp_success, temp_msg = temp_pipe.sync(update_df, check_existing=False, debug=debug)
         if not temp_success:
             return temp_success, temp_msg
+
         existing_cols = pipe.get_columns_types(debug=debug)
         join_cols = [
             col
@@ -1950,6 +1949,8 @@ def sync_pipe(
             upsert=upsert,
             schema=self.get_pipe_schema(pipe),
             patch_schema=self.internal_schema,
+            target_cols_types=pipe.get_columns_types(debug=debug),
+            patch_cols_types=_temp_columns_types,
             datetime_col=(dt_col if dt_col in update_df.columns else None),
             identity_insert=(autoincrement and primary_key in update_df.columns),
             null_indices=pipe.null_indices,
@@ -2237,13 +2238,13 @@ def sync_pipe_inplace(
 
     add_cols_queries = self.get_add_columns_queries(pipe, new_cols, debug=debug)
     if add_cols_queries:
-        _ = pipe.__dict__.pop('_columns_types', None)
-        _ = pipe.__dict__.pop('_columns_indices', None)
+        pipe._clear_cache_key('_columns_types', debug=debug)
+        pipe._clear_cache_key('_columns_indices', debug=debug)
         self.exec_queries(add_cols_queries, debug=debug)
 
     alter_cols_queries = self.get_alter_columns_queries(pipe, new_cols, debug=debug)
     if alter_cols_queries:
-        _ = pipe.__dict__.pop('_columns_types', None)
+        pipe._clear_cache_key('_columns_types', debug=debug)
         self.exec_queries(alter_cols_queries, debug=debug)
 
     insert_queries = [
@@ -2546,6 +2547,8 @@ def sync_pipe_inplace(
             upsert=upsert,
             schema=self.get_pipe_schema(pipe),
             patch_schema=internal_schema,
+            target_cols_types=pipe.get_columns_types(debug=debug),
+            patch_cols_types=delta_cols_types,
             datetime_col=pipe.columns.get('datetime', None),
             flavor=self.flavor,
             null_indices=pipe.null_indices,
@@ -3036,6 +3039,7 @@ def get_pipe_table(
     from meerschaum.utils.sql import get_sqlalchemy_table
     if not pipe.exists(debug=debug):
         return None
+
     return get_sqlalchemy_table(
         pipe.target,
         connector=self,
@@ -3093,9 +3097,11 @@ def get_pipe_columns_types(
         pipe_table = self.get_pipe_table(pipe, debug=debug)
         if pipe_table is None:
             return {}
+
         if debug:
-            dprint(f"Found columns:")
+            dprint("Found columns:")
             mrsm.pprint(dict(pipe_table.columns))
+
         for col in pipe_table.columns:
             table_columns[str(col.name)] = str(col.type)
     except Exception as e:
@@ -3127,6 +3133,7 @@ def get_pipe_columns_indices(
     """
     if pipe.__dict__.get('_skip_check_indices', False):
         return {}
+
     from meerschaum.utils.sql import get_table_cols_indices
     return get_table_cols_indices(
         pipe.target,
@@ -3181,7 +3188,6 @@ def get_add_columns_queries(
         get_db_type_from_pd_type,
     )
     from meerschaum.utils.misc import flatten_list
-    table_obj = self.get_pipe_table(pipe, debug=debug)
     is_dask = 'dask' in df.__module__ if not isinstance(df, dict) else False
     if is_dask:
         df = df.partitions[0].compute()
@@ -3205,9 +3211,6 @@ def get_add_columns_queries(
             elif isinstance(val, str):
                 df_cols_types[col] = 'str'
     db_cols_types = {
-        col: get_pd_type_from_db_type(str(typ.type))
-        for col, typ in table_obj.columns.items()
-    } if table_obj is not None else {
         col: get_pd_type_from_db_type(typ)
         for col, typ in get_table_cols_types(
             pipe.target,
@@ -3303,7 +3306,6 @@ def get_alter_columns_queries(
         get_db_type_from_pd_type,
     )
     from meerschaum.utils.misc import flatten_list, generate_password, items_str
-    table_obj = self.get_pipe_table(pipe, debug=debug)
     target = pipe.target
     session_id = generate_password(3)
     numeric_cols = (
@@ -3324,9 +3326,6 @@ def get_alter_columns_queries(
         else df
     )
     db_cols_types = {
-        col: get_pd_type_from_db_type(str(typ.type))
-        for col, typ in table_obj.columns.items()
-    } if table_obj is not None else {
         col: get_pd_type_from_db_type(typ)
         for col, typ in get_table_cols_types(
             pipe.target,
@@ -3335,7 +3334,7 @@ def get_alter_columns_queries(
             debug=debug,
         ).items()
     }
-    pipe_dtypes = pipe.dtypes
+    pipe_dtypes = pipe.get_dtypes(debug=debug)
     pipe_bool_cols = [col for col, typ in pipe_dtypes.items() if are_dtypes_equal(str(typ), 'bool')]
     pd_db_df_aliases = {
         'int': 'bool',
@@ -3347,6 +3346,7 @@ def get_alter_columns_queries(
         pd_db_df_aliases.update({
             'int': 'numeric',
             'date': 'datetime',
+            'numeric': 'int',
         })
 
     altered_cols = {
@@ -3357,14 +3357,32 @@ def get_alter_columns_queries(
     }
 
     if debug and altered_cols:
-        dprint(f"Columns to be altered:")
+        dprint("Columns to be altered:")
         mrsm.pprint(altered_cols)
 
+    ### NOTE: Special columns (numerics, bools, etc.) are captured and cached upon detection.
+    new_special_cols = pipe._get_cached_value('new_special_cols', debug=debug) or {}
+    new_special_db_cols_types = {
+        col: (db_cols_types.get(col, 'object'), typ)
+        for col, typ in new_special_cols.items()
+    }
+    if debug:
+        dprint("Cached new special columns:")
+        mrsm.pprint(new_special_cols)
+        dprint("New special columns db types:")
+        mrsm.pprint(new_special_db_cols_types)
+
+    altered_cols.update(new_special_db_cols_types)
+
     ### NOTE: Sometimes bools are coerced into ints or floats.
     altered_cols_to_ignore = set()
     for col, (db_typ, df_typ) in altered_cols.items():
         for db_alias, df_alias in pd_db_df_aliases.items():
-            if db_alias in db_typ.lower() and df_alias in df_typ.lower():
+            if (
+                db_alias in db_typ.lower()
+                and df_alias in df_typ.lower()
+                and col not in new_special_cols
+            ):
                 altered_cols_to_ignore.add(col)
 
     ### Oracle's bool handling sometimes mixes NUMBER and INT.
@@ -3387,7 +3405,7 @@ def get_alter_columns_queries(
             altered_cols_to_ignore.add(bool_col)
 
     if debug and altered_cols_to_ignore:
-        dprint(f"Ignoring the following altered columns (false positives).")
+        dprint("Ignoring the following altered columns (false positives).")
         mrsm.pprint(altered_cols_to_ignore)
 
     for col in altered_cols_to_ignore:
@@ -3434,12 +3452,12 @@ def get_alter_columns_queries(
             + sql_item_name(target, self.flavor, self.get_pipe_schema(pipe))
             + " (\n"
         )
-        for col_name, col_obj in table_obj.columns.items():
+        for col_name, col_typ in db_cols_types.items():
             create_query += (
                 sql_item_name(col_name, self.flavor, None)
                 + " "
                 + (
-                    str(col_obj.type)
+                    col_typ
                     if col_name not in altered_cols
                     else altered_cols_types[col_name]
                 )
@@ -3453,12 +3471,12 @@ def get_alter_columns_queries(
             + ' ('
             + ', '.join([
                 sql_item_name(col_name, self.flavor, None)
-                for col_name, _ in table_obj.columns.items()
+                for col_name in db_cols_types
             ])
             + ')'
             + "\nSELECT\n"
         )
-        for col_name, col_obj in table_obj.columns.items():
+        for col_name in db_cols_types:
             new_col_str = (
                 sql_item_name(col_name, self.flavor, None)
                 if col_name not in altered_cols
@@ -3471,6 +3489,7 @@ def get_alter_columns_queries(
                 )
             )
             insert_query += new_col_str + ",\n"
+
         insert_query = insert_query[:-2] + (
             f"\nFROM {sql_item_name(temp_table_name, self.flavor, self.get_pipe_schema(pipe))}"
         )

meerschaum/connectors/sql/_users.py

@@ -16,9 +16,15 @@ def get_users_pipe(self) -> mrsm.Pipe:
     """
     Return the internal metadata pipe for users management.
     """
-    return mrsm.Pipe(
+    if '_users_pipe' in self.__dict__:
+        return self._users_pipe
+
+    cache_connector = self.__dict__.get('_cache_connector', None)
+    self._users_pipe = mrsm.Pipe(
         'mrsm', 'users',
         temporary=True,
+        cache=True,
+        cache_connector_keys=cache_connector,
         static=True,
         null_indices=False,
         enforce=False,
@@ -36,6 +42,7 @@ def get_users_pipe(self) -> mrsm.Pipe:
             'unique': 'username',
         },
     )
+    return self._users_pipe
 
 
 def register_user(

meerschaum/connectors/sql/tables/__init__.py

@@ -24,6 +24,7 @@ _skip_index_names_flavors = {'mssql',}
 def get_tables(
     mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
     create: Optional[bool] = None,
+    refresh: bool = False,
     debug: bool = False,
 ) -> Union[Dict[str, 'sqlalchemy.Table'], bool]:
     """
@@ -37,6 +38,9 @@ def get_tables(
     create: Optional[bool], default None
         If `True`, create the tables if they don't exist.
 
+    refresh: bool, default False
+        If `True`, invalidate and rebuild any cache.
+
     debug: bool, default False
         Verbosity Toggle.
 
@@ -67,7 +71,15 @@ def get_tables(
     else: ### NOTE: mrsm_instance MUST BE a SQL Connector for this to work!
         conn = mrsm_instance
 
-    cache_expired = _check_create_cache(conn, debug=debug) if conn.type == 'sql' else False
+    cache_expired = refresh or (
+        (
+            _check_create_cache(conn, debug=debug)
+            if conn.flavor != 'sqlite'
+            else True
+        )
+        if conn.type == 'sql'
+        else False
+    )
     create = create or cache_expired
 
     ### Skip if the connector is not a SQL connector.
@@ -75,6 +87,10 @@ def get_tables(
         return {}
 
     conn_key = str(conn)
+
+    if refresh:
+        _ = connector_tables.pop(conn_key, None)
+
     if conn_key in connector_tables:
         return connector_tables[conn_key]
 
@@ -248,8 +264,9 @@ def get_tables(
 
                 _write_create_cache(mrsm.get_connector(str(mrsm_instance)), debug=debug)
 
-        with open(pickle_path, 'wb') as f:
-            pickle.dump(conn.metadata, f)
+        if conn.flavor != 'sqlite':
+            with open(pickle_path, 'wb') as f:
+                pickle.dump(conn.metadata, f)
 
     connector_tables[conn_key] = _tables
     return connector_tables[conn_key]

meerschaum/connectors/valkey/_ValkeyConnector.py

@@ -140,13 +140,13 @@ class ValkeyConnector(InstanceConnector):
 
         return uri
 
-    def set(self, key: str, value: Any, **kwargs: Any) -> None:
+    def set(self, key: str, value: Any, **kwargs: Any) -> bool:
         """
         Set the `key` to `value`.
         """
         return self.client.set(key, value, **kwargs)
 
-    def get(self, key: str) -> Union[str, None]:
+    def get(self, key: str, decode: bool = True) -> Union[str, None]:
         """
         Get the value for `key`.
         """
@@ -154,7 +154,7 @@ class ValkeyConnector(InstanceConnector):
         if val is None:
             return None
 
-        return val.decode('utf-8')
+        return val.decode('utf-8') if decode else val
 
     def test_connection(self) -> bool:
         """

meerschaum/connectors/valkey/_pipes.py

@@ -888,7 +888,9 @@ def fetch_pipes_keys(
     tags: Optional[List[str]] = None,
     params: Optional[Dict[str, Any]] = None,
     debug: bool = False
-) -> Optional[List[Tuple[str, str, Optional[str]]]]:
+) -> List[
+        Tuple[str, str, Union[str, None], Dict[str, Any]]
+    ]:
     """
     Return the keys for the registered pipes.
     """
@@ -919,6 +921,7 @@ def fetch_pipes_keys(
             doc['connector_keys'],
             doc['metric_key'],
             doc['location_key'],
+            doc.get('parameters', {})
         )
         for doc in df.to_dict(orient='records')
     ]
@@ -929,9 +932,8 @@ def fetch_pipes_keys(
     in_ex_tag_groups = [separate_negation_values(tag_group) for tag_group in tag_groups]
 
     filtered_keys = []
-    for ck, mk, lk in keys:
-        pipe = mrsm.Pipe(ck, mk, lk, instance=self)
-        pipe_tags = set(pipe.tags)
+    for ck, mk, lk, parameters in keys:
+        pipe_tags = set(parameters.get('tags', []))
         
         include_pipe = True
         for in_tags, ex_tags in in_ex_tag_groups:
@@ -943,6 +945,6 @@ def fetch_pipes_keys(
                 continue
 
         if include_pipe:
-            filtered_keys.append((ck, mk, lk))
+            filtered_keys.append((ck, mk, lk, parameters))
 
     return filtered_keys