meerschaum 2.1.0rc2__py3-none-any.whl → 2.1.1rc1__py3-none-any.whl

meerschaum/connectors/sql/_sql.py

@@ -167,7 +167,7 @@ def read(
     if debug:
         import time
         start = time.perf_counter()
-        dprint(query_or_table)
+        dprint(f"[{self}]\n{query_or_table}")
         dprint(f"[{self}] Fetching with chunksize: {chunksize}")
 
     ### This might be sqlalchemy object or the string of a table name.
@@ -432,9 +432,9 @@ def value(
         result, connection = self.exec(
             query,
             *args,
-            with_connection=True,
-            close=False,
-            commit=_commit,
+            with_connection = True,
+            close = False,
+            commit = _commit,
             **kw
         )
         first = result.first() if result is not None else None
@@ -547,7 +547,7 @@ def exec(
         if debug:
             dprint(f"[{self}] Failed to execute query:\n\n{query}\n\n{e}")
         if not silent:
-            warn(str(e))
+            warn(str(e), stacklevel=3)
         result = None
         if _commit:
             transaction.rollback()
@@ -563,7 +563,12 @@ def exec(
 
 def exec_queries(
         self,
-        queries: List[str],
+        queries: List[
+            Union[
+                str,
+                Tuple[str, Callable[['sqlalchemy.orm.session.Session'], List[str]]]
+            ]
+        ],
         break_on_error: bool = False,
         rollback: bool = True,
         silent: bool = False,
@@ -574,8 +579,16 @@ def exec_queries(
 
     Parameters
     ----------
-    queries: List[str]
+    queries: List[
+        Union[
+            str,
+            Tuple[str, Callable[[], List[str]]]
+        ]
+    ]
         The queries in the transaction to be executed.
+        If a query is a tuple, the second item of the tuple
+        will be considered a callable hook that returns a list of queries to be executed
+        before the next item in the list.
 
     break_on_error: bool, default True
         If `True`, stop executing when a query fails.
@@ -593,18 +606,27 @@ def exec_queries(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.packages import attempt_import
-    sqlalchemy = attempt_import('sqlalchemy')
+    sqlalchemy, sqlalchemy_orm = attempt_import('sqlalchemy', 'sqlalchemy.orm')
+    session = sqlalchemy_orm.Session(self.engine)
 
+    result = None
     results = []
-    with self.engine.begin() as connection:
+    with session.begin():
         for query in queries:
-            if debug:
-                dprint(f"[{self}]\n" + str(query))
+            hook = None
+            result = None
+
+            if isinstance(query, tuple):
+                query, hook = query
             if isinstance(query, str):
                 query = sqlalchemy.text(query)
 
+            if debug:
+                dprint(f"[{self}]\n" + str(query))
+ 
             try:
-                result = connection.execute(query)
+                result = session.execute(query)
+                session.flush()
             except Exception as e:
                 msg = (f"Encountered error while executing:\n{e}")
                 if not silent:
@@ -612,11 +634,24 @@ def exec_queries(
                 elif debug:
                     dprint(f"[{self}]\n" + str(msg))
                 result = None
-            results.append(result)
             if result is None and break_on_error:
                 if rollback:
-                    connection.rollback()
+                    session.rollback()
                 break
+            elif result is not None and hook is not None:
+                hook_queries = hook(session)
+                if hook_queries:
+                    hook_results = self.exec_queries(
+                        hook_queries,
+                        break_on_error = break_on_error,
+                        rollback = rollback,
+                        silent = silent,
+                        debug = debug,
+                    )
+                    result = (result, hook_results)
+
+            results.append(result)
+
     return results
 
 
@@ -875,6 +910,9 @@ def psql_insert_copy(
     data_iter: Iterable[Any]
         Iterable that iterates the values to be inserted
 
+    schema: Optional[str], default None
+        Optionally specify the schema of the table to be inserted into.
+
     Returns
     -------
     None

meerschaum/connectors/sql/tables/__init__.py

@@ -7,7 +7,8 @@ Define SQLAlchemy tables
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Optional, Dict, Union, InstanceConnector
+from meerschaum.utils.typing import Optional, Dict, Union, InstanceConnector, List
+from meerschaum.utils.warnings import error, warn
 
 ### store a tables dict for each connector
 connector_tables = {}
@@ -42,7 +43,6 @@ def get_tables(
     """
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.formatting import pprint
-    from meerschaum.utils.warnings import error
     from meerschaum.connectors.parse import parse_instance_keys
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.sql import json_flavors
@@ -99,15 +99,18 @@ def get_tables(
                 kw.update({'server_default': sequences[k].next_value()})
 
         _tables = {
-            'users' : sqlalchemy.Table(
-                'users',
+            'users': sqlalchemy.Table(
+                'mrsm_users',
                 conn.metadata,
                 sqlalchemy.Column(
                     *id_col_args['user_id'],
                     **id_col_kw['user_id'],
                 ),
                 sqlalchemy.Column(
-                    'username', sqlalchemy.String(256), index=index_names, nullable=False,
+                    'username',
+                    sqlalchemy.String(256),
+                    index = index_names,
+                    nullable = False,
                 ),
                 sqlalchemy.Column('password_hash', sqlalchemy.String(1024)),
                 sqlalchemy.Column('email', sqlalchemy.String(256)),
@@ -115,9 +118,9 @@ def get_tables(
                 sqlalchemy.Column('attributes', params_type),
                 extend_existing = True,
             ),
-            'plugins' : sqlalchemy.Table(
+            'plugins': sqlalchemy.Table(
                 *([
-                    'plugins',
+                    'mrsm_plugins',
                     conn.metadata,
                     sqlalchemy.Column(
                         *id_col_args['plugin_id'],
@@ -130,26 +133,37 @@ def get_tables(
                     sqlalchemy.Column('version', sqlalchemy.String(256)),
                     sqlalchemy.Column('attributes', params_type),
                 ] + ([
-                    sqlalchemy.ForeignKeyConstraint(['user_id'], ['users.user_id']),
+                    sqlalchemy.ForeignKeyConstraint(['user_id'], ['mrsm_users.user_id']),
                 ] if conn.flavor != 'duckdb' else [])),
                 extend_existing = True,
             ),
         }
 
         _tables['pipes'] = sqlalchemy.Table(
-            "pipes",
+            "mrsm_pipes",
             conn.metadata,
             sqlalchemy.Column(
                 *id_col_args['pipe_id'],
                 **id_col_kw['pipe_id'],
             ),
             sqlalchemy.Column(
-                "connector_keys", sqlalchemy.String(256), index=index_names, nullable=False
+                "connector_keys",
+                sqlalchemy.String(256),
+                index = index_names,
+                nullable = False,
             ),
             sqlalchemy.Column(
-                "metric_key", sqlalchemy.String(256), index=index_names, nullable=False
+                "metric_key",
+                sqlalchemy.String(256),
+                index = index_names,
+                nullable = False,
+            ),
+            sqlalchemy.Column(
+                "location_key",
+                sqlalchemy.String(256),
+                index = index_names,
+                nullable = True,
             ),
-            sqlalchemy.Column("location_key", sqlalchemy.String(256), index=index_names),
             sqlalchemy.Column("parameters", params_type),
             extend_existing = True,
         )
@@ -157,6 +171,11 @@ def get_tables(
         ### store the table dict for reuse (per connector)
         connector_tables[conn] = _tables
         if create:
+            create_schemas(
+                conn,
+                schemas = [conn.internal_schema],
+                debug = debug,
+            )
             create_tables(conn, tables=_tables)
 
     return connector_tables[conn]
@@ -169,7 +188,6 @@ def create_tables(
     """
     Create the tables on the database.
     """
-    from meerschaum.utils.warnings import warn
     _tables = tables if tables is not None else get_tables(conn)
     try:
         conn.metadata.create_all(bind=conn.engine)
@@ -181,3 +199,37 @@ def create_tables(
     return True
 
 
+def create_schemas(
+        conn: 'meerschaum.connectors.SQLConnector',
+        schemas: List[str],
+        debug: bool = False,
+    ) -> bool:
+    """
+    Create the internal Meerschaum schema on the database.
+    """
+    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.sql import sql_item_name, NO_SCHEMA_FLAVORS, SKIP_IF_EXISTS_FLAVORS
+    if conn.flavor in NO_SCHEMA_FLAVORS:
+        return True
+
+    sqlalchemy_schema = attempt_import('sqlalchemy.schema')
+    successes = {}
+    skip_if_not_exists = conn.flavor in SKIP_IF_EXISTS_FLAVORS
+    if_not_exists_str = ("IF NOT EXISTS " if not skip_if_not_exists else "")
+    with conn.engine.connect() as connection:
+        for schema in schemas:
+            if not schema:
+                continue
+            schema_name = sql_item_name(schema, conn.flavor)
+            schema_exists = conn.engine.dialect.has_schema(connection, schema)
+            if schema_exists:
+                continue
+
+            create_schema_query = f"CREATE SCHEMA {if_not_exists_str}{schema_name}"
+            try:
+                result = conn.exec(create_schema_query, debug=debug)
+                successes[schema] = (result is not None)
+            except Exception as e:
+                warn(f"Failed to create internal schema '{schema}':\n{e}")
+    return all(successes.values())

meerschaum/connectors/sql/tables/pipes.py

@@ -0,0 +1,9 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Define instance tables as pipes.
+"""
+
+

meerschaum/core/Pipe/__init__.py

@@ -140,7 +140,7 @@ class Pipe:
         metric: str = '',
         location: Optional[str] = None,
         parameters: Optional[Dict[str, Any]] = None,
-        columns: Optional[Dict[str, str]] = None,
+        columns: Union[Dict[str, str], List[str], None] = None,
         tags: Optional[List[str]] = None,
         target: Optional[str] = None,
         dtypes: Optional[Dict[str, str]] = None,

meerschaum/core/Pipe/_data.py

@@ -536,10 +536,9 @@ def get_chunk_interval(
         return timedelta(minutes=chunk_minutes)
 
     dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns]')
-    if 'datetime' in dt_dtype.lower():
-        return timedelta(minutes=chunk_minutes)
-
-    return chunk_minutes
+    if 'int' in dt_dtype.lower():
+        return chunk_minutes
+    return timedelta(minutes=chunk_minutes)
 
 
 def get_chunk_bounds(

meerschaum/core/Pipe/_delete.py

@@ -10,6 +10,7 @@ from meerschaum.utils.typing import SuccessTuple
 
 def delete(
         self,
+        drop: bool = True,
         debug: bool = False,
         **kw
     ) -> SuccessTuple:
@@ -18,7 +19,10 @@ def delete(
 
     Parameters
     ----------
-    debug : bool, default False:
+    drop: bool, default True
+        If `True`, drop the pipes' target table.
+
+    debug : bool, default False
         Verbosity toggle.
 
     Returns
@@ -49,11 +53,17 @@ def delete(
             except Exception as e:
                 warn(f"Could not delete cache file '{_cache_db_path}' for {self}:\n{e}")
 
+    if drop:
+        drop_success, drop_msg = self.drop(debug=debug)
+        if not drop_success:
+            warn(f"Failed to drop {self}:\n{drop_msg}")
+
     with Venv(get_connector_plugin(self.instance_connector)):
         result = self.instance_connector.delete_pipe(self, debug=debug, **kw)
 
     if not isinstance(result, tuple):
-        return False, f"Received unexpected result from '{self.instance_connector}': {result}"
+        return False, f"Received an unexpected result from '{self.instance_connector}': {result}"
+
     if result[0]:
         to_delete = ['_id']
         for member in to_delete:

meerschaum/core/Pipe/_sync.py

@@ -548,6 +548,7 @@ def filter_existing(
     )
     from meerschaum.utils.dtypes import (
         to_pandas_dtype,
+        none_if_null,
     )
     from meerschaum.config import get_config
     pd = import_pandas()
@@ -645,11 +646,7 @@ def filter_existing(
     _ = kw.pop('params', None)
     params = {
         col: [
-            (
-                val
-                if str(val).lower() not in ('none', 'na', 'nan')
-                else None
-            )
+            none_if_null(val)
             for val in unique_vals
         ]
         for col, unique_vals in unique_index_vals.items()

meerschaum/utils/dataframe.py

@@ -174,14 +174,22 @@ def filter_unseen_df(
             f"Was not able to cast the new DataFrame to the given dtypes.\n{e}"
         )
 
+    new_numeric_cols_existing = get_numeric_cols(new_df)
+    old_numeric_cols = get_numeric_cols(old_df)
     for col, typ in {k: v for k, v in dtypes.items()}.items():
         if not are_dtypes_equal(new_df_dtypes.get(col, 'None'), old_df_dtypes.get(col, 'None')):
             new_is_float = are_dtypes_equal(new_df_dtypes.get(col, 'None'), 'float')
             new_is_int = are_dtypes_equal(new_df_dtypes.get(col, 'None'), 'int')
+            new_is_numeric = col in new_numeric_cols_existing
             old_is_float = are_dtypes_equal(old_df_dtypes.get(col, 'None'), 'float')
             old_is_int = are_dtypes_equal(old_df_dtypes.get(col, 'None'), 'int')
+            old_is_numeric = col in old_numeric_cols
 
-            if (new_is_float or new_is_int) and (old_is_float or old_is_int):
+            if (
+                (new_is_float or new_is_int or new_is_numeric)
+                and
+                (old_is_float or old_is_int or old_is_numeric)
+            ):
                 dtypes[col] = attempt_cast_to_numeric
                 cast_cols = True
                 continue
@@ -199,7 +207,11 @@ def filter_unseen_df(
         for col, dtype in dtypes.items():
             if col in new_df.columns:
                 try:
-                    new_df[col] = new_df[col].astype(dtype)
+                    new_df[col] = (
+                        new_df[col].astype(dtype)
+                        if not callable(dtype)
+                        else new_df[col].apply(dtype)
+                    )
                 except Exception as e:
                     warn(f"Was not able to cast column '{col}' to dtype '{dtype}'.\n{e}")
 
@@ -213,7 +225,6 @@ def filter_unseen_df(
         new_df[json_col] = new_df[json_col].apply(serializer)
 
     new_numeric_cols = get_numeric_cols(new_df)
-    old_numeric_cols = get_numeric_cols(old_df)
     numeric_cols = set(new_numeric_cols + old_numeric_cols)
     for numeric_col in old_numeric_cols:
         old_df[numeric_col] = old_df[numeric_col].apply(
@@ -578,6 +589,7 @@ def enforce_dtypes(
         for col, typ in dtypes.items()
         if typ == 'numeric'
     ]
+    df_numeric_cols = get_numeric_cols(df)
     if debug:
         dprint(f"Desired data types:")
         pprint(dtypes)
@@ -658,7 +670,11 @@ def enforce_dtypes(
         mixed_numeric_types = (is_dtype_numeric(typ) and is_dtype_numeric(previous_typ))
         explicitly_float = are_dtypes_equal(dtypes.get(col, 'object'), 'float')
         explicitly_numeric = dtypes.get(col, 'numeric') == 'numeric'
-        cast_to_numeric = explicitly_numeric or (mixed_numeric_types and not explicitly_float)
+        cast_to_numeric = (
+            explicitly_numeric
+            or col in df_numeric_cols
+            or (mixed_numeric_types and not explicitly_float)
+        )
         if cast_to_numeric:
             common_dtypes[col] = attempt_cast_to_numeric
             common_diff_dtypes[col] = attempt_cast_to_numeric

meerschaum/utils/dtypes/__init__.py

@@ -170,13 +170,27 @@ def attempt_cast_to_numeric(value: Any) -> Any:
     try:
         return (
             Decimal(str(value))
-            if str(value).lower() not in ('none', 'nan', 'na', 'nat', '')
+            if not value_is_null(value)
             else Decimal('NaN')
         )
     except Exception as e:
         return value
 
 
+def value_is_null(value: Any) -> Any:
+    """
+    Determine if a value is a null-like string.
+    """
+    return str(value).lower() in ('none', 'nan', 'na', 'nat', '', '<na>')
+
+
+def none_if_null(value: Any) -> Any:
+    """
+    Return `None` if a value is a null-like string.
+    """
+    return (None if value_is_null(value) else value)
+
+
 def quantize_decimal(x: Decimal, scale: int, precision: int) -> Decimal:
     """
     Quantize a given `Decimal` to a known scale and precision.

meerschaum/utils/dtypes/sql.py

@@ -19,6 +19,7 @@ NUMERIC_PRECISION_FLAVORS: Dict[str, Tuple[int, int]] = {
 
 DB_TO_PD_DTYPES: Dict[str, Union[str, Dict[str, str]]] = {
     'FLOAT': 'float64[pyarrow]',
+    'REAL': 'float64[pyarrow]',
     'DOUBLE_PRECISION': 'float64[pyarrow]',
     'DOUBLE': 'float64[pyarrow]',
     'DECIMAL': 'numeric',