meerschaum 2.7.9__py3-none-any.whl → 2.8.0__py3-none-any.whl

meerschaum/connectors/sql/_pipes.py

@@ -7,6 +7,7 @@ Interact with Pipes metadata via SQLConnector.
 """
 from __future__ import annotations
 from datetime import datetime, date, timedelta
+
 import meerschaum as mrsm
 from meerschaum.utils.typing import (
     Union, Any, SuccessTuple, Tuple, Dict, Optional, List
@@ -1837,7 +1838,7 @@ def sync_pipe(
         and primary_key in unseen_df.columns
         and autoincrement
     )
-    stats = {'success': True, 'msg': 'Success'}
+    stats = {'success': True, 'msg': ''}
     if len(unseen_df) > 0:
         with self.engine.connect() as connection:
             with connection.begin():
@@ -1949,6 +1950,7 @@ def sync_pipe(
             datetime_col=(dt_col if dt_col in update_df.columns else None),
             identity_insert=(autoincrement and primary_key in update_df.columns),
             null_indices=pipe.null_indices,
+            cast_columns=pipe.enforce,
             debug=debug,
         )
         update_results = self.exec_queries(
@@ -1967,12 +1969,16 @@ def sync_pipe(
         if not update_success:
             warn(f"Failed to apply update to {pipe}.")
         stats['success'] = stats['success'] and update_success
-        stats['msg'] = (stats.get('msg', '') + f'\nFailed to apply update to {pipe}.').lstrip()
+        stats['msg'] = (
+            (stats.get('msg', '') + f'\nFailed to apply update to {pipe}.').lstrip()
+            if not update_success
+            else stats.get('msg', '')
+        )
 
     stop = time.perf_counter()
     success = stats['success']
     if not success:
-        return success, stats['msg']
+        return success, stats['msg'] or str(stats)
 
     unseen_count = len(unseen_df.index) if unseen_df is not None else 0
     update_count = len(update_df.index) if update_df is not None else 0
@@ -2529,6 +2535,7 @@ def sync_pipe_inplace(
             datetime_col=pipe.columns.get('datetime', None),
             flavor=self.flavor,
             null_indices=pipe.null_indices,
+            cast_columns=pipe.enforce,
             debug=debug,
         )
         if on_cols else []
@@ -2585,6 +2592,7 @@ def get_sync_time(
     pipe: 'mrsm.Pipe',
     params: Optional[Dict[str, Any]] = None,
     newest: bool = True,
+    remote: bool = False,
     debug: bool = False,
 ) -> Union[datetime, int, None]:
     """Get a Pipe's most recent datetime value.
@@ -2602,50 +2610,76 @@ def get_sync_time(
         If `True`, get the most recent datetime (honoring `params`).
         If `False`, get the oldest datetime (ASC instead of DESC).
 
+    remote: bool, default False
+        If `True`, return the sync time for the remote fetch definition.
+
     Returns
     -------
     A `datetime` object (or `int` if using an integer axis) if the pipe exists, otherwise `None`.
     """
-    from meerschaum.utils.sql import sql_item_name, build_where
-    table = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
+    from meerschaum.utils.sql import sql_item_name, build_where, wrap_query_with_cte
+    src_name = sql_item_name('src', self.flavor)
+    table_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
 
     dt_col = pipe.columns.get('datetime', None)
     if dt_col is None:
         return None
     dt_col_name = sql_item_name(dt_col, self.flavor, None)
 
+    if remote and pipe.connector.type != 'sql':
+        warn(f"Cannot get the remote sync time for {pipe}.")
+        return None
+
     ASC_or_DESC = "DESC" if newest else "ASC"
     existing_cols = pipe.get_columns_types(debug=debug)
     valid_params = {}
     if params is not None:
         valid_params = {k: v for k, v in params.items() if k in existing_cols}
+    flavor = self.flavor if not remote else pipe.connector.flavor
 
     ### If no bounds are provided for the datetime column,
     ### add IS NOT NULL to the WHERE clause.
     if dt_col not in valid_params:
         valid_params[dt_col] = '_None'
     where = "" if not valid_params else build_where(valid_params, self)
-    q = f"SELECT {dt_col_name}\nFROM {table}{where}\nORDER BY {dt_col_name} {ASC_or_DESC}\nLIMIT 1"
+    src_query = (
+        f"SELECT {dt_col_name}\nFROM {table_name}{where}"
+        if not remote
+        else self.get_pipe_metadef(pipe, params=params, begin=None, end=None)
+    )
+
+    base_query = (
+        f"SELECT {dt_col_name}\n"
+        f"FROM {src_name}{where}\n"
+        f"ORDER BY {dt_col_name} {ASC_or_DESC}\n"
+        f"LIMIT 1"
+    )
     if self.flavor == 'mssql':
-        q = f"SELECT TOP 1 {dt_col_name}\nFROM {table}{where}\nORDER BY {dt_col_name} {ASC_or_DESC}"
+        base_query = (
+            f"SELECT TOP 1 {dt_col_name}\n"
+            f"FROM {src_name}{where}\n"
+            f"ORDER BY {dt_col_name} {ASC_or_DESC}"
+        )
     elif self.flavor == 'oracle':
-        q = (
+        base_query = (
             "SELECT * FROM (\n"
-            + f"    SELECT {dt_col_name}\nFROM {table}{where}\n    "
-            + f"ORDER BY {dt_col_name} {ASC_or_DESC}\n"
-            + ") WHERE ROWNUM = 1"
+            f"    SELECT {dt_col_name}\n"
+            f"    FROM {src_name}{where}\n"
+            f"    ORDER BY {dt_col_name} {ASC_or_DESC}\n"
+            ") WHERE ROWNUM = 1"
         )
 
+    query = wrap_query_with_cte(src_query, base_query, flavor)
+
     try:
-        db_time = self.value(q, silent=True, debug=debug)
+        db_time = self.value(query, silent=True, debug=debug)
 
         ### No datetime could be found.
         if db_time is None:
             return None
         ### sqlite returns str.
         if isinstance(db_time, str):
-            from meerschaum.utils.packages import attempt_import
-            dateutil_parser = attempt_import('dateutil.parser')
+            dateutil_parser = mrsm.attempt_import('dateutil.parser')
             st = dateutil_parser.parse(db_time)
         ### Do nothing if a datetime object is returned.
         elif isinstance(db_time, datetime):
@@ -2743,7 +2777,7 @@ def get_pipe_rowcount(
     An `int` for the number of rows if the `pipe` exists, otherwise `None`.
 
     """
-    from meerschaum.utils.sql import dateadd_str, sql_item_name, wrap_query_with_cte
+    from meerschaum.utils.sql import dateadd_str, sql_item_name, wrap_query_with_cte, build_where
     from meerschaum.connectors.sql._fetch import get_pipe_query
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
     if remote:
@@ -2755,18 +2789,20 @@ def get_pipe_rowcount(
             error(msg)
             return None
 
-    _pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
 
+    flavor = self.flavor if not remote else pipe.connector.flavor
+    conn = self if not remote else pipe.connector
+    _pipe_name = sql_item_name(pipe.target, flavor, self.get_pipe_schema(pipe))
     dt_col = pipe.columns.get('datetime', None)
     dt_typ = pipe.dtypes.get(dt_col, 'datetime') if dt_col else None
-    dt_db_type = get_db_type_from_pd_type(dt_typ, self.flavor) if dt_typ else None
+    dt_db_type = get_db_type_from_pd_type(dt_typ, flavor) if dt_typ else None
     if not dt_col:
         dt_col = pipe.guess_datetime()
-        dt_name = sql_item_name(dt_col, self.flavor, None) if dt_col else None
+        dt_name = sql_item_name(dt_col, flavor, None) if dt_col else None
         is_guess = True
     else:
         dt_col = pipe.get_columns('datetime')
-        dt_name = sql_item_name(dt_col, self.flavor, None)
+        dt_name = sql_item_name(dt_col, flavor, None)
         is_guess = False
 
     if begin is not None or end is not None:
@@ -2786,32 +2822,15 @@ def get_pipe_rowcount(
                 )
 
 
-    _datetime_name = sql_item_name(
-        dt_col,
-        (
-            pipe.instance_connector.flavor
-            if not remote
-            else pipe.connector.flavor
-        ),
-        None,
-    )
+    _datetime_name = sql_item_name(dt_col, flavor)
     _cols_names = [
-        sql_item_name(
-            col,
-            (
-                pipe.instance_connector.flavor
-                if not remote
-                else pipe.connector.flavor
-            ),
-            None,
-        )
+        sql_item_name(col, flavor)
         for col in set(
             (
                 [dt_col]
                 if dt_col
                 else []
-            )
-            + (
+            ) + (
                 []
                 if params is None
                 else list(params.keys())
@@ -2826,34 +2845,33 @@ def get_pipe_rowcount(
         if not remote
         else get_pipe_query(pipe)
     )
-    parent_query = f"SELECT COUNT(*)\nFROM {sql_item_name('src', self.flavor)}"
-    query = wrap_query_with_cte(src, parent_query, self.flavor)
+    parent_query = f"SELECT COUNT(*)\nFROM {sql_item_name('src', flavor)}"
+    query = wrap_query_with_cte(src, parent_query, flavor)
     if begin is not None or end is not None:
         query += "\nWHERE"
     if begin is not None:
         query += (
             f"\n    {dt_name} >= "
-            + dateadd_str(self.flavor, datepart='minute', number=0, begin=begin, db_type=dt_db_type)
+            + dateadd_str(flavor, datepart='minute', number=0, begin=begin, db_type=dt_db_type)
         )
     if end is not None and begin is not None:
         query += "\n    AND"
     if end is not None:
         query += (
             f"\n    {dt_name} <  "
-            + dateadd_str(self.flavor, datepart='minute', number=0, begin=end, db_type=dt_db_type)
+            + dateadd_str(flavor, datepart='minute', number=0, begin=end, db_type=dt_db_type)
         )
     if params is not None:
-        from meerschaum.utils.sql import build_where
         existing_cols = pipe.get_columns_types(debug=debug)
         valid_params = {k: v for k, v in params.items() if k in existing_cols}
         if valid_params:
-            query += build_where(valid_params, self).replace('WHERE', (
+            query += build_where(valid_params, conn).replace('WHERE', (
                 'AND' if (begin is not None or end is not None)
                     else 'WHERE'
                 )
             )
 
-    result = self.value(query, debug=debug, silent=True)
+    result = conn.value(query, debug=debug, silent=True)
     try:
         return int(result)
     except Exception:
@@ -3634,7 +3652,6 @@ def deduplicate_pipe(
     if not pipe.exists(debug=debug):
         return False, f"Table {pipe_table_name} does not exist."
 
-    ### TODO: Handle deleting duplicates without a datetime axis.
     dt_col = pipe.columns.get('datetime', None)
     cols_types = pipe.get_columns_types(debug=debug)
     existing_cols = pipe.get_columns_types(debug=debug)
@@ -3738,9 +3755,8 @@ def deduplicate_pipe(
 
     session_id = generate_password(3)
 
-    dedup_table = '-' + session_id + f'_dedup_{pipe.target}'
-    temp_old_table = '-' + session_id + f"_old_{pipe.target}"
-
+    dedup_table = self.get_temporary_target(pipe.target, transact_id=session_id, label='dedup')
+    temp_old_table = self.get_temporary_target(pipe.target, transact_id=session_id, label='old')
     temp_old_table_name = sql_item_name(temp_old_table, self.flavor, self.get_pipe_schema(pipe))
 
     create_temporary_table_query = get_create_table_query(
@@ -3753,16 +3769,21 @@ def deduplicate_pipe(
     if_exists_str = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
     alter_queries = flatten_list([
         get_rename_table_queries(
-            pipe.target, temp_old_table, self.flavor, schema=self.get_pipe_schema(pipe)
+            pipe.target,
+            temp_old_table,
+            self.flavor,
+            schema=self.get_pipe_schema(pipe),
         ),
         get_rename_table_queries(
-            dedup_table, pipe.target, self.flavor, schema=self.get_pipe_schema(pipe)
+            dedup_table,
+            pipe.target,
+            self.flavor,
+            schema=self.get_pipe_schema(pipe),
         ),
-        f"""
-        DROP TABLE {if_exists_str} {temp_old_table_name}
-        """,
+        f"DROP TABLE {if_exists_str} {temp_old_table_name}",
     ])
 
+    self._log_temporary_tables_creation(temp_old_table, create=(not pipe.temporary), debug=debug)
     create_temporary_result = self.execute(create_temporary_table_query, debug=debug)
     if create_temporary_result is None:
         return False, f"Failed to deduplicate table {pipe_table_name}."
@@ -3794,8 +3815,7 @@ def deduplicate_pipe(
                 f"\nfrom {old_rowcount:,} to {new_rowcount:,} rows"
                 if old_rowcount != new_rowcount
                 else ''
-            )
-            + '.'
+            ) + '.'
         )
         if success
         else f"Failed to execute query:\n{fail_query}"

meerschaum/connectors/sql/_sql.py

@@ -773,7 +773,6 @@ def to_sql(
     """
     import time
     import json
-    from decimal import Decimal
     from datetime import timedelta
     from meerschaum.utils.warnings import error, warn
     import warnings
@@ -823,6 +822,7 @@ def to_sql(
 
     bytes_cols = get_bytes_cols(df)
     numeric_cols = get_numeric_cols(df)
+    ### NOTE: This excludes non-numeric serialized Decimals (e.g. SQLite).
     numeric_cols_dtypes = {
         col: typ
         for col, typ in kw.get('dtype', {}).items()
@@ -833,6 +833,27 @@ def to_sql(
         
     }
     numeric_cols.extend([col for col in numeric_cols_dtypes if col not in numeric_cols])
+    numeric_cols_precisions_scales = {
+        col: (
+            (typ.precision, typ.scale)
+            if hasattr(typ, 'precision')
+            else get_numeric_precision_scale(self.flavor)
+        )
+        for col, typ in numeric_cols_dtypes.items()
+    }
+    cols_pd_types = {
+        col: get_pd_type_from_db_type(str(typ))
+        for col, typ in kw.get('dtype', {}).items()
+    }
+    cols_pd_types.update({
+        col: f'numeric[{precision},{scale}]'
+        for col, (precision, scale) in numeric_cols_precisions_scales.items()
+        if precision and scale
+    })
+    cols_db_types = {
+        col: get_db_type_from_pd_type(typ, flavor=self.flavor)
+        for col, typ in cols_pd_types.items()
+    }
 
     enable_bulk_insert = mrsm.get_config(
         'system', 'connectors', 'sql', 'bulk_insert'
@@ -844,7 +865,7 @@ def to_sql(
     if method == "":
         if enable_bulk_insert:
             method = (
-                functools.partial(mssql_insert_json, debug=debug)
+                functools.partial(mssql_insert_json, cols_types=cols_db_types, debug=debug)
                 if self.flavor == 'mssql'
                 else functools.partial(psql_insert_copy, debug=debug)
             )
@@ -867,14 +888,10 @@ def to_sql(
 
     ### Check for numeric columns.
     for col in numeric_cols:
-        typ = numeric_cols_dtypes.get(col, None)
-
-        precision, scale = (
-            (typ.precision, typ.scale)
-            if hasattr(typ, 'precision')
-            else get_numeric_precision_scale(self.flavor)
+        precision, scale = numeric_cols_precisions_scales.get(
+            col,
+            get_numeric_precision_scale(self.flavor)
         )
-
         df[col] = df[col].apply(
             functools.partial(
                 serialize_decimal,

meerschaum/core/Pipe/__init__.py

@@ -92,6 +92,7 @@ class Pipe:
         _get_data_as_iterator,
         get_chunk_interval,
         get_chunk_bounds,
+        get_chunk_bounds_batches,
         parse_date_bounds,
     )
     from ._register import register
@@ -117,6 +118,7 @@ class Pipe:
         id,
         get_val_column,
         parents,
+        parent,
         children,
         target,
         _target_legacy,

meerschaum/core/Pipe/_attributes.py

@@ -590,7 +590,7 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
 
 
 @property
-def parents(self) -> List[meerschaum.Pipe]:
+def parents(self) -> List[mrsm.Pipe]:
     """
     Return a list of `meerschaum.Pipe` objects to be designated as parents.
     """
@@ -617,7 +617,18 @@ def parents(self) -> List[meerschaum.Pipe]:
 
 
 @property
-def children(self) -> List[meerschaum.Pipe]:
+def parent(self) -> Union[mrsm.Pipe, None]:
+    """
+    Return the first pipe in `self.parents` or `None`.
+    """
+    parents = self.parents
+    if not parents:
+        return None
+    return parents[0]
+
+
+@property
+def children(self) -> List[mrsm.Pipe]:
     """
     Return a list of `meerschaum.Pipe` objects to be designated as children.
     """

meerschaum/core/Pipe/_data.py

@@ -544,11 +544,33 @@ def get_rowcount(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
+    from meerschaum.utils.misc import filter_keywords
 
     begin, end = self.parse_date_bounds(begin, end)
     connector = self.instance_connector if not remote else self.connector
     try:
         with Venv(get_connector_plugin(connector)):
+            if not hasattr(connector, 'get_pipe_rowcount'):
+                warn(
+                    f"Connectors of type '{connector.type}' "
+                    "do not implement `get_pipe_rowcount()`.",
+                    stack=False,
+                )
+                return 0
+            kwargs = filter_keywords(
+                connector.get_pipe_rowcount,
+                begin=begin,
+                end=end,
+                params=params,
+                remote=remote,
+                debug=debug,
+            )
+            if remote and 'remote' not in kwargs:
+                warn(
+                    f"Connectors of type '{connector.type}' do not support remote rowcounts.",
+                    stack=False,
+                )
+                return 0
             rowcount = connector.get_pipe_rowcount(
                 self,
                 begin=begin,
@@ -651,12 +673,19 @@ def get_chunk_bounds(
     A list of chunk bounds (datetimes or integers).
     If unbounded, the first and last chunks will include `None`.
     """
+    from datetime import timedelta
+    from meerschaum.utils.dtypes import are_dtypes_equal
+    from meerschaum.utils.misc import interval_str
     include_less_than_begin = not bounded and begin is None
     include_greater_than_end = not bounded and end is None
     if begin is None:
         begin = self.get_sync_time(newest=False, debug=debug)
     if end is None:
         end = self.get_sync_time(newest=True, debug=debug)
+        if end is not None and hasattr(end, 'tzinfo'):
+            end += timedelta(minutes=1)
+        elif are_dtypes_equal(str(type(end)), 'int'):
+            end += 1
     if begin is None and end is None:
         return [(None, None)]
 
@@ -670,10 +699,17 @@ def get_chunk_bounds(
     ### Run `verify pipes --workers 1` to sync chunks in series.
     chunk_bounds = []
     begin_cursor = begin
+    num_chunks = 0
+    max_chunks = 1_000_000
     while begin_cursor < end:
         end_cursor = begin_cursor + chunk_interval
         chunk_bounds.append((begin_cursor, end_cursor))
         begin_cursor = end_cursor
+        num_chunks += 1
+        if num_chunks >= max_chunks:
+            raise ValueError(
+                f"Too many chunks of size '{interval_str(chunk_interval)}' between '{begin}' and '{end}'."
+            )
 
     ### The chunk interval might be too large.
     if not chunk_bounds and end >= begin:
@@ -695,6 +731,55 @@ def get_chunk_bounds(
     return chunk_bounds
 
 
+def get_chunk_bounds_batches(
+    self,
+    chunk_bounds: List[Tuple[Union[datetime, int, None], Union[datetime, int, None]]],
+    batchsize: Optional[int] = None,
+    workers: Optional[int] = None,
+    debug: bool = False,
+) -> List[
+    Tuple[
+        Tuple[
+            Union[datetime, int, None],
+            Union[datetime, int, None],
+        ], ...
+    ]
+]:
+    """
+    Return a list of tuples of chunk bounds of size `batchsize`.
+
+    Parameters
+    ----------
+    chunk_bounds: List[Tuple[Union[datetime, int, None], Union[datetime, int, None]]]
+        A list of chunk_bounds (see `Pipe.get_chunk_bounds()`).
+
+    batchsize: Optional[int], default None
+        How many chunks to include in a batch. Defaults to `Pipe.get_num_workers()`.
+
+    workers: Optional[int], default None
+        If `batchsize` is `None`, use this as the desired number of workers.
+        Passed to `Pipe.get_num_workers()`.
+
+    Returns
+    -------
+    A list of tuples of chunk bound tuples.
+    """
+    from meerschaum.utils.misc import iterate_chunks
+    
+    if batchsize is None:
+        batchsize = self.get_num_workers(workers=workers)
+
+    return [
+        tuple(
+            _batch_chunk_bounds
+            for _batch_chunk_bounds in batch
+            if _batch_chunk_bounds is not None
+        )
+        for batch in iterate_chunks(chunk_bounds, batchsize)
+        if batch
+    ]
+
+
 def parse_date_bounds(self, *dt_vals: Union[datetime, int, None]) -> Union[
     datetime,
     int,

meerschaum/core/Pipe/_deduplicate.py

@@ -110,13 +110,12 @@ def deduplicate(
         )
     if bounded and end is None:
         end = self.get_sync_time(newest=True, debug=debug)
-
-    if bounded and end is not None:
-        end += (
-            timedelta(minutes=1)
-            if isinstance(end, datetime)
-            else 1
-        )
+        if end is not None:
+            end += (
+                timedelta(minutes=1)
+                if hasattr(end, 'tzinfo')
+                else 1
+            )
 
     chunk_bounds = self.get_chunk_bounds(
         bounded=bounded,
@@ -129,7 +128,6 @@ def deduplicate(
     indices = [col for col in self.columns.values() if col]
     if not indices:
         return False, "Cannot deduplicate without index columns."
-    dt_col = self.columns.get('datetime', None)
 
     def process_chunk_bounds(bounds) -> Tuple[
         Tuple[