meerschaum 2.1.5__py3-none-any.whl → 2.1.7__py3-none-any.whl

meerschaum/_internal/shell/Shell.py

@@ -11,11 +11,7 @@ from copy import deepcopy
 from meerschaum.utils.typing import Union, SuccessTuple, Any, Callable, Optional, List, Dict
 from meerschaum.utils.packages import attempt_import
 from meerschaum.config import __doc__, __version__ as version, get_config
-cmd_import_name = get_config('shell', 'cmd')
-cmd_venv = None if cmd_import_name == 'cmd' else 'mrsm'
-cmd = attempt_import(cmd_import_name, venv=cmd_venv, warn=False, lazy=False)
-if cmd is None or isinstance(cmd, dict):
-    cmd = attempt_import('cmd', lazy=False, warn=False)
+import cmd
 _old_input = cmd.__builtins__['input']
 prompt_toolkit = attempt_import('prompt_toolkit', lazy=False, warn=False, install=True)
 (
@@ -53,7 +49,6 @@ hidden_commands = {
     'os',
     'sh',
     'pass',
-    'exit',
     'quit',
     'eof',
     'exit',

meerschaum/actions/api.py

@@ -44,7 +44,7 @@ def api(
         sysargs = []
     if len(action) == 0:
         info(api.__doc__)
-        return False, "Please provide a command to excecute (see above)."
+        return False, "Please provide a command to execute (see above)."
 
     boot_keywords = {'start', 'boot', 'init'}
     if action[0] in boot_keywords:

meerschaum/actions/sync.py

@@ -9,7 +9,7 @@ NOTE: `sync` required a SQL connection and is not intended for client use
 """
 
 from __future__ import annotations
-from datetime import timedelta
+from datetime import timedelta, datetime, timezone
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Tuple, Union
 
@@ -39,7 +39,7 @@ def sync(
 
 def _pipes_lap(
         workers: Optional[int] = None,
-        debug: bool = None,
+        debug: Optional[bool] = None,
         unblock: bool = False,
         force: bool = False,
         min_seconds: int = 1,
@@ -52,7 +52,7 @@ def _pipes_lap(
         nopretty: bool = False,
         _progress: Optional['rich.progress.Progress'] = None,
         **kw: Any
-    ) -> Tuple[List[meerschaum.Pipe], List[meerschaum.Pipe]]:
+    ) -> Tuple[List[mrsm.Pipe], List[mrsm.Pipe]]:
     """
     Do a lap of syncing pipes.
     """
@@ -402,11 +402,20 @@ def _wrap_pipe(
     Wrapper function for handling exceptions.
     """
     import time
+    import traceback
+    from datetime import datetime, timedelta, timezone
+    import meerschaum as mrsm
+    from meerschaum.utils.typing import is_success_tuple, SuccessTuple
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.venv import Venv
     from meerschaum.plugins import _pre_sync_hooks, _post_sync_hooks
     from meerschaum.utils.misc import filter_keywords
+    from meerschaum.utils.pool import get_pool
+    from meerschaum.utils.warnings import warn
+
+    pool = get_pool(workers=workers)
 
+    sync_timestamp = datetime.now(timezone.utc)
     sync_start = time.perf_counter()
     sync_kwargs = {k: v for k, v in kw.items() if k != 'blocking'}
     sync_kwargs.update({
@@ -415,8 +424,9 @@ def _wrap_pipe(
         'debug': debug,
         'min_seconds': min_seconds,
         'workers': workers,
-        'bounded': 'bounded',
+        'bounded': bounded,
         'chunk_interval': chunk_interval,
+        'sync_timestamp': sync_timestamp,
     })
     if not verify and not deduplicate:
         sync_method = pipe.sync
@@ -427,12 +437,32 @@ def _wrap_pipe(
         sync_kwargs['deduplicate'] = deduplicate
     sync_kwargs['sync_method'] = sync_method
 
-    for module_name, pre_sync_hooks in _pre_sync_hooks.items():
-        plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
+    def call_sync_hook(plugin_name: str, sync_hook) -> SuccessTuple:
         plugin = mrsm.Plugin(plugin_name) if plugin_name else None
-        with Venv(plugin):
-            for pre_sync_hook in pre_sync_hooks:
-                _ = pre_sync_hook(pipe, **filter_keywords(pre_sync_hook, **sync_kwargs))
+        with mrsm.Venv(plugin):
+            try:
+                sync_hook_result = sync_hook(pipe, **filter_keywords(sync_hook, **sync_kwargs))
+                if is_success_tuple(sync_hook_result):
+                    return sync_hook_result
+            except Exception as e:
+                msg = (
+                    f"Failed to execute sync hook '{sync_hook.__name__}' "
+                    + f"from plugin '{plugin}':\n{traceback.format_exc()}"
+                )
+                warn(msg, stack=False)
+                return False, msg
+        return True, "Success"
+
+    hook_results = []
+    def apply_hooks(is_pre_sync: bool):
+        _sync_hooks = (_pre_sync_hooks if is_pre_sync else _post_sync_hooks)
+        for module_name, sync_hooks in _sync_hooks.items():
+            plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
+            for sync_hook in sync_hooks:
+                hook_result = pool.apply_async(call_sync_hook, (plugin_name, sync_hook))
+                hook_results.append(hook_result)
+
+    apply_hooks(True)
    
     try:
         with Venv(get_connector_plugin(pipe.connector), debug=debug):
@@ -444,18 +474,16 @@ def _wrap_pipe(
         return_tuple = (False, f"Failed to sync {pipe} with exception:" + "\n" + str(e))
 
     duration = time.perf_counter() - sync_start
-    sync_kwargs['duration'] = duration
-    for module_name, post_sync_hooks in _post_sync_hooks.items():
-        plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
-        plugin = mrsm.Plugin(plugin_name) if plugin_name else None
-        with Venv(plugin):
-            for post_sync_hook in post_sync_hooks:
-                _ = post_sync_hook(
-                    pipe,
-                    return_tuple,
-                    **filter_keywords(post_sync_hook, **sync_kwargs)
-                )
- 
+    sync_kwargs.update({
+        'success_tuple': return_tuple,
+        'sync_duration': duration,
+        'sync_complete_timestamp': datetime.now(timezone.utc),
+    })
+    apply_hooks(False)
+    for hook_result in hook_results:
+        hook_success, hook_msg = hook_result.get()
+        mrsm.pprint((hook_success, hook_msg))
+
     return return_tuple
 
 

meerschaum/config/_shell.py

@@ -126,7 +126,6 @@ default_shell_config = {
     'timeout'          : 60,
     'max_history'      : 1000,
     'clear_screen'     : True,
-    'cmd'              : default_cmd,
     'bottom_toolbar'   : {
         'enabled'      : True,
     },

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.1.5"
+__version__ = "2.1.7"

meerschaum/connectors/sql/_fetch.py

@@ -174,9 +174,6 @@ def get_pipe_metadef(
                 )
 
 
-    if 'order by' in definition.lower() and 'over' not in definition.lower():
-        error("Cannot fetch with an ORDER clause in the definition")
-
     apply_backtrack = begin == '' and check_existing
     backtrack_interval = pipe.get_backtrack_interval(check_existing=check_existing, debug=debug)
     btm = (
@@ -308,9 +305,9 @@ def _simple_fetch_query(pipe, debug: bool=False, **kw) -> str:
     def_name = 'definition'
     definition = get_pipe_query(pipe)
     return (
-        f"WITH {def_name} AS ({definition}) SELECT * FROM {def_name}"
+        f"WITH {def_name} AS (\n{definition}\n) SELECT * FROM {def_name}"
         if pipe.connector.flavor not in ('mysql', 'mariadb')
-        else f"SELECT * FROM ({definition}) AS {def_name}"
+        else f"SELECT * FROM (\n{definition}\n) AS {def_name}"
     )
 
 def _join_fetch_query(
@@ -363,10 +360,10 @@ def _join_fetch_query(
             )
             + f") AS {id_remote_name}, "
             + dateadd_str(
-                flavor=pipe.connector.flavor,
-                begin=_st,
-                datepart='minute',
-                number=pipe.parameters.get('fetch', {}).get('backtrack_minutes', 0)
+                flavor = pipe.connector.flavor,
+                begin = _st,
+                datepart = 'minute',
+                number = pipe.parameters.get('fetch', {}).get('backtrack_minutes', 0)
             ) + " AS " + dt_remote_name + "\nUNION ALL\n"
         )
     _sync_times_q = _sync_times_q[:(-1 * len('UNION ALL\n'))] + ")"
@@ -374,13 +371,13 @@ def _join_fetch_query(
     definition = get_pipe_query(pipe)
     query = (
         f"""
-    WITH definition AS ({definition}){_sync_times_q}
+    WITH definition AS (\n{definition}\n){_sync_times_q}
     SELECT definition.*
     FROM definition"""
         if pipe.connector.flavor not in ('mysql', 'mariadb')
         else (
         f"""
-    SELECT * FROM ({definition}) AS definition"""
+    SELECT * FROM (\n{definition}\n) AS definition"""
         )
     ) + f"""
     LEFT OUTER JOIN {sync_times_remote_name} AS st

meerschaum/connectors/sql/_pipes.py

@@ -385,7 +385,13 @@ def get_create_index_queries(
     -------
     A dictionary of column names mapping to lists of queries.
     """
-    from meerschaum.utils.sql import sql_item_name, get_distinct_col_count, update_queries
+    from meerschaum.utils.sql import (
+        sql_item_name,
+        get_distinct_col_count,
+        update_queries,
+        get_null_replacement,
+        COALESCE_UNIQUE_INDEX_FLAVORS,
+    )
     from meerschaum.config import get_config
     index_queries = {}
 
@@ -497,15 +503,37 @@ def get_create_index_queries(
             if ix and ix in existing_cols_types
         ]
     )
+    coalesce_indices_cols_str = ', '.join(
+        [
+            (
+                "COALESCE("
+                + sql_item_name(ix, self.flavor)
+                + ", "
+                + get_null_replacement(existing_cols_types[ix], self.flavor)
+                + ") "
+            ) if ix_key != 'datetime' else (sql_item_name(ix, self.flavor))
+            for ix_key, ix in pipe.columns.items()
+            if ix and ix in existing_cols_types
+        ]
+    )
+    unique_index_name = sql_item_name(pipe.target + '_unique_index', self.flavor)
     constraint_name = sql_item_name(pipe.target + '_constraint', self.flavor)
-    constraint_query = (
+    add_constraint_query = (
         f"ALTER TABLE {_pipe_name} ADD CONSTRAINT {constraint_name} UNIQUE ({indices_cols_str})"
-        if self.flavor != 'sqlite'
-        else f"CREATE UNIQUE INDEX {constraint_name} ON {_pipe_name} ({indices_cols_str})"
     )
+    unique_index_cols_str = (
+        indices_cols_str
+        if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS
+        else coalesce_indices_cols_str
+    )
+    create_unique_index_query = (
+        f"CREATE UNIQUE INDEX {unique_index_name} ON {_pipe_name} ({unique_index_cols_str})"
+    )
+    constraint_queries = [create_unique_index_query]
+    if self.flavor != 'sqlite':
+        constraint_queries.append(add_constraint_query)
     if upsert and indices_cols_str:
-        index_queries[constraint_name] = [constraint_query]
-
+        index_queries[unique_index_name] = constraint_queries
     return index_queries
 
 
@@ -1074,7 +1102,7 @@ def get_pipe_attributes(
 def sync_pipe(
         self,
         pipe: mrsm.Pipe,
-        df: Union[pandas.DataFrame, str, Dict[Any, Any], None] = None,
+        df: Union[pd.DataFrame, str, Dict[Any, Any], None] = None,
         begin: Optional[datetime] = None,
         end: Optional[datetime] = None,
         chunksize: Optional[int] = -1,
@@ -1154,7 +1182,12 @@ def sync_pipe(
         dprint("Fetched data:\n" + str(df))
 
     if not isinstance(df, pd.DataFrame):
-        df = pipe.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+        df = pipe.enforce_dtypes(
+            df,
+            chunksize = chunksize,
+            safe_copy = kw.get('safe_copy', False),
+            debug = debug,
+        )
 
     ### if table does not exist, create it with indices
     is_new = False
@@ -1198,6 +1231,7 @@ def sync_pipe(
     upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in update_queries
     if upsert:
         check_existing = False
+    kw['safe_copy'] = kw.get('safe_copy', False)
 
     unseen_df, update_df, delta_df = (
         pipe.filter_existing(
@@ -1286,7 +1320,11 @@ def sync_pipe(
         temp_pipe = Pipe(
             pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
             instance = pipe.instance_keys,
-            columns = pipe.columns,
+            columns = {
+                ix_key: ix
+                for ix_key, ix in pipe.columns.items()
+                if ix and ix in update_df.columns
+            },
             dtypes = pipe.dtypes,
             target = temp_target,
             temporary = True,

meerschaum/connectors/sql/tables/__init__.py

@@ -214,22 +214,6 @@ def create_tables(
     from meerschaum.utils.sql import get_rename_table_queries, table_exists
     _tables = tables if tables is not None else get_tables(conn)
 
-    rename_queries = []
-    for table_key, table in _tables.items():
-        if table_exists(
-            table_key,
-            conn,
-            schema = conn.instance_schema,
-        ):
-            rename_queries.extend(get_rename_table_queries(
-                table_key,
-                table.name,
-                schema = conn.instance_schema,
-                flavor = conn.flavor,
-            ))
-    if rename_queries:
-        conn.exec_queries(rename_queries)
-
     try:
         conn.metadata.create_all(bind=conn.engine)
     except Exception as e:

meerschaum/core/Pipe/_data.py

@@ -8,7 +8,7 @@ Retrieve Pipes' data from instances.
 
 from __future__ import annotations
 from datetime import datetime, timedelta
-from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator, List, Tuple
+from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator, List, Tuple, Iterator
 from meerschaum.config import get_config
 
 def get_data(
@@ -247,7 +247,7 @@ def _get_data_as_iterator(
         fresh: bool = False,
         debug: bool = False,
         **kw: Any
-    ) -> Generator['pd.DataFrame']:
+    ) -> Iterator['pd.DataFrame']:
     """
     Return a pipe's data as a generator.
     """
@@ -267,16 +267,17 @@ def _get_data_as_iterator(
 
     _ = kw.pop('as_chunks', None)
     _ = kw.pop('as_iterator', None)
+    dt_col = self.columns.get('datetime', None)
     min_dt = (
         begin
         if begin is not None
         else self.get_sync_time(round_down=False, newest=False, params=params, debug=debug)
-    )
+    ) if dt_col else None
     max_dt = (
         end
         if end is not None
         else self.get_sync_time(round_down=False, newest=True, params=params, debug=debug)
-    )
+    ) if dt_col else None
 
     ### We want to search just past the maximum value.
     if end is None:

meerschaum/core/Pipe/_dtypes.py

@@ -14,6 +14,7 @@ def enforce_dtypes(
         self,
         df: 'pd.DataFrame',
         chunksize: Optional[int] = -1,
+        safe_copy: bool = True,
         debug: bool = False,
     ) -> 'pd.DataFrame':
     """
@@ -71,7 +72,7 @@ def enforce_dtypes(
             )
         return df
 
-    return _enforce_dtypes(df, pipe_dtypes, debug=debug)
+    return _enforce_dtypes(df, pipe_dtypes, safe_copy=safe_copy, debug=debug)
 
 
 def infer_dtypes(self, persist: bool=False, debug: bool=False) -> Dict[str, Any]:

meerschaum/core/Pipe/_sync.py

@@ -12,6 +12,7 @@ import json
 import time
 import threading
 import multiprocessing
+import functools
 from datetime import datetime, timedelta
 
 from meerschaum.utils.typing import (
@@ -518,6 +519,8 @@ def exists(
 def filter_existing(
         self,
         df: 'pd.DataFrame',
+        safe_copy: bool = True,
+        date_bound_only: bool = False,
         chunksize: Optional[int] = -1,
         debug: bool = False,
         **kw
@@ -530,6 +533,14 @@ def filter_existing(
     df: 'pd.DataFrame'
         The dataframe to inspect and filter.
         
+    safe_copy: bool, default True
+        If `True`, create a copy before comparing and modifying the dataframes.
+        Setting to `False` may mutate the DataFrames.
+        See `meerschaum.utils.dataframe.filter_unseen_df`.
+
+    date_bound_only: bool, default False
+        If `True`, only use the datetime index to fetch the sample dataframe.
+
     chunksize: Optional[int], default -1
         The `chunksize` used when fetching existing data.
 
@@ -567,7 +578,8 @@ def filter_existing(
     else:
         merge = pd.merge
         NA = pd.NA
-
+    if df is None:
+        return df, df, df
     if (df.empty if not is_dask else len(df) == 0):
         return df, df, df
 
@@ -617,7 +629,7 @@ def filter_existing(
         traceback.print_exc()
         max_dt = None
 
-    if not ('datetime' in str(type(max_dt))) or str(min_dt) == 'NaT':
+    if ('datetime' not in str(type(max_dt))) or str(min_dt) == 'NaT':
         if 'int' not in str(type(max_dt)).lower():
             max_dt = None
 
@@ -645,7 +657,7 @@ def filter_existing(
         col: df[col].unique()
         for col in self.columns
         if col in df.columns and col != dt_col
-    }
+    } if not date_bound_only else {}
     filter_params_index_limit = get_config('pipes', 'sync', 'filter_params_index_limit')
     _ = kw.pop('params', None)
     params = {
@@ -655,7 +667,7 @@ def filter_existing(
         ]
         for col, unique_vals in unique_index_vals.items()
         if len(unique_vals) <= filter_params_index_limit
-    }
+    } if not date_bound_only else {}
 
     if debug:
         dprint(f"Looking at data between '{begin}' and '{end}':", **kw)
@@ -698,18 +710,23 @@ def filter_existing(
                 col: to_pandas_dtype(typ)
                 for col, typ in self_dtypes.items()
             },
+            safe_copy = safe_copy,
             debug = debug
         ),
         on_cols_dtypes,
     )
 
     ### Cast dicts or lists to strings so we can merge.
+    serializer = functools.partial(json.dumps, sort_keys=True, separators=(',', ':'), default=str)
+    def deserializer(x):
+        return json.loads(x) if isinstance(x, str) else x
+
     unhashable_delta_cols = get_unhashable_cols(delta_df)
     unhashable_backtrack_cols = get_unhashable_cols(backtrack_df)
     for col in unhashable_delta_cols:
-        delta_df[col] = delta_df[col].apply(json.dumps)
+        delta_df[col] = delta_df[col].apply(serializer)
     for col in unhashable_backtrack_cols:
-        backtrack_df[col] = backtrack_df[col].apply(json.dumps)
+        backtrack_df[col] = backtrack_df[col].apply(serializer)
     casted_cols = set(unhashable_delta_cols + unhashable_backtrack_cols)
 
     joined_df = merge(
@@ -722,13 +739,9 @@ def filter_existing(
     ) if on_cols else delta_df
     for col in casted_cols:
         if col in joined_df.columns:
-            joined_df[col] = joined_df[col].apply(
-                lambda x: (
-                    json.loads(x)
-                    if isinstance(x, str)
-                    else x
-                )
-            )
+            joined_df[col] = joined_df[col].apply(deserializer)
+        if col in delta_df.columns:
+            delta_df[col] = delta_df[col].apply(deserializer)
 
     ### Determine which rows are completely new.
     new_rows_mask = (joined_df['_merge'] == 'left_only') if on_cols else None

meerschaum/utils/dataframe.py

@@ -7,9 +7,10 @@ Utility functions for working with DataFrames.
 """
 
 from __future__ import annotations
+from datetime import datetime
 from meerschaum.utils.typing import (
     Optional, Dict, Any, List, Hashable, Generator,
-    Iterator, Iterable, Union,
+    Iterator, Iterable, Union, Tuple,
 )
 
 
@@ -71,6 +72,7 @@ def add_missing_cols_to_df(df: 'pd.DataFrame', dtypes: Dict[str, Any]) -> pd.Dat
 def filter_unseen_df(
         old_df: 'pd.DataFrame',
         new_df: 'pd.DataFrame',
+        safe_copy: bool = True,
         dtypes: Optional[Dict[str, Any]] = None,
         debug: bool = False,
     ) -> 'pd.DataFrame':
@@ -84,6 +86,10 @@ def filter_unseen_df(
         
     new_df: 'pd.DataFrame'
         The fetched (source) dataframe. Rows that are contained in `old_df` are removed.
+
+    safe_copy: bool, default True
+        If `True`, create a copy before comparing and modifying the dataframes.
+        Setting to `False` may mutate the DataFrames.
         
     dtypes: Optional[Dict[str, Any]], default None
         Optionally specify the datatypes of the dataframe.
@@ -111,6 +117,10 @@ def filter_unseen_df(
     if old_df is None:
         return new_df
 
+    if safe_copy:
+        old_df = old_df.copy()
+        new_df = new_df.copy()
+
     import json
     import functools
     import traceback
@@ -118,6 +128,7 @@ def filter_unseen_df(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.packages import import_pandas, attempt_import
     from meerschaum.utils.dtypes import to_pandas_dtype, are_dtypes_equal, attempt_cast_to_numeric
+    from meerschaum.utils.debug import dprint
     pd = import_pandas(debug=debug)
     is_dask = 'dask' in new_df.__module__
     if is_dask:
@@ -243,12 +254,7 @@ def filter_unseen_df(
         indicator = True,
     )
     changed_rows_mask = (joined_df['_merge'] == 'left_only')
-
-    delta_df = joined_df[
-        list(new_df_dtypes.keys())
-    ][
-        changed_rows_mask
-    ].reset_index(drop=True)
+    delta_df = joined_df[list(new_df_dtypes.keys())][changed_rows_mask].reset_index(drop=True)
 
     for json_col in json_cols:
         if json_col not in delta_df.columns:
@@ -535,6 +541,8 @@ def get_numeric_cols(df: 'pd.DataFrame') -> List[str]:
 def enforce_dtypes(
         df: 'pd.DataFrame',
         dtypes: Dict[str, str],
+        safe_copy: bool = True,
+        coerce_numeric: bool = True,
         debug: bool = False,
     ) -> 'pd.DataFrame':
     """
@@ -548,6 +556,14 @@ def enforce_dtypes(
     dtypes: Dict[str, str]
         The data types to attempt to enforce on the DataFrame.
 
+    safe_copy: bool, default True
+        If `True`, create a copy before comparing and modifying the dataframes.
+        Setting to `False` may mutate the DataFrames.
+        See `meerschaum.utils.dataframe.filter_unseen_df`.
+
+    coerce_numeric: bool, default True
+        If `True`, convert float and int collisions to numeric.
+
     debug: bool, default False
         Verbosity toggle.
 
@@ -569,6 +585,8 @@ def enforce_dtypes(
         is_dtype_numeric,
         attempt_cast_to_numeric,
     )
+    if safe_copy:
+        df = df.copy()
     df_dtypes = {c: str(t) for c, t in df.dtypes.items()}
     if len(df_dtypes) == 0:
         if debug:
@@ -674,7 +692,7 @@ def enforce_dtypes(
             explicitly_numeric
             or col in df_numeric_cols
             or (mixed_numeric_types and not explicitly_float)
-        )
+        ) and coerce_numeric
         if cast_to_numeric:
             common_dtypes[col] = attempt_cast_to_numeric
             common_diff_dtypes[col] = attempt_cast_to_numeric
@@ -860,3 +878,160 @@ def get_first_valid_dask_partition(ddf: 'dask.dataframe.DataFrame') -> Union['pd
         if len(pdf) > 0:
             return pdf
     return ddf.compute()
+
+
+def query_df(
+        df: 'pd.DataFrame',
+        params: Optional[Dict[str, Any]] = None,
+        begin: Union[datetime, int, None] = None,
+        end: Union[datetime, int, None] = None,
+        datetime_column: Optional[str] = None,
+        select_columns: Optional[List[str]] = None,
+        omit_columns: Optional[List[str]] = None,
+        inplace: bool = False,
+        reset_index: bool = False,
+        debug: bool = False,
+    ) -> 'pd.DataFrame':
+    """
+    Query the dataframe with the params dictionary.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        The DataFrame to query against.
+
+    params: Optional[Dict[str, Any]], default None
+        The parameters dictionary to use for the query.
+
+    begin: Union[datetime, int, None], default None
+        If `begin` and `datetime_column` are provided, only return rows with a timestamp
+        greater than or equal to this value.
+
+    end: Union[datetime, int, None], default None
+        If `begin` and `datetime_column` are provided, only return rows with a timestamp
+        less than this value.
+
+    datetime_column: Optional[str], default None
+        A `datetime_column` must be provided to use `begin` and `end`.
+
+    select_columns: Optional[List[str]], default None
+        If provided, only return these columns.
+
+    omit_columns: Optional[List[str]], default None
+        If provided, do not include these columns in the result.
+
+    inplace: bool, default False
+        If `True`, modify the DataFrame inplace rather than creating a new DataFrame.
+
+    reset_index: bool, default True
+        If `True`, reset the index in the resulting DataFrame.
+
+    Returns
+    -------
+    A Pandas DataFrame query result.
+    """
+    if not params and not begin and not end:
+        return df
+
+    import json
+    import meerschaum as mrsm
+    from meerschaum.utils.debug import dprint
+    from meerschaum.utils.misc import get_in_ex_params
+    from meerschaum.utils.warnings import warn
+
+    dtypes = {col: str(typ) for col, typ in df.dtypes.items()}
+
+    if begin or end:
+        if not datetime_column or datetime_column not in df.columns:
+            warn(
+                f"The datetime column '{datetime_column}' is not present in the Dataframe, "
+                + "ignoring begin and end...",
+            )
+            begin, end = None, None
+
+    if debug:
+        dprint(f"Querying dataframe:\n{params=} {begin=} {end=} {datetime_column=}")
+
+    in_ex_params = get_in_ex_params(params)
+
+    def serialize(x: Any) -> str:
+        if isinstance(x, (dict, list, tuple)):
+            return json.dumps(x, sort_keys=True, separators=(',', ':'), default=str)
+        if hasattr(x, 'isoformat'):
+            return x.isoformat()
+        return str(x)
+
+    masks = [
+        (
+            (df[datetime_column] >= begin)
+            if begin is not None and datetime_column
+            else True
+        ) & (
+            (df[datetime_column] < end)
+            if end is not None and datetime_column
+            else True
+        )
+    ]
+
+    masks.extend([
+        (
+            (
+                df[col].apply(serialize).isin(
+                    [
+                        serialize(_in_val)
+                        for _in_val in in_vals
+                    ]
+                ) if in_vals else True
+            ) & (
+                ~df[col].apply(serialize).isin(
+                    [
+                        serialize(_ex_val)
+                        for _ex_val in ex_vals
+                    ]
+                ) if ex_vals else True
+            )
+        )
+        for col, (in_vals, ex_vals) in in_ex_params.items()
+        if col in df.columns
+    ])
+    query_mask = masks[0]
+    for mask in masks:
+        query_mask = query_mask & mask
+
+    if inplace:
+        df.where(query_mask, inplace=inplace)
+        df.dropna(how='all', inplace=inplace)
+        result_df = df
+    else:
+        result_df = df.where(query_mask).dropna(how='all')
+
+    if reset_index:
+        result_df.reset_index(drop=True, inplace=True)
+
+    result_df = enforce_dtypes(
+        result_df,
+        dtypes,
+        safe_copy = (not inplace),
+        debug = debug,
+        coerce_numeric = False,
+    )
+
+    if select_columns == ['*']:
+        select_columns = None
+
+    if not select_columns and not omit_columns:
+        return result_df
+
+    if select_columns:
+        for col in list(result_df.columns):
+            if col not in select_columns:
+                del result_df[col]
+        return result_df
+
+    if omit_columns:
+        for col in list(result_df.columns):
+            if col in omit_columns:
+                del result_df[col]
+    if debug:
+        dprint(f"{dtypes=}")
+    return result_df

meerschaum/utils/formatting/_pipes.py

@@ -9,7 +9,7 @@ Formatting functions for printing pipes
 from __future__ import annotations
 import json
 import meerschaum as mrsm
-from meerschaum.utils.typing import PipesDict, Dict, Union, Optional, SuccessTuple, Any
+from meerschaum.utils.typing import PipesDict, Dict, Union, Optional, SuccessTuple, Any, List
 from meerschaum.config import get_config
 
 def pprint_pipes(pipes: PipesDict) -> None:
@@ -481,22 +481,54 @@ def print_pipes_results(
         )
 
 
-def extract_stats_from_message(message: str) -> Dict[str, int]:
+def extract_stats_from_message(
+        message: str,
+        stat_keys: Optional[List[str]] = None,
+    ) -> Dict[str, int]:
     """
-    Given a sync message, return the insert, update stats from within.
+    Given a sync message, return the insert, update, upsert stats from within.
+
+    Parameters
+    ----------
+    message: str
+        The message to parse for statistics.
+
+    stat_keys: Optional[List[str]], default None
+        If provided, search for these words (case insensitive) in the message.
+        Defaults to `['inserted', 'updated', 'upserted']`.
+
+    Returns
+    -------
+    A dictionary mapping the stat keys to the total number of rows affected.
     """
-    stats = {
-        'inserted': 0,
-        'updated': 0,
-        'upserted': 0,
+    stat_keys = stat_keys or ['inserted', 'updated', 'upserted']
+    lines_stats = [extract_stats_from_line(line, stat_keys) for line in message.split('\n')]
+    message_stats = {
+        stat_key: sum(stats.get(stat_key, 0) for stats in lines_stats)
+        for stat_key in stat_keys
     }
+    return message_stats
 
-    for search_key in list(stats.keys()):
-        if search_key not in message.lower():
+
+def extract_stats_from_line(
+        line: str,
+        stat_keys: List[str],
+    ) -> Dict[str, int]:
+    """
+    Return the insert, update, upsert stats from a single line.
+    """
+    stats = {key: 0 for key in stat_keys}
+
+    for stat_key in stat_keys:
+        search_key = stat_key.lower()
+        if search_key not in line.lower():
             continue
 
         ### stat_text starts with the digits we want.
-        stat_text = message.lower().split(search_key + ' ')[1]
+        try:
+            stat_text = line.lower().split(search_key + ' ')[1]
+        except IndexError:
+            continue
 
         ### find the first non-digit value.
         end_of_num_ix = -1
@@ -504,6 +536,8 @@ def extract_stats_from_message(message: str) -> Dict[str, int]:
             if not char.isdigit():
                 end_of_num_ix = i
                 break
+            if i == len(stat_text) - 1:
+                end_of_num_ix = i + 1
         if end_of_num_ix == -1:
             continue
 

meerschaum/utils/misc.py

@@ -1234,7 +1234,7 @@ def truncate_string_sections(item: str, delimeter: str = '_', max_len: int = 128
 
 
 def separate_negation_values(
-        vals: List[str],
+        vals: Union[List[str], Tuple[str]],
         negation_prefix: Optional[str] = None,
     ) -> Tuple[List[str], List[str]]:
     """
@@ -1243,7 +1243,7 @@ def separate_negation_values(
 
     Parameters
     ----------
-    vals: List[str]
+    vals: Union[List[str], Tuple[str]]
         A list of strings to parse.
 
     negation_prefix: Optional[str], default None
@@ -1263,6 +1263,38 @@ def separate_negation_values(
     return _in_vals, _ex_vals
 
 
+def get_in_ex_params(params: Optional[Dict[str, Any]]) -> Dict[str, Tuple[List[Any], List[Any]]]:
+    """
+    Translate a params dictionary into lists of include- and exclude-values.
+
+    Parameters
+    ----------
+    params: Optional[Dict[str, Any]]
+        A params query dictionary.
+
+    Returns
+    -------
+    A dictionary mapping keys to a tuple of lists for include and exclude values.
+
+    Examples
+    --------
+    >>> get_in_ex_params({'a': ['b', 'c', '_d', 'e', '_f']})
+    {'a': (['b', 'c', 'e'], ['d', 'f'])}
+    """
+    if not params:
+        return {}
+    return {
+        col: separate_negation_values(
+            (
+                val
+                if isinstance(val, (list, tuple))
+                else [val]
+            )
+        )
+        for col, val in params.items()
+    }
+
+
 def flatten_list(list_: List[Any]) -> List[Any]:
     """
     Recursively flatten a list.

meerschaum/utils/packages/__init__.py

@@ -350,6 +350,7 @@ def determine_version(
     with _locks['import_versions']:
         if venv not in import_versions:
             import_versions[venv] = {}
+    import importlib.metadata
     import re, os
     old_cwd = os.getcwd()
     if debug:
@@ -1379,13 +1380,13 @@ def get_modules_from_package(
     Returns
     -------
     Either list of modules or tuple of lists.
-
     """
     from os.path import dirname, join, isfile, isdir, basename
     import glob
 
     pattern = '*' if recursive else '*.py'
-    module_names = glob.glob(join(dirname(package.__file__), pattern), recursive=recursive)
+    package_path = dirname(package.__file__ or package.__path__[0])
+    module_names = glob.glob(join(package_path, pattern), recursive=recursive)
     _all = [
         basename(f)[:-3] if isfile(f) else basename(f)
         for f in module_names
@@ -1410,7 +1411,7 @@ def get_modules_from_package(
             modules.append(m)
         except Exception as e:
             if debug:
-                dprint(e)
+                dprint(str(e))
         finally:
             if modules_venvs:
                 deactivate_venv(module_name.split('.')[-1], debug=debug)

meerschaum/utils/sql.py

@@ -38,6 +38,7 @@ version_queries = {
     'oracle': "SELECT version from PRODUCT_COMPONENT_VERSION WHERE rownum = 1",
 }
 SKIP_IF_EXISTS_FLAVORS = {'mssql', 'oracle'}
+COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'citus'}
 update_queries = {
     'default': """
         UPDATE {target_table_name} AS f
@@ -53,25 +54,25 @@ update_queries = {
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
         FROM {patch_table_name}
-        ON CONFLICT ({join_cols_str}) DO UPDATE {sets_subquery_none_excluded}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
     'postgresql-upsert': """
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
         FROM {patch_table_name}
-        ON CONFLICT ({join_cols_str}) DO UPDATE {sets_subquery_none_excluded}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
     'citus-upsert': """
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
         FROM {patch_table_name}
-        ON CONFLICT ({join_cols_str}) DO UPDATE {sets_subquery_none_excluded}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
     'cockroachdb-upsert': """
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
         FROM {patch_table_name}
-        ON CONFLICT ({join_cols_str}) DO UPDATE {sets_subquery_none_excluded}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
     'mysql': """
         UPDATE {target_table_name} AS f
@@ -122,7 +123,7 @@ update_queries = {
         SELECT {patch_cols_str}
         FROM {patch_table_name}
         WHERE true
-        ON CONFLICT ({join_cols_str}) DO UPDATE {sets_subquery_none_excluded}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
     'sqlite_delete_insert': [
         """
@@ -1084,7 +1085,7 @@ def get_update_queries(
             for col in patch_table_columns
         ]
     )
-    join_cols_str = ','.join(
+    join_cols_str = ', '.join(
         [
             sql_item_name(col, flavor)
             for col in join_cols
@@ -1109,10 +1110,27 @@ def get_update_queries(
     if debug:
         dprint(f"value_cols: {value_cols}")
 
-    if not value_cols or not join_cols_types:
+    if not join_cols_types:
+        return []
+    if not value_cols and not upsert:
         return []
 
+    coalesce_join_cols_str = ', '.join(
+        [
+            'COALESCE('
+            + sql_item_name(c_name, flavor)
+            + ', '
+            + get_null_replacement(c_type, flavor)
+            + ')'
+            for c_name, c_type in join_cols_types
+        ]
+    )
+
+    update_or_nothing = ('UPDATE' if value_cols else 'NOTHING')
+
     def sets_subquery(l_prefix: str, r_prefix: str):
+        if not value_cols:
+            return ''
         return 'SET ' + ',\n'.join([
             (
                 l_prefix + sql_item_name(c_name, flavor, None)
@@ -1169,6 +1187,8 @@ def get_update_queries(
             patch_cols_str = patch_cols_str,
             date_bounds_subquery = date_bounds_subquery,
             join_cols_str = join_cols_str,
+            coalesce_join_cols_str = coalesce_join_cols_str,
+            update_or_nothing = update_or_nothing,
         )
         for base_query in base_queries
     ]

meerschaum/utils/typing.py

@@ -87,3 +87,14 @@ PipesDict = Dict[
     ]
 ]
 WebState = Dict[str, str]
+
+def is_success_tuple(x: Any) -> bool:
+    """
+    Determine whether an object is a `SuccessTuple`.
+    """
+    return (
+        isinstance(x, tuple)
+        and len(x) == 2
+        and isinstance(x[0], bool)
+        and isinstance(x[1], str)
+    )

{meerschaum-2.1.5.dist-info → meerschaum-2.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.1.5
+Version: 2.1.7
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.1.5.dist-info → meerschaum-2.1.7.dist-info}/RECORD

@@ -12,7 +12,7 @@ meerschaum/_internal/gui/app/__init__.py,sha256=rKUa8hHk6Fai-PDF61tQcpT1myxKcfmv
 meerschaum/_internal/gui/app/_windows.py,sha256=-VHdjTzA3V596fVqnbmTxemONSp_80-sTNJ0CTB8FwU,2632
 meerschaum/_internal/gui/app/actions.py,sha256=rx37qXf3uoa7Ou0n1cISqNFZNL0nr4wO7vSUmWO8f2E,935
 meerschaum/_internal/gui/app/pipes.py,sha256=4nAQ0rrHb_2bNgDF0Ru2YlbPaCDDzAl5beOGU4Af-4A,1596
-meerschaum/_internal/shell/Shell.py,sha256=34lDd4FVTyvj17L4WdcSWwN60VXu0NpiVzY45IGTWq4,33102
+meerschaum/_internal/shell/Shell.py,sha256=zcgajt0FGH5ou0kqmOrYAvNfunOsn_CN6LHFYFXBBlI,32826
 meerschaum/_internal/shell/ShellCompleter.py,sha256=bbG-mExNXO4pltWBOXdbMp8P2wLgy8_BgipIr5aGp5s,3114
 meerschaum/_internal/shell/ValidAutoSuggest.py,sha256=bARjOWMidz0dvMelLUe6yRPto5l3gcEHYHqFDjoh22I,1280
 meerschaum/_internal/shell/__init__.py,sha256=vXQoQPEVlYiUYai1b5AwQAlTnja6A2cSABnqXhzlS7I,281
@@ -21,7 +21,7 @@ meerschaum/_internal/term/TermPageHandler.py,sha256=Rt5S47Pr_3HLJc8xIXpZUczYE_Dw
 meerschaum/_internal/term/__init__.py,sha256=xIwEWXyq1qaU7Rx-AryTtANJPdm__fy3XSMzxaFn0wU,1594
 meerschaum/_internal/term/tools.py,sha256=bpYexJBDCQXfzz6ESMvmpSHM1AIy4qWsrAHl95tSW2I,716
 meerschaum/actions/__init__.py,sha256=7CNoKEqkqqafqMcChspJX9cR9OdgEWk9ggj0000Jl98,11360
-meerschaum/actions/api.py,sha256=StCJN-gCLNTDJhGxgux_9aMdeSFDTl-Yzsx4o9imw_I,12757
+meerschaum/actions/api.py,sha256=mWhv4bn3Ap17_Gqf2Cx9bAsHKG-Zhy072pBbNzHLEJc,12756
 meerschaum/actions/bootstrap.py,sha256=JnIyJ4odw6cA4e0Cw7J8THkLavMcj68nRyGsQDAT8nc,13396
 meerschaum/actions/clear.py,sha256=OoFZE0bK5m8s3GLNZcixuVT0DMj1izXVxGCATcmUGbI,4851
 meerschaum/actions/copy.py,sha256=8g3ANXfVdvuyaoXcZjgTg3BxHTOhHGrzVDOOsTBrpSU,6213
@@ -43,7 +43,7 @@ meerschaum/actions/sql.py,sha256=wYofwk1vGO96U2ncigGEfMtYMZeprz2FR1PRRZhkAPI,431
 meerschaum/actions/stack.py,sha256=WMRMebyYwZGNlbnj6Ja09qvCSDNteFJOTa8_joHlnVo,5886
 meerschaum/actions/start.py,sha256=mNFWqxc_o9moavvDQWE4YoZF6b-SW2nKyw5MtwIj-90,18384
 meerschaum/actions/stop.py,sha256=KTBadAmJ6SbReqlltkwfqZW6EryB4kZXupl0ZyInI0Q,4311
-meerschaum/actions/sync.py,sha256=oMRZzT7q6eUs-7UrXcTpcem59EBwftu_6ug8XCen6dk,16235
+meerschaum/actions/sync.py,sha256=Pzj0p53Aa4ZjP3SnlUyB6OeNvVUoJ0iCfWffMIIsd0k,17266
 meerschaum/actions/tag.py,sha256=SJf5qFW0ccLXjqlTdkK_0MCcrCMdg6xhYrhKdco0hdA,3053
 meerschaum/actions/uninstall.py,sha256=2fUd5ZK45VGGCI8V4NLmSnavdKjOv7cGM22x2WlTStw,6068
 meerschaum/actions/upgrade.py,sha256=VQKyjCGioEF2FYbQmldHh21imDqApNl0xal0rhxzrJk,6302
@@ -132,9 +132,9 @@ meerschaum/config/_patch.py,sha256=21N30q1ANmWMDQ-2RUjpMx7KafWfPQ3lKx9rrMqg1s4,1
 meerschaum/config/_paths.py,sha256=ORzpEpYE5OM06H6Hw_nh51R8fkk45MyGPI7mDd1QzqI,7983
 meerschaum/config/_preprocess.py,sha256=-AEA8m_--KivZwTQ1sWN6LTn5sio_fUr2XZ51BO6wLs,1220
 meerschaum/config/_read_config.py,sha256=WFZKIXZMDe_ca0ES7ivgM_mnwShvFxLdoeisT_X5-h0,14720
-meerschaum/config/_shell.py,sha256=k6PH0BEr2imhgURLYlR5p6s5gXfYpWoyZSV29U-SsXk,3589
+meerschaum/config/_shell.py,sha256=s74cmJl8NrhM_Y1cB_P41_JDUYXV0g4WXnKFZWMtnrY,3551
 meerschaum/config/_sync.py,sha256=Q-sz5YcjL3CJS2Dyw4rVRQsz9th9GWa9o5F9D0Jrmn8,4120
-meerschaum/config/_version.py,sha256=5u15PU7pHH1p6K_hknjy0-rs4zQUg8VbND291EX5N4I,71
+meerschaum/config/_version.py,sha256=FZCUdRGYa70_q2wO0dpOp-Rp_YnYgZU3PX2U_SBZLB4,71
 meerschaum/config/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 meerschaum/config/stack/__init__.py,sha256=4a_up1oxkitwgIylWWk0vA4XkGhEpWazUaENOPEdYQI,9034
 meerschaum/config/stack/grafana/__init__.py,sha256=wzuoch_AK49lcn7lH2qTSJ_PPbSagF4lcweeipz_XiE,2010
@@ -163,31 +163,31 @@ meerschaum/connectors/sql/SQLConnector.py,sha256=dPMgJXLghzBmqHaT9qxvn8pzerQLpzk
 meerschaum/connectors/sql/__init__.py,sha256=xwSYhYuketTXhQLXyD9pZ0NNBPboW5Oqv9zrKfjx0Ic,175
 meerschaum/connectors/sql/_cli.py,sha256=XaWjWZzGIfhMiYoXAs2FrwHUGNyZpxIzH4g4xugLKsw,4123
 meerschaum/connectors/sql/_create_engine.py,sha256=PrHS5xCvqBMdCl0-AsoEw_9Dt2WrH3Wim5qKeTiLyp4,10382
-meerschaum/connectors/sql/_fetch.py,sha256=BLcZBGeE8JV6utnCgvq1TX_y6dTjX5KAvB4xm4YKxiQ,13210
+meerschaum/connectors/sql/_fetch.py,sha256=NYYWDoEd-aGIS337KwH-D9_3KVWVCZOHAspGLfdEuUE,13086
 meerschaum/connectors/sql/_instance.py,sha256=jAGq_qjz2WyBrjtnI7FyGRLOs2l3o630vonPM5Pp4Mg,6447
-meerschaum/connectors/sql/_pipes.py,sha256=L6ldoFLhhE0ap1Yf2aETzRIBby5TbkIWUb2_-B3_4-k,99687
+meerschaum/connectors/sql/_pipes.py,sha256=4v1-2AwW8QqOTRjNUEoDJJH8KsJa6FxMNSekEDWJu6o,100925
 meerschaum/connectors/sql/_plugins.py,sha256=hS0cuJQxwd6jUfY136AQ33dGQw_MigP_OFC84KdSMhA,8323
 meerschaum/connectors/sql/_sql.py,sha256=jNdIcwQC2ZMNKT9W6ugavQPtimUpoTr8cBt05z_vHIo,34262
 meerschaum/connectors/sql/_uri.py,sha256=0BrhQtqQdzg9mR04gWBZINs_BbPFtSlTECXT_TCUwik,3460
 meerschaum/connectors/sql/_users.py,sha256=JkD6lKYJO8JVnpVySMqPM20NWUr-XzD_JO-lLMcAD_c,10100
 meerschaum/connectors/sql/tools.py,sha256=jz8huOaRCwGlYdtGfAqAh7SoK8uydYBrasKQba9FT38,187
-meerschaum/connectors/sql/tables/__init__.py,sha256=a4eFvbvGBoL4ZaTNzl6RV1Rj90LTeHFcPPT-j51pzFo,9471
+meerschaum/connectors/sql/tables/__init__.py,sha256=e2dALTtThqbrq0soMNQ9QwgccyfTAjOrFkEClstLp3A,9001
 meerschaum/connectors/sql/tables/types.py,sha256=Jc_MTHIBM-KHpQt__Lckp39CeOo7tGOiAk5faDx-znY,1573
 meerschaum/core/__init__.py,sha256=tjASW10n9uLV6bYhcwP4rggh-ESXSJzgxpSBbVsuISs,251
 meerschaum/core/Pipe/__init__.py,sha256=1QOHqyUX2uJnfihVoGQV30BLm7JZZX2qjbsYIpCNFGI,16230
 meerschaum/core/Pipe/_attributes.py,sha256=XbSHfDsomaNymzf7-__UhbHqu6mlTTx20xprsw_L04I,13202
 meerschaum/core/Pipe/_bootstrap.py,sha256=sTbHUX8V0Kfn6vEErXzsVslSjQNfQ5MxXxxuRYslr4w,7613
 meerschaum/core/Pipe/_clear.py,sha256=hQVPztHiadzLB0c4_yFg6EETnf9MtFdJDCpO41Giuco,2261
-meerschaum/core/Pipe/_data.py,sha256=K4sZAPwBB8kb9mr5NqgQcCfgX1u-iYZ06uEPSJx5BqE,20783
+meerschaum/core/Pipe/_data.py,sha256=uafKY7GGD5uGQ__-YcKlWOnRatAp1ExSbvz06LWPTsA,20880
 meerschaum/core/Pipe/_deduplicate.py,sha256=hXeNhz8p6Zgz2Y-A2W5AZPkVFUgsLqNvTQKKL0JHJ0A,10267
 meerschaum/core/Pipe/_delete.py,sha256=1geNp9BgrocXP1gt76dMbnlJWKYFMuSNqPFA4K4-hXE,2118
 meerschaum/core/Pipe/_drop.py,sha256=uf3MvMkCw9tVfJ2fuo8LqZ4vvMNa3xC3YoFGEuc-hH8,1052
-meerschaum/core/Pipe/_dtypes.py,sha256=RIJ5KwPWawV1zIwmsLz3lF33NZ-6zICf7XPscmaigB8,3641
+meerschaum/core/Pipe/_dtypes.py,sha256=NcfVMJPyyA1VzIJOnPhyL36YTt2IXMRqQy4ZRDySSqg,3694
 meerschaum/core/Pipe/_edit.py,sha256=ZH2A0ZOpZKsVDnQxKzmXspNQKTEFUhkkZDjwOkmWtaY,8471
 meerschaum/core/Pipe/_fetch.py,sha256=zV3DzD7kfx08O6zda0I-9cX91m0Z_wO_on1ozHaqWnk,5234
 meerschaum/core/Pipe/_register.py,sha256=Sd5xaAW8H7uLTIoommcKb-6kHPRuHJLWNSbPnt2UbvA,2240
 meerschaum/core/Pipe/_show.py,sha256=nG50y8eBT9TVuKkRgAKtNDNIxysJvMNxfu__lkL1F9k,1352
-meerschaum/core/Pipe/_sync.py,sha256=761pNmLrH9wuemE2ZNqa1GtX-ZJeZ18aIEUQevB1Tvo,27330
+meerschaum/core/Pipe/_sync.py,sha256=48qk1xvkcKOqfzzYf3QdA7ojsP80yrso1YU8QIGJxwE,28038
 meerschaum/core/Pipe/_verify.py,sha256=KSnthUzImRLjt9fxyBaLvArqDuOLRpKBfk0tnseJClc,14262
 meerschaum/core/Plugin/__init__.py,sha256=UXg64EvJPgI1PCxkY_KM02-ZmBm4FZpLPIQR_uSJJDc,137
 meerschaum/core/User/_User.py,sha256=waVdpH4SFZSXNYBgX5KFQ8csbCSxRLI5T2efAzVONks,2448
@@ -196,18 +196,18 @@ meerschaum/plugins/_Plugin.py,sha256=f0RDPazwzYSTZmD21rxwF48abUeVyFWKggBC9cWR7sw
 meerschaum/plugins/__init__.py,sha256=pVRgbBk1UMlqLrM5p1s7_x_mN70epdDBZOa4vrt6C6w,20760
 meerschaum/utils/__init__.py,sha256=QrK1K9hIbPCRCM5k2nZGFqGnrqhA0Eh-iSmCU7FG6Cs,612
 meerschaum/utils/_get_pipes.py,sha256=dlPckpYYyM0IwRZ2VL0u9DiEeYhr5Ho9gkzvWxzNVwI,11460
-meerschaum/utils/dataframe.py,sha256=wqSmyD_UAnGJYWVYF9obD1T_zW0ioFE4tmbVwe4Okps,26479
+meerschaum/utils/dataframe.py,sha256=vxZ72ME7IWuadtktgjFZF5bc9fXW_0TuynjFlJljlLU,31955
 meerschaum/utils/debug.py,sha256=ry9UWf0ECelVIuBApwmKxPZ_IoL6UqjTSMpGNbjghVQ,3690
 meerschaum/utils/interactive.py,sha256=t-6jWozXSqL7lYGDHuwiOjTgr-UKhdcg61q_eR5mikI,3196
-meerschaum/utils/misc.py,sha256=aEohLCpf6u-eW1YWWFJed3077F7HJni1kXzeaeo3USg,42457
+meerschaum/utils/misc.py,sha256=H26hLtCP8QHwXoHlvkxjWu6cPTwudDbbsbRkGw6ultg,43296
 meerschaum/utils/networking.py,sha256=Sr_eYUGW8_UV9-k9LqRFf7xLtbUcsDucODyLCRsFRUc,1006
 meerschaum/utils/pool.py,sha256=vkE42af4fjrTEJTxf6Ek3xGucm1MtEkpsSEiaVzNKHs,2655
 meerschaum/utils/process.py,sha256=tbEutHAg_Kn5UetOI-fduRjsafGOYX5tkLvpzqosgvc,7098
 meerschaum/utils/prompt.py,sha256=0mBFbgi_l9rCou9UnC_6qKTHkqyl1Z_jSRzfmc0xRXM,16490
 meerschaum/utils/schedule.py,sha256=lYra4f7QpGlmV4vAhO1UvA5TWQSPdLqw7eG7ekKZtwg,1951
-meerschaum/utils/sql.py,sha256=mWGZO-JXmwfELSRyT7w3L6A-_HJ-k9aPwIgnMGj9Bqs,45982
+meerschaum/utils/sql.py,sha256=4sCNEpgUd6uFz6ySs4nnUMVaOT0YAvPM1ZlQYJTSF-0,46656
 meerschaum/utils/threading.py,sha256=fAXk7-FnbFvdU1FQ4vHKk5NeGbbTpTw7y9dRnlVayNI,2472
-meerschaum/utils/typing.py,sha256=mCWZ7QN5Oub5QtWtdLp3gL_PPbpbdRhI2DJJkv_dpTA,2535
+meerschaum/utils/typing.py,sha256=-g_LN1HzBdYZml7iZNSoB4EbpV27D0S0pM_PhuDSuzI,2784
 meerschaum/utils/warnings.py,sha256=0b5O2DBbhEAGnu6RAB1hlHSVmwL_hcR3EiMkExXmBJ0,6535
 meerschaum/utils/yaml.py,sha256=vbCrFjdapKsZ9wRRaI9Ih8dVUwZ-KHpSzfGhRcpDBgQ,3162
 meerschaum/utils/daemon/Daemon.py,sha256=_dxHUM4SEIvTin_k0xMIFjdXTDj97ZE1Do4A_AXcAHA,31961
@@ -218,19 +218,19 @@ meerschaum/utils/dtypes/__init__.py,sha256=5GJUHZSJBc0lrynmvc2vgKAN2ybYlCs-YEi0r
 meerschaum/utils/dtypes/sql.py,sha256=IkEOyB63je-rCLHM6WwFzGbCerYk1zobL1cXkWqmTa4,14638
 meerschaum/utils/formatting/__init__.py,sha256=Dn-8EUTjz0s11bFaAr1HjqlM8IogCTxlpulwp_Kn4ok,13778
 meerschaum/utils/formatting/_jobs.py,sha256=s1lVcdMkzNj5Bqw-GsUhcguUFtahi5nQ-kg1fbp0Idw,3294
-meerschaum/utils/formatting/_pipes.py,sha256=vixqKIUYNC7yDtU-MoeSXnHug9IGkK1N2yqujxvJ-i0,18414
+meerschaum/utils/formatting/_pipes.py,sha256=wy0iWJFsFl3X2VloaiA_gp9Yx9w6tD3FQZvAQAqef4A,19492
 meerschaum/utils/formatting/_pprint.py,sha256=tgrT3FyGyu5CWJYysqK3kX1xdZYorlbOk9fcU_vt9Qg,3096
 meerschaum/utils/formatting/_shell.py,sha256=ox75O7VHDAiwzSvdMSJZhXLadvAqYJVeihU6WeZ2Ogc,3677
-meerschaum/utils/packages/__init__.py,sha256=sUUtdkTNCXzMl21_8BN-fxuplVUuSx4gaKjXpC3XX6A,56445
+meerschaum/utils/packages/__init__.py,sha256=yyf-3rAyRy5XaQkNFHjOC85VIkZt0o00o7rG3ykG7XU,56534
 meerschaum/utils/packages/_packages.py,sha256=gEWZHxFbYV6FZfM2uV1JOzS4eBl5OVoBZftMkLsU-4g,7967
 meerschaum/utils/packages/lazy_loader.py,sha256=VHnph3VozH29R4JnSSBfwtA5WKZYZQFT_GeQSShCnuc,2540
 meerschaum/utils/venv/_Venv.py,sha256=sBnlmxHdAh2bx8btfVoD79-H9-cYsv5lP02IIXkyECs,3553
 meerschaum/utils/venv/__init__.py,sha256=sj-n8scWH2NPDJGAxfpqzsYqVUt2jMEr-7Uq9G7YUNQ,23183
-meerschaum-2.1.5.dist-info/LICENSE,sha256=jG2zQEdRNt88EgHUWPpXVWmOrOduUQRx7MnYV9YIPaw,11359
-meerschaum-2.1.5.dist-info/METADATA,sha256=ig4PzucKdxot5IvSY8VeKrh54OVd7ZM8grPLZixLgiw,23976
-meerschaum-2.1.5.dist-info/NOTICE,sha256=OTA9Fcthjf5BRvWDDIcBC_xfLpeDV-RPZh3M-HQBRtQ,114
-meerschaum-2.1.5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-meerschaum-2.1.5.dist-info/entry_points.txt,sha256=5YBVzibw-0rNA_1VjB16z5GABsOGf-CDhW4yqH8C7Gc,88
-meerschaum-2.1.5.dist-info/top_level.txt,sha256=bNoSiDj0El6buocix-FRoAtJOeq1qOF5rRm2u9i7Q6A,11
-meerschaum-2.1.5.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-meerschaum-2.1.5.dist-info/RECORD,,
+meerschaum-2.1.7.dist-info/LICENSE,sha256=jG2zQEdRNt88EgHUWPpXVWmOrOduUQRx7MnYV9YIPaw,11359
+meerschaum-2.1.7.dist-info/METADATA,sha256=AgC6CJaCcrlaqiGgfyG3pl2FtpqOX0ecLf8comQjt8w,23976
+meerschaum-2.1.7.dist-info/NOTICE,sha256=OTA9Fcthjf5BRvWDDIcBC_xfLpeDV-RPZh3M-HQBRtQ,114
+meerschaum-2.1.7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+meerschaum-2.1.7.dist-info/entry_points.txt,sha256=5YBVzibw-0rNA_1VjB16z5GABsOGf-CDhW4yqH8C7Gc,88
+meerschaum-2.1.7.dist-info/top_level.txt,sha256=bNoSiDj0El6buocix-FRoAtJOeq1qOF5rRm2u9i7Q6A,11
+meerschaum-2.1.7.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+meerschaum-2.1.7.dist-info/RECORD,,