meerschaum 2.1.5__tar.gz → 2.1.7__tar.gz

{meerschaum-2.1.5/meerschaum.egg-info → meerschaum-2.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.1.5
+Version: 2.1.7
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/_internal/shell/Shell.py

@@ -11,11 +11,7 @@ from copy import deepcopy
 from meerschaum.utils.typing import Union, SuccessTuple, Any, Callable, Optional, List, Dict
 from meerschaum.utils.packages import attempt_import
 from meerschaum.config import __doc__, __version__ as version, get_config
-cmd_import_name = get_config('shell', 'cmd')
-cmd_venv = None if cmd_import_name == 'cmd' else 'mrsm'
-cmd = attempt_import(cmd_import_name, venv=cmd_venv, warn=False, lazy=False)
-if cmd is None or isinstance(cmd, dict):
-    cmd = attempt_import('cmd', lazy=False, warn=False)
+import cmd
 _old_input = cmd.__builtins__['input']
 prompt_toolkit = attempt_import('prompt_toolkit', lazy=False, warn=False, install=True)
 (
@@ -53,7 +49,6 @@ hidden_commands = {
     'os',
     'sh',
     'pass',
-    'exit',
     'quit',
     'eof',
     'exit',

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/actions/api.py

@@ -44,7 +44,7 @@ def api(
         sysargs = []
     if len(action) == 0:
         info(api.__doc__)
-        return False, "Please provide a command to excecute (see above)."
+        return False, "Please provide a command to execute (see above)."
 
     boot_keywords = {'start', 'boot', 'init'}
     if action[0] in boot_keywords:

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/actions/sync.py

@@ -9,7 +9,7 @@ NOTE: `sync` required a SQL connection and is not intended for client use
 """
 
 from __future__ import annotations
-from datetime import timedelta
+from datetime import timedelta, datetime, timezone
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Tuple, Union
 
@@ -39,7 +39,7 @@ def sync(
 
 def _pipes_lap(
         workers: Optional[int] = None,
-        debug: bool = None,
+        debug: Optional[bool] = None,
         unblock: bool = False,
         force: bool = False,
         min_seconds: int = 1,
@@ -52,7 +52,7 @@ def _pipes_lap(
         nopretty: bool = False,
         _progress: Optional['rich.progress.Progress'] = None,
         **kw: Any
-    ) -> Tuple[List[meerschaum.Pipe], List[meerschaum.Pipe]]:
+    ) -> Tuple[List[mrsm.Pipe], List[mrsm.Pipe]]:
     """
     Do a lap of syncing pipes.
     """
@@ -402,11 +402,20 @@ def _wrap_pipe(
     Wrapper function for handling exceptions.
     """
     import time
+    import traceback
+    from datetime import datetime, timedelta, timezone
+    import meerschaum as mrsm
+    from meerschaum.utils.typing import is_success_tuple, SuccessTuple
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.venv import Venv
     from meerschaum.plugins import _pre_sync_hooks, _post_sync_hooks
     from meerschaum.utils.misc import filter_keywords
+    from meerschaum.utils.pool import get_pool
+    from meerschaum.utils.warnings import warn
+
+    pool = get_pool(workers=workers)
 
+    sync_timestamp = datetime.now(timezone.utc)
     sync_start = time.perf_counter()
     sync_kwargs = {k: v for k, v in kw.items() if k != 'blocking'}
     sync_kwargs.update({
@@ -415,8 +424,9 @@ def _wrap_pipe(
         'debug': debug,
         'min_seconds': min_seconds,
         'workers': workers,
-        'bounded': 'bounded',
+        'bounded': bounded,
         'chunk_interval': chunk_interval,
+        'sync_timestamp': sync_timestamp,
     })
     if not verify and not deduplicate:
         sync_method = pipe.sync
@@ -427,12 +437,32 @@ def _wrap_pipe(
         sync_kwargs['deduplicate'] = deduplicate
     sync_kwargs['sync_method'] = sync_method
 
-    for module_name, pre_sync_hooks in _pre_sync_hooks.items():
-        plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
+    def call_sync_hook(plugin_name: str, sync_hook) -> SuccessTuple:
         plugin = mrsm.Plugin(plugin_name) if plugin_name else None
-        with Venv(plugin):
-            for pre_sync_hook in pre_sync_hooks:
-                _ = pre_sync_hook(pipe, **filter_keywords(pre_sync_hook, **sync_kwargs))
+        with mrsm.Venv(plugin):
+            try:
+                sync_hook_result = sync_hook(pipe, **filter_keywords(sync_hook, **sync_kwargs))
+                if is_success_tuple(sync_hook_result):
+                    return sync_hook_result
+            except Exception as e:
+                msg = (
+                    f"Failed to execute sync hook '{sync_hook.__name__}' "
+                    + f"from plugin '{plugin}':\n{traceback.format_exc()}"
+                )
+                warn(msg, stack=False)
+                return False, msg
+        return True, "Success"
+
+    hook_results = []
+    def apply_hooks(is_pre_sync: bool):
+        _sync_hooks = (_pre_sync_hooks if is_pre_sync else _post_sync_hooks)
+        for module_name, sync_hooks in _sync_hooks.items():
+            plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
+            for sync_hook in sync_hooks:
+                hook_result = pool.apply_async(call_sync_hook, (plugin_name, sync_hook))
+                hook_results.append(hook_result)
+
+    apply_hooks(True)
    
     try:
         with Venv(get_connector_plugin(pipe.connector), debug=debug):
@@ -444,18 +474,16 @@ def _wrap_pipe(
         return_tuple = (False, f"Failed to sync {pipe} with exception:" + "\n" + str(e))
 
     duration = time.perf_counter() - sync_start
-    sync_kwargs['duration'] = duration
-    for module_name, post_sync_hooks in _post_sync_hooks.items():
-        plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
-        plugin = mrsm.Plugin(plugin_name) if plugin_name else None
-        with Venv(plugin):
-            for post_sync_hook in post_sync_hooks:
-                _ = post_sync_hook(
-                    pipe,
-                    return_tuple,
-                    **filter_keywords(post_sync_hook, **sync_kwargs)
-                )
- 
+    sync_kwargs.update({
+        'success_tuple': return_tuple,
+        'sync_duration': duration,
+        'sync_complete_timestamp': datetime.now(timezone.utc),
+    })
+    apply_hooks(False)
+    for hook_result in hook_results:
+        hook_success, hook_msg = hook_result.get()
+        mrsm.pprint((hook_success, hook_msg))
+
     return return_tuple
 
 

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/config/_shell.py

@@ -126,7 +126,6 @@ default_shell_config = {
     'timeout'          : 60,
     'max_history'      : 1000,
     'clear_screen'     : True,
-    'cmd'              : default_cmd,
     'bottom_toolbar'   : {
         'enabled'      : True,
     },

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.1.5"
+__version__ = "2.1.7"

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/connectors/sql/_fetch.py

@@ -174,9 +174,6 @@ def get_pipe_metadef(
                 )
 
 
-    if 'order by' in definition.lower() and 'over' not in definition.lower():
-        error("Cannot fetch with an ORDER clause in the definition")
-
     apply_backtrack = begin == '' and check_existing
     backtrack_interval = pipe.get_backtrack_interval(check_existing=check_existing, debug=debug)
     btm = (
@@ -308,9 +305,9 @@ def _simple_fetch_query(pipe, debug: bool=False, **kw) -> str:
     def_name = 'definition'
     definition = get_pipe_query(pipe)
     return (
-        f"WITH {def_name} AS ({definition}) SELECT * FROM {def_name}"
+        f"WITH {def_name} AS (\n{definition}\n) SELECT * FROM {def_name}"
         if pipe.connector.flavor not in ('mysql', 'mariadb')
-        else f"SELECT * FROM ({definition}) AS {def_name}"
+        else f"SELECT * FROM (\n{definition}\n) AS {def_name}"
     )
 
 def _join_fetch_query(
@@ -363,10 +360,10 @@ def _join_fetch_query(
             )
             + f") AS {id_remote_name}, "
             + dateadd_str(
-                flavor=pipe.connector.flavor,
-                begin=_st,
-                datepart='minute',
-                number=pipe.parameters.get('fetch', {}).get('backtrack_minutes', 0)
+                flavor = pipe.connector.flavor,
+                begin = _st,
+                datepart = 'minute',
+                number = pipe.parameters.get('fetch', {}).get('backtrack_minutes', 0)
             ) + " AS " + dt_remote_name + "\nUNION ALL\n"
         )
     _sync_times_q = _sync_times_q[:(-1 * len('UNION ALL\n'))] + ")"
@@ -374,13 +371,13 @@ def _join_fetch_query(
     definition = get_pipe_query(pipe)
     query = (
         f"""
-    WITH definition AS ({definition}){_sync_times_q}
+    WITH definition AS (\n{definition}\n){_sync_times_q}
     SELECT definition.*
     FROM definition"""
         if pipe.connector.flavor not in ('mysql', 'mariadb')
         else (
         f"""
-    SELECT * FROM ({definition}) AS definition"""
+    SELECT * FROM (\n{definition}\n) AS definition"""
         )
     ) + f"""
     LEFT OUTER JOIN {sync_times_remote_name} AS st

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/connectors/sql/_pipes.py

@@ -385,7 +385,13 @@ def get_create_index_queries(
     -------
     A dictionary of column names mapping to lists of queries.
     """
-    from meerschaum.utils.sql import sql_item_name, get_distinct_col_count, update_queries
+    from meerschaum.utils.sql import (
+        sql_item_name,
+        get_distinct_col_count,
+        update_queries,
+        get_null_replacement,
+        COALESCE_UNIQUE_INDEX_FLAVORS,
+    )
     from meerschaum.config import get_config
     index_queries = {}
 
@@ -497,15 +503,37 @@ def get_create_index_queries(
             if ix and ix in existing_cols_types
         ]
     )
+    coalesce_indices_cols_str = ', '.join(
+        [
+            (
+                "COALESCE("
+                + sql_item_name(ix, self.flavor)
+                + ", "
+                + get_null_replacement(existing_cols_types[ix], self.flavor)
+                + ") "
+            ) if ix_key != 'datetime' else (sql_item_name(ix, self.flavor))
+            for ix_key, ix in pipe.columns.items()
+            if ix and ix in existing_cols_types
+        ]
+    )
+    unique_index_name = sql_item_name(pipe.target + '_unique_index', self.flavor)
     constraint_name = sql_item_name(pipe.target + '_constraint', self.flavor)
-    constraint_query = (
+    add_constraint_query = (
         f"ALTER TABLE {_pipe_name} ADD CONSTRAINT {constraint_name} UNIQUE ({indices_cols_str})"
-        if self.flavor != 'sqlite'
-        else f"CREATE UNIQUE INDEX {constraint_name} ON {_pipe_name} ({indices_cols_str})"
     )
+    unique_index_cols_str = (
+        indices_cols_str
+        if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS
+        else coalesce_indices_cols_str
+    )
+    create_unique_index_query = (
+        f"CREATE UNIQUE INDEX {unique_index_name} ON {_pipe_name} ({unique_index_cols_str})"
+    )
+    constraint_queries = [create_unique_index_query]
+    if self.flavor != 'sqlite':
+        constraint_queries.append(add_constraint_query)
     if upsert and indices_cols_str:
-        index_queries[constraint_name] = [constraint_query]
-
+        index_queries[unique_index_name] = constraint_queries
     return index_queries
 
 
@@ -1074,7 +1102,7 @@ def get_pipe_attributes(
 def sync_pipe(
         self,
         pipe: mrsm.Pipe,
-        df: Union[pandas.DataFrame, str, Dict[Any, Any], None] = None,
+        df: Union[pd.DataFrame, str, Dict[Any, Any], None] = None,
         begin: Optional[datetime] = None,
         end: Optional[datetime] = None,
         chunksize: Optional[int] = -1,
@@ -1154,7 +1182,12 @@ def sync_pipe(
         dprint("Fetched data:\n" + str(df))
 
     if not isinstance(df, pd.DataFrame):
-        df = pipe.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+        df = pipe.enforce_dtypes(
+            df,
+            chunksize = chunksize,
+            safe_copy = kw.get('safe_copy', False),
+            debug = debug,
+        )
 
     ### if table does not exist, create it with indices
     is_new = False
@@ -1198,6 +1231,7 @@ def sync_pipe(
     upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in update_queries
     if upsert:
         check_existing = False
+    kw['safe_copy'] = kw.get('safe_copy', False)
 
     unseen_df, update_df, delta_df = (
         pipe.filter_existing(
@@ -1286,7 +1320,11 @@ def sync_pipe(
         temp_pipe = Pipe(
             pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
             instance = pipe.instance_keys,
-            columns = pipe.columns,
+            columns = {
+                ix_key: ix
+                for ix_key, ix in pipe.columns.items()
+                if ix and ix in update_df.columns
+            },
             dtypes = pipe.dtypes,
             target = temp_target,
             temporary = True,

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/connectors/sql/tables/__init__.py

@@ -214,22 +214,6 @@ def create_tables(
     from meerschaum.utils.sql import get_rename_table_queries, table_exists
     _tables = tables if tables is not None else get_tables(conn)
 
-    rename_queries = []
-    for table_key, table in _tables.items():
-        if table_exists(
-            table_key,
-            conn,
-            schema = conn.instance_schema,
-        ):
-            rename_queries.extend(get_rename_table_queries(
-                table_key,
-                table.name,
-                schema = conn.instance_schema,
-                flavor = conn.flavor,
-            ))
-    if rename_queries:
-        conn.exec_queries(rename_queries)
-
     try:
         conn.metadata.create_all(bind=conn.engine)
     except Exception as e:

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/core/Pipe/_data.py

@@ -8,7 +8,7 @@ Retrieve Pipes' data from instances.
 
 from __future__ import annotations
 from datetime import datetime, timedelta
-from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator, List, Tuple
+from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator, List, Tuple, Iterator
 from meerschaum.config import get_config
 
 def get_data(
@@ -247,7 +247,7 @@ def _get_data_as_iterator(
         fresh: bool = False,
         debug: bool = False,
         **kw: Any
-    ) -> Generator['pd.DataFrame']:
+    ) -> Iterator['pd.DataFrame']:
     """
     Return a pipe's data as a generator.
     """
@@ -267,16 +267,17 @@ def _get_data_as_iterator(
 
     _ = kw.pop('as_chunks', None)
     _ = kw.pop('as_iterator', None)
+    dt_col = self.columns.get('datetime', None)
     min_dt = (
         begin
         if begin is not None
         else self.get_sync_time(round_down=False, newest=False, params=params, debug=debug)
-    )
+    ) if dt_col else None
     max_dt = (
         end
         if end is not None
         else self.get_sync_time(round_down=False, newest=True, params=params, debug=debug)
-    )
+    ) if dt_col else None
 
     ### We want to search just past the maximum value.
     if end is None:

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/core/Pipe/_dtypes.py

@@ -14,6 +14,7 @@ def enforce_dtypes(
         self,
         df: 'pd.DataFrame',
         chunksize: Optional[int] = -1,
+        safe_copy: bool = True,
         debug: bool = False,
     ) -> 'pd.DataFrame':
     """
@@ -71,7 +72,7 @@ def enforce_dtypes(
             )
         return df
 
-    return _enforce_dtypes(df, pipe_dtypes, debug=debug)
+    return _enforce_dtypes(df, pipe_dtypes, safe_copy=safe_copy, debug=debug)
 
 
 def infer_dtypes(self, persist: bool=False, debug: bool=False) -> Dict[str, Any]:

{meerschaum-2.1.5 → meerschaum-2.1.7}/meerschaum/core/Pipe/_sync.py

@@ -12,6 +12,7 @@ import json
 import time
 import threading
 import multiprocessing
+import functools
 from datetime import datetime, timedelta
 
 from meerschaum.utils.typing import (
@@ -518,6 +519,8 @@ def exists(
 def filter_existing(
         self,
         df: 'pd.DataFrame',
+        safe_copy: bool = True,
+        date_bound_only: bool = False,
         chunksize: Optional[int] = -1,
         debug: bool = False,
         **kw
@@ -530,6 +533,14 @@ def filter_existing(
     df: 'pd.DataFrame'
         The dataframe to inspect and filter.
         
+    safe_copy: bool, default True
+        If `True`, create a copy before comparing and modifying the dataframes.
+        Setting to `False` may mutate the DataFrames.
+        See `meerschaum.utils.dataframe.filter_unseen_df`.
+
+    date_bound_only: bool, default False
+        If `True`, only use the datetime index to fetch the sample dataframe.
+
     chunksize: Optional[int], default -1
         The `chunksize` used when fetching existing data.
 
@@ -567,7 +578,8 @@ def filter_existing(
     else:
         merge = pd.merge
         NA = pd.NA
-
+    if df is None:
+        return df, df, df
     if (df.empty if not is_dask else len(df) == 0):
         return df, df, df
 
@@ -617,7 +629,7 @@ def filter_existing(
         traceback.print_exc()
         max_dt = None
 
-    if not ('datetime' in str(type(max_dt))) or str(min_dt) == 'NaT':
+    if ('datetime' not in str(type(max_dt))) or str(min_dt) == 'NaT':
         if 'int' not in str(type(max_dt)).lower():
             max_dt = None
 
@@ -645,7 +657,7 @@ def filter_existing(
         col: df[col].unique()
         for col in self.columns
         if col in df.columns and col != dt_col
-    }
+    } if not date_bound_only else {}
     filter_params_index_limit = get_config('pipes', 'sync', 'filter_params_index_limit')
     _ = kw.pop('params', None)
     params = {
@@ -655,7 +667,7 @@ def filter_existing(
         ]
         for col, unique_vals in unique_index_vals.items()
         if len(unique_vals) <= filter_params_index_limit
-    }
+    } if not date_bound_only else {}
 
     if debug:
         dprint(f"Looking at data between '{begin}' and '{end}':", **kw)
@@ -698,18 +710,23 @@ def filter_existing(
                 col: to_pandas_dtype(typ)
                 for col, typ in self_dtypes.items()
             },
+            safe_copy = safe_copy,
             debug = debug
         ),
         on_cols_dtypes,
     )
 
     ### Cast dicts or lists to strings so we can merge.
+    serializer = functools.partial(json.dumps, sort_keys=True, separators=(',', ':'), default=str)
+    def deserializer(x):
+        return json.loads(x) if isinstance(x, str) else x
+
     unhashable_delta_cols = get_unhashable_cols(delta_df)
     unhashable_backtrack_cols = get_unhashable_cols(backtrack_df)
     for col in unhashable_delta_cols:
-        delta_df[col] = delta_df[col].apply(json.dumps)
+        delta_df[col] = delta_df[col].apply(serializer)
     for col in unhashable_backtrack_cols:
-        backtrack_df[col] = backtrack_df[col].apply(json.dumps)
+        backtrack_df[col] = backtrack_df[col].apply(serializer)
     casted_cols = set(unhashable_delta_cols + unhashable_backtrack_cols)
 
     joined_df = merge(
@@ -722,13 +739,9 @@ def filter_existing(
     ) if on_cols else delta_df
     for col in casted_cols:
         if col in joined_df.columns:
-            joined_df[col] = joined_df[col].apply(
-                lambda x: (
-                    json.loads(x)
-                    if isinstance(x, str)
-                    else x
-                )
-            )
+            joined_df[col] = joined_df[col].apply(deserializer)
+        if col in delta_df.columns:
+            delta_df[col] = delta_df[col].apply(deserializer)
 
     ### Determine which rows are completely new.
     new_rows_mask = (joined_df['_merge'] == 'left_only') if on_cols else None