meerschaum 2.1.5__tar.gz → 2.1.6__tar.gz

{meerschaum-2.1.5/meerschaum.egg-info → meerschaum-2.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.1.5
+Version: 2.1.6
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.1.5 → meerschaum-2.1.6}/meerschaum/actions/sync.py

@@ -9,7 +9,7 @@ NOTE: `sync` required a SQL connection and is not intended for client use
 """
 
 from __future__ import annotations
-from datetime import timedelta
+from datetime import timedelta, datetime, timezone
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Tuple, Union
 
@@ -39,7 +39,7 @@ def sync(
 
 def _pipes_lap(
         workers: Optional[int] = None,
-        debug: bool = None,
+        debug: Optional[bool] = None,
         unblock: bool = False,
         force: bool = False,
         min_seconds: int = 1,
@@ -52,7 +52,7 @@ def _pipes_lap(
         nopretty: bool = False,
         _progress: Optional['rich.progress.Progress'] = None,
         **kw: Any
-    ) -> Tuple[List[meerschaum.Pipe], List[meerschaum.Pipe]]:
+    ) -> Tuple[List[mrsm.Pipe], List[mrsm.Pipe]]:
     """
     Do a lap of syncing pipes.
     """
@@ -402,11 +402,20 @@ def _wrap_pipe(
     Wrapper function for handling exceptions.
     """
     import time
+    import traceback
+    from datetime import datetime, timedelta, timezone
+    import meerschaum as mrsm
+    from meerschaum.utils.typing import is_success_tuple, SuccessTuple
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.venv import Venv
     from meerschaum.plugins import _pre_sync_hooks, _post_sync_hooks
     from meerschaum.utils.misc import filter_keywords
+    from meerschaum.utils.pool import get_pool
+    from meerschaum.utils.warnings import warn
+
+    pool = get_pool(workers=workers)
 
+    sync_timestamp = datetime.now(timezone.utc)
     sync_start = time.perf_counter()
     sync_kwargs = {k: v for k, v in kw.items() if k != 'blocking'}
     sync_kwargs.update({
@@ -415,8 +424,9 @@ def _wrap_pipe(
         'debug': debug,
         'min_seconds': min_seconds,
         'workers': workers,
-        'bounded': 'bounded',
+        'bounded': bounded,
         'chunk_interval': chunk_interval,
+        'sync_timestamp': sync_timestamp,
     })
     if not verify and not deduplicate:
         sync_method = pipe.sync
@@ -427,12 +437,32 @@ def _wrap_pipe(
         sync_kwargs['deduplicate'] = deduplicate
     sync_kwargs['sync_method'] = sync_method
 
-    for module_name, pre_sync_hooks in _pre_sync_hooks.items():
-        plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
+    def call_sync_hook(plugin_name: str, sync_hook) -> SuccessTuple:
         plugin = mrsm.Plugin(plugin_name) if plugin_name else None
-        with Venv(plugin):
-            for pre_sync_hook in pre_sync_hooks:
-                _ = pre_sync_hook(pipe, **filter_keywords(pre_sync_hook, **sync_kwargs))
+        with mrsm.Venv(plugin):
+            try:
+                sync_hook_result = sync_hook(pipe, **filter_keywords(sync_hook, **sync_kwargs))
+                if is_success_tuple(sync_hook_result):
+                    return sync_hook_result
+            except Exception as e:
+                msg = (
+                    f"Failed to execute sync hook '{sync_hook.__name__}' "
+                    + f"from plugin '{plugin}':\n{traceback.format_exc()}"
+                )
+                warn(msg, stack=False)
+                return False, msg
+        return True, "Success"
+
+    hook_results = []
+    def apply_hooks(is_pre_sync: bool):
+        _sync_hooks = (_pre_sync_hooks if is_pre_sync else _post_sync_hooks)
+        for module_name, sync_hooks in _sync_hooks.items():
+            plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
+            for sync_hook in sync_hooks:
+                hook_result = pool.apply_async(call_sync_hook, (plugin_name, sync_hook))
+                hook_results.append(hook_result)
+
+    apply_hooks(True)
    
     try:
         with Venv(get_connector_plugin(pipe.connector), debug=debug):
@@ -444,18 +474,16 @@ def _wrap_pipe(
         return_tuple = (False, f"Failed to sync {pipe} with exception:" + "\n" + str(e))
 
     duration = time.perf_counter() - sync_start
-    sync_kwargs['duration'] = duration
-    for module_name, post_sync_hooks in _post_sync_hooks.items():
-        plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
-        plugin = mrsm.Plugin(plugin_name) if plugin_name else None
-        with Venv(plugin):
-            for post_sync_hook in post_sync_hooks:
-                _ = post_sync_hook(
-                    pipe,
-                    return_tuple,
-                    **filter_keywords(post_sync_hook, **sync_kwargs)
-                )
- 
+    sync_kwargs.update({
+        'success_tuple': return_tuple,
+        'sync_duration': duration,
+        'sync_complete_timestamp': datetime.now(timezone.utc),
+    })
+    apply_hooks(False)
+    for hook_result in hook_results:
+        hook_success, hook_msg = hook_result.get()
+        mrsm.pprint((hook_success, hook_msg))
+
     return return_tuple
 
 

{meerschaum-2.1.5 → meerschaum-2.1.6}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.1.5"
+__version__ = "2.1.6"

{meerschaum-2.1.5 → meerschaum-2.1.6}/meerschaum/connectors/sql/_pipes.py

@@ -385,7 +385,13 @@ def get_create_index_queries(
     -------
     A dictionary of column names mapping to lists of queries.
     """
-    from meerschaum.utils.sql import sql_item_name, get_distinct_col_count, update_queries
+    from meerschaum.utils.sql import (
+        sql_item_name,
+        get_distinct_col_count,
+        update_queries,
+        get_null_replacement,
+        COALESCE_UNIQUE_INDEX_FLAVORS,
+    )
     from meerschaum.config import get_config
     index_queries = {}
 
@@ -497,15 +503,37 @@ def get_create_index_queries(
             if ix and ix in existing_cols_types
         ]
     )
+    coalesce_indices_cols_str = ', '.join(
+        [
+            (
+                "COALESCE("
+                + sql_item_name(ix, self.flavor)
+                + ", "
+                + get_null_replacement(existing_cols_types[ix], self.flavor)
+                + ") "
+            ) if ix_key != 'datetime' else (sql_item_name(ix, self.flavor))
+            for ix_key, ix in pipe.columns.items()
+            if ix and ix in existing_cols_types
+        ]
+    )
+    unique_index_name = sql_item_name(pipe.target + '_unique_index', self.flavor)
     constraint_name = sql_item_name(pipe.target + '_constraint', self.flavor)
-    constraint_query = (
+    add_constraint_query = (
         f"ALTER TABLE {_pipe_name} ADD CONSTRAINT {constraint_name} UNIQUE ({indices_cols_str})"
-        if self.flavor != 'sqlite'
-        else f"CREATE UNIQUE INDEX {constraint_name} ON {_pipe_name} ({indices_cols_str})"
     )
+    unique_index_cols_str = (
+        indices_cols_str
+        if self.flavor not in COALESCE_UNIQUE_INDEX_FLAVORS
+        else coalesce_indices_cols_str
+    )
+    create_unique_index_query = (
+        f"CREATE UNIQUE INDEX {unique_index_name} ON {_pipe_name} ({unique_index_cols_str})"
+    )
+    constraint_queries = [create_unique_index_query]
+    if self.flavor != 'sqlite':
+        constraint_queries.append(add_constraint_query)
     if upsert and indices_cols_str:
-        index_queries[constraint_name] = [constraint_query]
-
+        index_queries[unique_index_name] = constraint_queries
     return index_queries
 
 
@@ -1074,7 +1102,7 @@ def get_pipe_attributes(
 def sync_pipe(
         self,
         pipe: mrsm.Pipe,
-        df: Union[pandas.DataFrame, str, Dict[Any, Any], None] = None,
+        df: Union[pd.DataFrame, str, Dict[Any, Any], None] = None,
         begin: Optional[datetime] = None,
         end: Optional[datetime] = None,
         chunksize: Optional[int] = -1,
@@ -1286,7 +1314,11 @@ def sync_pipe(
         temp_pipe = Pipe(
             pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
             instance = pipe.instance_keys,
-            columns = pipe.columns,
+            columns = {
+                ix_key: ix
+                for ix_key, ix in pipe.columns.items()
+                if ix and ix in update_df.columns
+            },
             dtypes = pipe.dtypes,
             target = temp_target,
             temporary = True,

{meerschaum-2.1.5 → meerschaum-2.1.6}/meerschaum/connectors/sql/tables/__init__.py

@@ -214,22 +214,6 @@ def create_tables(
     from meerschaum.utils.sql import get_rename_table_queries, table_exists
     _tables = tables if tables is not None else get_tables(conn)
 
-    rename_queries = []
-    for table_key, table in _tables.items():
-        if table_exists(
-            table_key,
-            conn,
-            schema = conn.instance_schema,
-        ):
-            rename_queries.extend(get_rename_table_queries(
-                table_key,
-                table.name,
-                schema = conn.instance_schema,
-                flavor = conn.flavor,
-            ))
-    if rename_queries:
-        conn.exec_queries(rename_queries)
-
     try:
         conn.metadata.create_all(bind=conn.engine)
     except Exception as e:

{meerschaum-2.1.5 → meerschaum-2.1.6}/meerschaum/core/Pipe/_data.py

@@ -8,7 +8,7 @@ Retrieve Pipes' data from instances.
 
 from __future__ import annotations
 from datetime import datetime, timedelta
-from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator, List, Tuple
+from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator, List, Tuple, Iterator
 from meerschaum.config import get_config
 
 def get_data(
@@ -247,7 +247,7 @@ def _get_data_as_iterator(
         fresh: bool = False,
         debug: bool = False,
         **kw: Any
-    ) -> Generator['pd.DataFrame']:
+    ) -> Iterator['pd.DataFrame']:
     """
     Return a pipe's data as a generator.
     """
@@ -267,16 +267,17 @@ def _get_data_as_iterator(
 
     _ = kw.pop('as_chunks', None)
     _ = kw.pop('as_iterator', None)
+    dt_col = self.columns.get('datetime', None)
     min_dt = (
         begin
         if begin is not None
         else self.get_sync_time(round_down=False, newest=False, params=params, debug=debug)
-    )
+    ) if dt_col else None
     max_dt = (
         end
         if end is not None
         else self.get_sync_time(round_down=False, newest=True, params=params, debug=debug)
-    )
+    ) if dt_col else None
 
     ### We want to search just past the maximum value.
     if end is None:

{meerschaum-2.1.5 → meerschaum-2.1.6}/meerschaum/utils/sql.py

@@ -38,6 +38,7 @@ version_queries = {
     'oracle': "SELECT version from PRODUCT_COMPONENT_VERSION WHERE rownum = 1",
 }
 SKIP_IF_EXISTS_FLAVORS = {'mssql', 'oracle'}
+COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'citus'}
 update_queries = {
     'default': """
         UPDATE {target_table_name} AS f
@@ -53,25 +54,25 @@ update_queries = {
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
         FROM {patch_table_name}
-        ON CONFLICT ({join_cols_str}) DO UPDATE {sets_subquery_none_excluded}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
     'postgresql-upsert': """
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
         FROM {patch_table_name}
-        ON CONFLICT ({join_cols_str}) DO UPDATE {sets_subquery_none_excluded}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
     'citus-upsert': """
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
         FROM {patch_table_name}
-        ON CONFLICT ({join_cols_str}) DO UPDATE {sets_subquery_none_excluded}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
     'cockroachdb-upsert': """
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
         FROM {patch_table_name}
-        ON CONFLICT ({join_cols_str}) DO UPDATE {sets_subquery_none_excluded}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
     'mysql': """
         UPDATE {target_table_name} AS f
@@ -122,7 +123,7 @@ update_queries = {
         SELECT {patch_cols_str}
         FROM {patch_table_name}
         WHERE true
-        ON CONFLICT ({join_cols_str}) DO UPDATE {sets_subquery_none_excluded}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
     'sqlite_delete_insert': [
         """
@@ -1084,7 +1085,7 @@ def get_update_queries(
             for col in patch_table_columns
         ]
     )
-    join_cols_str = ','.join(
+    join_cols_str = ', '.join(
         [
             sql_item_name(col, flavor)
             for col in join_cols
@@ -1109,10 +1110,27 @@ def get_update_queries(
     if debug:
         dprint(f"value_cols: {value_cols}")
 
-    if not value_cols or not join_cols_types:
+    if not join_cols_types:
+        return []
+    if not value_cols and not upsert:
         return []
 
+    coalesce_join_cols_str = ', '.join(
+        [
+            'COALESCE('
+            + sql_item_name(c_name, flavor)
+            + ', '
+            + get_null_replacement(c_type, flavor)
+            + ')'
+            for c_name, c_type in join_cols_types
+        ]
+    )
+
+    update_or_nothing = ('UPDATE' if value_cols else 'NOTHING')
+
     def sets_subquery(l_prefix: str, r_prefix: str):
+        if not value_cols:
+            return ''
         return 'SET ' + ',\n'.join([
             (
                 l_prefix + sql_item_name(c_name, flavor, None)
@@ -1169,6 +1187,8 @@ def get_update_queries(
             patch_cols_str = patch_cols_str,
             date_bounds_subquery = date_bounds_subquery,
             join_cols_str = join_cols_str,
+            coalesce_join_cols_str = coalesce_join_cols_str,
+            update_or_nothing = update_or_nothing,
         )
         for base_query in base_queries
     ]

{meerschaum-2.1.5 → meerschaum-2.1.6}/meerschaum/utils/typing.py

@@ -87,3 +87,14 @@ PipesDict = Dict[
     ]
 ]
 WebState = Dict[str, str]
+
+def is_success_tuple(x: Any) -> bool:
+    """
+    Determine whether an object is a `SuccessTuple`.
+    """
+    return (
+        isinstance(x, tuple)
+        and len(x) == 2
+        and isinstance(x[0], bool)
+        and isinstance(x[1], str)
+    )

{meerschaum-2.1.5 → meerschaum-2.1.6/meerschaum.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.1.5
+Version: 2.1.6
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.1.5 → meerschaum-2.1.6}/tests/test_sync.py

@@ -689,3 +689,34 @@ def test_upsert_sync(flavor: str):
     assert 'na' in str(df['yes'][0]).lower()
     assert isinstance(df['num'][0], Decimal)
     assert df['num'][0] == Decimal('2')
+
+
+@pytest.mark.parametrize("flavor", get_flavors())
+def test_upsert_no_value_cols(flavor: str):
+    """
+    Test that setting `upsert` to `True` is able to sync without value columns.
+    """
+    conn = conns[flavor]
+    pipe = Pipe('test', 'upsert', 'no_vals', instance=conn)
+    pipe.delete()
+    pipe = Pipe(
+        'test', 'upsert', 'no_vals', instance=conn,
+        columns = {'datetime': 'dt', 'id': 'id'},
+        parameters = {'upsert': True},
+    )
+
+    docs = [
+        {'dt': '2023-01-01', 'id': 1},
+        {'dt': '2023-01-02', 'id': 2},
+        {'dt': '2023-01-03', 'id': 3},
+    ]
+    success, msg = pipe.sync(docs, debug=debug)
+    assert success, msg
+
+    docs = [
+        {'dt': '2023-01-03', 'id': 3},
+    ]
+    success, msg = pipe.sync(docs, debug=debug)
+    assert success, msg
+    return pipe
+    assert pipe.get_rowcount() == 3