meerschaum 2.6.1__tar.gz → 2.6.3__tar.gz

{meerschaum-2.6.1/meerschaum.egg-info → meerschaum-2.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.6.1
+Version: 2.6.3
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.6.1 → meerschaum-2.6.3}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.6.1"
+__version__ = "2.6.3"

{meerschaum-2.6.1 → meerschaum-2.6.3}/meerschaum/connectors/sql/_pipes.py

@@ -1980,19 +1980,20 @@ def sync_pipe_inplace(
             "SELECT\n"
             f"    MIN({dt_col_name}) AS {sql_item_name('min_dt', self.flavor)},\n"
             f"    MAX({dt_col_name}) AS {sql_item_name('max_dt', self.flavor)}\n"
-            f"FROM {temp_table_names['new']}\n"
+            f"FROM {temp_table_names['new' if not upsert else 'update']}\n"
             f"WHERE {dt_col_name} IS NOT NULL"
         ],
         with_results=True,
         debug=debug,
-    )
+    ) if not upsert else ((True, "Success"), None)
     if not new_dt_bounds_success:
         return (
             new_dt_bounds_success,
             f"Could not determine in-place datetime bounds:\n{new_dt_bounds_msg}"
         )
 
-    begin, end = new_dt_bounds_results[0].fetchone()
+    if not upsert:
+        begin, end = new_dt_bounds_results[0].fetchone()
 
     backtrack_def = self.get_pipe_data_query(
         pipe,
@@ -2010,7 +2011,7 @@ def sync_pipe_inplace(
         self.flavor,
         schema=internal_schema,
     )[0]
-    (create_backtrack_success, create_backtrack_msg), create_new_results = session_execute(
+    (create_backtrack_success, create_backtrack_msg), create_backtrack_results = session_execute(
         session,
         create_backtrack_query,
         with_results=True,

{meerschaum-2.6.1 → meerschaum-2.6.3}/meerschaum/utils/sql.py

@@ -338,7 +338,7 @@ columns_indices_queries = {
             AND kc.name = i.name
         WHERE
             t.name IN ('{table}', '{table_trunc}')
-            AND s.name = 'dbo'
+            AND s.name = '{schema}'
             AND i.type IN (1, 2)  -- 1 = CLUSTERED, 2 = NONCLUSTERED
     """,
     'oracle': """

{meerschaum-2.6.1 → meerschaum-2.6.3/meerschaum.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.6.1
+Version: 2.6.3
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.6.1 → meerschaum-2.6.3}/tests/test_pipes_dtypes.py

@@ -112,15 +112,14 @@ def test_infer_json_dtype(flavor: str):
     conn = conns[flavor]
     pipe = Pipe('foo', 'bar', session_id, instance=conn)
     _ = pipe.delete(debug=debug)
-    pipe = Pipe('foo', 'bar', session_id, instance=conn)
+    pipe = Pipe('foo', 'bar', session_id, instance=conn, columns=['id'])
     success, msg = pipe.sync([
-        {'a': ['b', 'c']},
-        {'a': {'b': 1}},
+        {'id': 1, 'a': ['b', 'c']},
+        {'id': 2, 'a': {'b': 1}},
     ])
     assert success, msg
     pprint(pipe.get_columns_types())
     df = pipe.get_data(debug=debug)
-    print(df)
     assert isinstance(df['a'][0], list)
     assert isinstance(df['a'][1], dict)
 

{meerschaum-2.6.1 → meerschaum-2.6.3}/tests/test_sync.py

@@ -536,6 +536,103 @@ def test_sync_inplace(flavor: str):
     assert df['a'][0] == 3
 
 
+@pytest.mark.parametrize("flavor", get_flavors())
+def test_sync_inplace_upsert(flavor: str):
+    """
+    Verify that in-place syncing works as expected.
+    """
+    from meerschaum.utils.sql import sql_item_name
+    conn = conns[flavor]
+    if conn.type != 'sql':
+        return
+    source_pipe = Pipe('test', 'inplace', 'upsert', instance=conn)
+    source_pipe.delete()
+    source_pipe = Pipe(
+        'test', 'inplace', 'upsert',
+        instance=conn,
+        columns={'datetime': 'dt', 'id': 'id'},
+        indices={'all': ['dt', 'id']},
+        upsert=True,
+    )
+    dest_pipe = Pipe(str(conn), 'inplace', 'upsert', instance=conn)
+    dest_pipe.delete()
+    query = f"""
+    WITH {sql_item_name('subquery', flavor)} AS (
+        SELECT *
+        FROM {sql_item_name(source_pipe.target, flavor)}
+    )
+    SELECT *
+    FROM {sql_item_name('subquery', flavor)}
+    """ if flavor not in ('mysql',) else f"""
+    SELECT *
+    FROM {sql_item_name(source_pipe.target, flavor)}
+    """
+    dest_pipe = Pipe(
+        str(conn), 'inplace', 'upsert',
+        instance=conn,
+        columns={'datetime': 'dt', 'id': 'id'},
+        dtypes={'a': 'int', 'id': 'uuid'},
+        upsert=True,
+        parameters={
+            "fetch": {
+                "definition": query,
+                "backtrack_minutes": 1440,
+            }
+        },
+    )
+
+    docs = [
+        {'dt': '2023-01-01 00:00:00', 'id': UUID('9f680a72-b5f7-4336-8f7c-30927ec21cb1')},
+        {'dt': '2023-01-01 00:01:00', 'id': UUID('335b0322-4b54-40aa-8019-07666cbefa52')},
+        {'dt': '2023-01-01 00:02:00', 'id': UUID('d7d42913-2dfe-47d6-b0e0-7f71e13e814e')},
+        {'dt': '2023-01-01 00:03:00', 'id': UUID('7e194a2c-26b4-4632-af02-e0a8b2c6ce1e')},
+        {'dt': '2023-01-01 00:04:00', 'id': UUID('31e5fd08-fb81-47f4-8a1c-0c9dcf08ac5e')},
+    ]
+    success, msg = source_pipe.sync(docs)
+    assert success, msg
+
+    success, msg = dest_pipe.sync(debug=debug)
+    assert success, msg
+    assert dest_pipe.get_rowcount(debug=debug) == len(docs)
+
+    success, msg = dest_pipe.sync(debug=debug)
+    assert success, msg
+    assert dest_pipe.get_rowcount(debug=debug) == len(docs)
+
+    new_docs = [
+        {'dt': '2023-01-02 00:00:00', 'id': UUID('afb0b31b-15dc-485e-ac6f-d8622b6d03d4')},
+        {'dt': '2023-01-02 00:01:00', 'id': UUID('8b7bf428-d0ed-40fa-951b-bb115a03eac5')},
+        {'dt': '2023-01-02 00:02:00', 'id': UUID('36aed9b4-4c7a-4566-a321-d1774ef1015a')},
+        {'dt': '2023-01-02 00:03:00', 'id': UUID('7a4ef6cc-37d8-4899-9ddb-07b4998d0b53')},
+        {'dt': '2023-01-02 00:04:00', 'id': UUID('59244211-fdb8-46f1-b14b-8631146758c0')},
+    ]
+    success, msg = source_pipe.sync(new_docs)
+    assert success, msg
+
+    success, msg = dest_pipe.sync(debug=debug)
+    assert success, msg
+    assert dest_pipe.get_rowcount(debug=debug) == len(docs) + len(new_docs)
+
+    update_docs = [
+        {'dt': '2023-01-01 00:00:00', 'id': UUID('9f680a72-b5f7-4336-8f7c-30927ec21cb1'), 'a': 1},
+        {'dt': '2023-01-01 00:01:00', 'id': UUID('335b0322-4b54-40aa-8019-07666cbefa52'), 'a': 2},
+        {'dt': '2023-01-01 00:02:00', 'id': UUID('d7d42913-2dfe-47d6-b0e0-7f71e13e814e'), 'a': 3},
+        {'dt': '2023-01-01 00:03:00', 'id': UUID('7e194a2c-26b4-4632-af02-e0a8b2c6ce1e'), 'a': 4},
+        {'dt': '2023-01-01 00:04:00', 'id': UUID('31e5fd08-fb81-47f4-8a1c-0c9dcf08ac5e'), 'a': 5},
+    ]
+    success, msg = source_pipe.sync(update_docs)
+    assert success, msg
+    assert source_pipe.get_rowcount(debug=debug) == len(docs) + len(new_docs)
+
+    success, msg = dest_pipe.verify(debug=debug)
+    assert success, msg
+    assert dest_pipe.get_rowcount(debug=debug) == len(docs) + len(new_docs)
+
+    df = dest_pipe.get_data(params={'id': 'd7d42913-2dfe-47d6-b0e0-7f71e13e814e'})
+    assert len(df) == 1
+    assert df['a'][0] == 3
+
+
 @pytest.mark.parametrize("flavor", get_flavors())
 def test_nested_chunks(flavor: str):
     """