meerschaum 2.6.3__tar.gz → 2.6.5__tar.gz

{meerschaum-2.6.3/meerschaum.egg-info → meerschaum-2.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.6.3
+Version: 2.6.5
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.6.3 → meerschaum-2.6.5}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.6.3"
+__version__ = "2.6.5"

{meerschaum-2.6.3 → meerschaum-2.6.5}/meerschaum/connectors/sql/_pipes.py

@@ -877,10 +877,21 @@ def get_pipe_data(
         attempt_cast_to_uuid,
         are_dtypes_equal,
     )
+    from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
     pd = import_pandas()
     is_dask = 'dask' in pd.__name__
 
-    dtypes = pipe.dtypes
+    cols_types = pipe.get_columns_types(debug=debug)
+    dtypes = {
+        **{
+            p_col: to_pandas_dtype(p_typ)
+            for p_col, p_typ in pipe.dtypes.items()
+        },
+        **{
+            col: get_pd_type_from_db_type(typ)
+            for col, typ in cols_types.items()
+        }
+    }
     if dtypes:
         if self.flavor == 'sqlite':
             if not pipe.columns.get('datetime', None):
@@ -1078,7 +1089,12 @@ def get_pipe_data_query(
     """
     from meerschaum.utils.misc import items_str
     from meerschaum.utils.sql import sql_item_name, dateadd_str
+    from meerschaum.utils.dtypes import coerce_timezone
+    from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
+
+    dt_col = pipe.columns.get('datetime', None)
     existing_cols = pipe.get_columns_types(debug=debug)
+    dt_typ = get_pd_type_from_db_type(existing_cols[dt_col]) if dt_col in existing_cols else None
     select_columns = (
         [col for col in existing_cols]
         if not select_columns
@@ -1097,6 +1113,10 @@ def get_pipe_data_query(
             begin -= backtrack_interval
 
     begin, end = pipe.parse_date_bounds(begin, end)
+    if isinstance(begin, datetime) and dt_typ:
+        begin = coerce_timezone(begin, strip_utc=('utc' not in dt_typ.lower()))
+    if isinstance(end, datetime) and dt_typ:
+        end = coerce_timezone(end, strip_utc=('utc' not in dt_typ.lower()))
 
     cols_names = [
         sql_item_name(col, self.flavor, None)

{meerschaum-2.6.3 → meerschaum-2.6.5}/meerschaum/connectors/sql/_sql.py

@@ -641,7 +641,7 @@ def exec_queries(
         will be considered a callable hook that returns a list of queries to be executed
         before the next item in the list.
 
-    break_on_error: bool, default True
+    break_on_error: bool, default False
         If `True`, stop executing when a query fails.
 
     rollback: bool, default True

{meerschaum-2.6.3 → meerschaum-2.6.5}/meerschaum/connectors/valkey/_pipes.py

@@ -416,7 +416,7 @@ def get_pipe_data(
     table_name = self.quote_table(pipe.target)
     indices = [col for col in pipe.columns.values() if col]
     ix_docs = [
-        string_to_dict(doc['ix'].replace(COLON, ':'))
+        string_to_dict(doc.get('ix', '').replace(COLON, ':'))
         for doc in self.read_docs(
             pipe.target,
             begin=begin,

{meerschaum-2.6.3 → meerschaum-2.6.5}/meerschaum/core/Pipe/_data.py

@@ -737,6 +737,8 @@ def parse_date_bounds(self, *dt_vals: Union[datetime, int, None]) -> Union[
 
         dt_col = self.columns.get('datetime', None)
         dt_typ = str(self.dtypes.get(dt_col, 'datetime64[ns, UTC]'))
+        if dt_typ == 'datetime':
+            dt_typ = 'datetime64[ns, UTC]'
         return coerce_timezone(dt_val, strip_utc=('utc' not in dt_typ.lower()))
 
     bounds = tuple(_parse_date_bound(dt_val) for dt_val in dt_vals)

{meerschaum-2.6.3 → meerschaum-2.6.5}/meerschaum/utils/dtypes/__init__.py

@@ -284,4 +284,7 @@ def coerce_timezone(
             return dt
         return dt.replace(tzinfo=timezone.utc)
 
-    return dt.astimezone(timezone.utc)
+    utc_dt = dt.astimezone(timezone.utc)
+    if strip_utc:
+        return utc_dt.replace(tzinfo=None)
+    return utc_dt

{meerschaum-2.6.3 → meerschaum-2.6.5/meerschaum.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.6.3
+Version: 2.6.5
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.6.3 → meerschaum-2.6.5}/tests/test_pipes_dtypes.py

@@ -136,10 +136,10 @@ def test_force_json_dtype(flavor: str):
     conn = conns[flavor]
     pipe = Pipe('foo', 'bar', session_id, instance=conn, dtypes={'a': 'json'})
     _ = pipe.delete(debug=debug)
-    pipe = Pipe('foo', 'bar', session_id, instance=conn, dtypes={'a': 'json'})
+    pipe = Pipe('foo', 'bar', session_id, instance=conn, dtypes={'a': 'json'}, columns=['id'])
     success, msg = pipe.sync([
-        {'a': json.dumps(['b', 'c'])},
-        {'a': json.dumps({'b': 1})},
+        {'id': 1, 'a': json.dumps(['b', 'c'])},
+        {'id': 2, 'a': json.dumps({'b': 1})},
     ])
     assert success, msg
     pprint(pipe.get_columns_types())
@@ -171,10 +171,10 @@ def test_infer_numeric_dtype(flavor: str):
     conn = conns[flavor]
     pipe = Pipe('infer', 'numeric', instance=conn)
     _ = pipe.delete(debug=debug)
-    pipe = Pipe('infer', 'numeric', instance=conn)
+    pipe = Pipe('infer', 'numeric', instance=conn, columns=['id'])
     success, msg = pipe.sync([
-        {'a': Decimal('1')},
-        {'a': Decimal(numeric_str)},
+        {'id': 1, 'a': Decimal('1')},
+        {'id': 2, 'a': Decimal(numeric_str)},
     ])
     assert success, msg
     pprint(pipe.get_columns_types())
@@ -195,11 +195,11 @@ def test_infer_uuid_dtype(flavor: str):
     conn = conns[flavor]
     pipe = Pipe('infer', 'uuid', instance=conn)
     _ = pipe.delete(debug=debug)
-    pipe = Pipe('infer', 'uuid', instance=conn)
+    pipe = Pipe('infer', 'uuid', instance=conn, columns=['id'])
     uuid_str = "e6f3a4ea-f1af-4e93-8da9-716b57672206"
     success, msg = pipe.sync(
         [
-            {'a': UUID(uuid_str)},
+            {'id': 1, 'a': UUID(uuid_str)},
         ],
         debug=debug,
     )
@@ -222,10 +222,10 @@ def test_infer_numeric_from_mixed_types(flavor: str):
     conn = conns[flavor]
     pipe = Pipe('foo', 'bar', session_id, instance=conn)
     _ = pipe.delete(debug=debug)
-    pipe = Pipe('foo', 'bar', session_id, instance=conn)
-    success, msg = pipe.sync([{'a': 1}])
+    pipe = Pipe('foo', 'bar', session_id, instance=conn, columns=['id'])
+    success, msg = pipe.sync([{'id': 1, 'a': 1}])
     assert success, msg
-    success, msg = pipe.sync([{'a': 2.1}])
+    success, msg = pipe.sync([{'id': 2, 'a': 2.1}])
     assert success, msg
     pprint(pipe.get_columns_types())
     df = pipe.get_data(debug=debug)
@@ -247,10 +247,10 @@ def test_force_numeric_dtype(flavor: str):
     conn = conns[flavor]
     pipe = Pipe('foo', 'bar', session_id, instance=conn, dtypes={'a': 'numeric'})
     _ = pipe.delete(debug=debug)
-    pipe = Pipe('foo', 'bar', session_id, instance=conn, dtypes={'a': 'numeric'})
+    pipe = Pipe('foo', 'bar', session_id, instance=conn, dtypes={'a': 'numeric'}, columns=['id'])
     success, msg = pipe.sync([
-        {'a': '1'},
-        {'a': '2.1'},
+        {'id': 1, 'a': '1'},
+        {'id': 2, 'a': '2.1'},
     ])
     assert success, msg
     pprint(pipe.get_columns_types())
@@ -706,3 +706,94 @@ def test_sync_uuids_simple_upsert(flavor: str):
     assert len(df) == 1
     assert isinstance(df['val'][0], UUID)
     assert df['val'][0] == UUID('d1cc1516-16e5-4471-8ab9-e969a1def655')
+
+
+@pytest.mark.parametrize("flavor", get_flavors())
+def test_mixed_timezone_aware_and_naive(flavor: str):
+    conn = conns[flavor]
+    if conn.type != 'sql':
+        return
+
+    pd = mrsm.attempt_import('pandas')
+    target = 'test_timezone_mix'
+    pipe = mrsm.Pipe('test', 'timezone', 'aware_naive', instance=conn)
+    pipe.delete()
+    pipe = mrsm.Pipe(
+        'test', 'timezone', 'aware_naive',
+        instance=conn,
+        columns={'datetime': 'ts'},
+        target=target,
+    )
+
+    src_df = pd.DataFrame([{'ts': datetime(2024, 1, 1), 'val': 2}])
+    conn.to_sql(src_df, target, debug=debug)
+
+    df = pipe.get_data(debug=debug)
+    unseen, update, delta = pipe.filter_existing(src_df, debug=debug)
+    assert len(unseen) == 0
+    assert len(update) == 0
+    assert len(delta) == 0
+
+    success, msg = pipe.sync([{'ts': '2024-01-01 05:00:00', 'val': 3}])
+    assert success, msg
+
+    df = pipe.get_data(begin='2024-01-01 05:00:00', debug=debug)
+    assert len(df) == 1
+    assert df['val'][0] == 3
+
+    success, msg = pipe.sync([{'ts': '2024-01-01 05:00:00+00:00', 'val': 4}])
+    assert success, msg
+
+    df = pipe.get_data(begin='2024-01-01 05:00:00+00:00', debug=debug)
+    assert len(df) == 1
+    assert df['val'][0] == 4
+
+
+@pytest.mark.parametrize("flavor", get_flavors())
+def test_parse_date_bounds(flavor: str):
+    """
+    Test that datetime bounds are parsed correctly.
+    """
+    conn = conns[flavor]
+    pipe = mrsm.Pipe('test', 'parse_date_bounds', 'tz', instance=conn)
+    pipe.delete()
+    pipe = mrsm.Pipe('test', 'parse_date_bounds', 'tz', instance=conn, columns={'datetime': 'ts'})
+    success, msg = pipe.sync([{'ts': '2024-01-01'}], debug=debug)
+    assert success, msg
+
+    naive_begin = datetime(2024, 1, 1)
+    naive_end = datetime(2024, 1, 2)
+    aware_begin = datetime(2024, 1, 1, tzinfo=timezone.utc)
+    aware_end = datetime(2024, 2, 1, tzinfo=timezone.utc)
+
+    begin, end = pipe.parse_date_bounds(naive_begin, naive_end)
+    assert begin.tzinfo is not None
+    assert end.tzinfo is not None
+    begin, end = pipe.parse_date_bounds(aware_begin, aware_end)
+    assert begin.tzinfo is not None
+    assert end.tzinfo is not None
+    begin, end = pipe.parse_date_bounds(naive_begin, aware_end)
+    assert begin.tzinfo is not None
+    assert end.tzinfo is not None
+
+    pipe = mrsm.Pipe('test', 'parse_date_bounds', 'naive', instance=conn)
+    pipe.delete()
+    pipe = mrsm.Pipe(
+        'test', 'parse_date_bounds', 'naive',
+        instance=conn,
+        columns={'datetime': 'ts'},
+        dtypes={'ts': 'datetime64[ns]'},
+    )
+
+    success, msg = pipe.sync([{'ts': '2024-01-01'}], debug=debug)
+    assert success, msg
+
+    begin, end = pipe.parse_date_bounds(naive_begin, naive_end)
+    assert begin.tzinfo is None
+    assert end.tzinfo is None
+    begin, end = pipe.parse_date_bounds(aware_begin, aware_end)
+    assert begin.tzinfo is None
+    assert end.tzinfo is None
+    begin, end = pipe.parse_date_bounds(naive_begin, aware_end)
+    assert begin.tzinfo is None
+    assert end.tzinfo is None