meerschaum 2.6.9__py3-none-any.whl → 2.6.10__py3-none-any.whl

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.6.9"
+__version__ = "2.6.10"

meerschaum/core/Pipe/_dtypes.py

@@ -56,7 +56,7 @@ def enforce_dtypes(
                 chunksize=chunksize,
                 debug=debug,
             )
-        else:
+        elif isinstance(df, (dict, list)):
             df = parse_df_datetimes(
                 df,
                 ignore_cols=[

meerschaum/core/Pipe/_sync.py

@@ -567,6 +567,7 @@ def filter_existing(
     safe_copy: bool = True,
     date_bound_only: bool = False,
     include_unchanged_columns: bool = False,
+    enforce_dtypes: bool = False,
     chunksize: Optional[int] = -1,
     debug: bool = False,
     **kw
@@ -591,6 +592,10 @@ def filter_existing(
         If `True`, include the backtrack columns which haven't changed in the update dataframe.
         This is useful if you can't update individual keys.
 
+    enforce_dtypes: bool, default False
+        If `True`, ensure the given and intermediate dataframes are enforced to the correct dtypes.
+        Setting `enforce_dtypes=True` may impact performance.
+
     chunksize: Optional[int], default -1
         The `chunksize` used when fetching existing data.
 
@@ -618,8 +623,9 @@ def filter_existing(
     from meerschaum.config import get_config
     pd = import_pandas()
     pandas = attempt_import('pandas')
-    df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
-    is_dask = 'dask' in df.__module__
+    if enforce_dtypes or 'dataframe' not in str(type(df)).lower():
+        df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+    is_dask = hasattr('df', '__module__') and 'dask' in df.__module__
     if is_dask:
         dd = attempt_import('dask.dataframe')
         merge = dd.merge
@@ -759,7 +765,8 @@ def filter_existing(
             dprint(f"No backtrack data was found for {self}.")
         return df, get_empty_df(), df
 
-    backtrack_df = self.enforce_dtypes(backtrack_df, chunksize=chunksize, debug=debug)
+    if enforce_dtypes:
+        backtrack_df = self.enforce_dtypes(backtrack_df, chunksize=chunksize, debug=debug)
 
     if debug:
         dprint(f"Existing data for {self}:\n" + str(backtrack_df), **kw)
@@ -796,7 +803,8 @@ def filter_existing(
         ),
         on_cols_dtypes,
     )
-    delta_df = self.enforce_dtypes(delta_df, chunksize=chunksize, debug=debug)
+    if enforce_dtypes:
+        delta_df = self.enforce_dtypes(delta_df, chunksize=chunksize, debug=debug)
 
     ### Cast dicts or lists to strings so we can merge.
     serializer = functools.partial(json.dumps, sort_keys=True, separators=(',', ':'), default=str)
@@ -820,7 +828,6 @@ def filter_existing(
         indicator=True,
         suffixes=('', '_old'),
     ) if on_cols else delta_df
-    joined_df = self.enforce_dtypes(joined_df, chunksize=chunksize, debug=debug)
     for col in casted_cols:
         if col in joined_df.columns:
             joined_df[col] = joined_df[col].apply(deserializer)
@@ -837,7 +844,6 @@ def filter_existing(
         .dropna(how='all')[cols]
         .reset_index(drop=True)
     ) if on_cols else delta_df
-    unseen_df = self.enforce_dtypes(unseen_df, chunksize=chunksize, debug=debug)
 
     ### Rows that have already been inserted but values have changed.
     update_df = (
@@ -846,7 +852,6 @@ def filter_existing(
         .dropna(how='all')[cols]
         .reset_index(drop=True)
     ) if on_cols else get_empty_df()
-    update_df = self.enforce_dtypes(update_df, chunksize=chunksize, debug=debug)
 
     if include_unchanged_columns and on_cols:
         unchanged_backtrack_cols = [
@@ -854,6 +859,8 @@ def filter_existing(
             for col in backtrack_df.columns
             if col in on_cols or col not in update_df.columns
         ]
+        if enforce_dtypes:
+            update_df = self.enforce_dtypes(update_df, chunksize=chunksize, debug=debug)
         update_df = merge(
             backtrack_df[unchanged_backtrack_cols],
             update_df,

meerschaum/utils/dataframe.py

@@ -1283,18 +1283,8 @@ def query_df(
                 if debug:
                     dprint(f"`end` will be cast to '{end}'.")
 
-            begin_tz = begin.tzinfo if begin is not None else None
-            end_tz = end.tzinfo if end is not None else None
-
-            if begin_tz is not None or end_tz is not None or df_tz is not None:
-                begin = coerce_timezone(begin, strip_utc=False)
-                end = coerce_timezone(end, strip_utc=False)
-                if df_tz is not None:
-                    if debug:
-                        dprint(f"Casting column '{datetime_column}' to UTC...")
-                    df[datetime_column] = coerce_timezone(df[datetime_column], strip_utc=False)
-                if debug:
-                    dprint(f"Using datetime bounds:\n{begin=}\n{end=}")
+            begin = coerce_timezone(begin, strip_utc=(df_tz is None)) if begin is not None else None
+            end = coerce_timezone(end, strip_utc=(df_tz is None)) if begin is not None else None
 
     in_ex_params = get_in_ex_params(params)
 
@@ -1340,15 +1330,24 @@ def query_df(
     ]
     for col in bool_cols:
         df[col] = df[col].astype('boolean[pyarrow]')
-    df['__mrsm_mask'] = query_mask.astype('boolean[pyarrow]')
 
-    if inplace:
-        df.where(query_mask, other=NA, inplace=True)
-        df.dropna(how='all', inplace=True)
-        result_df = df
+    if not isinstance(query_mask, bool):
+        df['__mrsm_mask'] = (
+            query_mask.astype('boolean[pyarrow]')
+            if hasattr(query_mask, 'astype')
+            else query_mask
+        )
+
+        if inplace:
+            df.where(query_mask, other=NA, inplace=True)
+            df.dropna(how='all', inplace=True)
+            result_df = df
+        else:
+            result_df = df.where(query_mask, other=NA)
+            result_df.dropna(how='all', inplace=True)
+
     else:
-        result_df = df.where(query_mask, other=NA)
-        result_df.dropna(how='all', inplace=True)
+        result_df = df
 
     if '__mrsm_mask' in df.columns:
         del df['__mrsm_mask']

meerschaum/utils/dtypes/__init__.py

@@ -267,8 +267,18 @@ def coerce_timezone(
 
     if dt_is_series:
         is_dask = 'dask' in dt.__module__
-        pandas = mrsm.attempt_import('pandas')
+        pandas = mrsm.attempt_import('pandas', lazy=False)
         dd = mrsm.attempt_import('dask.dataframe') if is_dask else None
+
+        if (
+            pandas.api.types.is_datetime64_any_dtype(dt) and (
+                (dt.dt.tz is not None and not strip_utc)
+                or
+                (dt.dt.tz is None and strip_utc)
+            )
+        ):
+            return dt
+
         dt_series = (
             pandas.to_datetime(dt, utc=True, format='ISO8601')
             if dd is None

{meerschaum-2.6.9.dist-info → meerschaum-2.6.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.6.9
+Version: 2.6.10
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.6.9.dist-info → meerschaum-2.6.10.dist-info}/RECORD

@@ -143,7 +143,7 @@ meerschaum/config/_preprocess.py,sha256=-AEA8m_--KivZwTQ1sWN6LTn5sio_fUr2XZ51BO6
 meerschaum/config/_read_config.py,sha256=RLC3HHi_1ndj7ITVDKLD9_uULY3caGRwSz3ATYE-ixA,15014
 meerschaum/config/_shell.py,sha256=46_m49Txc5q1rGfCgO49ca48BODx45DQJi8D0zz1R18,4245
 meerschaum/config/_sync.py,sha256=jHcWRkxd82_BgX8Xo8agsWvf7BSbv3qHLWmYl6ehp_0,4242
-meerschaum/config/_version.py,sha256=TPP8TjaIitWMVdjkOvsGLJlk5ixmTGvF8uEiF1nbPtA,71
+meerschaum/config/_version.py,sha256=QOPk_Pjr2KnGrSNrSaTJLqzoKYSFI0Wk__-Tb-BOgSQ,72
 meerschaum/config/paths.py,sha256=JjibeGN3YAdSNceRwsd42aNmeUrIgM6ndzC8qZAmNI0,621
 meerschaum/config/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 meerschaum/config/stack/__init__.py,sha256=2UukC0Lmk-aVL1o1qXzumqmuIrw3vu9fD7iCuz4XD4I,10544
@@ -200,12 +200,12 @@ meerschaum/core/Pipe/_data.py,sha256=HOvKlpD9NiBg599mSH1lzUQ_2g8-PsnMbGzMTOJx81E
 meerschaum/core/Pipe/_deduplicate.py,sha256=xthUdsDxGO2t3m0XGDm9K3F6dpaZoemtjNi8gyKm0e0,10177
 meerschaum/core/Pipe/_delete.py,sha256=1geNp9BgrocXP1gt76dMbnlJWKYFMuSNqPFA4K4-hXE,2118
 meerschaum/core/Pipe/_drop.py,sha256=qj5L0obDh2_dKAg7LD3WLI3_L0Q5YECTfTmPc8zZUxI,1135
-meerschaum/core/Pipe/_dtypes.py,sha256=VohahiZk44Uw3PNsTLyqH8m9wRdB-mkPgYvgCb5hlA0,4114
+meerschaum/core/Pipe/_dtypes.py,sha256=LNgZ47W5DMtZ2hwrcQamwapjWPWgtVQ42u67g4M0n3k,4143
 meerschaum/core/Pipe/_edit.py,sha256=HrKWe9vhqKaNOjOcJzW5BNbaUBPIbgNAhJEK8OMsy7c,8416
 meerschaum/core/Pipe/_fetch.py,sha256=Q_LncNi1nv-YwvRPbh1QK0hf6hflL7Hn9v9lT3oQgF4,5451
 meerschaum/core/Pipe/_register.py,sha256=Sd5xaAW8H7uLTIoommcKb-6kHPRuHJLWNSbPnt2UbvA,2240
 meerschaum/core/Pipe/_show.py,sha256=nG50y8eBT9TVuKkRgAKtNDNIxysJvMNxfu__lkL1F9k,1352
-meerschaum/core/Pipe/_sync.py,sha256=z24Y8nKiqwSQTZtuOK-UM1Jl-2IS4Uev_Y9_i7vORXs,34486
+meerschaum/core/Pipe/_sync.py,sha256=E2egt63tqhVpB0ZteMO36HTF4EYEMtdevKOtN1aLd9o,34753
 meerschaum/core/Pipe/_verify.py,sha256=c3HvsZd4QPydqozaV6cDDRtwYiNz4V91b0IcnKvcimA,14158
 meerschaum/core/Plugin/__init__.py,sha256=UXg64EvJPgI1PCxkY_KM02-ZmBm4FZpLPIQR_uSJJDc,137
 meerschaum/core/User/_User.py,sha256=JZ9Y1tsjZe-cgD24m9YfZ6ZwSOKn_sHc4rbQ7KblBz8,6592
@@ -219,7 +219,7 @@ meerschaum/plugins/__init__.py,sha256=6krcqaMKyzuVqesXMqEL0XEy2SJQ4xfNt2-oI_fJ6v
 meerschaum/plugins/bootstrap.py,sha256=VwjpZAuYdqPJW0YoVgAoM_taHkdQHqP902-8T7OWWCI,11339
 meerschaum/utils/__init__.py,sha256=QrK1K9hIbPCRCM5k2nZGFqGnrqhA0Eh-iSmCU7FG6Cs,612
 meerschaum/utils/_get_pipes.py,sha256=tu4xKPoDn79Dz2kWM13cXTP4DSCkn-3G9M8KiLftopw,11073
-meerschaum/utils/dataframe.py,sha256=RByfxVec8qv8-zSffTiLavGjG4FII0gehPFX0wcLhmM,43672
+meerschaum/utils/dataframe.py,sha256=R-TgFTV6qbJ3xPEl596Dw6m1lF8Lqz35n1PtFjCYmI0,43400
 meerschaum/utils/debug.py,sha256=GyIzJmunkoPnOcZNYVQdT4Sgd-aOb5MI2VbIgATOjIQ,3695
 meerschaum/utils/interactive.py,sha256=t-6jWozXSqL7lYGDHuwiOjTgr-UKhdcg61q_eR5mikI,3196
 meerschaum/utils/misc.py,sha256=soGmUooT216Dl15KbcUTzf8E-aC6uNM6Zvy1PiUT_Y4,47089
@@ -239,7 +239,7 @@ meerschaum/utils/daemon/RotatingFile.py,sha256=ePm_svjwyFDWh6V1k-bp1RHXCSWlyxDtl
 meerschaum/utils/daemon/StdinFile.py,sha256=J6tyUReM8NEp3bBQAxMfe8mjJG5mWi6CzHN4x86VQBI,3237
 meerschaum/utils/daemon/__init__.py,sha256=o9jWb4lRTIyny4EPt7fPXFgV_vIf1mUofsTwoE1ZecA,8751
 meerschaum/utils/daemon/_names.py,sha256=d2ZwTxBoTAqXZkCfZ5LuX2XrkQmLNUq1OTlUqfoH5dA,4515
-meerschaum/utils/dtypes/__init__.py,sha256=mswDCXBK8SKeP2wcCjdRPAEmdx4QvSMjBmwTmt3gtSw,8090
+meerschaum/utils/dtypes/__init__.py,sha256=ZG1Ccy8QmTNuR0IP5NSMtgWj7Cseh_VU4wxSQDw2m94,8352
 meerschaum/utils/dtypes/sql.py,sha256=IQihwQy4OKSbRjvJy6ky6SszFKR7W1iMs-ruZDsf2js,18701
 meerschaum/utils/formatting/__init__.py,sha256=GpJQWeqkdWw5IuDmW4Rgmapjzv-KkI4jhBZllJi4QIg,15999
 meerschaum/utils/formatting/_jobs.py,sha256=izsqPJhTtUkXUUtWnbXtReYsUYwulXtci3pBj72Ne64,6637
@@ -251,11 +251,11 @@ meerschaum/utils/packages/_packages.py,sha256=IFcQ4MzmTqjdWkqOsUa25xUNmG246TFqe2
 meerschaum/utils/packages/lazy_loader.py,sha256=VHnph3VozH29R4JnSSBfwtA5WKZYZQFT_GeQSShCnuc,2540
 meerschaum/utils/venv/_Venv.py,sha256=sBnlmxHdAh2bx8btfVoD79-H9-cYsv5lP02IIXkyECs,3553
 meerschaum/utils/venv/__init__.py,sha256=f3oi67lXYPLKJrnRW9lae7M3A8SFiC7DzaMoBdCVUFs,24609
-meerschaum-2.6.9.dist-info/LICENSE,sha256=jG2zQEdRNt88EgHUWPpXVWmOrOduUQRx7MnYV9YIPaw,11359
-meerschaum-2.6.9.dist-info/METADATA,sha256=MGzHL6_uLOLftJfakBs5_osOwEqgGkCNWTkMVAHWr-A,24757
-meerschaum-2.6.9.dist-info/NOTICE,sha256=OTA9Fcthjf5BRvWDDIcBC_xfLpeDV-RPZh3M-HQBRtQ,114
-meerschaum-2.6.9.dist-info/WHEEL,sha256=a7TGlA-5DaHMRrarXjVbQagU3Man_dCnGIWMJr5kRWo,91
-meerschaum-2.6.9.dist-info/entry_points.txt,sha256=5YBVzibw-0rNA_1VjB16z5GABsOGf-CDhW4yqH8C7Gc,88
-meerschaum-2.6.9.dist-info/top_level.txt,sha256=bNoSiDj0El6buocix-FRoAtJOeq1qOF5rRm2u9i7Q6A,11
-meerschaum-2.6.9.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-meerschaum-2.6.9.dist-info/RECORD,,
+meerschaum-2.6.10.dist-info/LICENSE,sha256=jG2zQEdRNt88EgHUWPpXVWmOrOduUQRx7MnYV9YIPaw,11359
+meerschaum-2.6.10.dist-info/METADATA,sha256=pJcfXtifDFIe38Cu39zlZOlgp0wZ1-IDGQPe7HE0cPU,24758
+meerschaum-2.6.10.dist-info/NOTICE,sha256=OTA9Fcthjf5BRvWDDIcBC_xfLpeDV-RPZh3M-HQBRtQ,114
+meerschaum-2.6.10.dist-info/WHEEL,sha256=a7TGlA-5DaHMRrarXjVbQagU3Man_dCnGIWMJr5kRWo,91
+meerschaum-2.6.10.dist-info/entry_points.txt,sha256=5YBVzibw-0rNA_1VjB16z5GABsOGf-CDhW4yqH8C7Gc,88
+meerschaum-2.6.10.dist-info/top_level.txt,sha256=bNoSiDj0El6buocix-FRoAtJOeq1qOF5rRm2u9i7Q6A,11
+meerschaum-2.6.10.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+meerschaum-2.6.10.dist-info/RECORD,,