pydiverse-common 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

pydiverse/common/dtypes.py

@@ -103,7 +103,10 @@ class Dtype:
                 type_, pd.core.dtypes.common.classes(np_dtype)
             )
 
-        if pd.api.types.is_signed_integer_dtype(pandas_type):
+        workaround = (
+            pandas_type is not np.floating
+        )  # see https://github.com/pandas-dev/pandas/issues/62018
+        if workaround and pd.api.types.is_signed_integer_dtype(pandas_type):
             if is_np_dtype(pandas_type, np.int64):
                 return Int64()
             elif is_np_dtype(pandas_type, np.int32):
@@ -113,7 +116,7 @@ class Dtype:
             elif is_np_dtype(pandas_type, np.int8):
                 return Int8()
             raise TypeError
-        if pd.api.types.is_unsigned_integer_dtype(pandas_type):
+        if workaround and pd.api.types.is_unsigned_integer_dtype(pandas_type):
             if is_np_dtype(pandas_type, np.uint64):
                 return UInt64()
             elif is_np_dtype(pandas_type, np.uint32):
@@ -123,8 +126,8 @@ class Dtype:
             elif is_np_dtype(pandas_type, np.uint8):
                 return UInt8()
             raise TypeError
-        if pd.api.types.is_float_dtype(pandas_type):
-            if is_np_dtype(pandas_type, np.float64):
+        if not workaround or pd.api.types.is_float_dtype(pandas_type):
+            if not workaround or is_np_dtype(pandas_type, np.float64):
                 return Float64()
             elif is_np_dtype(pandas_type, np.float32):
                 return Float32()

pydiverse/common/util/hashing.py

@@ -2,6 +2,14 @@
 # SPDX-License-Identifier: BSD-3-Clause
 import base64
 import hashlib
+import types
+import warnings
+
+try:
+    import polars as pl
+except ImportError:
+    pl = types.ModuleType("pl")
+    pl.DataFrame = None
 
 
 def stable_hash(*args: str) -> str:
@@ -30,3 +38,73 @@ def stable_hash(*args: str) -> str:
     hash_digest = combined_hash.digest()
     hash_str = base64.b32encode(hash_digest).decode("ascii").lower()
     return hash_str[:20]
+
+
+def hash_polars_dataframe(df: pl.DataFrame, use_init_repr=False) -> str:
+    if not use_init_repr:
+        try:
+
+            def unnest_all(df: pl.DataFrame) -> pl.DataFrame:
+                while struct_cols_and_dtypes := [
+                    (col, dtype)
+                    for col, dtype in df.schema.items()
+                    if dtype == pl.Struct
+                ]:
+                    df = df.with_columns(
+                        pl.col(struct_col_name).struct.rename_fields(
+                            [stable_hash(struct_col_name, struct_field_name)]
+                        )
+                        for struct_col_name, struct_field_name in struct_cols_and_dtypes
+                    ).unnest(
+                        struct_col_name for struct_col_name, _ in struct_cols_and_dtypes
+                    )
+                return df
+
+            schema_hash = stable_hash(repr(df.schema))
+            if df.is_empty():
+                content_hash = "empty"
+            else:
+                # Since we need to operate on all lists, we need to access them first
+                # if they are within a struct.
+                df = unnest_all(df)
+                array_columns = [
+                    col for col, dtype in df.schema.items() if dtype == pl.Array
+                ]
+                list_columns = [
+                    col for col, dtype in df.schema.items() if dtype == pl.List
+                ]
+                content_hash = str(
+                    df.lazy()
+                    .with_columns(pl.col(array_columns).reshape([-1]).implode())
+                    .with_columns(
+                        # Necessary because hash() does not work on lists of strings.
+                        # This can be removed when
+                        # https://github.com/pola-rs/polars/issues/21523 is resolved
+                        # in all supported versions of polars.
+                        pl.selectors.by_dtype(pl.List(pl.String)).list.eval(
+                            pl.element().hash()
+                        )
+                    )
+                    # Necessary because hash_rows() does not work on lists.
+                    # This can be removed when
+                    # https://github.com/pola-rs/polars/issues/24121 is resolved
+                    # in all supported versions of polars.
+                    .with_columns(pl.col(*list_columns, *array_columns).hash())
+                    .collect()
+                    .hash_rows()  # We get a Series of hashes, one for each row
+                    # Since polars only hashes rows, we need to implode the Series into
+                    # a single row to get a single hash
+                    .implode()
+                    .hash()
+                    .item()
+                )
+            return "0" + stable_hash(schema_hash, content_hash)
+        except Exception:
+            warnings.warn(
+                "Failed to compute hash for polars DataFrame in fast way. "
+                "Falling back to to_init_repr() method.",
+                stacklevel=1,
+            )
+
+    # fallback to to_init_repr string representation
+    return "1" + stable_hash(df.to_init_repr(len(df)))

pydiverse/common/util/structlog.py

@@ -80,7 +80,7 @@ class PydiverseConsoleRenderer(structlog.dev.ConsoleRenderer):
 
 def setup_logging(
     log_level=logging.INFO,
-    log_stream=sys.stderr,
+    log_stream=None,
     timestamp_format="%Y-%m-%d %H:%M:%S.%f",
 ):
     """Configures structlog and logging with sane defaults."""
@@ -92,6 +92,15 @@ def setup_logging(
             level=log_level,
             handlers=[StructlogHandler()],
         )
+        if log_stream is None:
+            try:
+                # hack to avoid dask pickling problems with pytest capture
+                import structlog._output
+
+                structlog._output.stderr = sys.stderr
+            finally:
+                pass
+            log_stream = sys.stderr
         # Configure structlog
         structlog.configure(
             processors=[

{pydiverse_common-0.3.5.dist-info → pydiverse_common-0.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydiverse-common
-Version: 0.3.5
+Version: 0.3.7
 Summary: Common functionality shared between pydiverse libraries
 Author: QuantCo, Inc.
 Author-email: Martin Trautmann <windiana@users.sf.net>, Finn Rudolph <finn.rudolph@t-online.de>

{pydiverse_common-0.3.5.dist-info → pydiverse_common-0.3.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 pydiverse/common/__init__.py,sha256=J7b4iStFyaEMYre_jdlZ4l_8dLyrMWCIpQdsMQcB8aI,806
-pydiverse/common/dtypes.py,sha256=Twna-Esfy0-4wzIxX37C1CzPTJz8MnesV_LNegndKzE,15787
+pydiverse/common/dtypes.py,sha256=LYZKaKYq_4uI4kUhoaCTTo5j1SRurswIOfN11Bkz25A,15986
 pydiverse/common/testing.py,sha256=FcivI5wn0X3gzJhwnysKvCOgjSTTXaN6FtSFJ72jfSg,341
 pydiverse/common/version.py,sha256=1IU_m4r76_Qq0u-Tyo2_bERZFOkh0ZFueVzDqcCfLO0,336
 pydiverse/common/errors/__init__.py,sha256=FNeEfVbUa23b9sHkFsmxHYhY6sRgjaZysPQmlovpJrI,262
@@ -8,10 +8,10 @@ pydiverse/common/util/computation_tracing.py,sha256=HeXRHRUI8vxpzQ27Xcpa0StndSTP
 pydiverse/common/util/deep_map.py,sha256=JtY5ViWMMelOiLzPF7ZjzruCfB-bETISGxCk37qETxg,2540
 pydiverse/common/util/deep_merge.py,sha256=bV5p5_lsC-9nFah28EiEyG2h6U3Z5AuTqSooxOgCHN0,1929
 pydiverse/common/util/disposable.py,sha256=4XoGz70YRWA9TAqnUBvRCTAdsOGBviFN0gzxU7veY9o,993
-pydiverse/common/util/hashing.py,sha256=6x77BKg-w61u59fuTe9di0BtU-kEKH6UTRcKsRoYJ84,1196
+pydiverse/common/util/hashing.py,sha256=TquRSYuArnupP9X_8G6C15L1WxvZT2TtFLyI3IIE7Do,4503
 pydiverse/common/util/import_.py,sha256=K7dSgz4YyrqEvqhoOzbwgD7D8HScMoO5XoSWtjbaoUs,4056
-pydiverse/common/util/structlog.py,sha256=g0d8yaXBzAxmGNGZYMnMP9dsSQ__jN44GAY8Mb0ABeI,3487
-pydiverse_common-0.3.5.dist-info/METADATA,sha256=E4Z7zT2kwRtIlGetzPlxFf5Oj5-b037lAkLikUbNgKk,3399
-pydiverse_common-0.3.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pydiverse_common-0.3.5.dist-info/licenses/LICENSE,sha256=AcE6SDVuAq6v9ZLE_8eOCe_NvSE0rAPR3NR7lSowYh4,1517
-pydiverse_common-0.3.5.dist-info/RECORD,,
+pydiverse/common/util/structlog.py,sha256=xxhauxMuyxcKXTVg1MiPTkuvPBj8Zcr4o_v8Bq59Nig,3778
+pydiverse_common-0.3.7.dist-info/METADATA,sha256=SUa7l3bVxbutbqpILoMMZOPs3SHSkYucd0c_2VZt75A,3399
+pydiverse_common-0.3.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pydiverse_common-0.3.7.dist-info/licenses/LICENSE,sha256=AcE6SDVuAq6v9ZLE_8eOCe_NvSE0rAPR3NR7lSowYh4,1517
+pydiverse_common-0.3.7.dist-info/RECORD,,