pydiverse-common 0.3.6__py3-none-any.whl → 0.3.7__py3-none-any.whl

pydiverse/common/util/hashing.py

@@ -2,6 +2,14 @@
 # SPDX-License-Identifier: BSD-3-Clause
 import base64
 import hashlib
+import types
+import warnings
+
+try:
+    import polars as pl
+except ImportError:
+    pl = types.ModuleType("pl")
+    pl.DataFrame = None
 
 
 def stable_hash(*args: str) -> str:
@@ -30,3 +38,73 @@ def stable_hash(*args: str) -> str:
     hash_digest = combined_hash.digest()
     hash_str = base64.b32encode(hash_digest).decode("ascii").lower()
     return hash_str[:20]
+
+
+def hash_polars_dataframe(df: pl.DataFrame, use_init_repr=False) -> str:
+    if not use_init_repr:
+        try:
+
+            def unnest_all(df: pl.DataFrame) -> pl.DataFrame:
+                while struct_cols_and_dtypes := [
+                    (col, dtype)
+                    for col, dtype in df.schema.items()
+                    if dtype == pl.Struct
+                ]:
+                    df = df.with_columns(
+                        pl.col(struct_col_name).struct.rename_fields(
+                            [stable_hash(struct_col_name, struct_field_name)]
+                        )
+                        for struct_col_name, struct_field_name in struct_cols_and_dtypes
+                    ).unnest(
+                        struct_col_name for struct_col_name, _ in struct_cols_and_dtypes
+                    )
+                return df
+
+            schema_hash = stable_hash(repr(df.schema))
+            if df.is_empty():
+                content_hash = "empty"
+            else:
+                # Since we need to operate on all lists, we need to access them first
+                # if they are within a struct.
+                df = unnest_all(df)
+                array_columns = [
+                    col for col, dtype in df.schema.items() if dtype == pl.Array
+                ]
+                list_columns = [
+                    col for col, dtype in df.schema.items() if dtype == pl.List
+                ]
+                content_hash = str(
+                    df.lazy()
+                    .with_columns(pl.col(array_columns).reshape([-1]).implode())
+                    .with_columns(
+                        # Necessary because hash() does not work on lists of strings.
+                        # This can be removed when
+                        # https://github.com/pola-rs/polars/issues/21523 is resolved
+                        # in all supported versions of polars.
+                        pl.selectors.by_dtype(pl.List(pl.String)).list.eval(
+                            pl.element().hash()
+                        )
+                    )
+                    # Necessary because hash_rows() does not work on lists.
+                    # This can be removed when
+                    # https://github.com/pola-rs/polars/issues/24121 is resolved
+                    # in all supported versions of polars.
+                    .with_columns(pl.col(*list_columns, *array_columns).hash())
+                    .collect()
+                    .hash_rows()  # We get a Series of hashes, one for each row
+                    # Since polars only hashes rows, we need to implode the Series into
+                    # a single row to get a single hash
+                    .implode()
+                    .hash()
+                    .item()
+                )
+            return "0" + stable_hash(schema_hash, content_hash)
+        except Exception:
+            warnings.warn(
+                "Failed to compute hash for polars DataFrame in fast way. "
+                "Falling back to to_init_repr() method.",
+                stacklevel=1,
+            )
+
+    # fallback to to_init_repr string representation
+    return "1" + stable_hash(df.to_init_repr(len(df)))

{pydiverse_common-0.3.6.dist-info → pydiverse_common-0.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydiverse-common
-Version: 0.3.6
+Version: 0.3.7
 Summary: Common functionality shared between pydiverse libraries
 Author: QuantCo, Inc.
 Author-email: Martin Trautmann <windiana@users.sf.net>, Finn Rudolph <finn.rudolph@t-online.de>

{pydiverse_common-0.3.6.dist-info → pydiverse_common-0.3.7.dist-info}/RECORD

@@ -8,10 +8,10 @@ pydiverse/common/util/computation_tracing.py,sha256=HeXRHRUI8vxpzQ27Xcpa0StndSTP
 pydiverse/common/util/deep_map.py,sha256=JtY5ViWMMelOiLzPF7ZjzruCfB-bETISGxCk37qETxg,2540
 pydiverse/common/util/deep_merge.py,sha256=bV5p5_lsC-9nFah28EiEyG2h6U3Z5AuTqSooxOgCHN0,1929
 pydiverse/common/util/disposable.py,sha256=4XoGz70YRWA9TAqnUBvRCTAdsOGBviFN0gzxU7veY9o,993
-pydiverse/common/util/hashing.py,sha256=6x77BKg-w61u59fuTe9di0BtU-kEKH6UTRcKsRoYJ84,1196
+pydiverse/common/util/hashing.py,sha256=TquRSYuArnupP9X_8G6C15L1WxvZT2TtFLyI3IIE7Do,4503
 pydiverse/common/util/import_.py,sha256=K7dSgz4YyrqEvqhoOzbwgD7D8HScMoO5XoSWtjbaoUs,4056
 pydiverse/common/util/structlog.py,sha256=xxhauxMuyxcKXTVg1MiPTkuvPBj8Zcr4o_v8Bq59Nig,3778
-pydiverse_common-0.3.6.dist-info/METADATA,sha256=TVaCnX9IArQwB64MGJ4wfVv1EE7KUDI51u9d0QnFXeY,3399
-pydiverse_common-0.3.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pydiverse_common-0.3.6.dist-info/licenses/LICENSE,sha256=AcE6SDVuAq6v9ZLE_8eOCe_NvSE0rAPR3NR7lSowYh4,1517
-pydiverse_common-0.3.6.dist-info/RECORD,,
+pydiverse_common-0.3.7.dist-info/METADATA,sha256=SUa7l3bVxbutbqpILoMMZOPs3SHSkYucd0c_2VZt75A,3399
+pydiverse_common-0.3.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pydiverse_common-0.3.7.dist-info/licenses/LICENSE,sha256=AcE6SDVuAq6v9ZLE_8eOCe_NvSE0rAPR3NR7lSowYh4,1517
+pydiverse_common-0.3.7.dist-info/RECORD,,