pydiverse-common 0.3.6__tar.gz → 0.3.8__tar.gz

{pydiverse_common-0.3.6 → pydiverse_common-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydiverse-common
-Version: 0.3.6
+Version: 0.3.8
 Summary: Common functionality shared between pydiverse libraries
 Author: QuantCo, Inc.
 Author-email: Martin Trautmann <windiana@users.sf.net>, Finn Rudolph <finn.rudolph@t-online.de>

{pydiverse_common-0.3.6 → pydiverse_common-0.3.8}/docs/source/changelog.md

@@ -1,9 +1,18 @@
 # Changelog
 
-## 0.3.5 (2025-08-01)
+## 0.3.8 (2025-08-19)
+- fixed util.hashing.hash_polars_dataframe for simple dataframe
+
+## 0.3.7 (2025-08-19)
+- support util.hashing.hash_polars_dataframe
+
+## 0.3.6 (2025-08-01)
 - hack structlog / dask / pytest capture incompatibility
 (structlog._output.stderr != sys.stderr leads to pickle error)
 
+## 0.3.5 (2025-06-27)
+- added enum type
+
 ## 0.3.4 (2025-06-10)
 - fixed pypi package dependencies
 

{pydiverse_common-0.3.6 → pydiverse_common-0.3.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pydiverse-common"
-version = "0.3.6"
+version = "0.3.8"
 description = "Common functionality shared between pydiverse libraries"
 authors = [
   { name = "QuantCo, Inc." },

pydiverse_common-0.3.8/src/pydiverse/common/util/hashing.py

@@ -0,0 +1,113 @@
+# Copyright (c) QuantCo and pydiverse contributors 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+import base64
+import hashlib
+import types
+import warnings
+
+try:
+    import polars as pl
+except ImportError:
+    pl = types.ModuleType("pl")
+    pl.DataFrame = None
+
+
+def stable_hash(*args: str) -> str:
+    """Compute a hash over a set of strings
+
+    :param args: Some strings from which to compute the cache key
+    :return: A sha256 base32 digest, trimmed to 20 char length
+    """
+
+    combined_hash = hashlib.sha256(b"PYDIVERSE")
+    for arg in args:
+        arg_bytes = str(arg).encode("utf8")
+        arg_bytes_len = len(arg_bytes).to_bytes(length=8, byteorder="big")
+
+        combined_hash.update(arg_bytes_len)
+        combined_hash.update(arg_bytes)
+
+    # Only take first 20 characters of base32 digest (100 bits). This
+    # provides 50 bits of collision resistance, which is more than enough.
+    # To illustrate: If you were to generate 1k hashes per second,
+    # you still would have to wait over 800k years until you encounter
+    # a collision.
+
+    # NOTE: Can't use base64 because it contains lower and upper case
+    #       letters; identifiers in pipedag are all lowercase
+    hash_digest = combined_hash.digest()
+    hash_str = base64.b32encode(hash_digest).decode("ascii").lower()
+    return hash_str[:20]
+
+
+def hash_polars_dataframe(df: pl.DataFrame, use_init_repr=False) -> str:
+    if not use_init_repr:
+        try:
+
+            def unnest_all(df: pl.DataFrame) -> pl.DataFrame:
+                while struct_cols_and_dtypes := [
+                    (col, dtype)
+                    for col, dtype in df.schema.items()
+                    if dtype == pl.Struct
+                ]:
+                    df = df.with_columns(
+                        pl.col(struct_col_name).struct.rename_fields(
+                            [stable_hash(struct_col_name, struct_field_name)]
+                        )
+                        for struct_col_name, struct_field_name in struct_cols_and_dtypes
+                    ).unnest(
+                        struct_col_name for struct_col_name, _ in struct_cols_and_dtypes
+                    )
+                return df
+
+            schema_hash = stable_hash(repr(df.schema))
+            if df.is_empty():
+                content_hash = "empty"
+            else:
+                # Since we need to operate on all lists, we need to access them first
+                # if they are within a struct.
+                df = unnest_all(df)
+                array_columns = [
+                    col for col, dtype in df.schema.items() if dtype == pl.Array
+                ]
+                list_columns = [
+                    col for col, dtype in df.schema.items() if dtype == pl.List
+                ]
+                lf = df.lazy()
+                if array_columns:
+                    lf = lf.with_columns(pl.col(array_columns).reshape([-1]).implode())
+                lf = lf.with_columns(
+                    # Necessary because hash() does not work on lists of strings.
+                    # This can be removed when
+                    # https://github.com/pola-rs/polars/issues/21523 is resolved
+                    # in all supported versions of polars.
+                    pl.selectors.by_dtype(pl.List(pl.String)).list.eval(
+                        pl.element().hash()
+                    )
+                )
+                if list_columns or array_columns:
+                    # Necessary because hash_rows() does not work on lists.
+                    # This can be removed when
+                    # https://github.com/pola-rs/polars/issues/24121 is resolved
+                    # in all supported versions of polars.
+                    lf = lf.with_columns(pl.col(*list_columns, *array_columns).hash())
+
+                content_hash = str(
+                    lf.collect()
+                    .hash_rows()  # We get a Series of hashes, one for each row
+                    # Since polars only hashes rows, we need to implode the Series into
+                    # a single row to get a single hash
+                    .implode()
+                    .hash()
+                    .item()
+                )
+            return "0" + stable_hash(schema_hash, content_hash)
+        except Exception:
+            warnings.warn(
+                "Failed to compute hash for polars DataFrame in fast way. "
+                "Falling back to to_init_repr() method.",
+                stacklevel=1,
+            )
+
+    # fallback to to_init_repr string representation
+    return "1" + stable_hash(df.to_init_repr(len(df)))

{pydiverse_common-0.3.6 → pydiverse_common-0.3.8}/tests/test_util.py

@@ -1,12 +1,20 @@
 # Copyright (c) QuantCo and pydiverse contributors 2025-2025
 # SPDX-License-Identifier: BSD-3-Clause
 import traceback
+import types
 from dataclasses import dataclass
 
 import pytest
 
 from pydiverse.common.errors import DisposedError
 from pydiverse.common.util import Disposable, deep_map, requires
+from pydiverse.common.util.hashing import hash_polars_dataframe
+
+try:
+    import polars as pl
+except ImportError:
+    pl = types.ModuleType("polars")
+    pl.DataFrame = None
 
 
 def test_requires():
@@ -112,3 +120,79 @@ def test_deep_map():
     res = deep_map([1, d.values()], lambda x: 2 if x == 1 else x)
     assert res[0] == 2
     assert list(res[1]) == list(d.values())
+
+
+def check_df_hashes(df1_a: pl.DataFrame, other_dfs: list[pl.DataFrame]) -> None:
+    assert hash_polars_dataframe(df1_a)[0] == "0"
+    assert hash_polars_dataframe(df1_a, use_init_repr=True)[0] == "1"
+    assert hash_polars_dataframe(df1_a) == hash_polars_dataframe(df1_a)
+    assert hash_polars_dataframe(df1_a, use_init_repr=True) == hash_polars_dataframe(
+        df1_a, use_init_repr=True
+    )
+    for df1_other in other_dfs:
+        assert hash_polars_dataframe(df1_other)[0] == "0"
+        assert hash_polars_dataframe(df1_other, use_init_repr=True)[0] == "1"
+        assert hash_polars_dataframe(df1_a) != hash_polars_dataframe(df1_other)
+        assert hash_polars_dataframe(
+            df1_a, use_init_repr=True
+        ) != hash_polars_dataframe(df1_other, use_init_repr=True)
+        assert hash_polars_dataframe(df1_other) == hash_polars_dataframe(df1_other)
+        assert hash_polars_dataframe(
+            df1_other, use_init_repr=True
+        ) == hash_polars_dataframe(df1_other, use_init_repr=True)
+
+
+@pytest.mark.skipif(pl.DataFrame is None, reason="requires polars")
+def test_hashing_basic():
+    df1_a = pl.DataFrame(dict(x=[1]))
+    df1_b = pl.DataFrame(dict(y=[1]))
+    df1_c = pl.DataFrame(dict(x=[2]))
+    df1_d = pl.DataFrame(dict(x=[1.0]))
+    df1_e = pl.DataFrame(dict(x=[]))
+
+    check_df_hashes(df1_a, [df1_b, df1_c, df1_d, df1_e])
+
+
+@pytest.mark.skipif(pl.DataFrame is None, reason="requires polars")
+def test_hashing():
+    df1_a = pl.DataFrame(
+        data=dict(x=[["foo", "bar"], [""]], y=[[1, 2], None], z=[1, 2])
+    ).with_columns(s=pl.struct("x", "y"))
+    df1_b = pl.DataFrame(
+        data=dict(x=[["foo", "bar"], [""]], z=[[1, 2], None], y=[1, 2])
+    ).with_columns(s=pl.struct("x", "y"))
+    df1_c = pl.DataFrame(
+        data=dict(x=[["foo", "baR"], [""]], y=[[1, 2], None], z=[1, 2])
+    ).with_columns(s=pl.struct("x", "y"))
+    df1_d = pl.DataFrame(
+        data=dict(x=[["foo", "bar"], [""]], y=[[1, 3], None], z=[1, 2])
+    ).with_columns(s=pl.struct("x", "y"))
+    df1_e = pl.DataFrame(
+        data=dict(x=[["foo", "bar"], [""]], y=[[1, 3], []], z=[1, 2])
+    ).with_columns(s=pl.struct("x", "y"))
+    df1_f = pl.DataFrame(
+        data=dict(x=[["foo", "bar"], [""]], y=[[1, 2], None], z=[1, 2])
+    ).with_columns(s=pl.struct("x", "z"))
+
+    check_df_hashes(df1_a, [df1_b, df1_c, df1_d, df1_e, df1_f])
+
+
+@pytest.mark.skipif(pl.DataFrame is None, reason="requires polars")
+def test_hashing_array():
+    df1_a = pl.DataFrame(
+        data=dict(x=[[[1], [2], [3]]]), schema=dict(x=pl.Array(pl.UInt16, shape=(3, 1)))
+    )
+    df1_b = pl.DataFrame(
+        data=dict(y=[[[1], [2], [3]]]), schema=dict(y=pl.Array(pl.UInt16, shape=(3, 1)))
+    )
+    df1_c = pl.DataFrame(
+        data=dict(x=[[[1], [3], [2]]]), schema=dict(x=pl.Array(pl.UInt16, shape=(3, 1)))
+    )
+    df1_d = pl.DataFrame(
+        data=dict(x=[[[1, 2, 3]]]), schema=dict(x=pl.Array(pl.UInt16, shape=(1, 3)))
+    )
+    df1_e = pl.DataFrame(
+        data=dict(x=[[1, 2, 3]]), schema=dict(x=pl.Array(pl.UInt16, shape=3))
+    )
+
+    check_df_hashes(df1_a, [df1_b, df1_c, df1_d, df1_e])

pydiverse_common-0.3.8/typos.toml

@@ -0,0 +1,2 @@
+[default]
+extend-ignore-words-re = ["^ba$"]

pydiverse_common-0.3.6/src/pydiverse/common/util/hashing.py

@@ -1,32 +0,0 @@
-# Copyright (c) QuantCo and pydiverse contributors 2025-2025
-# SPDX-License-Identifier: BSD-3-Clause
-import base64
-import hashlib
-
-
-def stable_hash(*args: str) -> str:
-    """Compute a hash over a set of strings
-
-    :param args: Some strings from which to compute the cache key
-    :return: A sha256 base32 digest, trimmed to 20 char length
-    """
-
-    combined_hash = hashlib.sha256(b"PYDIVERSE")
-    for arg in args:
-        arg_bytes = str(arg).encode("utf8")
-        arg_bytes_len = len(arg_bytes).to_bytes(length=8, byteorder="big")
-
-        combined_hash.update(arg_bytes_len)
-        combined_hash.update(arg_bytes)
-
-    # Only take first 20 characters of base32 digest (100 bits). This
-    # provides 50 bits of collision resistance, which is more than enough.
-    # To illustrate: If you were to generate 1k hashes per second,
-    # you still would have to wait over 800k years until you encounter
-    # a collision.
-
-    # NOTE: Can't use base64 because it contains lower and upper case
-    #       letters; identifiers in pipedag are all lowercase
-    hash_digest = combined_hash.digest()
-    hash_str = base64.b32encode(hash_digest).decode("ascii").lower()
-    return hash_str[:20]