datachain 0.34.4__tar.gz → 0.34.5__tar.gz

{datachain-0.34.4 → datachain-0.34.5}/.github/workflows/tests-studio.yml

@@ -32,7 +32,7 @@ jobs:
           POSTGRES_DB: database
           POSTGRES_HOST_AUTH_METHOD: trust
       clickhouse:
-        image: clickhouse/clickhouse-server:24.8
+        image: clickhouse/clickhouse-server:25.8
         ports:
           - 8123:8123
           - 9010:9000

{datachain-0.34.4 → datachain-0.34.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.34.4
+Version: 0.34.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.34.4 → datachain-0.34.5}/src/datachain/lib/udf.py

@@ -160,9 +160,15 @@ class UDFBase(AbstractUDF):
         """
         Creates SHA hash of this UDF function. It takes into account function,
         inputs and outputs.
+
+        For function-based UDFs, hashes self._func.
+        For class-based UDFs, hashes the process method.
         """
+        # Hash user code: either _func (function-based) or process method (class-based)
+        func_to_hash = self._func if self._func else self.process
+
         parts = [
-            hash_callable(self._func),
+            hash_callable(func_to_hash),
             self.params.hash() if self.params else "",
             self.output.hash(),
         ]

{datachain-0.34.4 → datachain-0.34.5}/src/datachain/query/dataset.py

@@ -982,18 +982,26 @@ class SQLUnion(Step):
 
         columns1, columns2 = _order_columns(q1.columns, q2.columns)
 
+        union_select = sqlalchemy.select(*columns1).union_all(
+            sqlalchemy.select(*columns2)
+        )
+        union_cte = union_select.cte()
+        regenerated = self.query1.catalog.warehouse._regenerate_system_columns(
+            union_cte
+        )
+        result_columns = tuple(regenerated.selected_columns)
+
         def q(*columns):
-            names = {c.name for c in columns}
-            col1 = [c for c in columns1 if c.name in names]
-            col2 = [c for c in columns2 if c.name in names]
-            res = sqlalchemy.select(*col1).union_all(sqlalchemy.select(*col2))
+            if not columns:
+                return regenerated
 
-            subquery = res.subquery()
-            return sqlalchemy.select(*subquery.c).select_from(subquery)
+            names = {c.name for c in columns}
+            selected = [c for c in result_columns if c.name in names]
+            return regenerated.with_only_columns(*selected)
 
         return step_result(
             q,
-            columns1,
+            result_columns,
             dependencies=self.query1.dependencies | self.query2.dependencies,
         )
 

{datachain-0.34.4 → datachain-0.34.5}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.34.4
+Version: 0.34.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.34.4 → datachain-0.34.5}/src/datachain.egg-info/SOURCES.txt

@@ -335,6 +335,7 @@ tests/func/test_studio_datetime_parsing.py
 tests/func/test_to_database.py
 tests/func/test_toolkit.py
 tests/func/test_udf.py
+tests/func/test_union.py
 tests/func/test_video.py
 tests/func/test_warehouse.py
 tests/func/data/Big_Buck_Bunny_360_10s_1MB.mp4

datachain-0.34.5/tests/func/test_union.py

@@ -0,0 +1,59 @@
+import datachain as dc
+
+
+def test_union_persist_no_duplication_large_session(test_session, monkeypatch):
+    # See https://github.com/iterative/datachain/issues/1356
+    # Lower insert batch size to keep the test fast and still cross the boundary.
+    monkeypatch.setattr(
+        "datachain.data_storage.warehouse.INSERT_BATCH_SIZE", 20, raising=False
+    )
+    monkeypatch.setattr("datachain.query.dataset.INSERT_BATCH_SIZE", 20, raising=False)
+    n = 20 + 7
+
+    x_ids = list(range(n))
+    y_ids = list(range(n, 2 * n))
+
+    x = dc.read_values(idx=x_ids, session=test_session)
+    y = dc.read_values(idx=y_ids, session=test_session)
+
+    xy = x.union(y)
+    assert xy.count() == 2 * n
+
+    xy_p = xy.persist()
+    assert xy_p.count() == 2 * n
+
+    distinct_idx = {v for (v,) in xy_p.select("idx").results()}
+    assert len(distinct_idx) == 2 * n
+
+    j = xy_p.merge(x, on="idx", inner=True)
+    assert j.count() == n
+
+
+def test_union_parallel_udf_ids_only_no_dup(test_session_tmpfile, monkeypatch):
+    # Validate that after union, running a parallel UDF that uses the ids-only
+    # path does not duplicate rows due to sys__id collisions across branches.
+    # This specifically exercises the parallel dispatch path where input rows
+    # are split by sys__id and fetched per worker using IN (...) filters.
+    # See https://github.com/iterative/datachain/issues/1356
+
+    # Make worker/ids fetch batches small to exercise splitting on tiny inputs.
+    monkeypatch.setattr("datachain.query.dispatch.DEFAULT_BATCH_SIZE", 5, raising=False)
+    n = 30
+
+    x_ids = list(range(n))
+    y_ids = list(range(n, 2 * n))
+
+    x = dc.read_values(idx=x_ids, session=test_session_tmpfile)
+    y = dc.read_values(idx=y_ids, session=test_session_tmpfile)
+
+    xy = x.union(y)
+    mapped = xy.settings(parallel=2).map(
+        out=lambda idx: idx, output=int, params=("idx",)
+    )
+
+    total = mapped.count()
+    distinct_idx = {v for (v,) in mapped.select("idx").results()}
+
+    assert total == 2 * n
+    assert len(distinct_idx) == 2 * n
+    assert total == len(distinct_idx)

{datachain-0.34.4 → datachain-0.34.5}/tests/unit/lib/test_datachain.py

@@ -3686,7 +3686,10 @@ def test_group_by_schema(test_session):
         "cnt": "int",
         "sum": "float",
     }
-    assert chain.to_records() == [
+    assert sorted(
+        chain.to_records(),
+        key=lambda row: (row["signal__name"], row["parent__signal__name"]),
+    ) == [
         {
             "signal__name": "a",
             "parent__signal__name": "a",

{datachain-0.34.4 → datachain-0.34.5}/tests/unit/test_datachain_hash.py

@@ -1,5 +1,6 @@
 from unittest.mock import patch
 
+import pandas as pd
 import pytest
 from pydantic import BaseModel
 
@@ -7,6 +8,11 @@ import datachain as dc
 from datachain import func
 from datachain.lib.dc import C
 
+DF_DATA = {
+    "first_name": ["Alice", "Bob", "Charlie", "David", "Eva"],
+    "age": [25, 30, 35, 40, 45],
+}
+
 
 class Person(BaseModel):
     name: str
@@ -55,13 +61,55 @@ def mock_get_listing():
 
 
 def test_read_values():
-    pytest.skip(
-        "Hash of the chain started with read_values is currently inconsistent,"
-        " meaning it produces different hash every time. This happens because we"
-        " create random name dataset in the process. Correct solution would be"
-        " to calculate hash of all those input values."
+    """
+    Hash of the chain started with read_values is currently inconsistent.
+    Goal of this test is just to check it doesn't break.
+    """
+    assert dc.read_values(num=[1, 2, 3]).hash() is not None
+
+
+def test_read_csv(test_session, tmp_dir):
+    """
+    Hash of the chain started with read_csv is currently inconsistent.
+    Goal of this test is just to check it doesn't break.
+    """
+    path = tmp_dir / "test.csv"
+    pd.DataFrame(DF_DATA).to_csv(path, index=False)
+    assert dc.read_csv(path.as_uri(), session=test_session).hash() is not None
+
+
+@pytest.mark.filterwarnings("ignore::pydantic.warnings.PydanticDeprecatedSince20")
+def test_read_json(test_session, tmp_dir):
+    """
+    Hash of the chain started with read_json is currently inconsistent.
+    Goal of this test is just to check it doesn't break.
+    """
+    path = tmp_dir / "test.jsonl"
+    dc.read_pandas(pd.DataFrame(DF_DATA), session=test_session).to_jsonl(path)
+    assert (
+        dc.read_json(path.as_uri(), format="jsonl", session=test_session).hash()
+        is not None
     )
-    assert dc.read_values(num=[1, 2, 3]).hash() == ""
+
+
+def test_read_pandas(test_session, tmp_dir):
+    """
+    Hash of the chain started with read_pandas is currently inconsistent.
+    Goal of this test is just to check it doesn't break.
+    """
+    df = pd.DataFrame(DF_DATA)
+    assert dc.read_pandas(df, session=test_session).hash() is not None
+
+
+def test_read_parquet(test_session, tmp_dir):
+    """
+    Hash of the chain started with read_parquet is currently inconsistent.
+    Goal of this test is just to check it doesn't break.
+    """
+    df = pd.DataFrame(DF_DATA)
+    path = tmp_dir / "test.parquet"
+    dc.read_pandas(df, session=test_session).to_parquet(path)
+    assert dc.read_parquet(path.as_uri(), session=test_session).hash() is not None
 
 
 def test_read_storage(mock_get_listing):

{datachain-0.34.4 → datachain-0.34.5}/tests/unit/test_query_steps_hash.py

@@ -75,6 +75,22 @@ def custom_feature_gen(m_fr):
     )
 
 
+# Class-based UDFs for testing hash calculation
+class DoubleMapper(Mapper):
+    """Class-based Mapper that overrides process()."""
+
+    def process(self, x):
+        return x * 2
+
+
+class TripleGenerator(Generator):
+    """Class-based Generator that overrides process()."""
+
+    def process(self, x):
+        yield x * 3
+        yield x * 3 + 1
+
+
 @pytest.fixture
 def numbers_dataset(test_session):
     """
@@ -394,6 +410,12 @@ def test_subtract_hash(test_session, numbers_dataset, on, _hash):
             {"x": CustomFeature},
             "b4edceaa18ed731085e1c433a6d21deabec8d92dfc338fb1d709ed7951977fc5",
         ),
+        (
+            DoubleMapper(),
+            ["x"],
+            {"double": int},
+            "7994436106fef0486b04078b02ee437be3aa73ade2d139fb8c020e2199515e26",
+        ),
     ],
 )
 def test_udf_mapper_hash(
@@ -428,6 +450,12 @@ def test_udf_mapper_hash(
             {"x": CustomFeature},
             "7ff702d242612cbb83cbd1777aa79d2792fb2a341db5ea406cd9fd3f42543b9c",
         ),
+        (
+            TripleGenerator(),
+            ["x"],
+            {"triple": int},
+            "02b4c6bf98ffa011b7c62f3374f219f21796ece5b001d99e4c2f69edf0a94f4a",
+        ),
     ],
 )
 def test_udf_generator_hash(