datachain 0.37.0__tar.gz → 0.37.1__tar.gz

{datachain-0.37.0 → datachain-0.37.1}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.14.0'
+    rev: 'v0.14.1'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.37.0 → datachain-0.37.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.37.0
+Version: 0.37.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -41,7 +41,7 @@ Requires-Dist: cloudpickle
 Requires-Dist: pydantic
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
-Requires-Dist: Pillow<12,>=10.0.0
+Requires-Dist: Pillow<13,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
 Requires-Dist: huggingface_hub

{datachain-0.37.0 → datachain-0.37.1}/pyproject.toml

@@ -45,7 +45,7 @@ dependencies = [
   "pydantic",
   "jmespath>=1.0",
   "datamodel-code-generator>=0.25",
-  "Pillow>=10.0.0,<12",
+  "Pillow>=10.0.0,<13",
   "msgpack>=1.0.4,<2",
   "psutil",
   "huggingface_hub",

{datachain-0.37.0 → datachain-0.37.1}/src/datachain/delta.py

@@ -200,7 +200,9 @@ def _get_source_info(
         indirect=False,
     )
 
-    source_ds_dep = next((d for d in dependencies if d.name == source_ds.name), None)
+    source_ds_dep = next(
+        (d for d in dependencies if d and d.name == source_ds.name), None
+    )
     if not source_ds_dep:
         # Starting dataset was removed, back off to normal dataset creation
         return None, None, None, None, None

{datachain-0.37.0 → datachain-0.37.1}/src/datachain/diff/__init__.py

@@ -103,8 +103,10 @@ def _compare(  # noqa: C901
     left = left.mutate(**{ldiff_col: 1})
     right = right.mutate(**{rdiff_col: 1})
 
-    if not compare:
+    if compare is None:
         modified_cond = True
+    elif len(compare) == 0:
+        modified_cond = False
     else:
         modified_cond = or_(  # type: ignore[assignment]
             *[

{datachain-0.37.0 → datachain-0.37.1}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.37.0
+Version: 0.37.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -41,7 +41,7 @@ Requires-Dist: cloudpickle
 Requires-Dist: pydantic
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
-Requires-Dist: Pillow<12,>=10.0.0
+Requires-Dist: Pillow<13,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
 Requires-Dist: huggingface_hub

{datachain-0.37.0 → datachain-0.37.1}/src/datachain.egg-info/requires.txt

@@ -23,7 +23,7 @@ cloudpickle
 pydantic
 jmespath>=1.0
 datamodel-code-generator>=0.25
-Pillow<12,>=10.0.0
+Pillow<13,>=10.0.0
 msgpack<2,>=1.0.4
 psutil
 huggingface_hub

{datachain-0.37.0 → datachain-0.37.1}/tests/func/test_delta.py

@@ -98,6 +98,70 @@ def test_delta_update_from_dataset(test_session, tmp_dir, tmp_path):
     create_delta_dataset(ds_name)
 
 
+def test_delta_falls_back_when_dependency_missing(test_session):
+    catalog = test_session.catalog
+
+    source_ds = "delta_removed_dep_source"
+    delta_ds = "delta_removed_dep_result"
+    process_log: list[int] = []
+
+    def record_processing(id: int) -> int:
+        process_log.append(id)
+        return id
+
+    # Create first source dataset and initial delta version that depends on it
+    dc.read_values(id=[1, 2], session=test_session).save(source_ds)
+    dc.read_dataset(
+        source_ds,
+        session=test_session,
+        delta=True,
+        delta_on="id",
+    ).map(processed_id=record_processing).save(delta_ds)
+
+    assert _get_dependencies(catalog, delta_ds, "1.0.0") == [(source_ds, "1.0.0")]
+    assert set(
+        dc.read_dataset(delta_ds, version="1.0.0", session=test_session).to_values("id")
+    ) == {1, 2}
+    assert sorted(process_log[:2]) == [1, 2]
+
+    dc.read_values(id=[1, 2, 10, 20, 30], session=test_session).save(source_ds)
+
+    # Drop the previous version so it is clear the dependency targets 1.0.1
+    dc.delete_dataset(source_ds, version="1.0.0", session=test_session)
+
+    with pytest.raises(DatasetNotFoundError):
+        dc.read_dataset(source_ds, session=test_session, version="1.0.0")
+
+    deps_after_removal = catalog.get_dataset_dependencies(
+        delta_ds,
+        "1.0.0",
+        namespace_name=catalog.metastore.default_project.namespace.name,
+        project_name=catalog.metastore.default_project.name,
+        indirect=False,
+    )
+    assert deps_after_removal == [None]
+
+    dc.read_dataset(
+        source_ds,
+        session=test_session,
+        delta=True,
+        delta_on="id",
+    ).map(processed_id=record_processing).save(delta_ds)
+
+    # Delta logic should fall back to rebuilding from scratch with the new dependency
+    assert _get_dependencies(catalog, delta_ds, "1.0.1") == [(source_ds, "1.0.1")]
+    assert set(
+        dc.read_dataset(delta_ds, version="1.0.1", session=test_session).to_values("id")
+    ) == {1, 2, 10, 20, 30}
+    # Previous version remains intact and still reflects the original source dataset
+    assert set(
+        dc.read_dataset(delta_ds, version="1.0.0", session=test_session).to_values("id")
+    ) == {1, 2}
+    # Fallback rebuilds the dataset, so ids 1 and 2 appear twice across both runs.
+    assert sorted(process_log[:2]) == [1, 2]
+    assert sorted(process_log[2:]) == [1, 2, 10, 20, 30]
+
+
 def test_delta_returns_correct_dataset_on_no_changes(test_session):
     catalog = test_session.catalog
 

{datachain-0.37.0 → datachain-0.37.1}/tests/unit/lib/test_checkpoints.py

@@ -35,7 +35,7 @@ def test_checkpoints(
     catalog = test_session.catalog
     metastore = catalog.metastore
 
-    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", reset_checkpoints)
+    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", str(reset_checkpoints))
 
     if with_delta:
         chain = dc.read_dataset(
@@ -75,8 +75,9 @@ def test_checkpoints(
     chain.save("nums3")
     second_job_id = test_session.get_or_create_job().id
 
-    assert len(catalog.get_dataset("nums1").versions) == 2 if reset_checkpoints else 1
-    assert len(catalog.get_dataset("nums2").versions) == 2 if reset_checkpoints else 1
+    expected_versions = 1 if with_delta or not reset_checkpoints else 2
+    assert len(catalog.get_dataset("nums1").versions) == expected_versions
+    assert len(catalog.get_dataset("nums2").versions) == expected_versions
     assert len(catalog.get_dataset("nums3").versions) == 1
 
     assert len(list(catalog.metastore.list_checkpoints(first_job_id))) == 2
@@ -88,7 +89,7 @@ def test_checkpoints_modified_chains(
     test_session, monkeypatch, nums_dataset, reset_checkpoints
 ):
     catalog = test_session.catalog
-    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", reset_checkpoints)
+    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", str(reset_checkpoints))
 
     chain = dc.read_dataset("nums", session=test_session)
 
@@ -120,7 +121,7 @@ def test_checkpoints_multiple_runs(
 ):
     catalog = test_session.catalog
 
-    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", reset_checkpoints)
+    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", str(reset_checkpoints))
 
     chain = dc.read_dataset("nums", session=test_session)
 
@@ -184,7 +185,7 @@ def test_checkpoints_check_valid_chain_is_returned(
     monkeypatch,
     nums_dataset,
 ):
-    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", False)
+    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", str(False))
     chain = dc.read_dataset("nums", session=test_session)
 
     # -------------- FIRST RUN -------------------
@@ -197,6 +198,7 @@ def test_checkpoints_check_valid_chain_is_returned(
 
     # checking that we return expected DataChain even though we skipped chain creation
     # because of the checkpoints
+    assert ds.dataset is not None
     assert ds.dataset.name == "nums1"
     assert len(ds.dataset.versions) == 1
     assert ds.order_by("num").to_list("num") == [(1,), (2,), (3,)]

{datachain-0.37.0 → datachain-0.37.1}/tests/unit/lib/test_diff.py

@@ -256,6 +256,24 @@ def test_diff_on_equal_datasets(test_session, on_self):
     assert diff.order_by("id").to_list(*collect_fields) == expected
 
 
+def test_diff_only_on_columns_treated_as_same(test_session):
+    ds1 = dc.read_values(
+        id=[1, 2],
+        session=test_session,
+    )
+    ds2 = dc.read_values(
+        id=[1, 2],
+        session=test_session,
+    )
+
+    diff = ds1.diff(ds2, on=["id"], same=True, status_col="diff")
+
+    assert diff.order_by("id").to_list("diff", "id") == [
+        (CompareStatus.SAME, 1),
+        (CompareStatus.SAME, 2),
+    ]
+
+
 def test_diff_multiple_columns(test_session, str_default):
     ds1 = dc.read_values(
         id=[1, 2, 4],
@@ -382,7 +400,7 @@ def test_diff_missing_on(test_session):
     ds2 = dc.read_values(id=[1, 2, 4], session=test_session)
 
     with pytest.raises(ValueError) as exc_info:
-        ds1.diff(ds2, on=None)
+        ds1.diff(ds2, on=None)  # type: ignore[arg-type]
 
     assert str(exc_info.value) == "'on' must be specified"
 

{datachain-0.37.0 → datachain-0.37.1}/tests/unit/test_datachain_hash.py

@@ -218,4 +218,4 @@ def test_diff(test_session):
             status_col="diff",
         )
         .hash()
-    ) == "4135f2deffa91702259de50b48076dd2f8cdf3be32c167332840209c137977f9"
+    ) == "8ffac19b12cf96e2916968914d357c4a9c1b81038c43ab5cf97ba1127fb86567"