datachain 0.37.0__tar.gz → 0.37.2__tar.gz

{datachain-0.37.0 → datachain-0.37.2}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.14.0'
+    rev: 'v0.14.1'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.37.0 → datachain-0.37.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.37.0
+Version: 0.37.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -41,7 +41,7 @@ Requires-Dist: cloudpickle
 Requires-Dist: pydantic
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
-Requires-Dist: Pillow<12,>=10.0.0
+Requires-Dist: Pillow<13,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
 Requires-Dist: huggingface_hub

{datachain-0.37.0 → datachain-0.37.2}/pyproject.toml

@@ -45,7 +45,7 @@ dependencies = [
   "pydantic",
   "jmespath>=1.0",
   "datamodel-code-generator>=0.25",
-  "Pillow>=10.0.0,<12",
+  "Pillow>=10.0.0,<13",
   "msgpack>=1.0.4,<2",
   "psutil",
   "huggingface_hub",

{datachain-0.37.0 → datachain-0.37.2}/src/datachain/delta.py

@@ -1,16 +1,12 @@
-import hashlib
 from collections.abc import Sequence
 from copy import copy
 from functools import wraps
 from typing import TYPE_CHECKING, TypeVar
 
-from attrs import frozen
-
 import datachain
 from datachain.dataset import DatasetDependency, DatasetRecord
 from datachain.error import DatasetNotFoundError
 from datachain.project import Project
-from datachain.query.dataset import Step, step_result
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -18,9 +14,7 @@ if TYPE_CHECKING:
 
     from typing_extensions import ParamSpec
 
-    from datachain.catalog import Catalog
     from datachain.lib.dc import DataChain
-    from datachain.query.dataset import QueryGenerator
 
     P = ParamSpec("P")
 
@@ -49,38 +43,11 @@ def delta_disabled(
     return _inner
 
 
-@frozen
-class _RegenerateSystemColumnsStep(Step):
-    catalog: "Catalog"
-
-    def hash_inputs(self) -> str:
-        return hashlib.sha256(b"regenerate_sys_columns").hexdigest()
-
-    def apply(self, query_generator: "QueryGenerator", temp_tables: list[str]):
-        selectable = query_generator.select()
-        regenerated = self.catalog.warehouse._regenerate_system_columns(
-            selectable,
-            keep_existing_columns=True,
-            regenerate_columns=None,
-        )
-
-        def q(*columns):
-            return regenerated.with_only_columns(*columns)
-
-        return step_result(q, regenerated.selected_columns)
-
-
 def _append_steps(dc: "DataChain", other: "DataChain"):
     """Returns cloned chain with appended steps from other chain.
     Steps are all those modification methods applied like filters, mappers etc.
     """
     dc = dc.clone()
-    dc._query.steps.append(
-        _RegenerateSystemColumnsStep(
-            catalog=dc.session.catalog,
-        )
-    )
-
     dc._query.steps += other._query.steps.copy()
     dc.signals_schema = other.signals_schema
     return dc
@@ -150,7 +117,9 @@ def _get_retry_chain(
         error_records = result_dataset.filter(C(delta_retry) != "")
         error_source_records = source_dc.merge(
             error_records, on=on, right_on=right_on, inner=True
-        ).select(*list(source_dc.signals_schema.values))
+        ).select(
+            *list(source_dc.signals_schema.clone_without_sys_signals().values.keys())
+        )
         retry_chain = error_source_records
 
     # Handle missing records if delta_retry is True
@@ -200,7 +169,9 @@ def _get_source_info(
         indirect=False,
     )
 
-    source_ds_dep = next((d for d in dependencies if d.name == source_ds.name), None)
+    source_ds_dep = next(
+        (d for d in dependencies if d and d.name == source_ds.name), None
+    )
     if not source_ds_dep:
         # Starting dataset was removed, back off to normal dataset creation
         return None, None, None, None, None

{datachain-0.37.0 → datachain-0.37.2}/src/datachain/diff/__init__.py

@@ -103,8 +103,10 @@ def _compare(  # noqa: C901
     left = left.mutate(**{ldiff_col: 1})
     right = right.mutate(**{rdiff_col: 1})
 
-    if not compare:
+    if compare is None:
         modified_cond = True
+    elif len(compare) == 0:
+        modified_cond = False
     else:
         modified_cond = or_(  # type: ignore[assignment]
             *[

{datachain-0.37.0 → datachain-0.37.2}/src/datachain/lib/dc/datachain.py

@@ -1697,14 +1697,13 @@ class DataChain:
         query.feature_schema = None
         ds = self._evolve(query=query)
 
+        # Note: merge drops sys signals from both sides, make sure to not include it
+        # in the resulting schema
         signals_schema = self.signals_schema.clone_without_sys_signals()
         right_signals_schema = right_ds.signals_schema.clone_without_sys_signals()
 
         ds.signals_schema = signals_schema.merge(right_signals_schema, rname)
 
-        if not full:
-            ds.signals_schema = SignalSchema({"sys": Sys}) | ds.signals_schema
-
         return ds
 
     @delta_disabled

{datachain-0.37.0 → datachain-0.37.2}/src/datachain/lib/dc/datasets.py

@@ -200,6 +200,10 @@ def read_dataset(
         signals_schema |= SignalSchema.deserialize(query.feature_schema)
     else:
         signals_schema |= SignalSchema.from_column_types(query.column_types or {})
+
+    if delta:
+        signals_schema = signals_schema.clone_without_sys_signals()
+
     chain = DataChain(query, _settings, signals_schema)
 
     if delta:

{datachain-0.37.0 → datachain-0.37.2}/src/datachain/lib/dc/storage.py

@@ -187,6 +187,12 @@ def read_storage(
             project=listing_project_name,
             session=session,
             settings=settings,
+            delta=delta,
+            delta_on=delta_on,
+            delta_result_on=delta_result_on,
+            delta_compare=delta_compare,
+            delta_retry=delta_retry,
+            delta_unsafe=delta_unsafe,
         )
         dc._query.update = update
         dc.signals_schema = dc.signals_schema.mutate({f"{column}": file_type})
@@ -252,13 +258,4 @@ def read_storage(
 
     assert storage_chain is not None
 
-    if delta:
-        storage_chain = storage_chain._as_delta(
-            on=delta_on,
-            right_on=delta_result_on,
-            compare=delta_compare,
-            delta_retry=delta_retry,
-            delta_unsafe=delta_unsafe,
-        )
-
     return storage_chain

{datachain-0.37.0 → datachain-0.37.2}/src/datachain/query/dataset.py

@@ -1065,7 +1065,7 @@ class SQLJoin(Step):
         q1 = self.get_query(self.query1, temp_tables)
         q2 = self.get_query(self.query2, temp_tables)
 
-        q1_columns = _drop_system_columns(q1.c) if self.full else list(q1.c)
+        q1_columns = _drop_system_columns(q1.c)
         q1_column_names = {c.name for c in q1_columns}
 
         q2_columns = []

{datachain-0.37.0 → datachain-0.37.2}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.37.0
+Version: 0.37.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -41,7 +41,7 @@ Requires-Dist: cloudpickle
 Requires-Dist: pydantic
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
-Requires-Dist: Pillow<12,>=10.0.0
+Requires-Dist: Pillow<13,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
 Requires-Dist: huggingface_hub

{datachain-0.37.0 → datachain-0.37.2}/src/datachain.egg-info/requires.txt

@@ -23,7 +23,7 @@ cloudpickle
 pydantic
 jmespath>=1.0
 datamodel-code-generator>=0.25
-Pillow<12,>=10.0.0
+Pillow<13,>=10.0.0
 msgpack<2,>=1.0.4
 psutil
 huggingface_hub

{datachain-0.37.0 → datachain-0.37.2}/tests/func/test_delta.py

@@ -98,6 +98,70 @@ def test_delta_update_from_dataset(test_session, tmp_dir, tmp_path):
     create_delta_dataset(ds_name)
 
 
+def test_delta_falls_back_when_dependency_missing(test_session):
+    catalog = test_session.catalog
+
+    source_ds = "delta_removed_dep_source"
+    delta_ds = "delta_removed_dep_result"
+    process_log: list[int] = []
+
+    def record_processing(id: int) -> int:
+        process_log.append(id)
+        return id
+
+    # Create first source dataset and initial delta version that depends on it
+    dc.read_values(id=[1, 2], session=test_session).save(source_ds)
+    dc.read_dataset(
+        source_ds,
+        session=test_session,
+        delta=True,
+        delta_on="id",
+    ).map(processed_id=record_processing).save(delta_ds)
+
+    assert _get_dependencies(catalog, delta_ds, "1.0.0") == [(source_ds, "1.0.0")]
+    assert set(
+        dc.read_dataset(delta_ds, version="1.0.0", session=test_session).to_values("id")
+    ) == {1, 2}
+    assert sorted(process_log[:2]) == [1, 2]
+
+    dc.read_values(id=[1, 2, 10, 20, 30], session=test_session).save(source_ds)
+
+    # Drop the previous version so it is clear the dependency targets 1.0.1
+    dc.delete_dataset(source_ds, version="1.0.0", session=test_session)
+
+    with pytest.raises(DatasetNotFoundError):
+        dc.read_dataset(source_ds, session=test_session, version="1.0.0")
+
+    deps_after_removal = catalog.get_dataset_dependencies(
+        delta_ds,
+        "1.0.0",
+        namespace_name=catalog.metastore.default_project.namespace.name,
+        project_name=catalog.metastore.default_project.name,
+        indirect=False,
+    )
+    assert deps_after_removal == [None]
+
+    dc.read_dataset(
+        source_ds,
+        session=test_session,
+        delta=True,
+        delta_on="id",
+    ).map(processed_id=record_processing).save(delta_ds)
+
+    # Delta logic should fall back to rebuilding from scratch with the new dependency
+    assert _get_dependencies(catalog, delta_ds, "1.0.1") == [(source_ds, "1.0.1")]
+    assert set(
+        dc.read_dataset(delta_ds, version="1.0.1", session=test_session).to_values("id")
+    ) == {1, 2, 10, 20, 30}
+    # Previous version remains intact and still reflects the original source dataset
+    assert set(
+        dc.read_dataset(delta_ds, version="1.0.0", session=test_session).to_values("id")
+    ) == {1, 2}
+    # Fallback rebuilds the dataset, so ids 1 and 2 appear twice across both runs.
+    assert sorted(process_log[:2]) == [1, 2]
+    assert sorted(process_log[2:]) == [1, 2, 10, 20, 30]
+
+
 def test_delta_returns_correct_dataset_on_no_changes(test_session):
     catalog = test_session.catalog
 
@@ -250,17 +314,66 @@ def test_delta_replay_regenerates_system_columns(test_session):
 
     build_chain(delta=False).save(result_name)
 
-    build_chain(delta=True).save(
-        result_name,
-        delta=True,
-        delta_on="measurement_id",
-    )
+    build_chain(delta=True).save(result_name)
 
     assert set(
         dc.read_dataset(result_name, session=test_session).to_values("measurement_id")
     ) == {1, 2}
 
 
+def test_storage_delta_replay_regenerates_system_columns(test_session, tmp_dir):
+    data_dir = tmp_dir / f"regen_storage_{uuid.uuid4().hex[:8]}"
+    data_dir.mkdir()
+    storage_uri = data_dir.as_uri()
+    result_name = f"regen_storage_result_{uuid.uuid4().hex[:8]}"
+
+    def write_payload(index: int) -> None:
+        (data_dir / f"item{index}.txt").write_text(f"payload-{index}")
+
+    write_payload(1)
+    write_payload(2)
+
+    def build_chain(delta: bool):
+        read_kwargs = {"session": test_session, "update": True}
+        if delta:
+            read_kwargs |= {
+                "delta": True,
+                "delta_on": ["file.path"],
+                "delta_result_on": ["file.path"],
+            }
+
+        def get_measurement_id(file: File) -> int:
+            match = re.search(r"item(\d+)\.txt$", file.path)
+            assert match
+            return int(match.group(1))
+
+        def get_num(file: File) -> int:
+            return get_measurement_id(file)
+
+        chain = dc.read_storage(storage_uri, **read_kwargs)
+        return (
+            chain.mutate(num=1)
+            .select_except("num")
+            .map(measurement_id=get_measurement_id)
+            .map(err=lambda file: "")
+            .map(num=get_num)
+            .filter(C.err == "")
+            .select_except("err")
+            .map(double=lambda num: num * 2, output=int)
+            .select_except("num")
+        )
+
+    build_chain(delta=False).save(result_name)
+
+    write_payload(3)
+
+    build_chain(delta=True).save(result_name)
+
+    assert set(
+        dc.read_dataset(result_name, session=test_session).to_values("measurement_id")
+    ) == {1, 2, 3}
+
+
 def test_delta_update_from_storage(test_session, tmp_dir, tmp_path):
     ds_name = "delta_ds"
     path = tmp_dir.as_uri()

{datachain-0.37.0 → datachain-0.37.2}/tests/unit/lib/test_checkpoints.py

@@ -35,7 +35,7 @@ def test_checkpoints(
     catalog = test_session.catalog
     metastore = catalog.metastore
 
-    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", reset_checkpoints)
+    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", str(reset_checkpoints))
 
     if with_delta:
         chain = dc.read_dataset(
@@ -75,8 +75,9 @@ def test_checkpoints(
     chain.save("nums3")
     second_job_id = test_session.get_or_create_job().id
 
-    assert len(catalog.get_dataset("nums1").versions) == 2 if reset_checkpoints else 1
-    assert len(catalog.get_dataset("nums2").versions) == 2 if reset_checkpoints else 1
+    expected_versions = 1 if with_delta or not reset_checkpoints else 2
+    assert len(catalog.get_dataset("nums1").versions) == expected_versions
+    assert len(catalog.get_dataset("nums2").versions) == expected_versions
     assert len(catalog.get_dataset("nums3").versions) == 1
 
     assert len(list(catalog.metastore.list_checkpoints(first_job_id))) == 2
@@ -88,7 +89,7 @@ def test_checkpoints_modified_chains(
     test_session, monkeypatch, nums_dataset, reset_checkpoints
 ):
     catalog = test_session.catalog
-    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", reset_checkpoints)
+    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", str(reset_checkpoints))
 
     chain = dc.read_dataset("nums", session=test_session)
 
@@ -120,7 +121,7 @@ def test_checkpoints_multiple_runs(
 ):
     catalog = test_session.catalog
 
-    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", reset_checkpoints)
+    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", str(reset_checkpoints))
 
     chain = dc.read_dataset("nums", session=test_session)
 
@@ -184,7 +185,7 @@ def test_checkpoints_check_valid_chain_is_returned(
     monkeypatch,
     nums_dataset,
 ):
-    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", False)
+    monkeypatch.setenv("DATACHAIN_CHECKPOINTS_RESET", str(False))
     chain = dc.read_dataset("nums", session=test_session)
 
     # -------------- FIRST RUN -------------------
@@ -197,6 +198,7 @@ def test_checkpoints_check_valid_chain_is_returned(
 
     # checking that we return expected DataChain even though we skipped chain creation
     # because of the checkpoints
+    assert ds.dataset is not None
     assert ds.dataset.name == "nums1"
     assert len(ds.dataset.versions) == 1
     assert ds.order_by("num").to_list("num") == [(1,), (2,), (3,)]

{datachain-0.37.0 → datachain-0.37.2}/tests/unit/lib/test_datachain_merge.py

@@ -140,7 +140,7 @@ def test_merge_similar_objects(test_session):
     rname = "qq"
     ch = ch1.merge(ch2, "emp.person.name", rname=rname)
 
-    assert list(ch.signals_schema.values.keys()) == ["sys", "emp", rname + "emp"]
+    assert list(ch.signals_schema.values.keys()) == ["emp", rname + "emp"]
 
     empl = list(ch.to_list())
     assert len(empl) == 4
@@ -175,7 +175,7 @@ def test_merge_similar_objects_in_memory():
     assert ch.session.catalog.metastore.db.db_file == ":memory:"
     assert ch.session.catalog.warehouse.db.db_file == ":memory:"
 
-    assert list(ch.signals_schema.values.keys()) == ["sys", "emp", rname + "emp"]
+    assert list(ch.signals_schema.values.keys()) == ["emp", rname + "emp"]
 
     empl = list(ch.to_list())
     assert len(empl) == 4
@@ -198,7 +198,6 @@ def test_merge_values(test_session):
     ch = ch1.merge(ch2, "id")
 
     assert list(ch.signals_schema.values.keys()) == [
-        "sys",
         "id",
         "descr",
         "right_id",
@@ -339,3 +338,42 @@ def test_merge_on_expression(test_session):
         count += 1
 
     assert count == len(team) * len(team)
+
+
+def test_merge_with_drops_sys_columns(test_session):
+    left = dc.read_values(id=[1, 1], lval=[10, 20], session=test_session)
+    right = dc.read_values(id=[1, 1], rval=["a", "b"], session=test_session)
+
+    merged = left.merge(right, on="id")
+
+    assert "sys" not in merged.signals_schema.values
+
+    cols = merged.settings(sys=True).to_pandas(flatten=True).columns
+    assert all(not str(col).startswith("sys") for col in cols)
+
+    ds_name = "merge_left_dups_sys_check_sys"
+    merged.save(ds_name)
+
+    df_with_sys = (
+        dc.read_dataset(ds_name, session=test_session)
+        .settings(sys=True)
+        .to_pandas(flatten=True)
+    )
+
+    sys_cols = [c for c in df_with_sys.columns if str(c).startswith("sys")]
+    assert sys_cols
+
+    def _col(name: str) -> str:
+        for col in df_with_sys.columns:
+            if str(col) == f"sys.{name}":
+                return str(col)
+        raise AssertionError(f"Missing sys column for {name}")
+
+    sys_id_col = _col("id")
+    sys_rand_col = _col("rand")
+
+    sys_ids = list(df_with_sys[sys_id_col])
+    assert len(sys_ids) == len(set(sys_ids))
+
+    sys_rand = list(df_with_sys[sys_rand_col])
+    assert len(sys_rand) == len(set(sys_rand))

{datachain-0.37.0 → datachain-0.37.2}/tests/unit/lib/test_diff.py

@@ -256,6 +256,24 @@ def test_diff_on_equal_datasets(test_session, on_self):
     assert diff.order_by("id").to_list(*collect_fields) == expected
 
 
+def test_diff_only_on_columns_treated_as_same(test_session):
+    ds1 = dc.read_values(
+        id=[1, 2],
+        session=test_session,
+    )
+    ds2 = dc.read_values(
+        id=[1, 2],
+        session=test_session,
+    )
+
+    diff = ds1.diff(ds2, on=["id"], same=True, status_col="diff")
+
+    assert diff.order_by("id").to_list("diff", "id") == [
+        (CompareStatus.SAME, 1),
+        (CompareStatus.SAME, 2),
+    ]
+
+
 def test_diff_multiple_columns(test_session, str_default):
     ds1 = dc.read_values(
         id=[1, 2, 4],
@@ -382,7 +400,7 @@ def test_diff_missing_on(test_session):
     ds2 = dc.read_values(id=[1, 2, 4], session=test_session)
 
     with pytest.raises(ValueError) as exc_info:
-        ds1.diff(ds2, on=None)
+        ds1.diff(ds2, on=None)  # type: ignore[arg-type]
 
     assert str(exc_info.value) == "'on' must be specified"
 

{datachain-0.37.0 → datachain-0.37.2}/tests/unit/test_datachain_hash.py

@@ -218,4 +218,4 @@ def test_diff(test_session):
             status_col="diff",
         )
         .hash()
-    ) == "4135f2deffa91702259de50b48076dd2f8cdf3be32c167332840209c137977f9"
+    ) == "8ffac19b12cf96e2916968914d357c4a9c1b81038c43ab5cf97ba1127fb86567"