datachain 0.30.5__tar.gz → 0.30.6__tar.gz

{datachain-0.30.5 → datachain-0.30.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.30.5
+Version: 0.30.6
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.30.5 → datachain-0.30.6}/docs/guide/delta.md

@@ -80,3 +80,23 @@ Delta processing can be combined with [retry processing](./retry.md) to create a
 
 1. Processes only new or changed records (delta)
 2. Reprocesses records with errors or that are missing (retry)
+
+## Using Delta with Restricted Methods
+
+By default, delta updates cannot be combined with the following methods:
+
+1. `merge`
+2. `union`
+3. `distinct`
+4. `agg`
+5. `group_by`
+
+These methods are restricted because they may produce **unexpected results** when used with delta processing. Delta runs the chain only on a subset of rows (new and changed records), while methods like `distinct`, `agg`, or `group_by` are designed to operate on the entire dataset.
+
+Similarly, combining delta with methods like `merge` or `union` may result in duplicated rows when merging with a static dataset.
+
+If you still need to use these methods together with delta, you can override this restriction by setting the additional flag:
+
+```python
+delta_unsafe=True
+```

{datachain-0.30.5 → datachain-0.30.6}/examples/get_started/json-csv-reader.py

@@ -1,3 +1,4 @@
+import os
 from typing import Optional
 
 import datachain as dc
@@ -39,7 +40,7 @@ def main():
     uri = "gs://datachain-demo/coco2017/annotations_captions/"
 
     # Print JSON schema in Pydantic format from main COCO annotation
-    chain = dc.read_storage(uri, anon="True").filter(dc.C("file.path").glob("*.json"))
+    chain = dc.read_storage(uri, anon=True).filter(dc.C("file.path").glob("*.json"))
     file = chain.limit(1).to_values("file")[0]
     print(gen_datamodel_code(file, jmespath="@", model_name="Coco"))
 
@@ -65,11 +66,12 @@ def main():
     dynamic_csv_ds.print_schema()
     dynamic_csv_ds.show()
 
-    print(
-        "Note: script might hang at the end due to https://github.com/apache/arrow/issues/43497"
-    )
-    print("Just press Ctrl+C to exit.")
-
 
 if __name__ == "__main__":
     main()
+
+    # Force exit without cleanup to avoid hanging due to arrow issue
+    print(
+        "Note: script might warn about leaked semaphore at the end due to https://github.com/apache/arrow/issues/43497"
+    )
+    os._exit(0)

{datachain-0.30.5 → datachain-0.30.6}/src/datachain/cli/commands/datasets.py

@@ -1,30 +1,41 @@
 import sys
-from typing import TYPE_CHECKING, Optional
+from collections.abc import Iterable, Iterator
+from typing import TYPE_CHECKING, Optional, Union
 
 from tabulate import tabulate
 
-if TYPE_CHECKING:
-    from datachain.catalog import Catalog
-
+from datachain import semver
 from datachain.catalog import is_namespace_local
 from datachain.cli.utils import determine_flavors
 from datachain.config import Config
 from datachain.error import DataChainError, DatasetNotFoundError
 from datachain.studio import list_datasets as list_datasets_studio
 
+if TYPE_CHECKING:
+    from datachain.catalog import Catalog
+
+
+def group_dataset_versions(
+    datasets: Iterable[tuple[str, str]], latest_only=True
+) -> dict[str, Union[str, list[str]]]:
+    grouped: dict[str, list[tuple[int, int, int]]] = {}
 
-def group_dataset_versions(datasets, latest_only=True):
-    grouped = {}
     # Sort to ensure groupby works as expected
     # (groupby expects consecutive items with the same key)
     for name, version in sorted(datasets):
-        grouped.setdefault(name, []).append(version)
+        grouped.setdefault(name, []).append(semver.parse(version))
 
     if latest_only:
         # For each dataset name, pick the highest version.
-        return {name: max(versions) for name, versions in grouped.items()}
+        return {
+            name: semver.create(*(max(versions))) for name, versions in grouped.items()
+        }
+
     # For each dataset name, return a sorted list of unique versions.
-    return {name: sorted(set(versions)) for name, versions in grouped.items()}
+    return {
+        name: [semver.create(*v) for v in sorted(set(versions))]
+        for name, versions in grouped.items()
+    }
 
 
 def list_datasets(
@@ -35,7 +46,7 @@ def list_datasets(
     team: Optional[str] = None,
     latest_only: bool = True,
     name: Optional[str] = None,
-):
+) -> None:
     token = Config().read().get("studio", {}).get("token")
     all, local, studio = determine_flavors(studio, local, all, token)
     if name:
@@ -95,27 +106,31 @@ def list_datasets(
     print(tabulate(rows, headers="keys"))
 
 
-def list_datasets_local(catalog: "Catalog", name: Optional[str] = None):
+def list_datasets_local(
+    catalog: "Catalog", name: Optional[str] = None
+) -> Iterator[tuple[str, str]]:
     if name:
         yield from list_datasets_local_versions(catalog, name)
         return
 
     for d in catalog.ls_datasets():
         for v in d.versions:
-            yield (d.full_name, v.version)
+            yield d.full_name, v.version
 
 
-def list_datasets_local_versions(catalog: "Catalog", name: str):
+def list_datasets_local_versions(
+    catalog: "Catalog", name: str
+) -> Iterator[tuple[str, str]]:
     namespace_name, project_name, name = catalog.get_full_dataset_name(name)
 
     ds = catalog.get_dataset(
         name, namespace_name=namespace_name, project_name=project_name
     )
     for v in ds.versions:
-        yield (name, v.version)
+        yield name, v.version
 
 
-def _datasets_tabulate_row(name, both, local_version, studio_version):
+def _datasets_tabulate_row(name, both, local_version, studio_version) -> dict[str, str]:
     row = {
         "Name": name,
     }
@@ -136,7 +151,7 @@ def rm_dataset(
     force: Optional[bool] = False,
     studio: Optional[bool] = False,
     team: Optional[str] = None,
-):
+) -> None:
     namespace_name, project_name, name = catalog.get_full_dataset_name(name)
 
     if studio:
@@ -166,7 +181,7 @@ def edit_dataset(
     description: Optional[str] = None,
     attrs: Optional[list[str]] = None,
     team: Optional[str] = None,
-):
+) -> None:
     from datachain.lib.dc.utils import is_studio
 
     namespace_name, project_name, name = catalog.get_full_dataset_name(name)

{datachain-0.30.5 → datachain-0.30.6}/src/datachain/delta.py

@@ -4,7 +4,7 @@ from functools import wraps
 from typing import TYPE_CHECKING, Callable, Optional, TypeVar, Union
 
 import datachain
-from datachain.dataset import DatasetDependency
+from datachain.dataset import DatasetDependency, DatasetRecord
 from datachain.error import DatasetNotFoundError
 from datachain.project import Project
 
@@ -30,9 +30,10 @@ def delta_disabled(
 
     @wraps(method)
     def _inner(self: T, *args: "P.args", **kwargs: "P.kwargs") -> T:
-        if self.delta:
+        if self.delta and not self._delta_unsafe:
             raise NotImplementedError(
-                f"Delta update cannot be used with {method.__name__}"
+                f"Cannot use {method.__name__} with delta datasets - may cause"
+                " inconsistency. Use delta_unsafe flag to allow this operation."
             )
         return method(self, *args, **kwargs)
 
@@ -124,10 +125,19 @@ def _get_retry_chain(
     # Subtract also diff chain since some items might be picked
     # up by `delta=True` itself (e.g. records got modified AND are missing in the
     # result dataset atm)
-    return retry_chain.subtract(diff_chain, on=on) if retry_chain else None
+    on = [on] if isinstance(on, str) else on
+
+    return (
+        retry_chain.diff(
+            diff_chain, on=on, added=True, same=True, modified=False, deleted=False
+        ).distinct(*on)
+        if retry_chain
+        else None
+    )
 
 
 def _get_source_info(
+    source_ds: DatasetRecord,
     name: str,
     namespace_name: str,
     project_name: str,
@@ -154,25 +164,23 @@ def _get_source_info(
         indirect=False,
     )
 
-    dep = dependencies[0]
-    if not dep:
+    source_ds_dep = next((d for d in dependencies if d.name == source_ds.name), None)
+    if not source_ds_dep:
         # Starting dataset was removed, back off to normal dataset creation
         return None, None, None, None, None
 
-    source_ds_project = catalog.metastore.get_project(dep.project, dep.namespace)
-    source_ds_name = dep.name
-    source_ds_version = dep.version
-    source_ds_latest_version = catalog.get_dataset(
-        source_ds_name,
-        namespace_name=source_ds_project.namespace.name,
-        project_name=source_ds_project.name,
-    ).latest_version
+    # Refresh starting dataset to have new versions if they are created
+    source_ds = catalog.get_dataset(
+        source_ds.name,
+        namespace_name=source_ds.project.namespace.name,
+        project_name=source_ds.project.name,
+    )
 
     return (
-        source_ds_name,
-        source_ds_project,
-        source_ds_version,
-        source_ds_latest_version,
+        source_ds.name,
+        source_ds.project,
+        source_ds_dep.version,
+        source_ds.latest_version,
         dependencies,
     )
 
@@ -244,7 +252,14 @@ def delta_retry_update(
         source_ds_version,
         source_ds_latest_version,
         dependencies,
-    ) = _get_source_info(name, namespace_name, project_name, latest_version, catalog)
+    ) = _get_source_info(
+        dc._query.starting_step.dataset,  # type: ignore[union-attr]
+        name,
+        namespace_name,
+        project_name,
+        latest_version,
+        catalog,
+    )
 
     # If source_ds_name is None, starting dataset was removed
     if source_ds_name is None:
@@ -267,8 +282,9 @@ def delta_retry_update(
     if dependencies:
         dependencies = copy(dependencies)
         dependencies = [d for d in dependencies if d is not None]
+        source_ds_dep = next(d for d in dependencies if d.name == source_ds_name)
         # Update to latest version
-        dependencies[0].version = source_ds_latest_version  # type: ignore[union-attr]
+        source_ds_dep.version = source_ds_latest_version  # type: ignore[union-attr]
 
     # Handle retry functionality if enabled
     if delta_retry:

{datachain-0.30.5 → datachain-0.30.6}/src/datachain/lib/dc/datachain.py

@@ -193,6 +193,7 @@ class DataChain:
         self._setup: dict = setup or {}
         self._sys = _sys
         self._delta = False
+        self._delta_unsafe = False
         self._delta_on: Optional[Union[str, Sequence[str]]] = None
         self._delta_result_on: Optional[Union[str, Sequence[str]]] = None
         self._delta_compare: Optional[Union[str, Sequence[str]]] = None
@@ -216,6 +217,7 @@ class DataChain:
         right_on: Optional[Union[str, Sequence[str]]] = None,
         compare: Optional[Union[str, Sequence[str]]] = None,
         delta_retry: Optional[Union[bool, str]] = None,
+        delta_unsafe: bool = False,
     ) -> "Self":
         """Marks this chain as delta, which means special delta process will be
         called on saving dataset for optimization"""
@@ -226,6 +228,7 @@ class DataChain:
         self._delta_result_on = right_on
         self._delta_compare = compare
         self._delta_retry = delta_retry
+        self._delta_unsafe = delta_unsafe
         return self
 
     @property
@@ -238,6 +241,10 @@ class DataChain:
         """Returns True if this chain is ran in "delta" update mode"""
         return self._delta
 
+    @property
+    def delta_unsafe(self) -> bool:
+        return self._delta_unsafe
+
     @property
     def schema(self) -> dict[str, DataType]:
         """Get schema of the chain."""
@@ -328,6 +335,7 @@ class DataChain:
                 right_on=self._delta_result_on,
                 compare=self._delta_compare,
                 delta_retry=self._delta_retry,
+                delta_unsafe=self._delta_unsafe,
             )
 
         return chain

{datachain-0.30.5 → datachain-0.30.6}/src/datachain/lib/dc/datasets.py

@@ -40,6 +40,7 @@ def read_dataset(
     delta_result_on: Optional[Union[str, Sequence[str]]] = None,
     delta_compare: Optional[Union[str, Sequence[str]]] = None,
     delta_retry: Optional[Union[bool, str]] = None,
+    delta_unsafe: bool = False,
     update: bool = False,
 ) -> "DataChain":
     """Get data from a saved Dataset. It returns the chain itself.
@@ -80,6 +81,8 @@ def read_dataset(
         update: If True always checks for newer versions available on Studio, even if
             some version of the dataset exists locally already. If False (default), it
             will only fetch the dataset from Studio if it is not found locally.
+        delta_unsafe: Allow restricted ops in delta: merge, agg, union, group_by,
+            distinct.
 
 
     Example:
@@ -205,6 +208,7 @@ def read_dataset(
             right_on=delta_result_on,
             compare=delta_compare,
             delta_retry=delta_retry,
+            delta_unsafe=delta_unsafe,
         )
 
     return chain

{datachain-0.30.5 → datachain-0.30.6}/src/datachain/lib/dc/storage.py

@@ -43,6 +43,7 @@ def read_storage(
     delta_result_on: Optional[Union[str, Sequence[str]]] = None,
     delta_compare: Optional[Union[str, Sequence[str]]] = None,
     delta_retry: Optional[Union[bool, str]] = None,
+    delta_unsafe: bool = False,
     client_config: Optional[dict] = None,
 ) -> "DataChain":
     """Get data from storage(s) as a list of file with all file attributes.
@@ -77,6 +78,9 @@ def read_storage(
               (error mode)
             - True: Reprocess records missing from the result dataset (missing mode)
             - None: No retry processing (default)
+        delta_unsafe: Allow restricted ops in delta: merge, agg, union, group_by,
+            distinct. Caller must ensure datasets are consistent and not partially
+            updated.
 
     Returns:
         DataChain: A DataChain object containing the file information.
@@ -218,6 +222,7 @@ def read_storage(
             right_on=delta_result_on,
             compare=delta_compare,
             delta_retry=delta_retry,
+            delta_unsafe=delta_unsafe,
         )
 
     return storage_chain

{datachain-0.30.5 → datachain-0.30.6}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.30.5
+Version: 0.30.6
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.30.5 → datachain-0.30.6}/src/datachain.egg-info/SOURCES.txt

@@ -348,6 +348,7 @@ tests/unit/test_asyn.py
 tests/unit/test_cache.py
 tests/unit/test_catalog.py
 tests/unit/test_catalog_loader.py
+tests/unit/test_cli_datasets.py
 tests/unit/test_cli_parsing.py
 tests/unit/test_client.py
 tests/unit/test_client_gcs.py

{datachain-0.30.5 → datachain-0.30.6}/tests/conftest.py

@@ -547,11 +547,9 @@ def is_studio():
 
 @pytest.fixture(autouse=True)
 def mock_is_studio(monkeypatch, is_studio):
-    if not is_studio:
-        yield
-    else:
-        monkeypatch.setenv("DATACHAIN_IS_STUDIO", True)
-        yield
+    if is_studio:
+        monkeypatch.setenv("DATACHAIN_IS_STUDIO", "True")
+    yield
 
 
 @pytest.fixture

{datachain-0.30.5 → datachain-0.30.6}/tests/func/test_delta.py

@@ -14,26 +14,16 @@ from datachain.lib.file import File, ImageFile
 def _get_dependencies(catalog, name, version) -> list[tuple[str, str]]:
     return sorted(
         [
-            (f"{d.namespace}.{d.project}.{d.name}", d.version)
+            (d.name, d.version)
             for d in catalog.get_dataset_dependencies(name, version, indirect=False)
         ]
     )
 
 
-@pytest.mark.parametrize("project", ("global.dev", ""))
-def test_delta_update_from_dataset(test_session, tmp_dir, tmp_path, project):
+def test_delta_update_from_dataset(test_session, tmp_dir, tmp_path):
     catalog = test_session.catalog
-    default_namespace_name = catalog.metastore.default_namespace_name
-    default_project_name = catalog.metastore.default_project_name
-
-    if project:
-        starting_ds_name = f"{project}.starting_ds"
-        dependency_ds_name = starting_ds_name
-    else:
-        starting_ds_name = "starting_ds"
-        dependency_ds_name = (
-            f"{default_namespace_name}.{default_project_name}.{starting_ds_name}"
-        )
+
+    starting_ds_name = "starting_ds"
     ds_name = "delta_ds"
 
     images = [
@@ -66,16 +56,12 @@ def test_delta_update_from_dataset(test_session, tmp_dir, tmp_path, project):
     create_image_dataset(starting_ds_name, images[:2])
     # first version of delta dataset
     create_delta_dataset(ds_name)
-    assert _get_dependencies(catalog, ds_name, "1.0.0") == [
-        (dependency_ds_name, "1.0.0")
-    ]
+    assert _get_dependencies(catalog, ds_name, "1.0.0") == [(starting_ds_name, "1.0.0")]
     # second version of starting dataset
     create_image_dataset(starting_ds_name, images[2:])
     # second version of delta dataset
     create_delta_dataset(ds_name)
-    assert _get_dependencies(catalog, ds_name, "1.0.1") == [
-        (dependency_ds_name, "1.0.1")
-    ]
+    assert _get_dependencies(catalog, ds_name, "1.0.1") == [(starting_ds_name, "1.0.1")]
 
     assert (dc.read_dataset(ds_name, version="1.0.0").order_by("file.path")).to_values(
         "file.path"
@@ -96,6 +82,66 @@ def test_delta_update_from_dataset(test_session, tmp_dir, tmp_path, project):
     create_delta_dataset(ds_name)
 
 
+def test_delta_update_unsafe(test_session):
+    catalog = test_session.catalog
+
+    starting_ds_name = "starting_ds"
+    merge_ds_name = "merge_ds"
+    ds_name = "delta_ds"
+
+    # create dataset which will be merged to delta one
+    merge_ds = dc.read_values(
+        id=[1, 2, 3, 4, 5, 6], value=[1, 2, 3, 4, 5, 6], session=test_session
+    ).save(merge_ds_name)
+
+    # first version of starting dataset
+    dc.read_values(id=[1, 2, 3], session=test_session).save(starting_ds_name)
+    # first version of delta dataset
+    dc.read_dataset(
+        starting_ds_name,
+        session=test_session,
+        delta_on="id",
+        delta=True,
+        delta_unsafe=True,
+    ).merge(merge_ds, on="id", inner=True).save(ds_name)
+
+    assert set(_get_dependencies(catalog, ds_name, "1.0.0")) == {
+        (starting_ds_name, "1.0.0"),
+        (merge_ds_name, "1.0.0"),
+    }
+
+    # second version of starting dataset
+    dc.read_values(id=[1, 2, 3, 4, 5, 6], session=test_session).save(starting_ds_name)
+    # second version of delta dataset
+    dc.read_dataset(
+        starting_ds_name,
+        session=test_session,
+        delta_on="id",
+        delta=True,
+        delta_unsafe=True,
+    ).merge(merge_ds, on="id", inner=True).save(ds_name)
+
+    assert set(_get_dependencies(catalog, ds_name, "1.0.1")) == {
+        (starting_ds_name, "1.0.1"),
+        (merge_ds_name, "1.0.0"),
+    }
+
+    assert set((dc.read_dataset(ds_name, version="1.0.0")).to_list("id", "value")) == {
+        (1, 1),
+        (2, 2),
+        (3, 3),
+    }
+
+    assert set((dc.read_dataset(ds_name, version="1.0.1")).to_list("id", "value")) == {
+        (1, 1),
+        (2, 2),
+        (3, 3),
+        (4, 4),
+        (5, 5),
+        (6, 6),
+    }
+
+
 def test_delta_update_from_storage(test_session, tmp_dir, tmp_path):
     ds_name = "delta_ds"
     path = tmp_dir.as_uri()
@@ -249,8 +295,6 @@ def test_delta_update_check_num_calls(test_session, tmp_dir, tmp_path, capsys):
 
 def test_delta_update_no_diff(test_session, tmp_dir, tmp_path):
     catalog = test_session.catalog
-    default_namespace_name = catalog.metastore.default_namespace_name
-    default_project_name = catalog.metastore.default_project_name
     ds_name = "delta_ds"
     path = tmp_dir.as_uri()
     tmp_dir = tmp_dir / "images"
@@ -301,7 +345,8 @@ def test_delta_update_no_diff(test_session, tmp_dir, tmp_path):
 
     assert str(exc_info.value) == (
         f"Dataset {ds_name} version 1.0.1 not found in namespace "
-        f"{default_namespace_name} and project {default_project_name}"
+        f"{catalog.metastore.default_namespace_name}"
+        f" and project {catalog.metastore.default_project_name}"
     )
 
 
@@ -325,11 +370,13 @@ def test_delta_update_union(test_session, file_dataset):
                 file_dataset.name,
                 session=test_session,
                 delta=True,
-                delta_on=["file.source", "file.path"],
             ).union(dc.read_dataset("numbers"), session=test_session)
         )
 
-    assert str(excinfo.value) == "Delta update cannot be used with union"
+    assert str(excinfo.value) == (
+        "Cannot use union with delta datasets - may cause inconsistency."
+        " Use delta_unsafe flag to allow this operation."
+    )
 
 
 def test_delta_update_merge(test_session, file_dataset):
@@ -341,11 +388,13 @@ def test_delta_update_merge(test_session, file_dataset):
                 file_dataset.name,
                 session=test_session,
                 delta=True,
-                delta_on=["file.source", "file.path"],
             ).merge(dc.read_dataset("numbers"), on="id", session=test_session)
         )
 
-    assert str(excinfo.value) == "Delta update cannot be used with merge"
+    assert str(excinfo.value) == (
+        "Cannot use merge with delta datasets - may cause inconsistency."
+        " Use delta_unsafe flag to allow this operation."
+    )
 
 
 def test_delta_update_distinct(test_session, file_dataset):
@@ -355,11 +404,13 @@ def test_delta_update_distinct(test_session, file_dataset):
                 file_dataset.name,
                 session=test_session,
                 delta=True,
-                delta_on=["file.source", "file.path"],
             ).distinct("file.path")
         )
 
-    assert str(excinfo.value) == "Delta update cannot be used with distinct"
+    assert str(excinfo.value) == (
+        "Cannot use distinct with delta datasets - may cause inconsistency."
+        " Use delta_unsafe flag to allow this operation."
+    )
 
 
 def test_delta_update_group_by(test_session, file_dataset):
@@ -369,11 +420,13 @@ def test_delta_update_group_by(test_session, file_dataset):
                 file_dataset.name,
                 session=test_session,
                 delta=True,
-                delta_on=["file.source", "file.path"],
             ).group_by(cnt=func.count(), partition_by="file.path")
         )
 
-    assert str(excinfo.value) == "Delta update cannot be used with group_by"
+    assert str(excinfo.value) == (
+        "Cannot use group_by with delta datasets - may cause inconsistency."
+        " Use delta_unsafe flag to allow this operation."
+    )
 
 
 def test_delta_update_agg(test_session, file_dataset):
@@ -383,8 +436,10 @@ def test_delta_update_agg(test_session, file_dataset):
                 file_dataset.name,
                 session=test_session,
                 delta=True,
-                delta_on=["file.source", "file.path"],
             ).agg(cnt=func.count(), partition_by="file.path")
         )
 
-    assert str(excinfo.value) == "Delta update cannot be used with agg"
+    assert str(excinfo.value) == (
+        "Cannot use agg with delta datasets - may cause inconsistency."
+        " Use delta_unsafe flag to allow this operation."
+    )

{datachain-0.30.5 → datachain-0.30.6}/tests/func/test_retry.py

@@ -1,3 +1,4 @@
+from collections.abc import Iterator
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING
 
@@ -425,3 +426,42 @@ def test_delta_and_delta_retry_no_duplicates(test_session):
     assert len(ids_in_result) == 4
     assert len(set(ids_in_result)) == 4  # No duplicate IDs
     assert set(ids_in_result) == {1, 2, 3, 4}
+
+
+def test_repeating_errors(test_session):
+    def run_delta():
+        def func(id) -> Iterator[tuple[int, str, str]]:
+            yield id, "name1", "error"
+            yield id, "name2", "error"
+
+        return (
+            dc.read_dataset(
+                "sample_data",
+                delta=True,
+                delta_on="id",
+                delta_result_on="id",
+                delta_retry="error",
+                session=test_session,
+            )
+            .gen(func, output={"id": int, "name": str, "error": str})
+            .save("processed_data")
+        )
+        return dc.read_dataset("processed_data")
+
+    _create_sample_data(
+        test_session, ids=list(range(1)), contents=[str(i) for i in range(1)]
+    )
+    ch1 = run_delta()
+    assert sorted(ch1.collect("id")) == [0, 0]
+
+    _create_sample_data(
+        test_session, ids=list(range(2)), contents=[str(i) for i in range(2)]
+    )
+    ch2 = run_delta()
+    assert sorted(ch2.collect("id")) == [0, 0, 1, 1]
+
+    _create_sample_data(
+        test_session, ids=list(range(3)), contents=[str(i) for i in range(3)]
+    )
+    ch3 = run_delta()
+    assert sorted(ch3.collect("id")) == [0, 0, 1, 1, 2, 2]