datachain 0.24.0__tar.gz → 0.24.1__tar.gz

{datachain-0.24.0 → datachain-0.24.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.24.0
+Version: 0.24.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.24.0 → datachain-0.24.1}/src/datachain/delta.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Callable, Optional, TypeVar, Union
 import datachain
 from datachain.dataset import DatasetDependency
 from datachain.error import DatasetNotFoundError
+from datachain.project import Project
 
 if TYPE_CHECKING:
     from typing_extensions import Concatenate, ParamSpec
@@ -50,15 +51,24 @@ def _append_steps(dc: "DataChain", other: "DataChain"):
 
 def _get_delta_chain(
     source_ds_name: str,
+    source_ds_project: Project,
     source_ds_version: str,
     source_ds_latest_version: str,
     on: Union[str, Sequence[str]],
     compare: Optional[Union[str, Sequence[str]]] = None,
 ) -> "DataChain":
     """Get delta chain for processing changes between versions."""
-    source_dc = datachain.read_dataset(source_ds_name, version=source_ds_version)
+    source_dc = datachain.read_dataset(
+        source_ds_name,
+        namespace=source_ds_project.namespace.name,
+        project=source_ds_project.name,
+        version=source_ds_version,
+    )
     source_dc_latest = datachain.read_dataset(
-        source_ds_name, version=source_ds_latest_version
+        source_ds_name,
+        namespace=source_ds_project.namespace.name,
+        project=source_ds_project.name,
+        version=source_ds_latest_version,
     )
 
     # Calculate diff between source versions
@@ -67,12 +77,15 @@ def _get_delta_chain(
 
 def _get_retry_chain(
     name: str,
+    project: Project,
     latest_version: str,
     source_ds_name: str,
-    source_ds_latest_version: str,
+    source_ds_project: Project,
+    source_ds_version: str,
     on: Union[str, Sequence[str]],
     right_on: Optional[Union[str, Sequence[str]]],
     delta_retry: Optional[Union[bool, str]],
+    diff_chain: "DataChain",
 ) -> Optional["DataChain"]:
     """Get retry chain for processing error records and missing records."""
     # Import here to avoid circular import
@@ -81,35 +94,49 @@ def _get_retry_chain(
     retry_chain = None
 
     # Read the latest version of the result dataset for retry logic
-    result_dataset = datachain.read_dataset(name, version=latest_version)
-    source_dc_latest = datachain.read_dataset(
-        source_ds_name, version=source_ds_latest_version
+    result_dataset = datachain.read_dataset(
+        name,
+        namespace=project.namespace.name,
+        project=project.name,
+        version=latest_version,
+    )
+    source_dc = datachain.read_dataset(
+        source_ds_name,
+        namespace=source_ds_project.namespace.name,
+        project=source_ds_project.name,
+        version=source_ds_version,
     )
 
     # Handle error records if delta_retry is a string (column name)
     if isinstance(delta_retry, str):
         error_records = result_dataset.filter(C(delta_retry) != "")
-        error_source_records = source_dc_latest.merge(
+        error_source_records = source_dc.merge(
             error_records, on=on, right_on=right_on, inner=True
-        ).select(*list(source_dc_latest.signals_schema.values))
+        ).select(*list(source_dc.signals_schema.values))
         retry_chain = error_source_records
 
     # Handle missing records if delta_retry is True
     elif delta_retry is True:
-        missing_records = source_dc_latest.subtract(
-            result_dataset, on=on, right_on=right_on
-        )
+        missing_records = source_dc.subtract(result_dataset, on=on, right_on=right_on)
         retry_chain = missing_records
 
-    return retry_chain
+    # Subtract also diff chain since some items might be picked
+    # up by `delta=True` itself (e.g. records got modified AND are missing in the
+    # result dataset atm)
+    return retry_chain.subtract(diff_chain, on=on) if retry_chain else None
 
 
 def _get_source_info(
     name: str,
+    project: Project,
     latest_version: str,
     catalog,
 ) -> tuple[
-    Optional[str], Optional[str], Optional[str], Optional[list[DatasetDependency]]
+    Optional[str],
+    Optional[Project],
+    Optional[str],
+    Optional[str],
+    Optional[list[DatasetDependency]],
 ]:
     """Get source dataset information and dependencies.
 
@@ -118,23 +145,34 @@ def _get_source_info(
         Returns (None, None, None, None) if source dataset was removed.
     """
     dependencies = catalog.get_dataset_dependencies(
-        name, latest_version, indirect=False
+        name, latest_version, project=project, indirect=False
     )
 
     dep = dependencies[0]
     if not dep:
         # Starting dataset was removed, back off to normal dataset creation
-        return None, None, None, None
+        return None, None, None, None, None
 
+    source_ds_project = catalog.metastore.get_project(dep.project, dep.namespace)
     source_ds_name = dep.name
     source_ds_version = dep.version
-    source_ds_latest_version = catalog.get_dataset(source_ds_name).latest_version
-
-    return source_ds_name, source_ds_version, source_ds_latest_version, dependencies
+    source_ds_latest_version = catalog.get_dataset(
+        source_ds_name, project=source_ds_project
+    ).latest_version
+
+    return (
+        source_ds_name,
+        source_ds_project,
+        source_ds_version,
+        source_ds_latest_version,
+        dependencies,
+    )
 
 
 def delta_retry_update(
     dc: "DataChain",
+    namespace_name: str,
+    project_name: str,
     name: str,
     on: Union[str, Sequence[str]],
     right_on: Optional[Union[str, Sequence[str]]] = None,
@@ -173,11 +211,12 @@ def delta_retry_update(
     """
 
     catalog = dc.session.catalog
+    project = catalog.metastore.get_project(project_name, namespace_name)
     dc._query.apply_listing_pre_step()
 
     # Check if dataset exists
     try:
-        dataset = catalog.get_dataset(name)
+        dataset = catalog.get_dataset(name, project=project)
         latest_version = dataset.latest_version
     except DatasetNotFoundError:
         # First creation of result dataset
@@ -189,19 +228,29 @@ def delta_retry_update(
     retry_chain = None
     processing_chain = None
 
-    source_ds_name, source_ds_version, source_ds_latest_version, dependencies = (
-        _get_source_info(name, latest_version, catalog)
-    )
+    (
+        source_ds_name,
+        source_ds_project,
+        source_ds_version,
+        source_ds_latest_version,
+        dependencies,
+    ) = _get_source_info(name, project, latest_version, catalog)
 
     # If source_ds_name is None, starting dataset was removed
     if source_ds_name is None:
         return None, None, True
 
+    assert source_ds_project
     assert source_ds_version
     assert source_ds_latest_version
 
     diff_chain = _get_delta_chain(
-        source_ds_name, source_ds_version, source_ds_latest_version, on, compare
+        source_ds_name,
+        source_ds_project,
+        source_ds_version,
+        source_ds_latest_version,
+        on,
+        compare,
     )
 
     # Filter out removed dep
@@ -215,12 +264,15 @@ def delta_retry_update(
     if delta_retry:
         retry_chain = _get_retry_chain(
             name,
+            project,
             latest_version,
             source_ds_name,
-            source_ds_latest_version,
+            source_ds_project,
+            source_ds_version,
             on,
             right_on,
             delta_retry,
+            diff_chain,
         )
 
     # Combine delta and retry chains
@@ -236,7 +288,12 @@ def delta_retry_update(
     if processing_chain is None or (processing_chain and processing_chain.empty):
         return None, None, False
 
-    latest_dataset = datachain.read_dataset(name, version=latest_version)
+    latest_dataset = datachain.read_dataset(
+        name,
+        namespace=project.namespace.name,
+        project=project.name,
+        version=latest_version,
+    )
     compared_chain = latest_dataset.diff(
         processing_chain,
         on=right_on or on,

{datachain-0.24.0 → datachain-0.24.1}/src/datachain/lib/dc/datachain.py

@@ -598,6 +598,8 @@ class DataChain:
 
             result_ds, dependencies, has_changes = delta_retry_update(
                 self,
+                namespace_name,
+                project_name,
                 name,
                 on=self._delta_on,
                 right_on=self._delta_result_on,

{datachain-0.24.0 → datachain-0.24.1}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.24.0
+Version: 0.24.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.24.0 → datachain-0.24.1}/tests/func/test_delta.py

@@ -14,15 +14,26 @@ from datachain.lib.file import File, ImageFile
 def _get_dependencies(catalog, name, version) -> list[tuple[str, str]]:
     return sorted(
         [
-            (d.name, d.version)
+            (f"{d.namespace}.{d.project}.{d.name}", d.version)
             for d in catalog.get_dataset_dependencies(name, version, indirect=False)
         ]
     )
 
 
-def test_delta_update_from_dataset(test_session, tmp_dir, tmp_path):
+@pytest.mark.parametrize("project", ("global.dev", ""))
+def test_delta_update_from_dataset(test_session, tmp_dir, tmp_path, project):
     catalog = test_session.catalog
-    starting_ds_name = "starting_ds"
+    default_namespace_name = catalog.metastore.default_namespace_name
+    default_project_name = catalog.metastore.default_project_name
+
+    if project:
+        starting_ds_name = f"{project}.starting_ds"
+        dependency_ds_name = starting_ds_name
+    else:
+        starting_ds_name = "starting_ds"
+        dependency_ds_name = (
+            f"{default_namespace_name}.{default_project_name}.{starting_ds_name}"
+        )
     ds_name = "delta_ds"
 
     images = [
@@ -55,12 +66,16 @@ def test_delta_update_from_dataset(test_session, tmp_dir, tmp_path):
     create_image_dataset(starting_ds_name, images[:2])
     # first version of delta dataset
     create_delta_dataset(ds_name)
-    assert _get_dependencies(catalog, ds_name, "1.0.0") == [(starting_ds_name, "1.0.0")]
+    assert _get_dependencies(catalog, ds_name, "1.0.0") == [
+        (dependency_ds_name, "1.0.0")
+    ]
     # second version of starting dataset
     create_image_dataset(starting_ds_name, images[2:])
     # second version of delta dataset
     create_delta_dataset(ds_name)
-    assert _get_dependencies(catalog, ds_name, "1.0.1") == [(starting_ds_name, "1.0.1")]
+    assert _get_dependencies(catalog, ds_name, "1.0.1") == [
+        (dependency_ds_name, "1.0.1")
+    ]
 
     assert (dc.read_dataset(ds_name, version="1.0.0").order_by("file.path")).to_values(
         "file.path"

{datachain-0.24.0 → datachain-0.24.1}/tests/func/test_retry.py

@@ -30,6 +30,23 @@ def _process_with_errors(id: int, content: str, attempt: int) -> ProcessingResul
     )
 
 
+def _create_sample_data(test_session, ids=None, contents=None):
+    """Helper function to create sample data for retry tests."""
+    ids = ids or [1, 2, 3, 4]
+    contents = contents or ["first item", "second item", "third item", "fourth item"]
+    dc.read_values(id=ids, content=contents, session=test_session).save("sample_data")
+
+
+def _simple_process(id: int, content: str, attempt: int = 1) -> ProcessingResult:
+    """Helper function for simple processing in retry tests."""
+    return ProcessingResult(
+        processed_content=content.upper(),
+        processed_at=datetime.now(tz=timezone.utc).isoformat(),
+        error="",
+        attempt=attempt,
+    )
+
+
 def test_retry_with_error_records(test_session):
     """Test retry functionality with records that have errors."""
 
@@ -48,13 +65,7 @@ def test_retry_with_error_records(test_session):
         )
 
     # First processing pass - some records will fail
-    sample_ids = [1, 2, 3, 4]
-    sample_contents = ["first item", "second item", "third item", "fourth item"]
-
-    dc.read_values(id=sample_ids, content=sample_contents, session=test_session).save(
-        "sample_data"
-    )
-
+    _create_sample_data(test_session)
     first_pass = _run_processing(1)
 
     # Check that some records failed
@@ -74,72 +85,103 @@ def test_retry_with_error_records(test_session):
 
 def test_retry_with_missing_records(test_session):
     """Test retry functionality with missing records."""
-    # Create source dataset
-    source_ids = [1, 2, 3]
-    source_contents = ["first", "second", "third"]
+    _create_sample_data(test_session)
 
-    dc.read_values(id=source_ids, content=source_contents, session=test_session).save(
-        "source_data"
+    # Process only first 2 records
+    # Create partial result dataset (missing id=3)
+    partial_result = (
+        dc.read_dataset("sample_data", session=test_session)
+        .setup(attempt=lambda: 1)
+        .filter(C("id") < 3)
+        .map(result=_simple_process)
+        .save("partial_result")
     )
 
-    def simple_process(id: int, content: str, attempt: int) -> ProcessingResult:
-        return ProcessingResult(
-            processed_content=content.upper(),
-            processed_at=datetime.now(tz=timezone.utc).isoformat(),
-            error="",
-            attempt=attempt,
+    assert partial_result.count() == 2
+
+    # Use retry with delta_retry=True to process missing records
+    retry_chain = (
+        dc.read_dataset(
+            "sample_data",
+            session=test_session,
+            delta=True,
+            delta_on="id",
+            delta_retry=True,
         )
+        .setup(attempt=lambda: 2)
+        .map(result=_simple_process)
+        .save("partial_result")
+    )
+
+    # Should now have all 4 records
+    assert retry_chain.count() == 4
+
+    # Verify all records are present
+    ids = set(retry_chain.to_values("id"))
+    assert ids == {1, 2, 3, 4}
+
+    final_first_attempts_count = retry_chain.filter(C("result.attempt") == 1).count()
+    final_missing_attempts_count = retry_chain.filter(C("result.attempt") == 2).count()
+
+    # Only missing records should have attempt 2
+    assert final_missing_attempts_count == 2
+    assert final_first_attempts_count == 2
+
+
+def test_retry_with_missing_and_new_records(test_session):
+    """Test retry functionality with missing records (e.g. ignored
+    in first pass since they failed). Also we add new records to the source
+    to test that retry and delta don't pick records twice."""
+    _create_sample_data(test_session)
 
     # Process only first 2 records
     # Create partial result dataset (missing id=3)
     partial_result = (
-        dc.read_dataset("source_data", session=test_session)
+        dc.read_dataset("sample_data", session=test_session)
         .setup(attempt=lambda: 1)
         .filter(C("id") < 3)
-        .map(result=simple_process)
+        .map(result=_simple_process)
         .save("partial_result")
     )
 
     assert partial_result.count() == 2
 
+    ids = [1, 2, 3, 4, 5]
+    contents = ["first item", "second item", "third item", "fourth item", "fifth item"]
+    _create_sample_data(test_session, ids, contents)
+
     # Use retry with delta_retry=True to process missing records
     retry_chain = (
         dc.read_dataset(
-            "source_data",
+            "sample_data",
             session=test_session,
             delta=True,
             delta_on="id",
             delta_retry=True,
         )
         .setup(attempt=lambda: 2)
-        .map(result=simple_process)
+        .map(result=_simple_process)
         .save("partial_result")
     )
 
     # Should now have all 3 records
-    assert retry_chain.count() == 3
+    assert retry_chain.count() == 5
 
     # Verify all records are present
     ids = set(retry_chain.to_values("id"))
-    assert ids == {1, 2, 3}
+    assert ids == {1, 2, 3, 4, 5}
 
     final_first_attempts_count = retry_chain.filter(C("result.attempt") == 1).count()
     final_missing_attempts_count = retry_chain.filter(C("result.attempt") == 2).count()
 
     # Only missing records should have attempt 2
-    assert final_missing_attempts_count == 1
+    assert final_missing_attempts_count == 3
     assert final_first_attempts_count == 2
 
 
 def test_retry_no_records_to_retry(test_session):
     """Test retry when no records need to be retried."""
-    # Create dataset with all successful records
-    source_ids = [1, 2]
-    source_contents = ["first", "second"]
-
-    dc.read_values(id=source_ids, content=source_contents, session=test_session).save(
-        "source_data"
-    )
+    _create_sample_data(test_session, ids=[1, 2], contents=["first", "second"])
 
     def successful_process(id: int, content: str) -> ProcessingResult:
         return ProcessingResult(
@@ -151,7 +193,7 @@ def test_retry_no_records_to_retry(test_session):
 
     # First pass - all succeed
     first_pass = (
-        dc.read_dataset("source_data", session=test_session)
+        dc.read_dataset("sample_data", session=test_session)
         .map(result=successful_process)
         .save("successful_data")
     )
@@ -162,7 +204,7 @@ def test_retry_no_records_to_retry(test_session):
     # Retry - should not create a new version since no records need retry
     (
         dc.read_dataset(
-            "source_data",
+            "sample_data",
             session=test_session,
             delta=True,
             delta_on="id",
@@ -179,32 +221,20 @@ def test_retry_no_records_to_retry(test_session):
 
 def test_retry_first_dataset_creation(test_session):
     """Test retry when dataset doesn't exist yet (first creation)."""
-    source_ids = [1, 2]
-    source_contents = ["first", "second"]
-
-    dc.read_values(id=source_ids, content=source_contents, session=test_session).save(
-        "source_data"
-    )
-
-    def simple_process(id: int, content: str) -> ProcessingResult:
-        return ProcessingResult(
-            processed_content=content.upper(),
-            processed_at=datetime.now(tz=timezone.utc).isoformat(),
-            error="",
-            attempt=1,
-        )
+    _create_sample_data(test_session, ids=[1, 2], contents=["first", "second"])
 
     # First run with retry enabled on non-existent dataset
     # Should process all records
     retry_chain = (
         dc.read_dataset(
-            "source_data",
+            "sample_data",
             session=test_session,
             delta=True,
             delta_on="id",
             delta_retry="result.error",
         )
-        .map(result=simple_process)
+        .setup(attempt=lambda: 1)
+        .map(result=_simple_process)
         .save("new_dataset")
     )
 
@@ -311,3 +341,87 @@ def test_retry_with_delta_functionality(test_session):
         (2, "", 1),
         (3, "", 2),
     }
+
+
+def test_delta_and_delta_retry_no_duplicates(test_session):
+    """Test that delta and delta_retry work together without creating duplicates
+    when the same records are picked up for different reasons:
+    - delta_retry=True picks up unprocessed records missing from result dataset
+    - delta=True picks up modified records from source dataset
+    """
+    _create_sample_data(test_session)
+
+    # First pass - process only records 1 and 2
+    partial_result = (
+        dc.read_dataset("sample_data", session=test_session)
+        .setup(attempt=lambda: 1)
+        .filter(C("id") < 3)  # Only process id=1,2, leaving id=3,4 unprocessed
+        .map(result=_simple_process)
+        .save("delta_retry_combined_result")
+    )
+
+    assert partial_result.count() == 2
+    initial_results = set(partial_result.to_iter("id", "result.attempt"))
+    assert initial_results == {(1, 1), (2, 1)}
+
+    # Modify the source data - update content for records 3 and 4
+    # This will make delta=True pick them up as "changed"
+    # But delta_retry=True will also pick them up as "missing from result"
+    modified_ids = [1, 2, 3, 4]
+    modified_contents = [
+        "first item",  # unchanged
+        "second item",  # unchanged
+        "MODIFIED third item",  # modified - delta will pick this up
+        "MODIFIED fourth item",  # modified - delta will pick this up
+    ]
+    _create_sample_data(test_session, modified_ids, modified_contents)
+
+    # Second pass with both delta=True and delta_retry=True
+    # Records 3,4 should be picked up by BOTH:
+    # - delta_retry=True (because they're missing from result dataset)
+    # - delta=True (because their content was modified in source)
+    # But they should only be processed ONCE (no duplicates)
+    combined_result = (
+        dc.read_dataset(
+            "sample_data",
+            session=test_session,
+            delta=True,
+            delta_on="id",
+            delta_retry=True,
+        )
+        .setup(attempt=lambda: 2)
+        .map(result=_simple_process)
+        .save("delta_retry_combined_result")
+    )
+
+    # Should have 4 total records: 2 from first pass + 2 newly processed
+    assert combined_result.count() == 4
+
+    # Get all results and verify no duplicates
+    all_results = set(
+        combined_result.to_iter("id", "result.attempt", "result.processed_content")
+    )
+
+    # Records 1,2 should have attempt=1 (from first pass)
+    # Records 3,4 should have attempt=2 (from second pass) and MODIFIED content
+    expected_results = {
+        (1, 1, "FIRST ITEM"),
+        (2, 1, "SECOND ITEM"),
+        (3, 2, "MODIFIED THIRD ITEM"),
+        (4, 2, "MODIFIED FOURTH ITEM"),
+    }
+
+    assert all_results == expected_results
+
+    # Verify counts by attempt
+    first_attempt_count = combined_result.filter(C("result.attempt") == 1).count()
+    second_attempt_count = combined_result.filter(C("result.attempt") == 2).count()
+
+    assert first_attempt_count == 2  # Records 1,2 from first pass
+    assert second_attempt_count == 2  # Records 3,4 from second pass (no duplicates)
+
+    # Verify that each id appears exactly once
+    ids_in_result = list(combined_result.to_values("id"))
+    assert len(ids_in_result) == 4
+    assert len(set(ids_in_result)) == 4  # No duplicate IDs
+    assert set(ids_in_result) == {1, 2, 3, 4}