datachain 0.18.1__tar.gz → 0.18.3__tar.gz

{datachain-0.18.1 → datachain-0.18.3}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.11.9'
+    rev: 'v0.11.10'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.18.1/src/datachain.egg-info → datachain-0.18.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.18.1
+Version: 0.18.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

datachain-0.18.3/examples/incremental_processing/delta.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+"""
+File Generator Script using DataChain Delta
+
+This script demonstrates:
+1. Creating numbered text files in a 'test' directory
+2. Using DataChain's delta flag for incremental dataset processing
+
+Each execution:
+- Creates a new numbered file in the 'test' directory
+- Updates a DataChain dataset to track these files incrementally
+"""
+
+import re
+import time
+
+from utils import generate_next_file
+
+import datachain as dc
+from datachain import C, File
+
+
+def extract_file_number(file: File) -> int:
+    """Extract file number from the filename."""
+    match = re.search(r"file-(\d+)\.txt", file.name)
+    if match:
+        return int(match.group(1))
+    return -1
+
+
+def process_files_with_delta():
+    """
+    Process files in the test directory using DataChain with delta mode.
+    This demonstrates incremental processing - only new files are processed.
+    """
+    chain = (
+        dc.read_storage("test/", update=True, delta=True, delta_on="file.path")
+        .filter(C("file.path").glob("*.txt"))
+        .map(file_number=extract_file_number)
+        .map(content=lambda file: file.read_text())
+        .map(processed_at=lambda: time.strftime("%Y-%m-%d %H:%M:%S"))
+        .save(name="test_files")
+    )
+
+    # Show information about the dataset
+    print(f"\nProcessed files. Total records: {chain.count()}")
+    print("\nDataset versions:")
+    test_dataset = dc.datasets().filter(C("name") == "test_files")
+
+    for version in test_dataset.collect("version"):
+        print(f"- Version: {version}")
+
+    # Show the last 3 records to demonstrate the incremental processing
+    print("\nLatest files processed:")
+    chain.order_by("file_number", descending=True).limit(3).show()
+
+
+if __name__ == "__main__":
+    # Generate a new file
+    new_file = generate_next_file()
+    print(f"Created new file: {new_file}")
+
+    # Process all new file with (delta update)
+    process_files_with_delta()

datachain-0.18.3/examples/incremental_processing/utils.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+File Generator Helper
+
+This helper creates numbered text files in a 'test' directory each time it runs.
+The files follow the naming pattern: file-0.txt, file-1.txt, file-2.txt, etc.
+
+Each execution, the script:
+
+1. Creates the 'test' directory if it doesn't exist
+2. Finds the highest numbered file currently present
+3. Creates a new file with the next number in sequence
+4. Adds timestamped content to the file
+"""
+
+import re
+import time
+from pathlib import Path
+
+
+def generate_next_file() -> Path:
+    """
+    Generate (appends) a new numbered text file in the 'test' directory.
+    """
+    test_dir = Path("test")
+    test_dir.mkdir(exist_ok=True)
+
+    max_num = -1
+    for file in test_dir.glob("file-*.txt"):
+        if file.is_file():
+            match = re.search(r"file-(\d+)\.txt", file.name)
+            if match:
+                max_num = max(max_num, int(match.group(1)))
+
+    next_num = max_num + 1
+    new_file_path = test_dir / f"file-{next_num}.txt"
+    timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    content = f"This is file number {next_num}\nCreated at: {timestamp}\n"
+    new_file_path.write_text(content)
+
+    return new_file_path

{datachain-0.18.1 → datachain-0.18.3}/src/datachain/func/array.py

@@ -178,6 +178,126 @@ def contains(arr: Union[str, Sequence, Func], elem: Any) -> Func:
     return Func("contains", inner=inner, cols=cols, args=args, result_type=int)
 
 
+def slice(
+    arr: Union[str, Sequence, Func],
+    offset: int,
+    length: Optional[int] = None,
+) -> Func:
+    """
+    Returns a slice of the array.
+
+    Args:
+        arr (str | Sequence | Func): Array to check for the element.
+            If a string is provided, it is assumed to be the name of the array column.
+            If a sequence is provided, it is assumed to be an array of values.
+            If a Func is provided, it is assumed to be a function returning an array.
+        offset (int): Offset to start the slice from.
+        length (int, optional): Length of the slice. If not provided, the slice will
+            continue to the end of the array.
+
+    Returns:
+        Func: A Func object that represents the slice function. Result of the
+            function will be a slice of the array starting from the offset
+            and with the given length.
+
+    Example:
+        ```py
+        dc.mutate(
+            contains1=func.array.slice("signal.values", 3),
+            contains2=func.array.slice([1, 2, 3, 4, 5], 1, 3),
+        )
+        ```
+    """
+
+    def inner(arg):
+        if length is not None:
+            return array.slice(arg, offset, length)
+        return array.slice(arg, offset)
+
+    def element_type(el):
+        if isinstance(el, list):
+            try:
+                return list[element_type(el[0])]
+            except IndexError:
+                # if the array is empty, return list[str] as default type
+                return list[str]
+        return type(el)
+
+    def type_from_args(arr, *_):
+        if isinstance(arr, list):
+            try:
+                return list[element_type(arr[0])]
+            except IndexError:
+                # if the array is empty, return list[str] as default type
+                return list[str]
+        return None
+
+    if isinstance(arr, (str, Func)):
+        cols = [arr]
+        args = None
+    else:
+        cols = None
+        args = [arr]
+
+    return Func(
+        "slice",
+        inner=inner,
+        cols=cols,
+        args=args,
+        from_array=True,
+        is_array=True,
+        type_from_args=type_from_args,
+    )
+
+
+def join(
+    arr: Union[str, Sequence, Func],
+    sep: str = "",
+) -> Func:
+    """
+    Returns a string that is the concatenation of the elements of the array,
+
+    Args:
+        arr (str | Sequence | Func): Array to check for the element.
+            If a string is provided, it is assumed to be the name of the array column.
+            If a sequence is provided, it is assumed to be an array of values.
+            If a Func is provided, it is assumed to be a function returning an array.
+        sep (str): Separator to use for the concatenation. Default is an empty string.
+
+    Returns:
+        Func: A Func object that represents the join function. Result of the
+            function will be a string that is the concatenation of the elements
+            of the array, separated by the given separator.
+
+    Example:
+        ```py
+        dc.mutate(
+            contains1=func.array.join("signal.values", ":"),
+            contains2=func.array.join(["1", "2", "3", "4", "5"], "/"),
+        )
+        ```
+    """
+
+    def inner(arg):
+        return array.join(arg, sep)
+
+    if isinstance(arr, (str, Func)):
+        cols = [arr]
+        args = None
+    else:
+        cols = None
+        args = [arr]
+
+    return Func(
+        "join",
+        inner=inner,
+        cols=cols,
+        args=args,
+        from_array=True,
+        result_type=str,
+    )
+
+
 def get_element(arg: Union[str, Sequence, Func], index: int) -> Func:
     """
     Returns the element at the given index from the array.

{datachain-0.18.1 → datachain-0.18.3}/src/datachain/func/func.py

@@ -108,18 +108,20 @@ class Func(Function):
                 )
 
         if self.from_array:
-            if get_origin(col_type) is list:
-                col_args = get_args(col_type)
-                if len(col_args) != 1:
-                    raise DataChainColumnError(
-                        str(self),
-                        "Array column must have a single type argument",
-                    )
-                return col_args[0]
-            raise DataChainColumnError(
-                str(self),
-                "Array column must be of type list",
-            )
+            if get_origin(col_type) is not list:
+                raise DataChainColumnError(
+                    str(self),
+                    "Array column must be of type list",
+                )
+            if self.is_array:
+                return col_type
+            col_args = get_args(col_type)
+            if len(col_args) != 1:
+                raise DataChainColumnError(
+                    str(self),
+                    "Array column must have a single type argument",
+                )
+            return col_args[0]
 
         return list[col_type] if self.is_array else col_type  # type: ignore[valid-type]
 

{datachain-0.18.1 → datachain-0.18.3}/src/datachain/lib/file.py

@@ -237,7 +237,7 @@ class File(DataModel):
     @field_validator("path", mode="before")
     @classmethod
     def validate_path(cls, path):
-        return Path(path).as_posix()
+        return Path(path).as_posix() if path else ""
 
     def model_dump_custom(self):
         res = self.model_dump()

{datachain-0.18.1 → datachain-0.18.3}/src/datachain/query/dataset.py

@@ -1675,13 +1675,27 @@ class DatasetQuery:
         return query
 
     def _add_dependencies(self, dataset: "DatasetRecord", version: str):
-        for dependency in self.dependencies:
-            ds_dependency_name, ds_dependency_version = dependency
+        dependencies: set[DatasetDependencyType] = set()
+        for dep_name, dep_version in self.dependencies:
+            if Session.is_temp_dataset(dep_name):
+                # temp dataset are created for optimization and they will be removed
+                # afterwards. Therefore, we should not put them as dependencies, but
+                # their own direct dependencies
+                for dep in self.catalog.get_dataset_dependencies(
+                    dep_name, dep_version, indirect=False
+                ):
+                    if dep:
+                        dependencies.add((dep.name, dep.version))
+            else:
+                dependencies.add((dep_name, dep_version))
+
+        for dep_name, dep_version in dependencies:
+            # ds_dependency_name, ds_dependency_version = dependency
             self.catalog.metastore.add_dataset_dependency(
                 dataset.name,
                 version,
-                ds_dependency_name,
-                ds_dependency_version,
+                dep_name,
+                dep_version,
             )
 
     def exec(self) -> "Self":

{datachain-0.18.1 → datachain-0.18.3}/src/datachain/query/session.py

@@ -195,5 +195,11 @@ class Session:
             Session.GLOBAL_SESSION_CTX.__exit__(None, None, None)
 
         for obj in gc.get_objects():  # Get all tracked objects
-            if isinstance(obj, Session):  # Cleanup temp dataset for session variables.
-                obj.__exit__(None, None, None)
+            try:
+                if isinstance(obj, Session):
+                    # Cleanup temp dataset for session variables.
+                    obj.__exit__(None, None, None)
+            except ReferenceError:
+                continue  # Object has been finalized already
+            except Exception as e:  # noqa: BLE001
+                logger.error(f"Exception while cleaning up session: {e}")  # noqa: G004

{datachain-0.18.1 → datachain-0.18.3}/src/datachain/sql/functions/array.py

@@ -1,6 +1,6 @@
 from sqlalchemy.sql.functions import GenericFunction
 
-from datachain.sql.types import Boolean, Float, Int64
+from datachain.sql.types import Boolean, Float, Int64, String
 from datachain.sql.utils import compiler_not_implemented
 
 
@@ -48,6 +48,27 @@ class contains(GenericFunction):  # noqa: N801
     inherit_cache = True
 
 
+class slice(GenericFunction):  # noqa: N801
+    """
+    Returns a slice of the array.
+    """
+
+    package = "array"
+    name = "slice"
+    inherit_cache = True
+
+
+class join(GenericFunction):  # noqa: N801
+    """
+    Returns the concatenation of the array elements.
+    """
+
+    type = String()
+    package = "array"
+    name = "join"
+    inherit_cache = True
+
+
 class get_element(GenericFunction):  # noqa: N801
     """
     Returns the element at the given index in the array.

{datachain-0.18.1 → datachain-0.18.3}/src/datachain/sql/sqlite/base.py

@@ -88,6 +88,8 @@ def setup():
     compiles(sql_path.file_ext, "sqlite")(compile_path_file_ext)
     compiles(array.length, "sqlite")(compile_array_length)
     compiles(array.contains, "sqlite")(compile_array_contains)
+    compiles(array.slice, "sqlite")(compile_array_slice)
+    compiles(array.join, "sqlite")(compile_array_join)
     compiles(array.get_element, "sqlite")(compile_array_get_element)
     compiles(string.length, "sqlite")(compile_string_length)
     compiles(string.split, "sqlite")(compile_string_split)
@@ -275,6 +277,15 @@ def register_user_defined_sql_functions() -> None:
         conn.create_function(
             "json_array_get_element", 2, py_json_array_get_element, deterministic=True
         )
+        conn.create_function(
+            "json_array_slice", 2, py_json_array_slice, deterministic=True
+        )
+        conn.create_function(
+            "json_array_slice", 3, py_json_array_slice, deterministic=True
+        )
+        conn.create_function(
+            "json_array_join", 2, py_json_array_join, deterministic=True
+        )
 
     _registered_function_creators["array_functions"] = create_array_functions
 
@@ -454,6 +465,20 @@ def py_json_array_get_element(val, idx):
         return None
 
 
+def py_json_array_slice(val, offset: int, length: Optional[int] = None):
+    arr = orjson.loads(val)
+    try:
+        return orjson.dumps(
+            list(arr[offset : offset + length] if length is not None else arr[offset:])
+        ).decode("utf-8")
+    except IndexError:
+        return None
+
+
+def py_json_array_join(val, sep: str):
+    return sep.join(orjson.loads(val))
+
+
 def compile_array_get_element(element, compiler, **kwargs):
     return compiler.process(
         func.json_array_get_element(*element.clauses.clauses), **kwargs
@@ -470,6 +495,14 @@ def compile_array_contains(element, compiler, **kwargs):
     )
 
 
+def compile_array_slice(element, compiler, **kwargs):
+    return compiler.process(func.json_array_slice(*element.clauses.clauses), **kwargs)
+
+
+def compile_array_join(element, compiler, **kwargs):
+    return compiler.process(func.json_array_join(*element.clauses.clauses), **kwargs)
+
+
 def compile_string_length(element, compiler, **kwargs):
     return compiler.process(func.length(*element.clauses.clauses), **kwargs)
 

{datachain-0.18.1 → datachain-0.18.3/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.18.1
+Version: 0.18.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.18.1 → datachain-0.18.3}/src/datachain.egg-info/SOURCES.txt

@@ -66,6 +66,8 @@ examples/get_started/torch-loader.py
 examples/get_started/udfs/parallel.py
 examples/get_started/udfs/simple.py
 examples/get_started/udfs/stateful.py
+examples/incremental_processing/delta.py
+examples/incremental_processing/utils.py
 examples/llm_and_nlp/claude-query.py
 examples/llm_and_nlp/hf-dataset-llm-eval.py
 examples/multimodal/clip_inference.py

{datachain-0.18.1 → datachain-0.18.3}/tests/examples/test_examples.py

@@ -12,6 +12,10 @@ llm_and_nlp_examples = sorted(glob.glob("examples/llm_and_nlp/**/*.py", recursiv
 
 multimodal_examples = sorted(glob.glob("examples/multimodal/**/*.py", recursive=True))
 
+incremental_processing_examples = sorted(
+    glob.glob("examples/incremental_processing/delta.py", recursive=True)
+)
+
 computer_vision_examples = sorted(
     [
         filename
@@ -86,6 +90,13 @@ def test_multimodal(example):
     )
 
 
+@pytest.mark.examples
+@pytest.mark.incremental_processing
+@pytest.mark.parametrize("example", incremental_processing_examples)
+def test_incremental_processing_examples(example):
+    smoke_test(example)
+
+
 @pytest.mark.examples
 @pytest.mark.computer_vision
 @pytest.mark.parametrize("example", computer_vision_examples)

{datachain-0.18.1 → datachain-0.18.3}/tests/func/test_datachain.py

@@ -239,6 +239,25 @@ def test_read_storage_dependencies(cloud_test_catalog, cloud_type):
     assert dependencies[0].name == dep_name
 
 
+def test_persist_not_affects_dependencies(tmp_dir, test_session):
+    for i in range(4):
+        (tmp_dir / f"file{i}.txt").write_text(f"file{i}")
+
+    uri = tmp_dir.as_uri()
+    dep_name, _, _ = parse_listing_uri(uri, test_session.catalog.client_config)
+    chain = dc.read_storage(uri, session=test_session)  # .persist()
+    # calling multiple persists to create temp datasets
+    chain = chain.persist()
+    chain = chain.persist()
+    chain = chain.persist()
+    chain.save("test-data")
+    dependencies = test_session.catalog.get_dataset_dependencies("test-data", "1.0.0")
+
+    assert len(dependencies) == 1
+    assert dependencies[0].name == dep_name
+    assert dependencies[0].type == DatasetDependencyType.STORAGE
+
+
 @pytest.mark.parametrize("use_cache", [True, False])
 @pytest.mark.parametrize("prefetch", [0, 2])
 def test_map_file(cloud_test_catalog, use_cache, prefetch, monkeypatch):