datachain 0.18.0__tar.gz → 0.18.2__tar.gz

{datachain-0.18.0 → datachain-0.18.2}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.11.9'
+    rev: 'v0.11.10'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.18.0/src/datachain.egg-info → datachain-0.18.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.18.0
+Version: 0.18.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

datachain-0.18.2/examples/incremental_processing/delta.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+"""
+File Generator Script using DataChain Delta
+
+This script demonstrates:
+1. Creating numbered text files in a 'test' directory
+2. Using DataChain's delta flag for incremental dataset processing
+
+Each execution:
+- Creates a new numbered file in the 'test' directory
+- Updates a DataChain dataset to track these files incrementally
+"""
+
+import re
+import time
+
+from utils import generate_next_file
+
+import datachain as dc
+from datachain import C, File
+
+
+def extract_file_number(file: File) -> int:
+    """Extract file number from the filename."""
+    match = re.search(r"file-(\d+)\.txt", file.name)
+    if match:
+        return int(match.group(1))
+    return -1
+
+
+def process_files_with_delta():
+    """
+    Process files in the test directory using DataChain with delta mode.
+    This demonstrates incremental processing - only new files are processed.
+    """
+    chain = (
+        dc.read_storage("test/", update=True, delta=True, delta_on="file.path")
+        .filter(C("file.path").glob("*.txt"))
+        .map(file_number=extract_file_number)
+        .map(content=lambda file: file.read_text())
+        .map(processed_at=lambda: time.strftime("%Y-%m-%d %H:%M:%S"))
+        .save(name="test_files")
+    )
+
+    # Show information about the dataset
+    print(f"\nProcessed files. Total records: {chain.count()}")
+    print("\nDataset versions:")
+    test_dataset = dc.datasets().filter(C("name") == "test_files")
+
+    for version in test_dataset.collect("version"):
+        print(f"- Version: {version}")
+
+    # Show the last 3 records to demonstrate the incremental processing
+    print("\nLatest files processed:")
+    chain.order_by("file_number", descending=True).limit(3).show()
+
+
+if __name__ == "__main__":
+    # Generate a new file
+    new_file = generate_next_file()
+    print(f"Created new file: {new_file}")
+
+    # Process all new file with (delta update)
+    process_files_with_delta()

datachain-0.18.2/examples/incremental_processing/utils.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+File Generator Helper
+
+This helper creates numbered text files in a 'test' directory each time it runs.
+The files follow the naming pattern: file-0.txt, file-1.txt, file-2.txt, etc.
+
+Each execution, the script:
+
+1. Creates the 'test' directory if it doesn't exist
+2. Finds the highest numbered file currently present
+3. Creates a new file with the next number in sequence
+4. Adds timestamped content to the file
+"""
+
+import re
+import time
+from pathlib import Path
+
+
+def generate_next_file() -> Path:
+    """
+    Generate (appends) a new numbered text file in the 'test' directory.
+    """
+    test_dir = Path("test")
+    test_dir.mkdir(exist_ok=True)
+
+    max_num = -1
+    for file in test_dir.glob("file-*.txt"):
+        if file.is_file():
+            match = re.search(r"file-(\d+)\.txt", file.name)
+            if match:
+                max_num = max(max_num, int(match.group(1)))
+
+    next_num = max_num + 1
+    new_file_path = test_dir / f"file-{next_num}.txt"
+    timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    content = f"This is file number {next_num}\nCreated at: {timestamp}\n"
+    new_file_path.write_text(content)
+
+    return new_file_path

{datachain-0.18.0 → datachain-0.18.2}/src/datachain/data_storage/metastore.py

@@ -674,7 +674,7 @@ class AbstractDBMetastore(AbstractMetastore):
             dv = self._datasets_versions
             self.db.execute(
                 self._datasets_versions_update()
-                .where(dv.c.dataset_id == dataset.id and dv.c.version == version)
+                .where(dv.c.dataset_id == dataset.id, dv.c.version == version)
                 .values(values),
                 conn=conn,
             )  # type: ignore [attr-defined]

{datachain-0.18.0 → datachain-0.18.2}/src/datachain/func/array.py

@@ -178,6 +178,126 @@ def contains(arr: Union[str, Sequence, Func], elem: Any) -> Func:
     return Func("contains", inner=inner, cols=cols, args=args, result_type=int)
 
 
+def slice(
+    arr: Union[str, Sequence, Func],
+    offset: int,
+    length: Optional[int] = None,
+) -> Func:
+    """
+    Returns a slice of the array.
+
+    Args:
+        arr (str | Sequence | Func): Array to check for the element.
+            If a string is provided, it is assumed to be the name of the array column.
+            If a sequence is provided, it is assumed to be an array of values.
+            If a Func is provided, it is assumed to be a function returning an array.
+        offset (int): Offset to start the slice from.
+        length (int, optional): Length of the slice. If not provided, the slice will
+            continue to the end of the array.
+
+    Returns:
+        Func: A Func object that represents the slice function. Result of the
+            function will be a slice of the array starting from the offset
+            and with the given length.
+
+    Example:
+        ```py
+        dc.mutate(
+            contains1=func.array.slice("signal.values", 3),
+            contains2=func.array.slice([1, 2, 3, 4, 5], 1, 3),
+        )
+        ```
+    """
+
+    def inner(arg):
+        if length is not None:
+            return array.slice(arg, offset, length)
+        return array.slice(arg, offset)
+
+    def element_type(el):
+        if isinstance(el, list):
+            try:
+                return list[element_type(el[0])]
+            except IndexError:
+                # if the array is empty, return list[str] as default type
+                return list[str]
+        return type(el)
+
+    def type_from_args(arr, *_):
+        if isinstance(arr, list):
+            try:
+                return list[element_type(arr[0])]
+            except IndexError:
+                # if the array is empty, return list[str] as default type
+                return list[str]
+        return None
+
+    if isinstance(arr, (str, Func)):
+        cols = [arr]
+        args = None
+    else:
+        cols = None
+        args = [arr]
+
+    return Func(
+        "slice",
+        inner=inner,
+        cols=cols,
+        args=args,
+        from_array=True,
+        is_array=True,
+        type_from_args=type_from_args,
+    )
+
+
+def join(
+    arr: Union[str, Sequence, Func],
+    sep: str = "",
+) -> Func:
+    """
+    Returns a string that is the concatenation of the elements of the array,
+
+    Args:
+        arr (str | Sequence | Func): Array to check for the element.
+            If a string is provided, it is assumed to be the name of the array column.
+            If a sequence is provided, it is assumed to be an array of values.
+            If a Func is provided, it is assumed to be a function returning an array.
+        sep (str): Separator to use for the concatenation. Default is an empty string.
+
+    Returns:
+        Func: A Func object that represents the join function. Result of the
+            function will be a string that is the concatenation of the elements
+            of the array, separated by the given separator.
+
+    Example:
+        ```py
+        dc.mutate(
+            contains1=func.array.join("signal.values", ":"),
+            contains2=func.array.join(["1", "2", "3", "4", "5"], "/"),
+        )
+        ```
+    """
+
+    def inner(arg):
+        return array.join(arg, sep)
+
+    if isinstance(arr, (str, Func)):
+        cols = [arr]
+        args = None
+    else:
+        cols = None
+        args = [arr]
+
+    return Func(
+        "join",
+        inner=inner,
+        cols=cols,
+        args=args,
+        from_array=True,
+        result_type=str,
+    )
+
+
 def get_element(arg: Union[str, Sequence, Func], index: int) -> Func:
     """
     Returns the element at the given index from the array.

{datachain-0.18.0 → datachain-0.18.2}/src/datachain/func/func.py

@@ -108,18 +108,20 @@ class Func(Function):
                 )
 
         if self.from_array:
-            if get_origin(col_type) is list:
-                col_args = get_args(col_type)
-                if len(col_args) != 1:
-                    raise DataChainColumnError(
-                        str(self),
-                        "Array column must have a single type argument",
-                    )
-                return col_args[0]
-            raise DataChainColumnError(
-                str(self),
-                "Array column must be of type list",
-            )
+            if get_origin(col_type) is not list:
+                raise DataChainColumnError(
+                    str(self),
+                    "Array column must be of type list",
+                )
+            if self.is_array:
+                return col_type
+            col_args = get_args(col_type)
+            if len(col_args) != 1:
+                raise DataChainColumnError(
+                    str(self),
+                    "Array column must have a single type argument",
+                )
+            return col_args[0]
 
         return list[col_type] if self.is_array else col_type  # type: ignore[valid-type]
 

{datachain-0.18.0 → datachain-0.18.2}/src/datachain/lib/file.py

@@ -237,7 +237,7 @@ class File(DataModel):
     @field_validator("path", mode="before")
     @classmethod
     def validate_path(cls, path):
-        return Path(path).as_posix()
+        return Path(path).as_posix() if path else ""
 
     def model_dump_custom(self):
         res = self.model_dump()

{datachain-0.18.0 → datachain-0.18.2}/src/datachain/query/session.py

@@ -195,5 +195,11 @@ class Session:
             Session.GLOBAL_SESSION_CTX.__exit__(None, None, None)
 
         for obj in gc.get_objects():  # Get all tracked objects
-            if isinstance(obj, Session):  # Cleanup temp dataset for session variables.
-                obj.__exit__(None, None, None)
+            try:
+                if isinstance(obj, Session):
+                    # Cleanup temp dataset for session variables.
+                    obj.__exit__(None, None, None)
+            except ReferenceError:
+                continue  # Object has been finalized already
+            except Exception as e:  # noqa: BLE001
+                logger.error(f"Exception while cleaning up session: {e}")  # noqa: G004

{datachain-0.18.0 → datachain-0.18.2}/src/datachain/sql/functions/array.py

@@ -1,6 +1,6 @@
 from sqlalchemy.sql.functions import GenericFunction
 
-from datachain.sql.types import Boolean, Float, Int64
+from datachain.sql.types import Boolean, Float, Int64, String
 from datachain.sql.utils import compiler_not_implemented
 
 
@@ -48,6 +48,27 @@ class contains(GenericFunction):  # noqa: N801
     inherit_cache = True
 
 
+class slice(GenericFunction):  # noqa: N801
+    """
+    Returns a slice of the array.
+    """
+
+    package = "array"
+    name = "slice"
+    inherit_cache = True
+
+
+class join(GenericFunction):  # noqa: N801
+    """
+    Returns the concatenation of the array elements.
+    """
+
+    type = String()
+    package = "array"
+    name = "join"
+    inherit_cache = True
+
+
 class get_element(GenericFunction):  # noqa: N801
     """
     Returns the element at the given index in the array.

{datachain-0.18.0 → datachain-0.18.2}/src/datachain/sql/sqlite/base.py

@@ -88,6 +88,8 @@ def setup():
     compiles(sql_path.file_ext, "sqlite")(compile_path_file_ext)
     compiles(array.length, "sqlite")(compile_array_length)
     compiles(array.contains, "sqlite")(compile_array_contains)
+    compiles(array.slice, "sqlite")(compile_array_slice)
+    compiles(array.join, "sqlite")(compile_array_join)
     compiles(array.get_element, "sqlite")(compile_array_get_element)
     compiles(string.length, "sqlite")(compile_string_length)
     compiles(string.split, "sqlite")(compile_string_split)
@@ -275,6 +277,15 @@ def register_user_defined_sql_functions() -> None:
         conn.create_function(
             "json_array_get_element", 2, py_json_array_get_element, deterministic=True
         )
+        conn.create_function(
+            "json_array_slice", 2, py_json_array_slice, deterministic=True
+        )
+        conn.create_function(
+            "json_array_slice", 3, py_json_array_slice, deterministic=True
+        )
+        conn.create_function(
+            "json_array_join", 2, py_json_array_join, deterministic=True
+        )
 
     _registered_function_creators["array_functions"] = create_array_functions
 
@@ -454,6 +465,20 @@ def py_json_array_get_element(val, idx):
         return None
 
 
+def py_json_array_slice(val, offset: int, length: Optional[int] = None):
+    arr = orjson.loads(val)
+    try:
+        return orjson.dumps(
+            list(arr[offset : offset + length] if length is not None else arr[offset:])
+        ).decode("utf-8")
+    except IndexError:
+        return None
+
+
+def py_json_array_join(val, sep: str):
+    return sep.join(orjson.loads(val))
+
+
 def compile_array_get_element(element, compiler, **kwargs):
     return compiler.process(
         func.json_array_get_element(*element.clauses.clauses), **kwargs
@@ -470,6 +495,14 @@ def compile_array_contains(element, compiler, **kwargs):
     )
 
 
+def compile_array_slice(element, compiler, **kwargs):
+    return compiler.process(func.json_array_slice(*element.clauses.clauses), **kwargs)
+
+
+def compile_array_join(element, compiler, **kwargs):
+    return compiler.process(func.json_array_join(*element.clauses.clauses), **kwargs)
+
+
 def compile_string_length(element, compiler, **kwargs):
     return compiler.process(func.length(*element.clauses.clauses), **kwargs)
 

{datachain-0.18.0 → datachain-0.18.2/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.18.0
+Version: 0.18.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.18.0 → datachain-0.18.2}/src/datachain.egg-info/SOURCES.txt

@@ -66,6 +66,8 @@ examples/get_started/torch-loader.py
 examples/get_started/udfs/parallel.py
 examples/get_started/udfs/simple.py
 examples/get_started/udfs/stateful.py
+examples/incremental_processing/delta.py
+examples/incremental_processing/utils.py
 examples/llm_and_nlp/claude-query.py
 examples/llm_and_nlp/hf-dataset-llm-eval.py
 examples/multimodal/clip_inference.py

{datachain-0.18.0 → datachain-0.18.2}/tests/examples/test_examples.py

@@ -12,6 +12,10 @@ llm_and_nlp_examples = sorted(glob.glob("examples/llm_and_nlp/**/*.py", recursiv
 
 multimodal_examples = sorted(glob.glob("examples/multimodal/**/*.py", recursive=True))
 
+incremental_processing_examples = sorted(
+    glob.glob("examples/incremental_processing/delta.py", recursive=True)
+)
+
 computer_vision_examples = sorted(
     [
         filename
@@ -86,6 +90,13 @@ def test_multimodal(example):
     )
 
 
+@pytest.mark.examples
+@pytest.mark.incremental_processing
+@pytest.mark.parametrize("example", incremental_processing_examples)
+def test_incremental_processing_examples(example):
+    smoke_test(example)
+
+
 @pytest.mark.examples
 @pytest.mark.computer_vision
 @pytest.mark.parametrize("example", computer_vision_examples)

datachain-0.18.2/tests/func/test_func.py

@@ -0,0 +1,223 @@
+import math
+
+import datachain as dc
+from datachain import func
+from datachain.sql.types import Float, Int, String
+
+
+def values_almost_equal(a, b):
+    """Compare two values, treating NaNs as equal."""
+    if (
+        isinstance(a, float)
+        and isinstance(b, float)
+        and math.isnan(a)
+        and math.isnan(b)
+    ):
+        return True
+    return a == b
+
+
+def tuples_almost_equal(t1, t2, path=""):
+    """Compare two tuples, treating NaN floats as equal."""
+    if len(t1) != len(t2):
+        raise AssertionError(
+            f"Tuple length mismatch at {path or 'root'}: {len(t1)} != {len(t2)}\n"
+            f"  Left ({type(t1)}): {t1}\n"
+            f"  Right ({type(t2)}): {t2}"
+        )
+
+    for i, (x, y) in enumerate(zip(t1, t2)):
+        subpath = f"{path}[{i}]"
+        if isinstance(x, tuple) and isinstance(y, tuple):
+            tuples_almost_equal(x, y, path=subpath)
+        elif not values_almost_equal(x, y):
+            raise AssertionError(
+                f"Mismatch at {subpath}:\n"
+                f"  Left ({type(x)}): {x}\n"
+                f"  Right ({type(y)}): {y}"
+            )
+
+
+def test_array_slice(test_session):
+    class Arr(dc.DataModel):
+        i: list[int]
+        f: list[float]
+        s: list[str]
+
+    ds = list(
+        dc.read_values(
+            id=[1, 2, 3],
+            arr=(
+                Arr(i=[10, 20, 30], f=[1.0, 2.0, 3.0], s=["a", "b", "c"]),
+                Arr(i=[40, 50, 60], f=[4.0, 5.0, 6.0], s=["d", "e", "f"]),
+                Arr(i=[50], f=[5.0], s=["g"]),
+            ),
+            session=test_session,
+        )
+        .mutate(
+            t1=func.array.slice("arr.i", 1),
+            t2=func.array.slice("arr.i", 100),
+            t3=func.array.slice("arr.f", 0),
+            t4=func.array.slice("arr.f", 1, 1),
+            t5=func.array.slice("arr.s", 2),
+            t6=func.array.slice("arr.s", 1, 10),
+            t7=func.array.slice([9.0], 0),
+            t8=func.array.slice([17], 5),
+            t9=func.array.slice(["a", "b", "c", "d"], 1, 5),
+            t10=func.array.slice(["a", "b", "c", "d"], 100),
+            t11=func.array.slice([], 0),
+        )
+        .order_by("id")
+        .collect("t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9", "t10", "t11")
+    )
+
+    assert tuple(ds) == (
+        (
+            [20, 30],
+            [],
+            [1.0, 2.0, 3.0],
+            [2.0],
+            ["c"],
+            ["b", "c"],
+            [9.0],
+            [],
+            ["b", "c", "d"],
+            [],
+            [],
+        ),
+        (
+            [50, 60],
+            [],
+            [4.0, 5.0, 6.0],
+            [5.0],
+            ["f"],
+            ["e", "f"],
+            [9.0],
+            [],
+            ["b", "c", "d"],
+            [],
+            [],
+        ),
+        (
+            [],
+            [],
+            [5.0],
+            [],
+            [],
+            [],
+            [9.0],
+            [],
+            ["b", "c", "d"],
+            [],
+            [],
+        ),
+    )
+
+
+def test_array_join(test_session):
+    class Arr(dc.DataModel):
+        s: list[str]
+
+    ds = list(
+        dc.read_values(
+            id=[1, 2, 3],
+            arr=(
+                Arr(s=["a", "b", "c"]),
+                Arr(s=["d"]),
+                Arr(s=[]),
+            ),
+            session=test_session,
+        )
+        .mutate(
+            t1=func.array.join("arr.s", "/"),
+            t2=func.array.join("arr.s", ","),
+            t3=func.array.join("arr.s"),
+            t4=func.array.join(["a", "b", "c", "d"], ":"),
+            t5=func.array.join(["1", "2"], ","),
+            t6=func.array.join([]),
+        )
+        .order_by("id")
+        .collect("t1", "t2", "t3", "t4", "t5", "t6")
+    )
+
+    assert tuple(ds) == (
+        ("a/b/c", "a,b,c", "abc", "a:b:c:d", "1,2", ""),
+        ("d", "d", "d", "a:b:c:d", "1,2", ""),
+        ("", "", "", "a:b:c:d", "1,2", ""),
+    )
+
+
+def test_array_get_element(test_session):
+    db_dialect = test_session.catalog.warehouse.db.dialect
+
+    class Arr(dc.DataModel):
+        i: list[int]
+        f: list[float]
+
+    ds = list(
+        dc.read_values(
+            id=[1, 2, 3],
+            arr=(
+                Arr(i=[10, 20, 30], f=[1.0, 2.0, 3.0]),
+                Arr(i=[40, 50, 60], f=[4.0, 5.0, 6.0]),
+                Arr(i=[50], f=[5.0]),
+            ),
+            session=test_session,
+        )
+        .mutate(
+            t1=func.array.get_element("arr.i", 0),
+            t2=func.array.get_element("arr.i", 1),
+            t3=func.array.get_element("arr.i", 100),
+            t4=func.array.get_element("arr.f", 0),
+            t5=func.array.get_element("arr.f", 1),
+            t6=func.array.get_element([9.0], 0),
+            t7=func.array.get_element(["a", "b", "c", "d"], 0),
+            t8=func.array.get_element(["a", "b", "c", "d"], 1),
+            t9=func.array.get_element(["a", "b", "c", "d"], 100),
+            t10=func.array.get_element([], 0),
+        )
+        .order_by("id")
+        .collect("t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9", "t10")
+    )
+
+    tuples_almost_equal(
+        tuple(ds),
+        (
+            (
+                10,
+                20,
+                Int.default_value(db_dialect),
+                1.0,
+                2.0,
+                9.0,
+                "a",
+                "b",
+                String.default_value(db_dialect),
+                String.default_value(db_dialect),
+            ),
+            (
+                40,
+                50,
+                Int.default_value(db_dialect),
+                4.0,
+                5.0,
+                9.0,
+                "a",
+                "b",
+                String.default_value(db_dialect),
+                String.default_value(db_dialect),
+            ),
+            (
+                50,
+                Int.default_value(db_dialect),
+                Int.default_value(db_dialect),
+                5.0,
+                Float.default_value(db_dialect),
+                9.0,
+                "a",
+                "b",
+                String.default_value(db_dialect),
+                String.default_value(db_dialect),
+            ),
+        ),
+    )