datachain 0.18.1__py3-none-any.whl → 0.18.3__py3-none-any.whl

datachain/func/array.py

@@ -178,6 +178,126 @@ def contains(arr: Union[str, Sequence, Func], elem: Any) -> Func:
     return Func("contains", inner=inner, cols=cols, args=args, result_type=int)
 
 
+def slice(
+    arr: Union[str, Sequence, Func],
+    offset: int,
+    length: Optional[int] = None,
+) -> Func:
+    """
+    Returns a slice of the array.
+
+    Args:
+        arr (str | Sequence | Func): Array to check for the element.
+            If a string is provided, it is assumed to be the name of the array column.
+            If a sequence is provided, it is assumed to be an array of values.
+            If a Func is provided, it is assumed to be a function returning an array.
+        offset (int): Offset to start the slice from.
+        length (int, optional): Length of the slice. If not provided, the slice will
+            continue to the end of the array.
+
+    Returns:
+        Func: A Func object that represents the slice function. Result of the
+            function will be a slice of the array starting from the offset
+            and with the given length.
+
+    Example:
+        ```py
+        dc.mutate(
+            contains1=func.array.slice("signal.values", 3),
+            contains2=func.array.slice([1, 2, 3, 4, 5], 1, 3),
+        )
+        ```
+    """
+
+    def inner(arg):
+        if length is not None:
+            return array.slice(arg, offset, length)
+        return array.slice(arg, offset)
+
+    def element_type(el):
+        if isinstance(el, list):
+            try:
+                return list[element_type(el[0])]
+            except IndexError:
+                # if the array is empty, return list[str] as default type
+                return list[str]
+        return type(el)
+
+    def type_from_args(arr, *_):
+        if isinstance(arr, list):
+            try:
+                return list[element_type(arr[0])]
+            except IndexError:
+                # if the array is empty, return list[str] as default type
+                return list[str]
+        return None
+
+    if isinstance(arr, (str, Func)):
+        cols = [arr]
+        args = None
+    else:
+        cols = None
+        args = [arr]
+
+    return Func(
+        "slice",
+        inner=inner,
+        cols=cols,
+        args=args,
+        from_array=True,
+        is_array=True,
+        type_from_args=type_from_args,
+    )
+
+
+def join(
+    arr: Union[str, Sequence, Func],
+    sep: str = "",
+) -> Func:
+    """
+    Returns a string that is the concatenation of the elements of the array,
+
+    Args:
+        arr (str | Sequence | Func): Array to check for the element.
+            If a string is provided, it is assumed to be the name of the array column.
+            If a sequence is provided, it is assumed to be an array of values.
+            If a Func is provided, it is assumed to be a function returning an array.
+        sep (str): Separator to use for the concatenation. Default is an empty string.
+
+    Returns:
+        Func: A Func object that represents the join function. Result of the
+            function will be a string that is the concatenation of the elements
+            of the array, separated by the given separator.
+
+    Example:
+        ```py
+        dc.mutate(
+            contains1=func.array.join("signal.values", ":"),
+            contains2=func.array.join(["1", "2", "3", "4", "5"], "/"),
+        )
+        ```
+    """
+
+    def inner(arg):
+        return array.join(arg, sep)
+
+    if isinstance(arr, (str, Func)):
+        cols = [arr]
+        args = None
+    else:
+        cols = None
+        args = [arr]
+
+    return Func(
+        "join",
+        inner=inner,
+        cols=cols,
+        args=args,
+        from_array=True,
+        result_type=str,
+    )
+
+
 def get_element(arg: Union[str, Sequence, Func], index: int) -> Func:
     """
     Returns the element at the given index from the array.

datachain/func/func.py

@@ -108,18 +108,20 @@ class Func(Function):
                 )
 
         if self.from_array:
-            if get_origin(col_type) is list:
-                col_args = get_args(col_type)
-                if len(col_args) != 1:
-                    raise DataChainColumnError(
-                        str(self),
-                        "Array column must have a single type argument",
-                    )
-                return col_args[0]
-            raise DataChainColumnError(
-                str(self),
-                "Array column must be of type list",
-            )
+            if get_origin(col_type) is not list:
+                raise DataChainColumnError(
+                    str(self),
+                    "Array column must be of type list",
+                )
+            if self.is_array:
+                return col_type
+            col_args = get_args(col_type)
+            if len(col_args) != 1:
+                raise DataChainColumnError(
+                    str(self),
+                    "Array column must have a single type argument",
+                )
+            return col_args[0]
 
         return list[col_type] if self.is_array else col_type  # type: ignore[valid-type]
 

datachain/lib/file.py

@@ -237,7 +237,7 @@ class File(DataModel):
     @field_validator("path", mode="before")
     @classmethod
     def validate_path(cls, path):
-        return Path(path).as_posix()
+        return Path(path).as_posix() if path else ""
 
     def model_dump_custom(self):
         res = self.model_dump()

datachain/query/dataset.py

@@ -1675,13 +1675,27 @@ class DatasetQuery:
         return query
 
     def _add_dependencies(self, dataset: "DatasetRecord", version: str):
-        for dependency in self.dependencies:
-            ds_dependency_name, ds_dependency_version = dependency
+        dependencies: set[DatasetDependencyType] = set()
+        for dep_name, dep_version in self.dependencies:
+            if Session.is_temp_dataset(dep_name):
+                # temp dataset are created for optimization and they will be removed
+                # afterwards. Therefore, we should not put them as dependencies, but
+                # their own direct dependencies
+                for dep in self.catalog.get_dataset_dependencies(
+                    dep_name, dep_version, indirect=False
+                ):
+                    if dep:
+                        dependencies.add((dep.name, dep.version))
+            else:
+                dependencies.add((dep_name, dep_version))
+
+        for dep_name, dep_version in dependencies:
+            # ds_dependency_name, ds_dependency_version = dependency
             self.catalog.metastore.add_dataset_dependency(
                 dataset.name,
                 version,
-                ds_dependency_name,
-                ds_dependency_version,
+                dep_name,
+                dep_version,
             )
 
     def exec(self) -> "Self":

datachain/query/session.py

@@ -195,5 +195,11 @@ class Session:
             Session.GLOBAL_SESSION_CTX.__exit__(None, None, None)
 
         for obj in gc.get_objects():  # Get all tracked objects
-            if isinstance(obj, Session):  # Cleanup temp dataset for session variables.
-                obj.__exit__(None, None, None)
+            try:
+                if isinstance(obj, Session):
+                    # Cleanup temp dataset for session variables.
+                    obj.__exit__(None, None, None)
+            except ReferenceError:
+                continue  # Object has been finalized already
+            except Exception as e:  # noqa: BLE001
+                logger.error(f"Exception while cleaning up session: {e}")  # noqa: G004

datachain/sql/functions/array.py

@@ -1,6 +1,6 @@
 from sqlalchemy.sql.functions import GenericFunction
 
-from datachain.sql.types import Boolean, Float, Int64
+from datachain.sql.types import Boolean, Float, Int64, String
 from datachain.sql.utils import compiler_not_implemented
 
 
@@ -48,6 +48,27 @@ class contains(GenericFunction):  # noqa: N801
     inherit_cache = True
 
 
+class slice(GenericFunction):  # noqa: N801
+    """
+    Returns a slice of the array.
+    """
+
+    package = "array"
+    name = "slice"
+    inherit_cache = True
+
+
+class join(GenericFunction):  # noqa: N801
+    """
+    Returns the concatenation of the array elements.
+    """
+
+    type = String()
+    package = "array"
+    name = "join"
+    inherit_cache = True
+
+
 class get_element(GenericFunction):  # noqa: N801
     """
     Returns the element at the given index in the array.

datachain/sql/sqlite/base.py

@@ -88,6 +88,8 @@ def setup():
     compiles(sql_path.file_ext, "sqlite")(compile_path_file_ext)
     compiles(array.length, "sqlite")(compile_array_length)
     compiles(array.contains, "sqlite")(compile_array_contains)
+    compiles(array.slice, "sqlite")(compile_array_slice)
+    compiles(array.join, "sqlite")(compile_array_join)
     compiles(array.get_element, "sqlite")(compile_array_get_element)
     compiles(string.length, "sqlite")(compile_string_length)
     compiles(string.split, "sqlite")(compile_string_split)
@@ -275,6 +277,15 @@ def register_user_defined_sql_functions() -> None:
         conn.create_function(
             "json_array_get_element", 2, py_json_array_get_element, deterministic=True
         )
+        conn.create_function(
+            "json_array_slice", 2, py_json_array_slice, deterministic=True
+        )
+        conn.create_function(
+            "json_array_slice", 3, py_json_array_slice, deterministic=True
+        )
+        conn.create_function(
+            "json_array_join", 2, py_json_array_join, deterministic=True
+        )
 
     _registered_function_creators["array_functions"] = create_array_functions
 
@@ -454,6 +465,20 @@ def py_json_array_get_element(val, idx):
         return None
 
 
+def py_json_array_slice(val, offset: int, length: Optional[int] = None):
+    arr = orjson.loads(val)
+    try:
+        return orjson.dumps(
+            list(arr[offset : offset + length] if length is not None else arr[offset:])
+        ).decode("utf-8")
+    except IndexError:
+        return None
+
+
+def py_json_array_join(val, sep: str):
+    return sep.join(orjson.loads(val))
+
+
 def compile_array_get_element(element, compiler, **kwargs):
     return compiler.process(
         func.json_array_get_element(*element.clauses.clauses), **kwargs
@@ -470,6 +495,14 @@ def compile_array_contains(element, compiler, **kwargs):
     )
 
 
+def compile_array_slice(element, compiler, **kwargs):
+    return compiler.process(func.json_array_slice(*element.clauses.clauses), **kwargs)
+
+
+def compile_array_join(element, compiler, **kwargs):
+    return compiler.process(func.json_array_join(*element.clauses.clauses), **kwargs)
+
+
 def compile_string_length(element, compiler, **kwargs):
     return compiler.process(func.length(*element.clauses.clauses), **kwargs)
 

{datachain-0.18.1.dist-info → datachain-0.18.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.18.1
+Version: 0.18.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.18.1.dist-info → datachain-0.18.3.dist-info}/RECORD

@@ -58,10 +58,10 @@ datachain/fs/reference.py,sha256=A8McpXF0CqbXPqanXuvpKu50YLB3a2ZXA3YAPxtBXSM,914
 datachain/fs/utils.py,sha256=s-FkTOCGBk-b6TT3toQH51s9608pofoFjUSTc1yy7oE,825
 datachain/func/__init__.py,sha256=CjNLHfJkepdXdRZ6HjJBjNSIjOeFMuMkwPDaPUrM75g,1270
 datachain/func/aggregate.py,sha256=UfxENlw56Qv3UEkj2sZ-JZHmr9q8Rnic9io9_63gF-E,10942
-datachain/func/array.py,sha256=OmmjdK5AQyBXa_7NXFUPY_m3lFlRK4Um4J9NCtYwvak,8394
+datachain/func/array.py,sha256=xWiXYc2OgmndRoNtvczvkbpa9qNQalzFS0S3408jbSA,11872
 datachain/func/base.py,sha256=wA0sBQAVyN9LPxoo7Ox83peS0zUVnyuKxukwAcjGLfY,534
 datachain/func/conditional.py,sha256=HkNamQr9dLyIMDEbIeO6CZR0emQoDqeaWrZ1fECod4M,8062
-datachain/func/func.py,sha256=jzEvnc2iN0BAly-uzxhaoMntL_xF4j94DrFuRi2ADSw,17321
+datachain/func/func.py,sha256=11z24FnhM_U6KQca0er9OSkd4xOTzO3W0tRs0yFg2mM,17375
 datachain/func/numeric.py,sha256=gMe1Ks0dqQKHkjcpvj7I5S-neECzQ_gltPQLNoaWOyo,5632
 datachain/func/path.py,sha256=mqN_mfkwv44z2II7DMTp_fGGw95hmTCNls_TOFNpr4k,3155
 datachain/func/random.py,sha256=pENOLj9rSmWfGCnOsUIaCsVC5486zQb66qfQvXaz9Z4,452
@@ -72,7 +72,7 @@ datachain/lib/arrow.py,sha256=mFO_6wRqzpEzBhXf7Xn1aeLUvaiHcC6XQ-8as9sbcgY,10253
 datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
 datachain/lib/data_model.py,sha256=ZwBXELtqROEdLL4DmxTipnwUZmhQvMz_UVDzyf7nQ9Y,2899
 datachain/lib/dataset_info.py,sha256=d-jz6zeDU5DEgYtyeSF5nK0MU-40FV5km_iOCh4pXzo,3179
-datachain/lib/file.py,sha256=HLQXS_WULm7Y-fkHMy0WpibVAcrkLPRS6CrZy6rwFe0,30450
+datachain/lib/file.py,sha256=0oFm1MWU7AatXplxRj-6Xbjjb6A_AvM_awwk9mYb0hc,30466
 datachain/lib/hf.py,sha256=gjxuStZBlKtNk3-4yYSlWZDv9zBGblOdvEy_Lwap5hA,5882
 datachain/lib/image.py,sha256=butvUY_33PVEYPKX2nVCPeJjJVcBaptZwsE9REQsTS8,3247
 datachain/lib/listing.py,sha256=lFG1Ms6jwm_uqlOcsBUpkmyeSO9TZdtNd820PEpAHP4,7077
@@ -121,13 +121,13 @@ datachain/model/ultralytics/pose.py,sha256=gXAWfAk4OWZl93hKcQPKZvqJa3nIrECB4RM8K
 datachain/model/ultralytics/segment.py,sha256=koq1HASo29isf0in6oSlzmU4IzsmOXe87F1ajQQVfh4,2911
 datachain/query/__init__.py,sha256=7DhEIjAA8uZJfejruAVMZVcGFmvUpffuZJwgRqNwe-c,263
 datachain/query/batch.py,sha256=-goxLpE0EUvaDHu66rstj53UnfHpYfBUGux8GSpJ93k,4306
-datachain/query/dataset.py,sha256=ac4mameklmZ-mnL3ZSzv5n8teaPnoXT8aWCdGlgkZE8,59904
+datachain/query/dataset.py,sha256=3c3MAiIl7ZnCii_0dZA-Om73ornNMSKkna32JX3H05E,60587
 datachain/query/dispatch.py,sha256=15M3zlTUFKM6D2ijITX4o5QxCkRe2klkODsIDi3aQOg,15544
 datachain/query/metrics.py,sha256=DOK5HdNVaRugYPjl8qnBONvTkwjMloLqAr7Mi3TjCO0,858
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
 datachain/query/queue.py,sha256=v0UeK4ilmdiRoJ5OdjB5qpnHTYDxRP4vhVp5Iw_toaI,3512
 datachain/query/schema.py,sha256=b_KnVy6B26Ol4nYG0LqNNpeQ1QYPk95YRGUjXfdaQWs,6606
-datachain/query/session.py,sha256=3nyOvPmLiA86IdHc3BL6Dt_himtHVvaDz_I1h3hZ_gI,6512
+datachain/query/session.py,sha256=6_ydvPasurmc5tR11dzFj51DpUAo4NxXP9p4ltoTauc,6792
 datachain/query/udf.py,sha256=e753bDJzTNjGFQn1WGTvOAWSwjDbrFI1-_DDWkWN2ls,1343
 datachain/query/utils.py,sha256=HaSDNH_XGvp_NIcXjcB7j4vJRPi4_tbztDWclYelHY4,1208
 datachain/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -140,22 +140,22 @@ datachain/sql/default/__init__.py,sha256=XQ2cEZpzWiABqjV-6yYHUBGI9vN_UHxbxZENESm
 datachain/sql/default/base.py,sha256=QD-31C6JnyOXzogyDx90sUhm7QvgXIYpeHEASH84igU,628
 datachain/sql/functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/sql/functions/aggregate.py,sha256=3AQdA8YHPFdtCEfwZKQXTT8SlQWdG9gD5PBtGN3Odqs,944
-datachain/sql/functions/array.py,sha256=LkVDq1Iu3pF2vz9opjxcS0oSvEepoLNYVXcmnOzsmY0,1679
+datachain/sql/functions/array.py,sha256=eRWpDRItwIG87-AU7jb8WuiR-MGuhklVxWwR7t97GvY,2050
 datachain/sql/functions/conditional.py,sha256=q7YUKfunXeEldXaxgT-p5pUTcOEVU_tcQ2BJlquTRPs,207
 datachain/sql/functions/numeric.py,sha256=BK2KCiPSgM2IveCq-9M_PG3CtPBlztaS9TTn1LGzyLs,1250
 datachain/sql/functions/path.py,sha256=zixpERotTFP6LZ7I4TiGtyRA8kXOoZmH1yzH9oRW0mg,1294
 datachain/sql/functions/random.py,sha256=vBwEEj98VH4LjWixUCygQ5Bz1mv1nohsCG0-ZTELlVg,271
 datachain/sql/functions/string.py,sha256=E-T9OIzUR-GKaLgjZsEtg5CJrY_sLf1lt1awTvY7w2w,1426
 datachain/sql/sqlite/__init__.py,sha256=TAdJX0Bg28XdqPO-QwUVKy8rg78cgMileHvMNot7d04,166
-datachain/sql/sqlite/base.py,sha256=SktpdtyZmxG9ip_UX_0WL3YxP9o66CYTeMfriRrZzaE,20281
+datachain/sql/sqlite/base.py,sha256=ldyWoMBvrYH789GK8xE4wbBNYxz4B-g4BmQAK6_KjR0,21398
 datachain/sql/sqlite/types.py,sha256=cH6oge2E_YWFy22wY-txPJH8gxoQFSpCthtZR8PZjpo,1849
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.18.1.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.18.1.dist-info/METADATA,sha256=kzo8enR6CguWxicuamrqzFQm2bqiYITORyVQvt08sHE,11331
-datachain-0.18.1.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-datachain-0.18.1.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.18.1.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.18.1.dist-info/RECORD,,
+datachain-0.18.3.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.18.3.dist-info/METADATA,sha256=DmUXOFYb6qA6yM_O7SvetNEBLbVyJc-qcGqF6DQuV_I,11331
+datachain-0.18.3.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+datachain-0.18.3.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.18.3.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.18.3.dist-info/RECORD,,

{datachain-0.18.1.dist-info → datachain-0.18.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.1)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any