datachain 0.3.7__py3-none-any.whl → 0.3.8__py3-none-any.whl

datachain/catalog/catalog.py

@@ -1540,87 +1540,6 @@ class Catalog:
         dataset = self.get_dataset(name)
         return self.update_dataset(dataset, **update_data)
 
-    def merge_datasets(
-        self,
-        src: DatasetRecord,
-        dst: DatasetRecord,
-        src_version: int,
-        dst_version: Optional[int] = None,
-    ) -> DatasetRecord:
-        """
-        Merges records from source to destination dataset.
-        It will create new version
-        of a dataset with records merged from old version and the source, unless
-        existing version is specified for destination in which case it must
-        be in non final status as datasets are immutable
-        """
-        if (
-            dst_version
-            and not dst.is_valid_next_version(dst_version)
-            and dst.get_version(dst_version).is_final_status()
-        ):
-            raise DatasetInvalidVersionError(
-                f"Version {dst_version} must be higher than the current latest one"
-            )
-
-        src_dep = self.get_dataset_dependencies(src.name, src_version)
-        dst_dep = self.get_dataset_dependencies(
-            dst.name,
-            dst.latest_version,  # type: ignore[arg-type]
-        )
-
-        if dst.has_version(dst_version):  # type: ignore[arg-type]
-            # case where we don't create new version, but append to the existing one
-            self.warehouse.merge_dataset_rows(
-                src,
-                dst,
-                src_version,
-                dst_version=dst_version,  # type: ignore[arg-type]
-            )
-            merged_schema = src.serialized_schema | dst.serialized_schema
-            self.update_dataset(dst, schema=merged_schema)
-            self.update_dataset_version_with_warehouse_info(
-                dst,
-                dst_version,  # type: ignore[arg-type]
-                schema=merged_schema,
-            )
-            for dep in src_dep:
-                if dep and dep not in dst_dep:
-                    self.metastore.add_dependency(
-                        dep,
-                        dst.name,
-                        dst_version,  # type: ignore[arg-type]
-                    )
-        else:
-            # case where we create new version of merged results
-            src_dr = self.warehouse.dataset_rows(src, src_version)
-            dst_dr = self.warehouse.dataset_rows(dst)
-
-            merge_result_columns = list(
-                {
-                    c.name: c for c in list(src_dr.table.c) + list(dst_dr.table.c)
-                }.values()
-            )
-
-            dst_version = dst_version or dst.next_version
-            dst = self.create_new_dataset_version(
-                dst,
-                dst_version,
-                columns=merge_result_columns,
-            )
-            self.warehouse.merge_dataset_rows(
-                src,
-                dst,
-                src_version,
-                dst_version,
-            )
-            self.update_dataset_version_with_warehouse_info(dst, dst_version)
-            for dep in set(src_dep + dst_dep):
-                if dep:
-                    self.metastore.add_dependency(dep, dst.name, dst_version)
-
-        return dst
-
     def get_file_signals(
         self, dataset_name: str, dataset_version: int, row: RowDict
     ) -> Optional[dict]:

datachain/cli.py

@@ -336,36 +336,6 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         help="Display size using powers of 1000 not 1024",
     )
 
-    parse_merge_datasets = subp.add_parser(
-        "merge-datasets", parents=[parent_parser], description="Merges datasets"
-    )
-    parse_merge_datasets.add_argument(
-        "--src",
-        action="store",
-        default=None,
-        help="Source dataset name",
-    )
-    parse_merge_datasets.add_argument(
-        "--dst",
-        action="store",
-        default=None,
-        help="Destination dataset name",
-    )
-    parse_merge_datasets.add_argument(
-        "--src-version",
-        action="store",
-        default=None,
-        type=int,
-        help="Source dataset version",
-    )
-    parse_merge_datasets.add_argument(
-        "--dst-version",
-        action="store",
-        default=None,
-        type=int,
-        help="Destination dataset version",
-    )
-
     parse_ls = subp.add_parser(
         "ls", parents=[parent_parser], description="List storage contents"
     )
@@ -996,13 +966,6 @@ def main(argv: Optional[list[str]] = None) -> int:  # noqa: C901, PLR0912, PLR09
                 new_name=args.new_name,
                 labels=args.labels,
             )
-        elif args.command == "merge-datasets":
-            catalog.merge_datasets(
-                catalog.get_dataset(args.src),
-                catalog.get_dataset(args.dst),
-                args.src_version,
-                dst_version=args.dst_version,
-            )
         elif args.command == "ls":
             ls(
                 args.sources,

datachain/lib/arrow.py

@@ -95,7 +95,7 @@ def schema_to_output(schema: pa.Schema, col_names: Optional[Sequence[str]] = Non
         if not column:
             column = f"c{default_column}"
             default_column += 1
-        dtype = _arrow_type_mapper(field.type)  # type: ignore[assignment]
+        dtype = arrow_type_mapper(field.type)  # type: ignore[assignment]
         if field.nullable:
             dtype = Optional[dtype]  # type: ignore[assignment]
         output[column] = dtype
@@ -103,7 +103,7 @@ def schema_to_output(schema: pa.Schema, col_names: Optional[Sequence[str]] = Non
     return output
 
 
-def _arrow_type_mapper(col_type: pa.DataType) -> type:  # noqa: PLR0911
+def arrow_type_mapper(col_type: pa.DataType) -> type:  # noqa: PLR0911
     """Convert pyarrow types to basic types."""
     from datetime import datetime
 
@@ -122,11 +122,11 @@ def _arrow_type_mapper(col_type: pa.DataType) -> type:  # noqa: PLR0911
     if pa.types.is_string(col_type) or pa.types.is_large_string(col_type):
         return str
     if pa.types.is_list(col_type):
-        return list[_arrow_type_mapper(col_type.value_type)]  # type: ignore[return-value, misc]
+        return list[arrow_type_mapper(col_type.value_type)]  # type: ignore[return-value, misc]
     if pa.types.is_struct(col_type) or pa.types.is_map(col_type):
         return dict
     if isinstance(col_type, pa.lib.DictionaryType):
-        return _arrow_type_mapper(col_type.value_type)  # type: ignore[return-value]
+        return arrow_type_mapper(col_type.value_type)  # type: ignore[return-value]
     raise TypeError(f"{col_type!r} datatypes not supported")
 
 

datachain/lib/clip.py

@@ -1,5 +1,5 @@
 import inspect
-from typing import TYPE_CHECKING, Any, Callable, Literal, Union
+from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Union
 
 import torch
 from transformers.modeling_utils import PreTrainedModel
@@ -39,6 +39,7 @@ def clip_similarity_scores(
     tokenizer: Callable,
     prob: bool = False,
     image_to_text: bool = True,
+    device: Optional[Union[str, torch.device]] = None,
 ) -> list[list[float]]:
     """
     Calculate CLIP similarity scores between one or more images and/or text.
@@ -52,6 +53,7 @@ def clip_similarity_scores(
         prob : Compute softmax probabilities.
         image_to_text : Whether to compute for image-to-text or text-to-image. Ignored
             if only one of images or text provided.
+        device : Device to use. Defaults is None - use model's device.
 
 
     Example:
@@ -130,17 +132,26 @@ def clip_similarity_scores(
         ```
     """
 
+    if device is None:
+        if hasattr(model, "device"):
+            device = model.device
+        else:
+            device = next(model.parameters()).device
+    else:
+        model = model.to(device)
     with torch.no_grad():
         if images is not None:
             encoder = _get_encoder(model, "image")
             image_features = convert_images(
-                images, transform=preprocess, encoder=encoder
+                images, transform=preprocess, encoder=encoder, device=device
             )
             image_features /= image_features.norm(dim=-1, keepdim=True)  # type: ignore[union-attr]
 
         if text is not None:
             encoder = _get_encoder(model, "text")
-            text_features = convert_text(text, tokenizer, encoder=encoder)
+            text_features = convert_text(
+                text, tokenizer, encoder=encoder, device=device
+            )
             text_features /= text_features.norm(dim=-1, keepdim=True)  # type: ignore[union-attr]
 
         if images is not None and text is not None:

datachain/lib/convert/python_to_sql.py

@@ -73,6 +73,9 @@ def python_to_sql(typ):  # noqa: PLR0911
         if len(args) == 2 and (type(None) in args):
             return python_to_sql(args[0])
 
+        if _is_union_str_literal(orig, args):
+            return String
+
         if _is_json_inside_union(orig, args):
             return JSON
 
@@ -94,3 +97,9 @@ def _is_json_inside_union(orig, args) -> bool:
         if any(inspect.isclass(arg) and issubclass(arg, BaseModel) for arg in args):
             return True
     return False
+
+
+def _is_union_str_literal(orig, args) -> bool:
+    if orig != Union:
+        return False
+    return all(arg is str or get_origin(arg) in (Literal, LiteralEx) for arg in args)

datachain/lib/data_model.py

@@ -2,7 +2,7 @@ from collections.abc import Sequence
 from datetime import datetime
 from typing import ClassVar, Union, get_args, get_origin
 
-from pydantic import BaseModel
+from pydantic import BaseModel, create_model
 
 from datachain.lib.model_store import ModelStore
 
@@ -57,3 +57,12 @@ def is_chain_type(t: type) -> bool:
         return is_chain_type(args[0])
 
     return False
+
+
+def dict_to_data_model(name: str, data_dict: dict[str, DataType]) -> type[BaseModel]:
+    fields = {name: (anno, ...) for name, anno in data_dict.items()}
+    return create_model(
+        name,
+        __base__=(DataModel,),  # type: ignore[call-overload]
+        **fields,
+    )  # type: ignore[call-overload]

datachain/lib/dc.py

@@ -18,14 +18,13 @@ from typing import (
 
 import pandas as pd
 import sqlalchemy
-from pydantic import BaseModel, create_model
+from pydantic import BaseModel
 from sqlalchemy.sql.functions import GenericFunction
 from sqlalchemy.sql.sqltypes import NullType
 
-from datachain import DataModel
 from datachain.lib.convert.python_to_sql import python_to_sql
 from datachain.lib.convert.values_to_tuples import values_to_tuples
-from datachain.lib.data_model import DataType
+from datachain.lib.data_model import DataModel, DataType, dict_to_data_model
 from datachain.lib.dataset_info import DatasetInfo
 from datachain.lib.file import ExportPlacement as FileExportPlacement
 from datachain.lib.file import File, IndexedFile, get_file
@@ -55,6 +54,8 @@ from datachain.utils import inside_notebook
 if TYPE_CHECKING:
     from typing_extensions import Concatenate, ParamSpec, Self
 
+    from datachain.lib.hf import HFDatasetType
+
     P = ParamSpec("P")
 
 C = Column
@@ -77,12 +78,12 @@ def resolve_columns(
     @wraps(method)
     def _inner(self: D, *args: "P.args", **kwargs: "P.kwargs") -> D:
         resolved_args = self.signals_schema.resolve(
-            *[arg for arg in args if not isinstance(arg, GenericFunction)]
+            *[arg for arg in args if not isinstance(arg, GenericFunction)]  # type: ignore[arg-type]
         ).db_signals()
 
         for idx, arg in enumerate(args):
             if isinstance(arg, GenericFunction):
-                resolved_args.insert(idx, arg)
+                resolved_args.insert(idx, arg)  # type: ignore[arg-type]
 
         return method(self, *resolved_args, **kwargs)
 
@@ -208,23 +209,28 @@ class DataChain(DatasetQuery):
         "size": 0,
     }
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, settings: Optional[dict] = None, **kwargs):
         """This method needs to be redefined as a part of Dataset and DataChain
         decoupling.
         """
-        super().__init__(
+        super().__init__(  # type: ignore[misc]
             *args,
             **kwargs,
             indexing_column_types=File._datachain_column_types,
         )
-        self._settings = Settings()
-        self._setup = {}
+        if settings:
+            self._settings = Settings(**settings)
+        else:
+            self._settings = Settings()
+        self._setup: dict = {}
 
         self.signals_schema = SignalSchema({"sys": Sys})
         if self.feature_schema:
             self.signals_schema |= SignalSchema.deserialize(self.feature_schema)
         else:
-            self.signals_schema |= SignalSchema.from_column_types(self.column_types)
+            self.signals_schema |= SignalSchema.from_column_types(
+                self.column_types or {}
+            )
 
         self._sys = False
 
@@ -309,6 +315,7 @@ class DataChain(DatasetQuery):
         *,
         type: Literal["binary", "text", "image"] = "binary",
         session: Optional[Session] = None,
+        settings: Optional[dict] = None,
         in_memory: bool = False,
         recursive: Optional[bool] = True,
         object_name: str = "file",
@@ -336,6 +343,7 @@ class DataChain(DatasetQuery):
             cls(
                 path,
                 session=session,
+                settings=settings,
                 recursive=recursive,
                 update=update,
                 in_memory=in_memory,
@@ -489,6 +497,7 @@ class DataChain(DatasetQuery):
     def datasets(
         cls,
         session: Optional[Session] = None,
+        settings: Optional[dict] = None,
         in_memory: bool = False,
         object_name: str = "dataset",
     ) -> "DataChain":
@@ -513,6 +522,7 @@ class DataChain(DatasetQuery):
 
         return cls.from_values(
             session=session,
+            settings=settings,
             in_memory=in_memory,
             output={object_name: DatasetInfo},
             **{object_name: datasets},  # type: ignore[arg-type]
@@ -895,7 +905,7 @@ class DataChain(DatasetQuery):
             if isinstance(value, Column):
                 # renaming existing column
                 for signal in schema.db_signals(name=value.name, as_columns=True):
-                    mutated[signal.name.replace(value.name, name, 1)] = signal
+                    mutated[signal.name.replace(value.name, name, 1)] = signal  # type: ignore[union-attr]
             else:
                 # adding new signal
                 mutated[name] = value
@@ -1086,7 +1096,7 @@ class DataChain(DatasetQuery):
             )
 
         signals_schema = self.signals_schema.clone_without_sys_signals()
-        on_columns = signals_schema.resolve(*on).db_signals()
+        on_columns: list[str] = signals_schema.resolve(*on).db_signals()  # type: ignore[assignment]
 
         right_signals_schema = right_ds.signals_schema.clone_without_sys_signals()
         if right_on is not None:
@@ -1105,7 +1115,9 @@ class DataChain(DatasetQuery):
                     on, right_on, "'on' and 'right_on' must have the same length'"
                 )
 
-            right_on_columns = right_signals_schema.resolve(*right_on).db_signals()
+            right_on_columns: list[str] = right_signals_schema.resolve(
+                *right_on
+            ).db_signals()  # type: ignore[assignment]
 
             if len(right_on_columns) != len(on_columns):
                 on_str = ", ".join(right_on_columns)
@@ -1169,7 +1181,7 @@ class DataChain(DatasetQuery):
                 "'on' cannot be empty",
             )
         else:
-            signals = self.signals_schema.resolve(*on).db_signals()
+            signals = self.signals_schema.resolve(*on).db_signals()  # type: ignore[assignment]
         return super()._subtract(other, signals)  # type: ignore[arg-type]
 
     @classmethod
@@ -1177,6 +1189,7 @@ class DataChain(DatasetQuery):
         cls,
         ds_name: str = "",
         session: Optional[Session] = None,
+        settings: Optional[dict] = None,
         in_memory: bool = False,
         output: OutputType = None,
         object_name: str = "",
@@ -1195,10 +1208,13 @@ class DataChain(DatasetQuery):
             yield from tuples
 
         chain = DataChain.from_records(
-            DataChain.DEFAULT_FILE_RECORD, session=session, in_memory=in_memory
+            DataChain.DEFAULT_FILE_RECORD,
+            session=session,
+            settings=settings,
+            in_memory=in_memory,
         )
         if object_name:
-            output = {object_name: DataChain._dict_to_data_model(object_name, output)}  # type: ignore[arg-type]
+            output = {object_name: dict_to_data_model(object_name, output)}  # type: ignore[arg-type]
         return chain.gen(_func_fr, output=output)
 
     @classmethod
@@ -1207,6 +1223,7 @@ class DataChain(DatasetQuery):
         df: "pd.DataFrame",
         name: str = "",
         session: Optional[Session] = None,
+        settings: Optional[dict] = None,
         in_memory: bool = False,
         object_name: str = "",
     ) -> "DataChain":
@@ -1236,7 +1253,12 @@ class DataChain(DatasetQuery):
                 )
 
         return cls.from_values(
-            name, session, object_name=object_name, in_memory=in_memory, **fr_map
+            name,
+            session,
+            settings=settings,
+            object_name=object_name,
+            in_memory=in_memory,
+            **fr_map,
         )
 
     def to_pandas(self, flatten=False) -> "pd.DataFrame":
@@ -1306,6 +1328,59 @@ class DataChain(DatasetQuery):
         if len(df) == limit:
             print(f"\n[Limited by {len(df)} rows]")
 
+    @classmethod
+    def from_hf(
+        cls,
+        dataset: Union[str, "HFDatasetType"],
+        *args,
+        session: Optional[Session] = None,
+        settings: Optional[dict] = None,
+        object_name: str = "",
+        model_name: str = "",
+        **kwargs,
+    ) -> "DataChain":
+        """Generate chain from huggingface hub dataset.
+
+        Parameters:
+            dataset : Path or name of the dataset to read from Hugging Face Hub,
+                or an instance of `datasets.Dataset`-like object.
+            session : Session to use for the chain.
+            settings : Settings to use for the chain.
+            object_name : Generated object column name.
+            model_name : Generated model name.
+            kwargs : Parameters to pass to datasets.load_dataset.
+
+        Example:
+            Load from Hugging Face Hub:
+            ```py
+            DataChain.from_hf("beans", split="train")
+            ```
+
+            Generate chain from loaded dataset:
+            ```py
+            from datasets import load_dataset
+            ds = load_dataset("beans", split="train")
+            DataChain.from_hf(ds)
+            ```
+        """
+        from datachain.lib.hf import HFGenerator, get_output_schema, stream_splits
+
+        output: dict[str, DataType] = {}
+        ds_dict = stream_splits(dataset, *args, **kwargs)
+        if len(ds_dict) > 1:
+            output = {"split": str}
+
+        model_name = model_name or object_name or ""
+        output = output | get_output_schema(next(iter(ds_dict.values())), model_name)
+        model = dict_to_data_model(model_name, output)
+        if object_name:
+            output = {object_name: model}
+
+        chain = DataChain.from_values(
+            split=list(ds_dict.keys()), session=session, settings=settings
+        )
+        return chain.gen(HFGenerator(dataset, model, *args, **kwargs), output=output)
+
     def parse_tabular(
         self,
         output: OutputType = None,
@@ -1367,7 +1442,7 @@ class DataChain(DatasetQuery):
 
         if isinstance(output, dict):
             model_name = model_name or object_name or ""
-            model = DataChain._dict_to_data_model(model_name, output)
+            model = dict_to_data_model(model_name, output)
         else:
             model = output  # type: ignore[assignment]
 
@@ -1384,17 +1459,6 @@ class DataChain(DatasetQuery):
             ArrowGenerator(schema, model, source, nrows, **kwargs), output=output
         )
 
-    @staticmethod
-    def _dict_to_data_model(
-        name: str, data_dict: dict[str, DataType]
-    ) -> type[BaseModel]:
-        fields = {name: (anno, ...) for name, anno in data_dict.items()}
-        return create_model(
-            name,
-            __base__=(DataModel,),  # type: ignore[call-overload]
-            **fields,
-        )  # type: ignore[call-overload]
-
     @classmethod
     def from_csv(
         cls,
@@ -1543,6 +1607,7 @@ class DataChain(DatasetQuery):
         cls,
         to_insert: Optional[Union[dict, list[dict]]],
         session: Optional[Session] = None,
+        settings: Optional[dict] = None,
         in_memory: bool = False,
         schema: Optional[dict[str, DataType]] = None,
     ) -> "DataChain":
@@ -1597,7 +1662,7 @@ class DataChain(DatasetQuery):
         insert_q = dr.get_table().insert()
         for record in to_insert:
             db.execute(insert_q.values(**record))
-        return DataChain(name=dsr.name)
+        return DataChain(name=dsr.name, settings=settings)
 
     def sum(self, fr: DataType):  # type: ignore[override]
         """Compute the sum of a column."""

datachain/lib/hf.py

@@ -0,0 +1,166 @@
+try:
+    from datasets import (
+        Array2D,
+        Array3D,
+        Array4D,
+        Array5D,
+        Audio,
+        ClassLabel,
+        Dataset,
+        DatasetDict,
+        Image,
+        IterableDataset,
+        IterableDatasetDict,
+        Sequence,
+        Value,
+        load_dataset,
+    )
+    from datasets.features.features import string_to_arrow
+    from datasets.features.image import image_to_bytes
+
+except ImportError as exc:
+    raise ImportError(
+        "Missing dependencies for huggingface datasets:\n"
+        "To install run:\n\n"
+        "  pip install 'datachain[hf]'\n"
+    ) from exc
+
+from io import BytesIO
+from typing import TYPE_CHECKING, Any, Union
+
+import PIL
+from tqdm import tqdm
+
+from datachain.lib.arrow import arrow_type_mapper
+from datachain.lib.data_model import DataModel, DataType, dict_to_data_model
+from datachain.lib.udf import Generator
+
+if TYPE_CHECKING:
+    from pydantic import BaseModel
+
+
+HFDatasetType = Union[DatasetDict, Dataset, IterableDatasetDict, IterableDataset]
+
+
+class HFClassLabel(DataModel):
+    string: str
+    integer: int
+
+    def read(self):
+        return self.integer
+
+
+class HFImage(DataModel):
+    img: bytes
+
+    def read(self):
+        return PIL.Image.open(BytesIO(self.img))
+
+
+class HFAudio(DataModel):
+    path: str
+    array: list[float]
+    sampling_rate: int
+
+
+class HFGenerator(Generator):
+    def __init__(
+        self,
+        ds: Union[str, HFDatasetType],
+        output_schema: type["BaseModel"],
+        *args,
+        **kwargs,
+    ):
+        super().__init__()
+        self.ds = ds
+        self.output_schema = output_schema
+        self.args = args
+        self.kwargs = kwargs
+
+    def setup(self):
+        self.ds_dict = stream_splits(self.ds, *self.args, **self.kwargs)
+
+    def process(self, split: str = ""):
+        desc = "Parsed Hugging Face dataset"
+        ds = self.ds_dict[split]
+        if split:
+            desc += f" split '{split}'"
+        with tqdm(desc=desc, unit=" rows") as pbar:
+            for row in ds:
+                output_dict = {}
+                if split:
+                    output_dict["split"] = split
+                for name, feat in ds.features.items():
+                    anno = self.output_schema.model_fields[name].annotation
+                    output_dict[name] = _convert_feature(row[name], feat, anno)
+                yield self.output_schema(**output_dict)
+                pbar.update(1)
+
+
+def stream_splits(ds: Union[str, HFDatasetType], *args, **kwargs):
+    if isinstance(ds, str):
+        ds = load_dataset(ds, *args, streaming=True, **kwargs)
+    if isinstance(ds, (DatasetDict, IterableDatasetDict)):
+        return ds
+    return {"": ds}
+
+
+def _convert_feature(val: Any, feat: Any, anno: Any) -> Any:
+    if isinstance(feat, (Value, Array2D, Array3D, Array4D, Array5D)):
+        return val
+    if isinstance(feat, ClassLabel):
+        return HFClassLabel(string=feat.names[val], integer=val)
+    if isinstance(feat, Sequence):
+        if isinstance(feat.feature, dict):
+            sdict = {}
+            for sname in val:
+                sfeat = feat.feature[sname]
+                sanno = anno.model_fields[sname].annotation
+                sdict[sname] = [_convert_feature(v, sfeat, sanno) for v in val[sname]]
+            return anno(**sdict)
+        return val
+    if isinstance(feat, Image):
+        return HFImage(img=image_to_bytes(val))
+    if isinstance(feat, Audio):
+        return HFAudio(**val)
+
+
+def get_output_schema(
+    ds: Union[Dataset, IterableDataset], model_name: str = ""
+) -> dict[str, DataType]:
+    fields_dict = {}
+    for name, val in ds.features.items():
+        fields_dict[name] = _feature_to_chain_type(name, val)  # type: ignore[assignment]
+    return fields_dict  # type: ignore[return-value]
+
+
+def _feature_to_chain_type(name: str, val: Any) -> type:  # noqa: PLR0911
+    if isinstance(val, Value):
+        return arrow_type_mapper(val.pa_type)
+    if isinstance(val, ClassLabel):
+        return HFClassLabel
+    if isinstance(val, Sequence):
+        if isinstance(val.feature, dict):
+            sequence_dict = {}
+            for sname, sval in val.feature.items():
+                dtype = _feature_to_chain_type(sname, sval)
+                sequence_dict[sname] = list[dtype]  # type: ignore[valid-type]
+            return dict_to_data_model(name, sequence_dict)  # type: ignore[arg-type]
+        return list[_feature_to_chain_type(name, val.feature)]  # type: ignore[arg-type,misc,return-value]
+    if isinstance(val, Array2D):
+        dtype = arrow_type_mapper(string_to_arrow(val.dtype))
+        return list[list[dtype]]  # type: ignore[valid-type]
+    if isinstance(val, Array3D):
+        dtype = arrow_type_mapper(string_to_arrow(val.dtype))
+        return list[list[list[dtype]]]  # type: ignore[valid-type]
+    if isinstance(val, Array4D):
+        dtype = arrow_type_mapper(string_to_arrow(val.dtype))
+        return list[list[list[list[dtype]]]]  # type: ignore[valid-type]
+    if isinstance(val, Array5D):
+        dtype = arrow_type_mapper(string_to_arrow(val.dtype))
+        return list[list[list[list[list[dtype]]]]]  # type: ignore[valid-type]
+    if isinstance(val, Image):
+        return HFImage
+    if isinstance(val, Audio):
+        return HFAudio
+    raise TypeError(f"Unknown huggingface datasets type {type(val)}")

datachain/lib/image.py

@@ -10,6 +10,7 @@ def convert_image(
     size: Optional[tuple[int, int]] = None,
     transform: Optional[Callable] = None,
     encoder: Optional[Callable] = None,
+    device: Optional[Union[str, torch.device]] = None,
 ) -> Union[Image.Image, torch.Tensor]:
     """
     Resize, transform, and otherwise convert an image.
@@ -20,6 +21,7 @@ def convert_image(
         size (tuple[int, int]): Size in (width, height) pixels for resizing.
         transform (Callable): Torchvision transform or huggingface processor to apply.
         encoder (Callable): Encode image using model.
+        device (str or torch.device): Device to use.
     """
     if mode:
         img = img.convert(mode)
@@ -35,6 +37,8 @@ def convert_image(
                 img = torch.tensor(img.pixel_values[0])  # type: ignore[assignment,attr-defined]
         except ImportError:
             pass
+        if device:
+            img = img.to(device)  # type: ignore[attr-defined]
         if encoder:
             img = img.unsqueeze(0)  # type: ignore[attr-defined]
     if encoder:
@@ -48,6 +52,7 @@ def convert_images(
     size: Optional[tuple[int, int]] = None,
     transform: Optional[Callable] = None,
     encoder: Optional[Callable] = None,
+    device: Optional[Union[str, torch.device]] = None,
 ) -> Union[list[Image.Image], torch.Tensor]:
     """
     Resize, transform, and otherwise convert one or more images.
@@ -58,11 +63,14 @@ def convert_images(
         size (tuple[int, int]): Size in (width, height) pixels for resizing.
         transform (Callable): Torchvision transform or huggingface processor to apply.
         encoder (Callable): Encode image using model.
+        device (str or torch.device): Device to use.
     """
     if isinstance(images, Image.Image):
         images = [images]
 
-    converted = [convert_image(img, mode, size, transform) for img in images]
+    converted = [
+        convert_image(img, mode, size, transform, device=device) for img in images
+    ]
 
     if isinstance(converted[0], torch.Tensor):
         converted = torch.stack(converted)  # type: ignore[assignment,arg-type]

datachain/lib/pytorch.py

@@ -10,7 +10,6 @@ from torchvision.transforms import v2
 
 from datachain.catalog import Catalog, get_catalog
 from datachain.lib.dc import DataChain
-from datachain.lib.file import File
 from datachain.lib.text import convert_text
 
 if TYPE_CHECKING:
@@ -97,7 +96,7 @@ class PytorchDataset(IterableDataset):
         for row_features in ds.collect():
             row = []
             for fr in row_features:
-                if isinstance(fr, File):
+                if hasattr(fr, "read"):
                     row.append(fr.read())  # type: ignore[unreachable]
                 else:
                     row.append(fr)

datachain/lib/signal_schema.py

@@ -1,4 +1,5 @@
 import copy
+import warnings
 from collections.abc import Iterator, Sequence
 from dataclasses import dataclass
 from datetime import datetime
@@ -42,6 +43,8 @@ NAMES_TO_TYPES = {
     "dict": dict,
     "bytes": bytes,
     "datetime": datetime,
+    "Literal": Literal,
+    "Union": Union,
 }
 
 
@@ -49,6 +52,10 @@ class SignalSchemaError(DataChainParamsError):
     pass
 
 
+class SignalSchemaWarning(RuntimeWarning):
+    pass
+
+
 class SignalResolvingError(SignalSchemaError):
     def __init__(self, path: Optional[list[str]], msg: str):
         name = " '" + ".".join(path) + "'" if path else ""
@@ -69,6 +76,28 @@ class SignalResolvingTypeError(SignalResolvingError):
         )
 
 
+def create_feature_model(
+    name: str, fields: dict[str, Union[type, tuple[type, Any]]]
+) -> type[BaseModel]:
+    """
+    This gets or returns a dynamic feature model for use in restoring a model
+    from the custom_types stored within a serialized SignalSchema. This is useful
+    when using a custom feature model where the original definition is not available.
+    This happens in Studio and if a custom model is used in a dataset, then that dataset
+    is used in a DataChain in a separate script where that model is not declared.
+    """
+    name = name.replace("@", "_")
+    return create_model(
+        name,
+        __base__=DataModel,  # type: ignore[call-overload]
+        # These are tuples for each field of: annotation, default (if any)
+        **{
+            field_name: anno if isinstance(anno, tuple) else (anno, None)
+            for field_name, anno in fields.items()
+        },
+    )
+
+
 @dataclass
 class SignalSchema:
     values: dict[str, DataType]
@@ -117,40 +146,115 @@ class SignalSchema:
                 )
         return SignalSchema(signals)
 
-    def serialize(self) -> dict[str, str]:
-        signals = {}
+    @staticmethod
+    def _get_name_original_type(fr_type: type) -> tuple[str, type]:
+        """Returns the name of and the original type for the given type,
+        based on whether the type is Optional or not."""
+        orig = get_origin(fr_type)
+        args = get_args(fr_type)
+        # Check if fr_type is Optional
+        if orig == Union and len(args) == 2 and (type(None) in args):
+            fr_type = args[0]
+            orig = get_origin(fr_type)
+        if orig in (Literal, LiteralEx):
+            # Literal has no __name__ in Python 3.9
+            type_name = "Literal"
+        elif orig == Union:
+            # Union also has no __name__ in Python 3.9
+            type_name = "Union"
+        else:
+            type_name = str(fr_type.__name__)  # type: ignore[union-attr]
+        return type_name, fr_type
+
+    @staticmethod
+    def serialize_custom_model_fields(
+        name: str, fr: type, custom_types: dict[str, Any]
+    ) -> str:
+        """This serializes any custom type information to the provided custom_types
+        dict, and returns the name of the type provided."""
+        if hasattr(fr, "__origin__") or not issubclass(fr, BaseModel):
+            # Don't store non-feature types.
+            return name
+        version_name = ModelStore.get_name(fr)
+        if version_name in custom_types:
+            # This type is already stored in custom_types.
+            return version_name
+        fields = {}
+        for field_name, info in fr.model_fields.items():
+            field_type = info.annotation
+            # All fields should be typed.
+            assert field_type
+            field_type_name, field_type = SignalSchema._get_name_original_type(
+                field_type
+            )
+            # Serialize this type to custom_types if it is a custom type as well.
+            fields[field_name] = SignalSchema.serialize_custom_model_fields(
+                field_type_name, field_type, custom_types
+            )
+        custom_types[version_name] = fields
+        return version_name
+
+    def serialize(self) -> dict[str, Any]:
+        signals: dict[str, Any] = {}
+        custom_types: dict[str, Any] = {}
         for name, fr_type in self.values.items():
             if (fr := ModelStore.to_pydantic(fr_type)) is not None:
                 ModelStore.register(fr)
                 signals[name] = ModelStore.get_name(fr)
+                type_name, fr_type = SignalSchema._get_name_original_type(fr)
             else:
-                orig = get_origin(fr_type)
-                args = get_args(fr_type)
-                # Check if fr_type is Optional
-                if orig == Union and len(args) == 2 and (type(None) in args):
-                    fr_type = args[0]
-                signals[name] = str(fr_type.__name__)  # type: ignore[union-attr]
+                type_name, fr_type = SignalSchema._get_name_original_type(fr_type)
+                signals[name] = type_name
+            self.serialize_custom_model_fields(type_name, fr_type, custom_types)
+        if custom_types:
+            signals["_custom_types"] = custom_types
         return signals
 
     @staticmethod
-    def deserialize(schema: dict[str, str]) -> "SignalSchema":
+    def _resolve_type(type_name: str, custom_types: dict[str, Any]) -> Optional[type]:
+        """Convert a string-based type back into a python type."""
+        fr = NAMES_TO_TYPES.get(type_name)
+        if fr:
+            return fr  # type: ignore[return-value]
+
+        model_name, version = ModelStore.parse_name_version(type_name)
+        fr = ModelStore.get(model_name, version)
+        if fr:
+            return fr
+
+        if type_name in custom_types:
+            fields = custom_types[type_name]
+            fields = {
+                field_name: SignalSchema._resolve_type(field_type_str, custom_types)
+                for field_name, field_type_str in fields.items()
+            }
+            return create_feature_model(type_name, fields)
+        return None
+
+    @staticmethod
+    def deserialize(schema: dict[str, Any]) -> "SignalSchema":
         if not isinstance(schema, dict):
             raise SignalSchemaError(f"cannot deserialize signal schema: {schema}")
 
         signals: dict[str, DataType] = {}
+        custom_types: dict[str, Any] = schema.get("_custom_types", {})
         for signal, type_name in schema.items():
+            if signal == "_custom_types":
+                # This entry is used as a lookup for custom types,
+                # and is not an actual field.
+                continue
             try:
-                fr = NAMES_TO_TYPES.get(type_name)
-                if not fr:
-                    type_name, version = ModelStore.parse_name_version(type_name)
-                    fr = ModelStore.get(type_name, version)
-
-                    if not fr:
-                        raise SignalSchemaError(
-                            f"cannot deserialize '{signal}': "
-                            f"unknown type '{type_name}'."
-                            f" Try to add it with `ModelStore.register({type_name})`."
-                        )
+                fr = SignalSchema._resolve_type(type_name, custom_types)
+                if fr is None:
+                    # Skip if the type is not found, so all data can be displayed.
+                    warnings.warn(
+                        f"In signal '{signal}': "
+                        f"unknown type '{type_name}'."
+                        f" Try to add it with `ModelStore.register({type_name})`.",
+                        SignalSchemaWarning,
+                        stacklevel=2,
+                    )
+                    continue
             except TypeError as err:
                 raise SignalSchemaError(
                     f"cannot deserialize '{signal}': {err}"

datachain/lib/text.py

@@ -9,6 +9,7 @@ def convert_text(
     tokenizer: Optional[Callable] = None,
     tokenizer_kwargs: Optional[dict[str, Any]] = None,
     encoder: Optional[Callable] = None,
+    device: Optional[Union[str, torch.device]] = None,
 ) -> Union[str, list[str], torch.Tensor]:
     """
     Tokenize and otherwise transform text.
@@ -18,6 +19,7 @@ def convert_text(
         tokenizer (Callable): Tokenizer to use to tokenize objects.
         tokenizer_kwargs (dict): Additional kwargs to pass when calling tokenizer.
         encoder (Callable): Encode text using model.
+        device (str or torch.device): Device to use.
     """
     if not tokenizer:
         return text
@@ -32,6 +34,8 @@ def convert_text(
 
     tokens = res.input_ids if isinstance(tokenizer, PreTrainedTokenizerBase) else res
     tokens = torch.tensor(tokens)
+    if device:
+        tokens = tokens.to(device)
 
     if not encoder:
         return tokens

datachain/lib/udf.py

@@ -242,26 +242,8 @@ class UDFBase(AbstractUDF):
         if not self.is_output_batched:
             result_objs = [result_objs]
 
-        if len(self.output.values) > 1:
-            res = []
-            for tuple_ in result_objs:
-                flat = []
-                for obj in tuple_:
-                    if isinstance(obj, BaseModel):
-                        flat.extend(flatten(obj))
-                    else:
-                        flat.append(obj)
-                res.append(tuple(flat))
-        else:
-            # Generator expression is required, otherwise the value will be materialized
-            res = (
-                flatten(obj)
-                if isinstance(obj, BaseModel)
-                else obj
-                if isinstance(obj, tuple)
-                else (obj,)
-                for obj in result_objs
-            )
+        # Generator expression is required, otherwise the value will be materialized
+        res = (self._flatten_row(row) for row in result_objs)
 
         if not self.is_output_batched:
             res = list(res)
@@ -282,6 +264,18 @@ class UDFBase(AbstractUDF):
 
         return res
 
+    def _flatten_row(self, row):
+        if len(self.output.values) > 1 and not isinstance(row, BaseModel):
+            flat = []
+            for obj in row:
+                flat.extend(self._obj_to_list(obj))
+            return tuple(flat)
+        return row if isinstance(row, tuple) else tuple(self._obj_to_list(row))
+
+    @staticmethod
+    def _obj_to_list(obj):
+        return flatten(obj) if isinstance(obj, BaseModel) else [obj]
+
     def _parse_rows(self, rows, cache, download_cb):
         objs = []
         for row in rows:

datachain/query/dataset.py

@@ -24,6 +24,7 @@ from typing import (
 )
 
 import attrs
+import psutil
 import sqlalchemy
 import sqlalchemy as sa
 from attrs import frozen
@@ -383,7 +384,7 @@ def process_udf_outputs(
     udf_table: "Table",
     udf_results: Iterator[Iterable["UDFResult"]],
     udf: UDFBase,
-    batch_size=INSERT_BATCH_SIZE,
+    batch_size: int = INSERT_BATCH_SIZE,
     cb: Callback = DEFAULT_CALLBACK,
 ) -> None:
     rows: list[UDFResult] = []
@@ -396,7 +397,9 @@ def process_udf_outputs(
         for row in udf_output:
             cb.relative_update()
             rows.append(adjust_outputs(warehouse, row, udf_col_types))
-            if len(rows) >= batch_size:
+            if len(rows) >= batch_size or (
+                len(rows) % 10 == 0 and psutil.virtual_memory().percent > 80
+            ):
                 for row_chunk in batched(rows, batch_size):
                     warehouse.insert_rows(udf_table, row_chunk)
                 rows.clear()
@@ -1775,6 +1778,10 @@ def query_wrapper(dataset_query: DatasetQuery) -> DatasetQuery:
     save = bool(os.getenv("DATACHAIN_QUERY_SAVE"))
     save_as = os.getenv("DATACHAIN_QUERY_SAVE_AS")
 
+    is_session_temp_dataset = dataset_query.name and dataset_query.name.startswith(
+        dataset_query.session.get_temp_prefix()
+    )
+
     if save_as:
         if dataset_query.attached:
             dataset_name = dataset_query.name
@@ -1801,7 +1808,7 @@ def query_wrapper(dataset_query: DatasetQuery) -> DatasetQuery:
             )
         else:
             dataset_query = dataset_query.save(save_as)
-    elif save and not dataset_query.attached:
+    elif save and (is_session_temp_dataset or not dataset_query.attached):
         name = catalog.generate_query_dataset_name()
         dataset_query = dataset_query.save(name)
 

datachain/query/session.py

@@ -74,11 +74,13 @@ class Session:
             self.catalog.id_generator.close_on_exit()
 
     def generate_temp_dataset_name(self) -> str:
-        tmp_table_uid = uuid4().hex[: self.TEMP_TABLE_UUID_LEN]
-        return f"{self.DATASET_PREFIX}{self.name}_{tmp_table_uid}"
+        return self.get_temp_prefix() + uuid4().hex[: self.TEMP_TABLE_UUID_LEN]
+
+    def get_temp_prefix(self) -> str:
+        return f"{self.DATASET_PREFIX}{self.name}_"
 
     def _cleanup_temp_datasets(self) -> None:
-        prefix = f"{self.DATASET_PREFIX}{self.name}"
+        prefix = self.get_temp_prefix()
         try:
             for dataset in list(self.catalog.metastore.list_datasets_by_prefix(prefix)):
                 self.catalog.remove_dataset(dataset.name, force=True)

{datachain-0.3.7.dist-info → datachain-0.3.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.7
+Version: 0.3.8
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -41,10 +41,11 @@ Requires-Dist: jmespath >=1.0
 Requires-Dist: datamodel-code-generator >=0.25
 Requires-Dist: Pillow <11,>=10.0.0
 Requires-Dist: msgpack <2,>=1.0.4
+Requires-Dist: psutil
 Requires-Dist: numpy <2,>=1 ; sys_platform == "win32"
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests] ; extra == 'dev'
-Requires-Dist: mypy ==1.11.1 ; extra == 'dev'
+Requires-Dist: mypy ==1.11.2 ; extra == 'dev'
 Requires-Dist: types-python-dateutil ; extra == 'dev'
 Requires-Dist: types-pytz ; extra == 'dev'
 Requires-Dist: types-PyYAML ; extra == 'dev'
@@ -64,11 +65,14 @@ Requires-Dist: accelerate ; extra == 'examples'
 Requires-Dist: unstructured[pdf] ; extra == 'examples'
 Requires-Dist: pdfplumber ==0.11.4 ; extra == 'examples'
 Requires-Dist: huggingface-hub[hf_transfer] ; extra == 'examples'
+Provides-Extra: hf
+Requires-Dist: numba >=0.60.0 ; extra == 'hf'
+Requires-Dist: datasets[audio,vision] ; extra == 'hf'
 Provides-Extra: remote
 Requires-Dist: lz4 ; extra == 'remote'
 Requires-Dist: requests >=2.22.0 ; extra == 'remote'
 Provides-Extra: tests
-Requires-Dist: datachain[remote,torch,vector] ; extra == 'tests'
+Requires-Dist: datachain[hf,remote,torch,vector] ; extra == 'tests'
 Requires-Dist: pytest <9,>=8 ; extra == 'tests'
 Requires-Dist: pytest-sugar >=0.9.6 ; extra == 'tests'
 Requires-Dist: pytest-cov >=4.1.0 ; extra == 'tests'
@@ -83,6 +87,7 @@ Requires-Dist: hypothesis ; extra == 'tests'
 Requires-Dist: open-clip-torch ; extra == 'tests'
 Requires-Dist: aiotools >=1.7.0 ; extra == 'tests'
 Requires-Dist: requests-mock ; extra == 'tests'
+Requires-Dist: scipy ; extra == 'tests'
 Provides-Extra: torch
 Requires-Dist: torch >=2.1.0 ; extra == 'torch'
 Requires-Dist: torchvision ; extra == 'torch'

{datachain-0.3.7.dist-info → datachain-0.3.8.dist-info}/RECORD

@@ -2,7 +2,7 @@ datachain/__init__.py,sha256=GeyhE-5LgfJav2OKYGaieP2lBvf2Gm-ihj7thnK9zjI,800
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=biF8M8fQujtj5xs0VLi8S16eBtzG6kceWlO_NILbCsg,8197
 datachain/cache.py,sha256=wznC2pge6RhlPTaJfBVGjmBc6bxWCPThu4aTFMltvFU,4076
-datachain/cli.py,sha256=DbmI1sXs7-KCQz6RdLE_JAp3XO3yrTSRJ71LdUzx-XE,33099
+datachain/cli.py,sha256=otR2eN0JL-JhZ9SOTPcPwt_-_TiT-vHifx2h4YzD6Tg,32052
 datachain/cli_utils.py,sha256=jrn9ejGXjybeO1ur3fjdSiAyCHZrX0qsLLbJzN9ErPM,2418
 datachain/config.py,sha256=PfC7W5yO6HFO6-iMB4YB-0RR88LPiGmD6sS_SfVbGso,1979
 datachain/dataset.py,sha256=MZezyuJWNj_3PEtzr0epPMNyWAOTrhTSPI5FmemV6L4,14470
@@ -17,7 +17,7 @@ datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/storage.py,sha256=RiSJLYdHUjnrEWkLBKPcETHpAxld_B2WxLg711t0aZI,3733
 datachain/utils.py,sha256=ROVCLwb37VmFRzgTlSGUDw4eJNgYGiQ4yMX581HfUX8,12988
 datachain/catalog/__init__.py,sha256=g2iAAFx_gEIrqshXlhSEbrc8qDaEH11cjU40n3CHDz4,409
-datachain/catalog/catalog.py,sha256=dSEpktnwnpx1yY_QMvUexZVvvn6085olV7bnyImPM_k,81280
+datachain/catalog/catalog.py,sha256=6S4AnDos4sGYGhy4wNSyV2pKPQNXvo819cd3Dl8Htgg,78271
 datachain/catalog/datasource.py,sha256=D-VWIVDCM10A8sQavLhRXdYSCG7F4o4ifswEF80_NAQ,1412
 datachain/catalog/loader.py,sha256=-6VelNfXUdgUnwInVyA8g86Boxv2xqhTh9xNS-Zlwig,8242
 datachain/catalog/subclass.py,sha256=B5R0qxeTYEyVAAPM1RutBPSoXZc8L5mVVZeSGXki9Sw,2096
@@ -38,21 +38,22 @@ datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2kru
 datachain/data_storage/sqlite.py,sha256=jLgkvikYkENQUO_ykoNFfsBc2ofZXwFHLMa1nyWP3aw,28316
 datachain/data_storage/warehouse.py,sha256=cvlfa-nyIxqrrpSRtCdeVjlTwhn7rcIoWjOq91HhItU,33668
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/lib/arrow.py,sha256=D8N7zCppRdc5sTYT1hNIbROc-sKA_8FN5J_m-KjD3Us,4929
-datachain/lib/clip.py,sha256=16u4b_y2Y15nUS2UN_8ximMo6r_-_4IQpmct2ol-e-g,5730
-datachain/lib/data_model.py,sha256=ZvtMRMcPpBxI-rOhkXb-ry1PkGYcEFFK1w1wH12vs4g,1718
+datachain/lib/arrow.py,sha256=W8bIxMIe_b3dqMFYKGWmfbC_7Xe0gV3UiJjQ2i4EYLA,4925
+datachain/lib/clip.py,sha256=33RL11OIqfbwyhvBgiMGM8rDAnZx1IRmxk9dY89ls3Q,6130
+datachain/lib/data_model.py,sha256=gHIjlow84GMRDa78yLL1Ud-N18or21fnTyPEwsatpXY,2045
 datachain/lib/dataset_info.py,sha256=lONGr71ozo1DS4CQEhnpKORaU4qFb6Ketv8Xm8CVm2U,2188
-datachain/lib/dc.py,sha256=DkVhbjlxpl-HgHenIK1msofU2tUwsSiKPtNim5ai6OE,60136
+datachain/lib/dc.py,sha256=wdMzFLglOhwWKHwh4qcLA0ezMrjuRJq2il2WnkHjyag,62490
 datachain/lib/file.py,sha256=ZHpdilDPYCob8uqtwUPtBvBNxVvQRq4AC_0IGg5m-G4,12003
-datachain/lib/image.py,sha256=TgYhRhzd4nkytfFMeykQkPyzqb5Le_-tU81unVMPn4Q,2328
+datachain/lib/hf.py,sha256=mYaHFPS4CW2-stRZHBMWW-NKN4dhrnhjZobBgRocnvo,5317
+datachain/lib/image.py,sha256=WbcwSaFzuyqjg4x4hH5CUogeUQjkZFjQHqw_oDEV1nA,2655
 datachain/lib/listing.py,sha256=nXLmGae_oQke4hnurzzWiHTEjHjWiqqHdB41Wb-hMTk,3521
 datachain/lib/meta_formats.py,sha256=Hels85LJmNCz1aYVJvhymNdAt3qdJ2-qoxsIiUezrow,7198
 datachain/lib/model_store.py,sha256=c4USXsBBjrGH8VOh4seIgOiav-qHOwdoixtxfLgU63c,2409
-datachain/lib/pytorch.py,sha256=9PsypKseyKfIimTmTQOgb-pbNXgeeAHLdlWx0qRPULY,5660
+datachain/lib/pytorch.py,sha256=vK3GbWCy7kunN7ubul6w1hrWmJLja56uTCiMG_7XVQA,5623
 datachain/lib/settings.py,sha256=39thOpYJw-zPirzeNO6pmRC2vPrQvt4eBsw1xLWDFsw,2344
-datachain/lib/signal_schema.py,sha256=MS8qkOIl-3Qh3KyYTCtuSgF9nP5PeaGccbtGqfWo2wI,15902
-datachain/lib/text.py,sha256=dVe2Ilc_gW2EV0kun0UwegiCkapWcd20cef7CgINWHU,1083
-datachain/lib/udf.py,sha256=n3x6No-7l5LAciPJPWwZbA8WtTnGUU7d0wRL6CyfZh8,11847
+datachain/lib/signal_schema.py,sha256=rW1R6nIzdtmqWzpXk7aNAfrQD58_gbvkvEGyNTQ4WNM,20099
+datachain/lib/text.py,sha256=vqs1SQdsw1vCzfvOanIeT4xY2R2TmPonElBgYDVeZmY,1241
+datachain/lib/udf.py,sha256=nG7DDuPgZ5ZuijwvDoCq-OZMxlDM8vFNzyxMmik0Y1c,11716
 datachain/lib/udf_signature.py,sha256=gMStcEeYJka5M6cg50Z9orC6y6HzCAJ3MkFqqn1fjZg,7137
 datachain/lib/utils.py,sha256=5-kJlAZE0D9nXXweAjo7-SP_AWGo28feaDByONYaooQ,463
 datachain/lib/vfile.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -60,20 +61,20 @@ datachain/lib/webdataset.py,sha256=SsjCKLSKEkHRRfeTHQhjoGqNPqIWw_SCWQcUwgUWWP0,8
 datachain/lib/webdataset_laion.py,sha256=PQP6tQmUP7Xu9fPuAGK1JDBYA6T5UufYMUTGaxgspJA,2118
 datachain/lib/convert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/convert/flatten.py,sha256=Uebc5CeqCsacp-nr6IG9i6OGuUavXqdqnoGctZBk3RQ,1384
-datachain/lib/convert/python_to_sql.py,sha256=4gplGlr_Kg-Z40OpJUzJiarDWj7pwbUOk-dPOYYCJ9Q,2629
+datachain/lib/convert/python_to_sql.py,sha256=40SAOdoOgikZRhn8iomCPDRoxC3RFxjJLivEAA9MHDU,2880
 datachain/lib/convert/sql_to_python.py,sha256=lGnKzSF_tz9Y_5SSKkrIU95QEjpcDzvOxIRkEKTQag0,443
 datachain/lib/convert/unflatten.py,sha256=Ogvh_5wg2f38_At_1lN0D_e2uZOOpYEvwvB2xdq56Tw,2012
 datachain/lib/convert/values_to_tuples.py,sha256=YOdbjzHq-uj6-cV2Qq43G72eN2avMNDGl4x5t6yQMl8,3931
 datachain/query/__init__.py,sha256=tv-spkjUCYamMN9ys_90scYrZ8kJ7C7d1MTYVmxGtk4,325
 datachain/query/batch.py,sha256=-vlpINJiertlnaoUVv1C95RatU0F6zuhpIYRufJRo1M,3660
 datachain/query/builtins.py,sha256=EmKPYsoQ46zwdyOn54MuCzvYFmfsBn5F8zyF7UBUfrc,2550
-datachain/query/dataset.py,sha256=4F_Q101Lbpc0YxOAcP3rc3GtKv8HwxpqF9lpJ0OoUEk,60818
+datachain/query/dataset.py,sha256=G6xA3ItIGUJTXhizdAb6S3L1zFwTf8I0w0jHa1A6F4A,61103
 datachain/query/dispatch.py,sha256=GBh3EZHDp5AaXxrjOpfrpfsuy7Umnqxu-MAXcK9X3gc,12945
 datachain/query/metrics.py,sha256=vsECqbZfoSDBnvC3GQlziKXmISVYDLgHP1fMPEOtKyo,640
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
 datachain/query/queue.py,sha256=waqM_KzavU8C-G95-4211Nd4GXna_u2747Chgwtgz2w,3839
 datachain/query/schema.py,sha256=BvHipN79CnSTbVFcfIEwzo1npe7HmThnk0iY-CSLEkM,7899
-datachain/query/session.py,sha256=PkOLANS0s8KPz4wO17tAab-CMzIt7FK8RPzJiibExds,4290
+datachain/query/session.py,sha256=UPH5Z4fzCDsvj81ji0e8GA6Mgra3bOAEpVq4htqOtis,4317
 datachain/query/udf.py,sha256=j3NhmKK5rYG5TclcM2Sr0LhS1tmYLMjzMugx9G9iFLM,8100
 datachain/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/remote/studio.py,sha256=f5s6qSZ9uB4URGUoU_8_W1KZRRQQVSm6cgEBkBUEfuE,7226
@@ -94,9 +95,9 @@ datachain/sql/sqlite/base.py,sha256=WLPHBhZbXbiqPoRV1VgDrXJqku4UuvJpBhYeQ0k5rI8,
 datachain/sql/sqlite/types.py,sha256=yzvp0sXSEoEYXs6zaYC_2YubarQoZH-MiUNXcpuEP4s,1573
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.3.7.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.3.7.dist-info/METADATA,sha256=m7NZ31iEMCD2xOF8HZNp8YvGu05TmF_3UiZQQPUVmmc,16719
-datachain-0.3.7.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
-datachain-0.3.7.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.3.7.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.3.7.dist-info/RECORD,,
+datachain-0.3.8.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.3.8.dist-info/METADATA,sha256=ivteXQrJgp8dKgIO2pdwUj6Qdg96rbI3Gq0kx5fyxtk,16903
+datachain-0.3.8.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+datachain-0.3.8.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.3.8.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.3.8.dist-info/RECORD,,

{datachain-0.3.7.dist-info → datachain-0.3.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (73.0.1)
+Generator: setuptools (74.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any