datachain 0.1.10__py3-none-any.whl → 0.1.11__py3-none-any.whl

datachain/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.10'
-__version_tuple__ = version_tuple = (0, 1, 10)
+__version__ = version = '0.1.11'
+__version_tuple__ = version_tuple = (0, 1, 11)

datachain/catalog/catalog.py

@@ -1580,10 +1580,54 @@ class Catalog:
 
         return dst
 
-    def open_object(self, row: RowDict, use_cache: bool = True, **config: Any):
+    def get_file_signals(
+        self, dataset_name: str, dataset_version: int, row: RowDict
+    ) -> Optional[dict]:
+        """
+        Function that returns file signals from dataset row.
+        Note that signal names are without prefix, so if there was 'laion__file__source'
+        in original row, result will have just 'source'
+        Example output:
+            {
+                "source": "s3://ldb-public",
+                "parent": "animals/dogs",
+                "name": "dog.jpg",
+                ...
+            }
+        """
+        from datachain.lib.signal_schema import SignalSchema
+
+        version = self.get_dataset(dataset_name).get_version(dataset_version)
+
+        file_signals_values = SignalSchema.deserialize(
+            version.feature_schema
+        ).get_file_signals_values(row)
+        if not file_signals_values:
+            return None
+
+        # there can be multiple file signals in a schema, but taking the first
+        # one for now. In future we might add ability to choose from which one
+        # to open object
+        return next(iter(file_signals_values.values()))
+
+    def open_object(
+        self,
+        dataset_name: str,
+        dataset_version: int,
+        row: RowDict,
+        use_cache: bool = True,
+        **config: Any,
+    ):
+        file_signals = self.get_file_signals(dataset_name, dataset_version, row)
+        if not file_signals:
+            raise RuntimeError("Cannot open object without file signals")
+
         config = config or self.client_config
-        client = self.get_client(row["source"], **config)
-        return client.open_object(self._get_row_uid(row), use_cache=use_cache)
+        client = self.get_client(file_signals["source"], **config)
+        return client.open_object(
+            self._get_row_uid(file_signals),  # type: ignore [arg-type]
+            use_cache=use_cache,
+        )
 
     def _get_row_uid(self, row: RowDict) -> UniqueId:
         return UniqueId(

datachain/data_storage/metastore.py

@@ -1142,6 +1142,8 @@ class AbstractDBMetastore(AbstractMetastore):
                 if field == "schema":
                     dataset_version.update(**{field: DatasetRecord.parse_schema(value)})
                     values[field] = json.dumps(value) if value else None
+                elif field == "feature_schema":
+                    values[field] = json.dumps(value) if value else None
                 elif field == "preview" and isinstance(value, list):
                     values[field] = json.dumps(value, cls=JSONSerialize)
                 else:

datachain/dataset.py

@@ -157,7 +157,7 @@ class DatasetVersion:
     dataset_id: int
     version: int
     status: int
-    feature_schema: Optional[str]
+    feature_schema: dict
     created_at: datetime
     finished_at: Optional[datetime]
     error_message: str
@@ -199,7 +199,7 @@ class DatasetVersion:
             dataset_id,
             version,
             status,
-            feature_schema,
+            json.loads(feature_schema) if feature_schema else {},
             created_at,
             finished_at,
             error_message,
@@ -263,9 +263,9 @@ class DatasetRecord:
     labels: list[str]
     shadow: bool
     schema: dict[str, Union[SQLType, type[SQLType]]]
+    feature_schema: dict
     versions: list[DatasetVersion]
     status: int = DatasetStatus.CREATED
-    feature_schema: Optional[dict] = None
     created_at: Optional[datetime] = None
     finished_at: Optional[datetime] = None
     error_message: str = ""
@@ -320,8 +320,6 @@ class DatasetRecord:
         version_job_id: Optional[str] = None,
         version_is_job_result: bool = False,
     ) -> "DatasetRecord":
-        fr_schema = json.loads(feature_schema) if feature_schema else {}
-
         labels_lst: list[str] = json.loads(labels) if labels else []
         schema_dct: dict[str, Any] = json.loads(schema) if schema else {}
         version_schema_dct: dict[str, str] = (
@@ -333,7 +331,7 @@ class DatasetRecord:
             version_dataset_id,
             version,
             version_status,
-            fr_schema,
+            version_feature_schema,
             version_created_at,
             version_finished_at,
             version_error_message,
@@ -356,9 +354,9 @@ class DatasetRecord:
             labels_lst,
             bool(shadow),
             cls.parse_schema(schema_dct),  # type: ignore[arg-type]
+            json.loads(feature_schema) if feature_schema else {},
             [dataset_version],
             status,
-            fr_schema,
             created_at,
             finished_at,
             error_message,

datachain/lib/dc.py

@@ -6,6 +6,7 @@ import sqlalchemy
 from datachain.lib.feature import Feature, FeatureType
 from datachain.lib.feature_utils import features_to_tuples
 from datachain.lib.file import File, get_file
+from datachain.lib.meta_formats import read_meta
 from datachain.lib.settings import Settings
 from datachain.lib.signal_schema import SignalSchema
 from datachain.lib.udf import (
@@ -219,6 +220,89 @@ class DataChain(DatasetQuery):
         """
         return DataChain(name=name, version=version)
 
+    @classmethod
+    def from_csv(
+        cls,
+        path,
+        type: Literal["binary", "text", "image"] = "text",
+        anon: bool = False,
+        spec: Optional[FeatureType] = None,
+        schema_from: Optional[str] = "auto",
+        show_schema: Optional[bool] = False,
+    ) -> "DataChain":
+        """Get data from CSV. It returns the chain itself.
+
+        Parameters
+        ----------
+        path : storage URI with directory. URI must start with storage prefix such
+               as `s3://`, `gs://`, `az://` or "file:///"
+        type : read file as "binary", "text", or "image" data. Default is "binary".
+        anon : use anonymous mode to access the storage.
+        spec : optional Data Model
+        schema_from : path to sample to infer spec from
+        show_schema : print auto-generated schema
+
+        Examples
+        --------
+
+        >>> chain = DataChain.from_csv("gs://csv")
+        """
+        if schema_from == "auto":
+            schema_from = path
+
+        chain = DataChain.from_storage(path=path, type=type, anon=anon)
+        return chain.gen(
+            csv=read_meta(
+                schema_from=schema_from,
+                meta_type="csv",
+                spec=spec,
+                show_schema=show_schema,
+            )
+        )
+
+    @classmethod
+    def from_json(
+        cls,
+        path,
+        type: Literal["binary", "text", "image"] = "text",
+        anon: bool = False,
+        spec: Optional[FeatureType] = None,
+        schema_from: Optional[str] = "auto",
+        jmespath: Optional[str] = None,
+        show_schema: Optional[bool] = False,
+    ) -> "DataChain":
+        """Get data from CSV. It returns the chain itself.
+
+        Parameters
+        ----------
+        path : storage URI with directory. URI must start with storage prefix such
+               as `s3://`, `gs://`, `az://` or "file:///"
+        type : read file as "binary", "text", or "image" data. Default is "binary".
+        anon : use anonymous mode to access the storage.
+        spec : optional Data Model
+        schema_from : path to sample to infer spec from
+        show_schema : print auto-generated schema
+        jmespath : JMESPATH expression to reduce JSON
+        name : return object name
+        Examples
+        --------
+
+        >>> chain = DataChain.from_json("gs://json")
+        """
+        if schema_from == "auto":
+            schema_from = path
+
+        chain = DataChain.from_storage(path=path, type=type, anon=anon)
+        return chain.gen(
+            json=read_meta(
+                schema_from=schema_from,
+                meta_type="json",
+                spec=spec,
+                show_schema=show_schema,
+                jmespath=jmespath,
+            )
+        )
+
     def save(  # type: ignore[override]
         self, name: Optional[str] = None, version: Optional[int] = None
     ) -> "DataChain":
@@ -408,7 +492,7 @@ class DataChain(DatasetQuery):
         chain.signals_schema = new_schema
         return chain
 
-    def get_values(self) -> Iterator[Sequence]:
+    def get_values(self) -> Iterator[list]:
         """Iterate over rows, getting feature values and applying reader calls."""
         for features in self.iterate():
             yield [fr.get_value() if isinstance(fr, Feature) else fr for fr in features]
@@ -607,3 +691,35 @@ class DataChain(DatasetQuery):
 
     def max(self, fr: FeatureType):  # type: ignore[override]
         return self._extend_features("max", fr)
+
+    @detach
+    def gen_random(self) -> "DataChain":
+        from random import getrandbits
+
+        from datachain.data_storage.warehouse import RANDOM_BITS
+
+        if "random" not in self.signals_schema.values:
+            chain = self.map(random=lambda: getrandbits(RANDOM_BITS), output=int).save()
+            return chain.select_except("random")
+
+        return self
+
+    @detach
+    def shuffle(self) -> "DataChain":
+        """Return results in deterministic random order."""
+        chain = self.gen_random()
+        return DatasetQuery.shuffle(chain)
+
+    @detach
+    def chunk(self, index: int, total: int) -> "DataChain":
+        """Split a query into smaller chunks for e.g. parallelization.
+        Example:
+            >>> dc = DataChain(...)
+            >>> chunk_1 = dc._chunk(0, 2)
+            >>> chunk_2 = dc._chunk(1, 2)
+        Note:
+            Bear in mind that `index` is 0-indexed but `total` isn't.
+            Use 0/3, 1/3 and 2/3, not 1/3, 2/3 and 3/3.
+        """
+        chain = self.gen_random()
+        return DatasetQuery.chunk(chain, index, total)

datachain/lib/feature.py

@@ -78,16 +78,6 @@ DATACHAIN_TO_TYPE = {
     JSON: dict,
 }
 
-NAMES_TO_TYPES = {
-    "int": int,
-    "str": str,
-    "float": float,
-    "bool": bool,
-    "list": list,
-    "dict": dict,
-    "bytes": bytes,
-    "datetime": datetime,
-}
 
 NUMPY_TO_DATACHAIN = {
     np.dtype("int8"): Int,

datachain/lib/meta_formats.py

@@ -0,0 +1,164 @@
+# pip install datamodel-code-generator
+# pip install jmespath
+#
+import csv
+import io
+import json
+import subprocess
+import sys
+import uuid
+from collections.abc import Iterator
+from typing import Any, Callable
+
+import jmespath as jsp
+
+from datachain.lib.feature_utils import pydantic_to_feature  # noqa: F401
+from datachain.lib.file import File
+
+# from datachain.lib.dc import C, DataChain
+
+
+def generate_uuid():
+    return uuid.uuid4()  # Generates a random UUID.
+
+
+# JSON decoder
+def load_json_from_string(json_string):
+    try:
+        data = json.loads(json_string)
+        print("Successfully parsed JSON", file=sys.stderr)
+        return data
+    except json.JSONDecodeError:
+        print("Failed to decode JSON: The string is not formatted correctly.")
+    return None
+
+
+# Read valid JSON and return a data object sample
+def process_json(data_string, jmespath):
+    json_dict = load_json_from_string(data_string)
+    if jmespath:
+        json_dict = jsp.search(jmespath, json_dict)
+        # we allow non-list JSONs here to print the root schema
+        # but if jmespath expression is given, we assume a list
+        if not isinstance(json_dict, list):
+            raise ValueError("JMESPATH expression must resolve to a list")
+            return None
+        json_dict = json_dict[0]  # sample the first object
+    return json.dumps(json_dict)
+
+
+# Print a dynamic datamodel-codegen output from JSON or CSV on stdout
+def read_schema(source_file, data_type="csv", expr=None):
+    data_string = ""
+    uid_str = str(generate_uuid()).replace("-", "")  # comply with Python class names
+    # using uiid to get around issue #1617
+    model_name = f"Model{uid_str}"
+    try:
+        with source_file.open() as fd:  # CSV can be larger than memory
+            if data_type == "csv":
+                data_string += fd.readline().decode("utf-8", "ignore").replace("\r", "")
+                data_string += fd.readline().decode("utf-8", "ignore").replace("\r", "")
+            else:
+                data_string = fd.read()  # other meta must fit into RAM
+    except OSError as e:
+        print(f"An unexpected file error occurred: {e}")
+        return
+    if data_type == "json":
+        data_string = process_json(data_string, expr)
+    command = [
+        "datamodel-codegen",
+        "--input-file-type",
+        data_type,
+        "--class-name",
+        model_name,
+    ]
+    try:
+        result = subprocess.run(
+            command,  # noqa: S603
+            input=data_string,
+            text=True,
+            capture_output=True,
+            check=True,
+        )
+        model_output = (
+            result.stdout
+        )  # This will contain the output from datamodel-codegen
+    except subprocess.CalledProcessError as e:
+        model_output = f"An error occurred in datamodel-codegen: {e.stderr}"
+    print(f"{model_output}")
+    print("\n" + f"spec=pydantic_to_feature({model_name})" + "\n")
+
+
+#
+# UDF mapper which calls chain in the setup to infer the dynamic schema
+#
+def read_meta(
+    spec=None, schema_from=None, meta_type="json", jmespath=None, show_schema=False
+) -> Callable:
+    from datachain.lib.dc import DataChain
+
+    # ugly hack: datachain is run redirecting printed outputs to a variable
+    if schema_from:
+        captured_output = io.StringIO()
+        current_stdout = sys.stdout
+        sys.stdout = captured_output
+        try:
+            chain = (
+                DataChain.from_storage(schema_from)
+                .limit(1)
+                .map(  # dummy column created (#1615)
+                    meta_schema=lambda file: read_schema(
+                        file, data_type=meta_type, expr=jmespath
+                    ),
+                    output=str,
+                )
+            )
+            # dummy executor (#1616)
+            chain.save()
+        finally:
+            sys.stdout = current_stdout
+        model_output = captured_output.getvalue()
+        captured_output.close()
+        if show_schema:
+            print(f"{model_output}")
+        # Below 'spec' should be a dynamically converted Feature from Pydantic datamodel
+        if not spec:
+            local_vars: dict[str, Any] = {}
+            exec(model_output, globals(), local_vars)  # noqa: S102
+            spec = local_vars["spec"]
+
+    if not (spec) and not (schema_from):
+        raise ValueError(
+            "Must provide a static schema in spec: or metadata sample in schema_from:"
+        )
+
+    #
+    # UDF mapper parsing a JSON or CSV file using schema spec
+    #
+    def parse_data(
+        file: File, data_model=spec, meta_type=meta_type, jmespath=jmespath
+    ) -> Iterator[spec]:
+        if meta_type == "csv":
+            with (
+                file.open() as fd
+            ):  # TODO: if schema is statically given, should allow CSV without headers
+                reader = csv.DictReader(fd)
+                for row in reader:  # CSV can be larger than memory
+                    json_string = json.dumps(row)
+                    yield data_model.model_validate_json(json_string)
+        if meta_type == "json":
+            try:
+                with file.open() as fd:  # JSON must fit into RAM
+                    data_string = fd.read()
+            except OSError as e:
+                print(f"An unexpected file error occurred: {e}")
+            json_object = load_json_from_string(data_string)
+            if jmespath:
+                json_object = jsp.search(jmespath, json_object)
+            if not isinstance(json_object, list):
+                raise ValueError("JSON expression must resolve in a list of objects")
+            for json_dict in json_object:
+                json_string = json.dumps(json_dict)
+                yield data_model.model_validate_json(json_string)
+
+    return parse_data

datachain/lib/pytorch.py

@@ -1,6 +1,6 @@
 import logging
 from collections.abc import Iterator
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Callable, Optional
 
 from torch import float32
 from torch.distributed import get_rank, get_world_size
@@ -8,6 +8,7 @@ from torch.utils.data import IterableDataset, get_worker_info
 
 from datachain.catalog import Catalog, get_catalog
 from datachain.lib.dc import DataChain
+from datachain.lib.text import convert_text
 
 if TYPE_CHECKING:
     from torchvision.transforms.v2 import Transform
@@ -17,14 +18,25 @@ logger = logging.getLogger("datachain")
 
 
 try:
+    from PIL import Image
     from torchvision.transforms import v2
 
     DEFAULT_TRANSFORM = v2.Compose([v2.ToImage(), v2.ToDtype(float32, scale=True)])
 except ImportError:
-    logger.warning("Missing dependency torchvision for computer vision transforms.")
+    logger.warning(
+        "Missing dependencies for computer vision:\n"
+        "To install run:\n\n"
+        "  pip install 'datachain[cv]'\n"
+    )
+    Image = None  # type: ignore[assignment]
+    v2 = None
     DEFAULT_TRANSFORM = None
 
 
+def label_to_int(value: str, classes: list) -> int:
+    return classes.index(value)
+
+
 class PytorchDataset(IterableDataset):
     def __init__(
         self,
@@ -32,6 +44,8 @@ class PytorchDataset(IterableDataset):
         version: Optional[int] = None,
         catalog: Optional["Catalog"] = None,
         transform: Optional["Transform"] = DEFAULT_TRANSFORM,
+        tokenizer: Optional[Callable] = None,
+        tokenizer_kwargs: Optional[dict[str, Any]] = None,
         num_samples: int = 0,
     ):
         """
@@ -41,13 +55,17 @@ class PytorchDataset(IterableDataset):
             name (str): Name of DataChain dataset to stream.
             version (int): Version of DataChain dataset to stream.
             catalog (Catalog): DataChain catalog to which dataset belongs.
-            transform (Transform): Torchvision v2 transforms to apply to the dataset.
+            transform (Transform): Torchvision transforms to apply to the dataset.
+            tokenizer (Callable): Tokenizer to use to tokenize text values.
+            tokenizer_kwargs (dict): Additional kwargs to pass when calling tokenizer.
             num_samples (int): Number of random samples to draw for each epoch.
                 This argument is ignored if `num_samples=0` (the default).
         """
         self.name = name
         self.version = version
         self.transform = transform
+        self.tokenizer = tokenizer
+        self.tokenizer_kwargs = tokenizer_kwargs or {}
         self.num_samples = num_samples
         if catalog is None:
             catalog = get_catalog()
@@ -87,10 +105,21 @@ class PytorchDataset(IterableDataset):
             # Apply transforms
             if self.transform:
                 try:
-                    row = self.transform(row)
+                    if v2 and isinstance(self.transform, v2.Transform):
+                        row = self.transform(row)
+                    elif Image:
+                        for i, val in enumerate(row):
+                            if isinstance(val, Image.Image):
+                                row[i] = self.transform(val)
                 except ValueError:
                     logger.warning("Skipping transform due to unsupported data types.")
                     self.transform = None
+            if self.tokenizer:
+                for i, val in enumerate(row):
+                    if isinstance(val, str):
+                        row[i] = convert_text(
+                            val, self.tokenizer, self.tokenizer_kwargs
+                        )
             yield row
 
     @staticmethod

datachain/lib/signal_schema.py

@@ -1,19 +1,45 @@
 import copy
 from collections.abc import Sequence
-from typing import Any, Optional, Union, get_args, get_origin
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Optional, Union, get_args, get_origin
 
-from datachain.catalog import Catalog
 from datachain.lib.feature import (
     DATACHAIN_TO_TYPE,
     DEFAULT_DELIMITER,
-    NAMES_TO_TYPES,
     Feature,
     FeatureType,
     convert_type_to_datachain,
 )
 from datachain.lib.feature_registry import Registry
-from datachain.lib.file import File
+from datachain.lib.file import File, ImageFile, TextFile
 from datachain.lib.utils import DataChainParamsError
+from datachain.lib.webdataset import TarStream, WDSAllFile, WDSBasic
+from datachain.lib.webdataset_laion import Laion, LaionParquet, WDSLaion
+
+if TYPE_CHECKING:
+    from datachain.catalog import Catalog
+
+
+# TODO fix hardcoded Feature class names with://github.com/iterative/dvcx/issues/1625
+NAMES_TO_TYPES = {
+    "int": int,
+    "str": str,
+    "float": float,
+    "bool": bool,
+    "list": list,
+    "dict": dict,
+    "bytes": bytes,
+    "datetime": datetime,
+    "WDSLaion": WDSLaion,
+    "Laion": Laion,
+    "LaionParquet": LaionParquet,
+    "File": File,
+    "ImageFile": ImageFile,
+    "TextFile": TextFile,
+    "TarStream": TarStream,
+    "WDSBasic": WDSBasic,
+    "WDSAllFile": WDSAllFile,
+}
 
 
 class SignalSchemaError(DataChainParamsError):
@@ -74,7 +100,7 @@ class SignalSchema:
         signals: dict[str, FeatureType] = {}
         for signal, type_name in schema.items():
             try:
-                fr = NAMES_TO_TYPES.get(type_name, None)
+                fr = NAMES_TO_TYPES.get(type_name)
                 if not fr:
                     type_name, version = Registry.parse_name_version(type_name)
                     fr = Registry.get(type_name, version)
@@ -137,7 +163,7 @@ class SignalSchema:
     def slice(self, keys: Sequence[str]) -> "SignalSchema":
         return SignalSchema({k: v for k, v in self.values.items() if k in keys})
 
-    def row_to_features(self, row: Sequence, catalog: Catalog) -> list[FeatureType]:
+    def row_to_features(self, row: Sequence, catalog: "Catalog") -> list[FeatureType]:
         res = []
         pos = 0
         for fr_cls in self.values.values():
@@ -279,3 +305,34 @@ class SignalSchema:
                 for signal in signals:
                     res.append(".".join(signal))
         return res
+
+    def get_file_signals_values(self, row: dict[str, Any]) -> dict[str, Any]:
+        """
+        Method that returns values with clean field names (without prefix) for
+        all file signals found in this schema for some row
+        Output example:
+        {
+            laion.file: {
+                "source": "s3://ldb-public",
+                "name": "dog.jpg",
+                ...
+            },
+            meta.file: {
+                "source": "s3://datacomp",
+                "name": "cat.jpg",
+                ...
+            }
+        }
+        """
+        res = {}
+
+        for file_signals in self.get_file_signals():
+            prefix = file_signals.replace(".", DEFAULT_DELIMITER) + DEFAULT_DELIMITER
+            res[file_signals] = {
+                c_name.removeprefix(prefix): c_value
+                for c_name, c_value in row.items()
+                if c_name.startswith(prefix)
+                and DEFAULT_DELIMITER not in c_name.removeprefix(prefix)
+            }
+
+        return res

datachain/query/dataset.py

@@ -1462,7 +1462,7 @@ class DatasetQuery:
 
         return cls.from_dataframe(pd_df, *args, **kwargs)
 
-    def shuffle(self) -> "DatasetQuery":
+    def shuffle(self) -> "Self":
         # ToDo: implement shaffle based on seed and/or generating random column
         return self.order_by(C.random)
 

{datachain-0.1.10.dist-info → datachain-0.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.1.10
+Version: 0.1.11
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -35,6 +35,8 @@ Requires-Dist: multiprocess ==0.70.16
 Requires-Dist: dill ==0.3.8
 Requires-Dist: ujson >=5.9.0
 Requires-Dist: pydantic <3,>=2
+Requires-Dist: jmespath >=1.0
+Requires-Dist: datamodel-code-generator >=0.25
 Requires-Dist: numpy <2,>=1 ; sys_platform == "win32"
 Provides-Extra: cv
 Requires-Dist: Pillow <11,>=10.0.0 ; extra == 'cv'

{datachain-0.1.10.dist-info → datachain-0.1.11.dist-info}/RECORD

@@ -1,12 +1,12 @@
 datachain/__init__.py,sha256=9a0qX6tqyA9KC3ahLmGarqlRTZJXhM7HijAWpfUaOnQ,102
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
-datachain/_version.py,sha256=0iLmzkTe5cfY4SBtaPpUzHn9tXwbwplszcfp5pHW6nU,413
+datachain/_version.py,sha256=HreDwlLXV189L3kiBj3huM_kqWD1usijlC8LN1YXcCM,413
 datachain/asyn.py,sha256=opARBVZJxTKU3EGYd-8gcpNXoshuCfVz_b0ut3oxC50,7641
 datachain/cache.py,sha256=FaPWrqWznPffmskTb1pdPkt2jAMMf__9FC2zEnP0vDU,4022
 datachain/cli.py,sha256=1mBozBJS9Nq-EeahxwyKH8ef64E2v93o0CAEzxjcbkY,32209
 datachain/cli_utils.py,sha256=jrn9ejGXjybeO1ur3fjdSiAyCHZrX0qsLLbJzN9ErPM,2418
 datachain/config.py,sha256=PfC7W5yO6HFO6-iMB4YB-0RR88LPiGmD6sS_SfVbGso,1979
-datachain/dataset.py,sha256=VojzbJxxmGQmL38kxp-hQNVPv2drZIR2SD2oHEo4Cqo,14512
+datachain/dataset.py,sha256=4ksFJlfo_CEmt5xqXPca-hhQL1syFpKxCl_ZOhTS30s,14506
 datachain/error.py,sha256=GY9KYTmb7GHXn2gGHV9X-PBhgwLj3i7VpK7tGHtAoGM,1279
 datachain/listing.py,sha256=-Cm74Ne2Q36QuCpA22feDA_v-7uPqkwAOg-QzkiZAGQ,8243
 datachain/node.py,sha256=jCBvwiEUYSKQa27Tb6RORgaUjoiz7mOX63NQmP7JQY0,5703
@@ -17,7 +17,7 @@ datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/storage.py,sha256=RiSJLYdHUjnrEWkLBKPcETHpAxld_B2WxLg711t0aZI,3733
 datachain/utils.py,sha256=FW1LR5qCL5BtCYk-B-6LUCCMq8zOobkKKMrLqfFfCAg,13535
 datachain/catalog/__init__.py,sha256=Gkto1V7rUbVjJmgMEnB_VpVeHOfV47IQh1fSjEKnit4,409
-datachain/catalog/catalog.py,sha256=fSs4RDMA4Hl9svy3GoVBo-DMIwPJP6HUw_YndKRwYQY,77109
+datachain/catalog/catalog.py,sha256=7ZqCsyr7W4enOIX6jiLJbBfFZvjkqjI1E_NOyL3V3AA,78585
 datachain/catalog/datasource.py,sha256=D-VWIVDCM10A8sQavLhRXdYSCG7F4o4ifswEF80_NAQ,1412
 datachain/catalog/loader.py,sha256=FTI9s1b8iX0_TffSAx1mwm-ucsRV14NHX-F1xtTXRSE,7310
 datachain/catalog/subclass.py,sha256=B5R0qxeTYEyVAAPM1RutBPSoXZc8L5mVVZeSGXki9Sw,2096
@@ -32,7 +32,7 @@ datachain/data_storage/__init__.py,sha256=arlkQIj2J0ozcT_GvNDxm6PLT9NeabHvIsxPND
 datachain/data_storage/db_engine.py,sha256=mxOoWP4ntBMgLeTAk4dlEeIJArAz4x_tFrHytcAfLpo,3341
 datachain/data_storage/id_generator.py,sha256=VlDALKijggegAnNMJwuMETJgnLoPYxpkrkld5DNTPQw,3839
 datachain/data_storage/job.py,sha256=w-7spowjkOa1P5fUVtJou3OltT0L48P0RYWZ9rSJ9-s,383
-datachain/data_storage/metastore.py,sha256=K-bW_um9qkDAZUCs9DJV5W3FlpeblQ9f4ulqgWQ4Isg,53528
+datachain/data_storage/metastore.py,sha256=GnJH2NlFngdj30aK9CSaimJNnh_x_pSjntWUnvQuI2A,53649
 datachain/data_storage/schema.py,sha256=pF3KBi-8Pz3n5jRYoJpDR3gF8qUFdyAu2XR58J4Fyuo,8724
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
 datachain/data_storage/sqlite.py,sha256=eHTiJ0VIxU-chnhKNTN14EsaSnw5LAaxTLi9aMCZpl4,24978
@@ -40,8 +40,8 @@ datachain/data_storage/warehouse.py,sha256=sQLOrv6DH8UcWH1aqlg3YJKmaHr696XkVafBx
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/cached_stream.py,sha256=BQI6gpJ2y7_-jqQo_0VB9ntbkOVISvj9wlDwGDQbqw8,3537
 datachain/lib/claude.py,sha256=iAauA1zNVNONpLzUo1t0QN5PZ5Ot6cZkfib7Ka_c638,1969
-datachain/lib/dc.py,sha256=9lL6fNEkPxzkPE8fZqksCmRk3NBbmqMMZFi55j6OXGU,22224
-datachain/lib/feature.py,sha256=7ZZzGkafxKeYUfPN84hgQgdf8LzX54ikrne7itbTreI,14369
+datachain/lib/dc.py,sha256=kyuSg-l7HciqFaunqPx41WKyAeuJ2H2tpWJplCXhZJc,26086
+datachain/lib/feature.py,sha256=C5lxQ_Ef4rL0-mef4A4EeoqB0rcNZ0ExRE26ehx20RM,14196
 datachain/lib/feature_registry.py,sha256=hg_S_9JPEYaQ-8PI64mU0sEhSJ-rcrKtwQk5TPBotEw,1570
 datachain/lib/feature_utils.py,sha256=6wbKZ2xq08b751EFBRJy1OZLqWYd_gxq9A_Em_aMFk4,4713
 datachain/lib/file.py,sha256=ZNGzmJSq7PNVxLhGLNdR9YSYkP-1ZeqY_yhDMcDNfkI,8586
@@ -51,11 +51,12 @@ datachain/lib/hf_pipeline.py,sha256=f0AH_XCziOF1OKN3d1w1swTBLaeajMJ8xgdsX37i5-o,
 datachain/lib/image.py,sha256=gb-My4rx5zMwOlDkcu_2G8GtRAMfsRvd7-QWUBErDw8,3486
 datachain/lib/image_transform.py,sha256=NXWtnVOcofWBgl_YMxb4ABpaT7JTBMx7tLKvErH1IC4,3024
 datachain/lib/iptc_exif_xmp.py,sha256=xrbxFeY-wRP6T5JsUgE3EXfTxKvZVymRaRD_VIfxD0A,2236
+datachain/lib/meta_formats.py,sha256=-JAS47NOO6rx1vmr0Cy-G_txxmTvMflXfzJiFD7rWlQ,5742
 datachain/lib/parquet.py,sha256=_MbRBzcgLLLegjKZNGF9Rm9IkYRSy0IqOksVjL1nntg,917
-datachain/lib/pytorch.py,sha256=fdclv6ZqbCgzCDdMZFv3IBrRQT3V1nI8xRaGF4Oa44c,4224
+datachain/lib/pytorch.py,sha256=oU16XXAyAmiiabe1IoQoID00-u3uZ5GhCN48uAl6WDs,5421
 datachain/lib/reader.py,sha256=rPXXNoTUdm6PQwkAlaU-nOBreP_q4ett_EjFStrA_W0,1727
 datachain/lib/settings.py,sha256=mVtzyA_y9JA-6chMv1baggDvgeFsaUszySp660Gu4gw,2854
-datachain/lib/signal_schema.py,sha256=ST6lw5YGAwOWjZlwFxw1Qjfx0WTXt0lvCpb2Lk9Kn1E,10039
+datachain/lib/signal_schema.py,sha256=WPKHzgZ6HatbDQ2IN_L0JPi46n6acfHpkq91DYdlgSg,11753
 datachain/lib/text.py,sha256=EEZrYohADi5rAGg3aLLRwtvyAV9js_yWAGhr2C3QbwI,2424
 datachain/lib/udf.py,sha256=PeZ-UbprfxlmgVbzH4FtNib3kIhTi9C869QM8RuM5dw,6292
 datachain/lib/udf_signature.py,sha256=1cOMcGXHbdBjyBRkvNxIEt9A_CoyiADxio2wkYu8U5M,7140
@@ -67,7 +68,7 @@ datachain/lib/webdataset_laion.py,sha256=tHn3Zhqx7Eb5Ywy_mobs6jDI0o_pFUbsuHqv0W_
 datachain/query/__init__.py,sha256=tv-spkjUCYamMN9ys_90scYrZ8kJ7C7d1MTYVmxGtk4,325
 datachain/query/batch.py,sha256=sOMxXbaNii7lVyFIEZ2noqbhy_S8qtZ-WWxrka72shc,3474
 datachain/query/builtins.py,sha256=RyVEPZEuC7K1vlulrsaUjATLG_tZEvYYW7N5i6Fg-tQ,2781
-datachain/query/dataset.py,sha256=shLAtpERSu-ZbeV0EWtE32oa0i7d4O3ma8WL38i5ba8,66869
+datachain/query/dataset.py,sha256=2DZAaEwX9gQlQgrRY3t-ymXN9SUkN_3XN0AfMFT6Mto,66861
 datachain/query/dispatch.py,sha256=9zcwKkLIuK5-xyRSQNw3yTqYLMHVbuZIn6KcB0g_ZBQ,13107
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
 datachain/query/schema.py,sha256=CGu9NBIFvX4iHQnaThLLxwWndxqkyUtYmo2JBgnZ4YQ,7660
@@ -91,9 +92,9 @@ datachain/sql/sqlite/__init__.py,sha256=TAdJX0Bg28XdqPO-QwUVKy8rg78cgMileHvMNot7
 datachain/sql/sqlite/base.py,sha256=XVxn4pB-N4pPfiby5uVvfH7feNzRKlBNzsc5eyKPvhI,10965
 datachain/sql/sqlite/types.py,sha256=oP93nLfTBaYnN0z_4Dsv-HZm8j9rrUf1esMM-z3JLbg,1754
 datachain/sql/sqlite/vector.py,sha256=stBeEW6fbVbILmAtV4khjXdJIGT13HkRWJeCoqIOk50,315
-datachain-0.1.10.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.1.10.dist-info/METADATA,sha256=PwiflznodH7Q2esyrf7GsTY45-O3fqWxjBbHzjVfLIk,13895
-datachain-0.1.10.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
-datachain-0.1.10.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.1.10.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.1.10.dist-info/RECORD,,
+datachain-0.1.11.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.1.11.dist-info/METADATA,sha256=BFTmlt8_vtCHF80AHQcIQkE9YMCigp7k1jcAZV1D7j4,13972
+datachain-0.1.11.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
+datachain-0.1.11.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.1.11.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.1.11.dist-info/RECORD,,