datachain 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl

datachain/__init__.py

@@ -0,0 +1,34 @@
+from datachain.lib.dc import C, DataChain
+from datachain.lib.feature import Feature
+from datachain.lib.feature_utils import pydantic_to_feature
+from datachain.lib.file import File, FileError, FileFeature, IndexedFile, TarVFile
+from datachain.lib.image import ImageFile, convert_images
+from datachain.lib.text import convert_text
+from datachain.lib.udf import Aggregator, Generator, Mapper
+from datachain.lib.utils import AbstractUDF, DataChainError
+from datachain.query.dataset import UDF as BaseUDF  # noqa: N811
+from datachain.query.schema import Column
+from datachain.query.session import Session
+
+__all__ = [
+    "AbstractUDF",
+    "Aggregator",
+    "BaseUDF",
+    "C",
+    "Column",
+    "DataChain",
+    "DataChainError",
+    "Feature",
+    "File",
+    "FileError",
+    "FileFeature",
+    "Generator",
+    "ImageFile",
+    "IndexedFile",
+    "Mapper",
+    "Session",
+    "TarVFile",
+    "convert_images",
+    "convert_text",
+    "pydantic_to_feature",
+]

datachain/lib/dc.py

@@ -137,6 +137,7 @@ class DataChain(DatasetQuery):
             indexing_column_types=File._datachain_column_types,
         )
         self._settings = Settings()
+        self._setup = {}
 
         if self.feature_schema:
             self.signals_schema = SignalSchema.deserialize(self.feature_schema)
@@ -536,9 +537,9 @@ class DataChain(DatasetQuery):
         name = self.name or ""
 
         sign = UdfSignature.parse(name, signal_map, func, params, output, is_generator)
-        params_schema = self.signals_schema.slice(sign.params)
+        params_schema = self.signals_schema.slice(sign.params, self._setup)
 
-        return UDFBase._create(target_class, sign, params_schema, self.catalog)
+        return UDFBase._create(target_class, sign, params_schema)
 
     def _extend_features(self, method_name, *args, **kwargs):
         super_func = getattr(super(), method_name)
@@ -569,18 +570,6 @@ class DataChain(DatasetQuery):
         chain.signals_schema = new_schema
         return chain
 
-    def get_values(self, *cols: str) -> Iterator[list]:
-        """Iterate over rows, getting feature values and applying reader calls.
-
-        If columns are specified - limit them to specified columns.
-        """
-        for features in self.iterate(*cols):
-            yield [fr.get_value() if isinstance(fr, Feature) else fr for fr in features]  # type: ignore[union-attr,call-arg]
-
-    def get_one_value(self, col: str) -> Iterator:
-        for item in self.get_values(col):
-            yield item[0]
-
     def iterate(self, *cols: str) -> Iterator[list[FeatureType]]:
         """Iterate over rows.
 
@@ -604,12 +593,6 @@ class DataChain(DatasetQuery):
     def collect_one(self, col: str) -> list[FeatureType]:
         return list(self.iterate_one(col))
 
-    def collect_values(self, *cols: str) -> list[list]:
-        return list(self.get_values(*cols))
-
-    def collect_one_value(self, col: str) -> list:
-        return list(self.get_one_value(col))
-
     def to_pytorch(self, **kwargs):
         """Convert to pytorch dataset format."""
 
@@ -931,3 +914,12 @@ class DataChain(DatasetQuery):
 
     def max(self, fr: FeatureType):  # type: ignore[override]
         return self._extend_features("max", fr)
+
+    def setup(self, **kwargs) -> "Self":
+        intersection = set(self._setup.keys()) & set(kwargs.keys())
+        if intersection:
+            keys = ", ".join(intersection)
+            raise DatasetPrepareError(self.name, f"this value(s) already setup: {keys}")
+
+        self._setup = self._setup | kwargs
+        return self

datachain/lib/feature.py

@@ -4,6 +4,7 @@ import re
 import warnings
 from collections.abc import Iterable, Sequence
 from datetime import datetime
+from enum import Enum
 from functools import lru_cache
 from types import GenericAlias
 from typing import (
@@ -63,6 +64,7 @@ TYPE_TO_DATACHAIN = {
     str: String,
     Literal: String,
     LiteralEx: String,
+    Enum: String,
     float: Float,
     bool: Boolean,
     datetime: DateTime,  # Note, list of datetime is not supported yet
@@ -364,8 +366,11 @@ def _resolve(cls, name, field_info, prefix: list[str]):
 
 
 def convert_type_to_datachain(typ):  # noqa: PLR0911
-    if inspect.isclass(typ) and issubclass(typ, SQLType):
-        return typ
+    if inspect.isclass(typ):
+        if issubclass(typ, SQLType):
+            return typ
+        if issubclass(typ, Enum):
+            return str
 
     res = TYPE_TO_DATACHAIN.get(typ)
     if res:

datachain/lib/feature_utils.py

@@ -1,5 +1,7 @@
+import inspect
 import string
 from collections.abc import Sequence
+from enum import Enum
 from typing import Any, Union, get_args, get_origin
 
 from pydantic import BaseModel, create_model
@@ -12,9 +14,6 @@ from datachain.lib.feature import (
     convert_type_to_datachain,
 )
 from datachain.lib.utils import DataChainParamsError
-from datachain.query.schema import Column
-
-FeatureLike = Union[type["Feature"], Column, str]
 
 AUTO_FEATURE_PREFIX = "_auto_fr"
 SUFFIX_SYMBOLS = string.digits + string.ascii_lowercase
@@ -38,23 +37,7 @@ def pydantic_to_feature(data_cls: type[BaseModel]) -> type[Feature]:
     for name, field_info in data_cls.model_fields.items():
         anno = field_info.annotation
         if anno not in TYPE_TO_DATACHAIN:
-            orig = get_origin(anno)
-            if orig is list:
-                anno = get_args(anno)  # type: ignore[assignment]
-                if isinstance(anno, Sequence):
-                    anno = anno[0]  # type: ignore[unreachable]
-                is_list = True
-            else:
-                is_list = False
-
-            try:
-                convert_type_to_datachain(anno)
-            except TypeError:
-                if not Feature.is_feature(anno):  # type: ignore[arg-type]
-                    anno = pydantic_to_feature(anno)  # type: ignore[arg-type]
-
-            if is_list:
-                anno = list[anno]  # type: ignore[valid-type]
+            anno = _to_feature_type(anno)
         fields[name] = (anno, field_info.default)
 
     cls = create_model(
@@ -66,6 +49,38 @@ def pydantic_to_feature(data_cls: type[BaseModel]) -> type[Feature]:
     return cls
 
 
+def _to_feature_type(anno):
+    if inspect.isclass(anno) and issubclass(anno, Enum):
+        return str
+
+    orig = get_origin(anno)
+    if orig is list:
+        anno = get_args(anno)  # type: ignore[assignment]
+        if isinstance(anno, Sequence):
+            anno = anno[0]  # type: ignore[unreachable]
+        is_list = True
+    else:
+        is_list = False
+
+    try:
+        convert_type_to_datachain(anno)
+    except TypeError:
+        if not Feature.is_feature(anno):  # type: ignore[arg-type]
+            orig = get_origin(anno)
+            if orig in TYPE_TO_DATACHAIN:
+                anno = _to_feature_type(anno)
+            else:
+                if orig == Union:
+                    args = get_args(anno)
+                    if len(args) == 2 and (type(None) in args):
+                        return _to_feature_type(args[0])
+
+                anno = pydantic_to_feature(anno)  # type: ignore[arg-type]
+    if is_list:
+        anno = list[anno]  # type: ignore[valid-type]
+    return anno
+
+
 def features_to_tuples(
     ds_name: str = "",
     output: Union[None, FeatureType, Sequence[str], dict[str, FeatureType]] = None,

datachain/lib/file.py

@@ -238,9 +238,6 @@ class File(FileFeature):
         return self._catalog.get_client(self.source).fs
 
 
-BinaryFile = File
-
-
 class TextFile(File):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)

datachain/lib/pytorch.py

@@ -8,6 +8,7 @@ from torch.utils.data import IterableDataset, get_worker_info
 
 from datachain.catalog import Catalog, get_catalog
 from datachain.lib.dc import DataChain
+from datachain.lib.feature import Feature
 from datachain.lib.text import convert_text
 
 if TYPE_CHECKING:
@@ -100,8 +101,14 @@ class PytorchDataset(IterableDataset):
         if self.num_samples > 0:
             ds = ds.sample(self.num_samples)
         ds = ds.chunk(total_rank, total_workers)
-        stream = ds.get_values()
-        for row in stream:
+        stream = ds.iterate()
+        for row_features in stream:
+            row = []
+            for fr in row_features:
+                if isinstance(fr, Feature):
+                    row.append(fr.get_value())  # type: ignore[unreachable]
+                else:
+                    row.append(fr)
             # Apply transforms
             if self.transform:
                 try:

datachain/lib/signal_schema.py

@@ -1,7 +1,7 @@
 import copy
 from collections.abc import Iterator, Sequence
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Optional, Union, get_args, get_origin
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union, get_args, get_origin
 
 from pydantic import create_model
 
@@ -42,6 +42,11 @@ class SignalResolvingError(SignalSchemaError):
         super().__init__(f"cannot resolve signal name{name}: {msg}")
 
 
+class SetupError(SignalSchemaError):
+    def __init__(self, name: str, msg: str):
+        super().__init__(f"cannot setup value '{name}': {msg}")
+
+
 class SignalResolvingTypeError(SignalResolvingError):
     def __init__(self, method: str, field):
         super().__init__(
@@ -52,9 +57,31 @@ class SignalResolvingTypeError(SignalResolvingError):
 
 
 class SignalSchema:
-    def __init__(self, values: dict[str, FeatureType]):
+    def __init__(
+        self,
+        values: dict[str, FeatureType],
+        setup: Optional[dict[str, Callable]] = None,
+    ):
         self.values = values
-        self.tree = self._build_tree()
+        self.tree = self._build_tree(values)
+
+        self.setup_func = setup or {}
+        self.setup_values = None
+        for key, func in self.setup_func.items():
+            if not callable(func):
+                raise SetupError(key, "value must be function or callable class")
+
+    def _init_setup_values(self):
+        if self.setup_values is not None:
+            return self.setup_values
+
+        res = {}
+        for key, func in self.setup_func.items():
+            try:
+                res[key] = func()
+            except Exception as ex:
+                raise SetupError(key, f"error when call function: '{ex}'") from ex
+        self.setup_values = res
 
     @staticmethod
     def from_column_types(col_types: dict[str, Any]) -> "SignalSchema":
@@ -111,16 +138,22 @@ class SignalSchema:
     def to_udf_spec(self) -> dict[str, Any]:
         res = {}
         for path, type_, has_subtree, _ in self.get_flat_tree():
+            if path[0] in self.setup_func:
+                continue
             if not has_subtree:
                 db_name = DEFAULT_DELIMITER.join(path)
                 res[db_name] = convert_type_to_datachain(type_)
         return res
 
     def row_to_objs(self, row: Sequence[Any]) -> list[FeatureType]:
+        self._init_setup_values()
+
         objs = []
         pos = 0
-        for fr_type in self.values.values():
-            if Feature.is_feature(fr_type):
+        for name, fr_type in self.values.items():
+            if val := self.setup_values.get(name, None):  # type: ignore[attr-defined]
+                objs.append(val)
+            elif Feature.is_feature(fr_type):
                 j, pos = fr_type._unflatten_to_json_pos(row, pos)  # type: ignore[union-attr]
                 objs.append(fr_type(**j))
             else:
@@ -135,8 +168,14 @@ class SignalSchema:
             if Feature.is_feature(fr)
         )
 
-    def slice(self, keys: Sequence[str]) -> "SignalSchema":
-        return SignalSchema({k: self.values[k] for k in keys if k in self.values})
+    def slice(
+        self, keys: Sequence[str], setup: Optional[dict[str, Callable]] = None
+    ) -> "SignalSchema":
+        setup = setup or {}
+        setup_no_types = dict.fromkeys(setup.keys(), str)
+        union = self.values | setup_no_types
+        schema = {k: union[k] for k in keys if k in union}
+        return SignalSchema(schema, setup)
 
     def row_to_features(self, row: Sequence, catalog: "Catalog") -> list[FeatureType]:
         res = []
@@ -235,10 +274,11 @@ class SignalSchema:
             **fields,
         )
 
-    def _build_tree(self) -> dict[str, Any]:
+    @staticmethod
+    def _build_tree(values: dict[str, FeatureType]) -> dict[str, Any]:
         res = {}
 
-        for name, val in self.values.items():
+        for name, val in values.items():
             subtree = val.build_tree() if Feature.is_feature(val) else None  # type: ignore[union-attr]
             res[name] = (val, subtree)
 

datachain/lib/udf.py

@@ -64,7 +64,6 @@ class UDFBase(AbstractUDF):
         target_class: type["UDFBase"],
         sign: UdfSignature,
         params: SignalSchema,
-        catalog,
     ) -> "UDFBase":
         if isinstance(sign.func, AbstractUDF):
             if not isinstance(sign.func, target_class):  # type: ignore[unreachable]

datachain-0.2.6.dist-info/METADATA

@@ -0,0 +1,429 @@
+Metadata-Version: 2.1
+Name: datachain
+Version: 0.2.6
+Summary: Wrangle unstructured AI data at scale
+Author-email: Dmitry Petrov <support@dvc.org>
+License: Apache-2.0
+Project-URL: Documentation, https://datachain.dvc.ai
+Project-URL: Issues, https://github.com/iterative/dvcx/issues
+Project-URL: Source, https://github.com/iterative/dvcx
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Development Status :: 2 - Pre-Alpha
+Requires-Python: >=3.9
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: pyyaml
+Requires-Dist: tomlkit
+Requires-Dist: tqdm
+Requires-Dist: numpy
+Requires-Dist: pandas >=2.0.0
+Requires-Dist: pyarrow
+Requires-Dist: typing-extensions
+Requires-Dist: python-dateutil >=2
+Requires-Dist: attrs >=21.3.0
+Requires-Dist: s3fs >=2024.2.0
+Requires-Dist: gcsfs >=2024.2.0
+Requires-Dist: adlfs >=2024.2.0
+Requires-Dist: dvc-data <4,>=3.10
+Requires-Dist: dvc-objects <6,>=4
+Requires-Dist: shtab <2,>=1.3.4
+Requires-Dist: sqlalchemy >=2
+Requires-Dist: multiprocess ==0.70.16
+Requires-Dist: dill ==0.3.8
+Requires-Dist: ujson >=5.9.0
+Requires-Dist: pydantic <3,>=2
+Requires-Dist: jmespath >=1.0
+Requires-Dist: datamodel-code-generator >=0.25
+Requires-Dist: numpy <2,>=1 ; sys_platform == "win32"
+Provides-Extra: cv
+Requires-Dist: Pillow <11,>=10.0.0 ; extra == 'cv'
+Requires-Dist: torch >=2.1.0 ; extra == 'cv'
+Requires-Dist: torchvision ; extra == 'cv'
+Requires-Dist: transformers >=4.36.0 ; extra == 'cv'
+Provides-Extra: dev
+Requires-Dist: datachain[docs,tests] ; extra == 'dev'
+Requires-Dist: mypy ==1.10.1 ; extra == 'dev'
+Requires-Dist: types-python-dateutil ; extra == 'dev'
+Requires-Dist: types-PyYAML ; extra == 'dev'
+Requires-Dist: types-requests ; extra == 'dev'
+Requires-Dist: types-ujson ; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs >=1.5.2 ; extra == 'docs'
+Requires-Dist: mkdocs-gen-files >=0.5.0 ; extra == 'docs'
+Requires-Dist: mkdocs-material >=9.3.1 ; extra == 'docs'
+Requires-Dist: mkdocs-section-index >=0.3.6 ; extra == 'docs'
+Requires-Dist: mkdocstrings-python >=1.6.3 ; extra == 'docs'
+Requires-Dist: mkdocs-literate-nav >=0.6.1 ; extra == 'docs'
+Provides-Extra: remote
+Requires-Dist: lz4 ; extra == 'remote'
+Requires-Dist: msgpack <2,>=1.0.4 ; extra == 'remote'
+Requires-Dist: requests >=2.22.0 ; extra == 'remote'
+Provides-Extra: tests
+Requires-Dist: datachain[cv,remote,vector] ; extra == 'tests'
+Requires-Dist: pytest <9,>=8 ; extra == 'tests'
+Requires-Dist: pytest-sugar >=0.9.6 ; extra == 'tests'
+Requires-Dist: pytest-cov >=4.1.0 ; extra == 'tests'
+Requires-Dist: pytest-mock >=3.12.0 ; extra == 'tests'
+Requires-Dist: pytest-servers[all] >=0.5.4 ; extra == 'tests'
+Requires-Dist: pytest-benchmark[histogram] ; extra == 'tests'
+Requires-Dist: pytest-asyncio >=0.23.2 ; extra == 'tests'
+Requires-Dist: pytest-xdist >=3.3.1 ; extra == 'tests'
+Requires-Dist: virtualenv ; extra == 'tests'
+Requires-Dist: dulwich ; extra == 'tests'
+Requires-Dist: hypothesis ; extra == 'tests'
+Requires-Dist: open-clip-torch ; extra == 'tests'
+Requires-Dist: aiotools >=1.7.0 ; extra == 'tests'
+Requires-Dist: requests-mock ; extra == 'tests'
+Provides-Extra: vector
+Requires-Dist: usearch ; extra == 'vector'
+
+|PyPI| |Python Version| |Codecov| |Tests|
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/datachain.svg
+   :target: https://pypi.org/project/datachain/
+   :alt: PyPI
+.. |Python Version| image:: https://img.shields.io/pypi/pyversions/datachain
+   :target: https://pypi.org/project/datachain
+   :alt: Python Version
+.. |Codecov| image:: https://codecov.io/gh/iterative/dvcx/branch/main/graph/badge.svg?token=VSCP2T9R5X
+   :target: https://app.codecov.io/gh/iterative/dvcx
+   :alt: Codecov
+.. |Tests| image:: https://github.com/iterative/dvcx/workflows/Tests/badge.svg
+   :target: https://github.com/iterative/dvcx/actions?workflow=Tests
+   :alt: Tests
+
+AI 🔗 DataChain
+----------------
+
+DataChain is an open-source Python data processing library for wrangling unstructured AI data at scale.
+
+Datachain enables multimodal API calls and local AI inferences to run in parallel over many samples as chained operations. The resulting datasets can be saved, versioned, and sent directly to PyTorch and TensorFlow for training. Datachain can persist features of Python objects returned by AI models, and enables vectorized analytical operations over them.
+
+The typical use cases are data curation, LLM analytics and validation, image segmentation, pose detection, and GenAI alignment. Datachain is especially helpful if batch operations can be optimized – for instance, when synchronous API calls can be parallelized  or where an LLM API offers batch processing.
+
+.. code:: console
+
+   $ pip install datachain
+
+Operation basics
+----------------
+
+DataChain is built by composing wrangling operations.
+
+For example, let us consider a dataset from Karlsruhe Institute of Technology detailing dialogs between users and customer service chatbots. We can use the chain to read data from the cloud, map it onto the parallel API calls for LLM evaluation, and organize the output into a dataset :
+
+.. code:: py
+
+      # pip install mistralai
+      # this example requires a free Mistral API key, get yours at https://console.mistral.ai
+      # add the key to your shell environment: $ export MISTRAL_API_KEY= your key
+
+      # pip install mistralai
+      # this example requires a free Mistral API key, get yours at https://console.mistral.ai
+      # add the key to your shell environment: $ export MISTRAL_API_KEY= your key
+
+      import os
+
+      from mistralai.client import MistralClient
+      from mistralai.models.chat_completion import ChatMessage
+
+      from datachain.lib.dc import DataChain, Column
+
+      PROMPT = "Was this bot dialog successful? Describe the 'result' as 'Yes' or 'No' in a short JSON"
+
+      model = "mistral-large-latest"
+      api_key = os.environ["MISTRAL_API_KEY"]
+
+      chain = (
+          DataChain.from_storage("gs://datachain-demo/chatbot-KiT/")
+          .limit(5)
+          .settings(cache=True, parallel=5)
+          .map(
+              mistral_response=lambda file: MistralClient(api_key=api_key)
+              .chat(
+                  model=model,
+                  response_format={"type": "json_object"},
+                  messages=[
+                      ChatMessage(role="user", content=f"{PROMPT}: {file.get_value()}")
+                  ],
+              )
+              .choices[0]
+              .message.content,
+          )
+          .save()
+      )
+
+      try:
+          print(chain.select("mistral_response").results())
+      except Exception as e:
+          print(f"do you have the right Mistral API key? {e}")
+
+
+.. code:: shell
+
+      [('{"result": "Yes"}',), ('{"result": "No"}',), ... , ('{"result": "Yes"}',)]
+
+Now we have parallel-processed an LLM API-based query over cloud data and persisted the results.
+
+Vectorized analytics
+--------------------
+
+Datachain internally represents datasets as tables, so analytical queries on the chain are automatically vectorized:
+
+.. code:: py
+
+      failed_dialogs = chain.filter(Column("mistral_response") == '{"result": "No"}')
+      success_rate = failed_dialogs.count() / chain.count()
+      print(f"Chatbot dialog success rate: {100*success_rate:.2f}%")
+
+
+.. code:: shell
+
+      "40.00%"
+
+Note that DataChain represents file samples as pointers into their respective storage locations. This means a newly created dataset version does not duplicate files in storage, and storage remains the single source of truth for the original samples
+
+Handling Python objects
+-----------------------
+In addition to storing primitive Python data types, chain is also capable of using data models.
+
+For example, instead of collecting just a text response from Mistral API, we might be interested in more fields of the Mistral response object. For this task, we can define a Pydantic-like model and populate it from the API replies:
+
+.. code:: py
+
+      import os
+
+      from mistralai.client import MistralClient
+      from mistralai.models.chat_completion import ChatMessage
+
+      from datachain.lib.dc import DataChain
+      from datachain.lib.feature import Feature
+
+
+      PROMPT = (
+          "Was this dialog successful? Describe the 'result' as 'Yes' or 'No' in a short JSON"
+      )
+
+      model = "mistral-large-latest"
+      api_key = os.environ["MISTRAL_API_KEY"]
+
+
+      ## define the data model ###
+      class Usage(Feature):
+          prompt_tokens: int = 0
+          completion_tokens: int = 0
+
+
+      class MyChatMessage(Feature):
+          role: str = ""
+          content: str = ""
+
+
+      class CompletionResponseChoice(Feature):
+          message: MyChatMessage = MyChatMessage()
+
+
+      class MistralModel(Feature):
+          id: str = ""
+          choices: list[CompletionResponseChoice]
+          usage: Usage = Usage()
+
+
+      ## Populate model instances ###
+      chain = (
+          DataChain.from_storage("gs://datachain-demo/chatbot-KiT/")
+          .limit(5)
+          .settings(cache=True, parallel=5)
+          .map(
+              mistral_response=lambda file: MistralModel(
+                  **MistralClient(api_key=api_key)
+                  .chat(
+                      model=model,
+                      response_format={"type": "json_object"},
+                      messages=[
+                          ChatMessage(role="user", content=f"{PROMPT}: {file.get_value()}")
+                      ],
+                  )
+                  .dict()
+              ),
+              output=MistralModel,
+          )
+          .save("dialog-eval")
+      )
+
+After the chain execution, we can collect the objects:
+
+.. code:: py
+
+      for obj in responses:
+          assert isinstance(obj, MistralModel)
+          print(obj.dict())
+
+.. code:: shell
+
+      {'choices': [{'message': {'role': 'assistant', 'content': '{"result": "Yes"}'}}], 'usage': {'prompt_tokens': 610, 'completion_tokens': 6}}
+      {'choices': [{'message': {'role': 'assistant', 'content': '{"result": "No"}'}}], 'usage': {'prompt_tokens': 3983, 'completion_tokens': 6}}
+      {'choices': [{'message': {'role': 'assistant', 'content': '{"result": "Yes"}'}}], 'usage': {'prompt_tokens': 706, 'completion_tokens': 6}}
+      {'choices': [{'message': {'role': 'assistant', 'content': '{"result": "No"}'}}], 'usage': {'prompt_tokens': 1250, 'completion_tokens': 6}}
+      {'choices': [{'message': {'role': 'assistant', 'content': '{"result": "Yes"}'}}], 'usage': {'prompt_tokens': 1217, 'completion_tokens': 6}}
+
+
+Dataset persistence
+--------------------
+
+The “save” operation makes chain dataset persistent in the current (working) directory of the query. A hidden folder .datachain/ holds the records. A persistent dataset can be accessed later to start a derivative chain:
+
+.. code:: py
+
+         DataChain.from_dataset("dialog-eval").limit(2).save("dialog-eval")
+
+Persistent datasets are immutable and automatically versioned. Versions can be listed from shell:
+
+.. code:: shell
+
+      $ datachain ls-datasets
+
+      dialog-rate (v1)
+      dialog-rate (v2)
+
+By default, when a persistent dataset is loaded, the latest version is fetched but another version can be requested:
+
+.. code:: py
+
+      ds = DataChain.from_dataset("dialog-eval", version = 1)
+
+Chain optimization and execution
+--------------------------------
+
+Datachain avoids redundant operations. Execution is triggered only when a downstream operation requests the processed results. However, it would be inefficient to run, say, LLM queries again every time you just want to collect several objects.
+
+“Save” operation nails execution results and automatically refers to them every time the downstream functions ask for data. Saving without an explicit name generates an auto-named dataset which serves the same purpose.
+
+
+Matching data with metadata
+----------------------------
+It is common for AI data to come with pre-computed metadata (annotations, classes, etc).
+
+DataChain library understands common metadata formats (JSON, CSV and parquet), and can unite data samples from storage with side-loaded metadata. The schema for metadata can be set explicitly or be inferred.
+
+Here is an example of reading a CSV file where schema is heuristically derived from the header:
+
+.. code:: py
+
+      from datachain.lib.dc import DataChain
+      csv_dataset = DataChain.from_csv("gs://datachain-demo/chatbot-csv/")
+
+      print(csv_dataset.to_pandas())
+
+Reading metadata from JSON format is a more complicated scenario because a JSON-annotated dataset typically references data samples (e.g. images) in annotation arrays somewhere within JSON files.
+
+Here is an example from MS COCO “captions” JSON which employs separate sections for image meta and captions:
+
+.. code:: json
+
+
+      {
+        "images": [
+          {
+            "license": 4,
+            "file_name": "000000397133.jpg",
+            "coco_url": "http://images.cocodataset.org/val2017/000000397133.jpg",
+            "height": 427,
+            "width": 640,
+            "date_captured": "2013-11-14 17:02:52",
+            "flickr_url": "http://farm7.staticflickr.com/6116/6255196340_da26cf2c9e_z.jpg",
+            "id": 397133
+          },
+          ...
+        ],
+        "annotations": [
+          {
+            "image_id"	:	"179765",
+            "id"	:	38,
+            "caption"	:	"A black Honda motorcycle parked in front of a garage."
+          },
+          ...
+        ],
+        ...
+      }
+
+To deal with this layout, we can take the following steps:
+
+1. Generate a dataset of raw image files from storage
+2. Generate a meta-information dataset from the JSON section “images”
+3. Join these datasets via the matching id keys
+
+.. code:: python
+
+
+   from datachain.lib.dc import DataChain
+
+   images = DataChain.from_storage("gs://datachain-demo/coco2017/images/val/")
+   meta = DataChain.from_json("gs://datachain-demo/coco2017/annotations_captions", jmespath = "images")
+
+   images_with_meta = images.merge(meta, on="file.name", right_on="images.file_name")
+
+   print(images_with_meta.limit(1).results())
+
+.. code:: shell
+
+
+      Processed: 5000 rows [00:00, 15481.66 rows/s]
+      Processed: 1 rows [00:00, 1291.75 rows/s]
+      Processed: 1 rows [00:00,  4.70 rows/s]
+      Generated: 5000 rows [00:00, 27128.67 rows/s]
+      [(1, 2336066478558845549, '', 0, 'coco2017/images/val', '000000000139.jpg', 'CNvXoemj8IYDEAE=', '1719096046021595', 1, datetime.datetime(2024, 6, 22, 22, 40, 46, 70000, tzinfo=datetime.timezone.utc), 161811, '', '', None, 'gs://datachain-demo', 'gs://datachain-demo', 'coco2017/images/val', '000000000139.jpg', 161811, '1719096046021595', 'CNvXoemj8IYDEAE=', 1, datetime.datetime(1970, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), None, '', 4146, 6967063844996569113, 2, '000000000139.jpg', 'http://images.cocodataset.org/val2017/000000000139.jpg', 426, 640, '2013-11-21 01:34:01', 'http://farm9.staticflickr.com/8035/8024364858_9c41dc1666_z.jpg', 139)]
+
+Passing data to training
+------------------------
+
+Chain results can be exported or passed directly to Pytorch dataloader. For example, if we are interested in passing three columns to training, the following Pytorch code will do it:
+
+.. code:: py
+
+      ds = train.select("file", "caption_choices", "label_ind").to_pytorch(
+          transform=preprocess,
+          tokenizer=clip.tokenize,
+      )
+
+      loader = DataLoader(ds, batch_size=2)
+      optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
+      train(loader, model, optimizer)
+
+Tutorials
+------------------
+
+* `Multimodal <examples/multimodal/clip_fine_tuning.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/dvclive/blob/main/examples/multimodal/clip_fine_tuning.ipynb>`__)
+
+Contributions
+--------------------
+
+Contributions are very welcome.
+To learn more, see the `Contributor Guide`_.
+
+
+License
+-------
+
+Distributed under the terms of the `Apache 2.0 license`_,
+*DataChain* is free and open source software.
+
+
+Issues
+------
+
+If you encounter any problems,
+please `file an issue`_ along with a detailed description.
+
+
+.. _Apache 2.0 license: https://opensource.org/licenses/Apache-2.0
+.. _PyPI: https://pypi.org/
+.. _file an issue: https://github.com/iterative/dvcx/issues
+.. _pip: https://pip.pypa.io/
+.. github-only
+.. _Contributor Guide: CONTRIBUTING.rst
+.. _Pydantic: https://github.com/pydantic/pydantic

{datachain-0.2.4.dist-info → datachain-0.2.6.dist-info}/RECORD

@@ -1,4 +1,4 @@
-datachain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datachain/__init__.py,sha256=wvf70BnoqaeYdxZYyZIw_wkykA7ZbFwvOZ2gRk-ZY5o,959
 datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
 datachain/asyn.py,sha256=CKCFQJ0CbB3r04S7mUTXxriKzPnOvdUaVPXjM8vCtJw,7644
 datachain/cache.py,sha256=FaPWrqWznPffmskTb1pdPkt2jAMMf__9FC2zEnP0vDU,4022
@@ -41,11 +41,11 @@ datachain/lib/arrow.py,sha256=FF3WWUOjB6Prw8ygfiLsrVfrdob0S01lPzEazuGqoO8,2556
 datachain/lib/cached_stream.py,sha256=t2ifK0hZVZiVn0MQ8D3FaFK1-qK84TwJW2Dw1SRsw9g,1066
 datachain/lib/claude.py,sha256=iAauA1zNVNONpLzUo1t0QN5PZ5Ot6cZkfib7Ka_c638,1969
 datachain/lib/clip.py,sha256=rDeZlFGs0DXBlpmh5ZQJhR9Sz13bWAZGQjfYm1hsUI4,5388
-datachain/lib/dc.py,sha256=Sf99R0oOqf7tlS2gieaG56z3bF7YVcMjhJOZrFRfFs8,34778
-datachain/lib/feature.py,sha256=QDloA9HE7URf9J_veKrguYBvSg-0cbXZFTswNxrKsB8,12135
+datachain/lib/dc.py,sha256=D3cgib-U0Mo0x5wEK1_NfgymAldHqCvooZwtyohi53Q,34426
+datachain/lib/feature.py,sha256=iMwbMyQUyjRUeB-vhAucnx59kNSVvX_xEChTW5B9klY,12244
 datachain/lib/feature_registry.py,sha256=K3jGQzBp2HZDjR9hdGe1BZaXOAne8RpkCRRQdTVjkTs,1622
-datachain/lib/feature_utils.py,sha256=F4ZENO6tTQvd36a-O1AurYjFSUpoyZaT4qgXsKjQDts,4650
-datachain/lib/file.py,sha256=TdhsPYmG0Atkd_QAO997oA8AuM854wNbjjLLT1uiD2M,8346
+datachain/lib/feature_utils.py,sha256=2yLdZd9o4AJ5QQX7kqgbCxCT78aT7HE12CLxQ6QRpbc,4982
+datachain/lib/file.py,sha256=LGBwC7tFU7VcSWk5kjPpEWPBQas5me69L2uTDNvYXGM,8326
 datachain/lib/gpt4_vision.py,sha256=idyXVZVWzltstGaVIu5RYE5UNbdqcPEjIWy81O1MwkM,2922
 datachain/lib/hf_image_to_text.py,sha256=HiPSWzJRDT-vnz9DXJbJBNCMNl9wmpxiSS3PbbVz8SE,3310
 datachain/lib/hf_pipeline.py,sha256=f0AH_XCziOF1OKN3d1w1swTBLaeajMJ8xgdsX37i5-o,2287
@@ -53,11 +53,11 @@ datachain/lib/image.py,sha256=ZYfDqr9p-RRmWBeWFQwXLS1J3vQS616ykfMUvQVpqBY,2717
 datachain/lib/image_transform.py,sha256=NXWtnVOcofWBgl_YMxb4ABpaT7JTBMx7tLKvErH1IC4,3024
 datachain/lib/iptc_exif_xmp.py,sha256=xrbxFeY-wRP6T5JsUgE3EXfTxKvZVymRaRD_VIfxD0A,2236
 datachain/lib/meta_formats.py,sha256=wIVVLRLp45Zk4vjZRd_P1UtD24vpDCb-vILWtcsACwk,6630
-datachain/lib/pytorch.py,sha256=Z7iZCsqJzUT0PynVo23Xu4Fx7qIuuEZyH83R1tR5mfI,5561
+datachain/lib/pytorch.py,sha256=Ea1sXhborF6zcywQjLpXgKnbr1lTez4Bfu3m0Gr78FI,5843
 datachain/lib/settings.py,sha256=6Nkoh8riETrftYwDp3aniK53Dsjc07MdztL8N0cW1D8,2849
-datachain/lib/signal_schema.py,sha256=KTegbx-yMvtaKEoUxLgDx5MxMA8De-nmdtqnV1932N8,10151
+datachain/lib/signal_schema.py,sha256=hD56hyO1H3A5H2oyTUwPcNu6UOQ_XY0DeA0nrXBqFaU,11492
 datachain/lib/text.py,sha256=PUT1O0jNJoQGsuhff2LgDpzTWk2eMdwIKqEDBrE448M,1307
-datachain/lib/udf.py,sha256=kMlOsHCVybnnq4AMtYqjylZH7x2tGE62FsDPOu9qhWM,6612
+datachain/lib/udf.py,sha256=axMvqYz4tdyg_C3nyuOcDsu3Aqr19jWv2vl54U_8LQM,6595
 datachain/lib/udf_signature.py,sha256=CUKgoVpM_N8CgvMncpAw2RYchoiJdAGdDSdluoP0hIk,7161
 datachain/lib/unstructured.py,sha256=9Y6rAelXdYqkNbPaqz6DhXjhS8d6qXcP0ieIsWkzvkk,1143
 datachain/lib/utils.py,sha256=5-kJlAZE0D9nXXweAjo7-SP_AWGo28feaDByONYaooQ,463
@@ -92,9 +92,9 @@ datachain/sql/sqlite/__init__.py,sha256=TAdJX0Bg28XdqPO-QwUVKy8rg78cgMileHvMNot7
 datachain/sql/sqlite/base.py,sha256=nPMF6_FF04hclDNZev_YfxMgbJAsWEdF-rU2pUhqBtc,12048
 datachain/sql/sqlite/types.py,sha256=oP93nLfTBaYnN0z_4Dsv-HZm8j9rrUf1esMM-z3JLbg,1754
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
-datachain-0.2.4.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.2.4.dist-info/METADATA,sha256=rWswQ1xeEbhMXJ3xRTiFhWjgEqjhSX4ay-ashGNxf8o,14399
-datachain-0.2.4.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-datachain-0.2.4.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.2.4.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.2.4.dist-info/RECORD,,
+datachain-0.2.6.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.2.6.dist-info/METADATA,sha256=j3Pq4f0toq25yDr9FAQhJaygkE7St8BTZVcigAw47t4,16475
+datachain-0.2.6.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
+datachain-0.2.6.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.2.6.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.2.6.dist-info/RECORD,,

datachain-0.2.4.dist-info/METADATA

@@ -1,371 +0,0 @@
-Metadata-Version: 2.1
-Name: datachain
-Version: 0.2.4
-Summary: Wrangle unstructured AI data at scale
-Author-email: Dmitry Petrov <support@dvc.org>
-License: Apache-2.0
-Project-URL: Documentation, https://datachain.dvc.ai
-Project-URL: Issues, https://github.com/iterative/dvcx/issues
-Project-URL: Source, https://github.com/iterative/dvcx
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Development Status :: 2 - Pre-Alpha
-Requires-Python: >=3.9
-Description-Content-Type: text/x-rst
-License-File: LICENSE
-Requires-Dist: pyyaml
-Requires-Dist: tomlkit
-Requires-Dist: tqdm
-Requires-Dist: numpy
-Requires-Dist: pandas >=2.0.0
-Requires-Dist: pyarrow
-Requires-Dist: typing-extensions
-Requires-Dist: python-dateutil >=2
-Requires-Dist: attrs >=21.3.0
-Requires-Dist: s3fs >=2024.2.0
-Requires-Dist: gcsfs >=2024.2.0
-Requires-Dist: adlfs >=2024.2.0
-Requires-Dist: dvc-data <4,>=3.10
-Requires-Dist: dvc-objects <6,>=4
-Requires-Dist: shtab <2,>=1.3.4
-Requires-Dist: sqlalchemy >=2
-Requires-Dist: multiprocess ==0.70.16
-Requires-Dist: dill ==0.3.8
-Requires-Dist: ujson >=5.9.0
-Requires-Dist: pydantic <3,>=2
-Requires-Dist: jmespath >=1.0
-Requires-Dist: datamodel-code-generator >=0.25
-Requires-Dist: numpy <2,>=1 ; sys_platform == "win32"
-Provides-Extra: cv
-Requires-Dist: Pillow <11,>=10.0.0 ; extra == 'cv'
-Requires-Dist: torch >=2.1.0 ; extra == 'cv'
-Requires-Dist: torchvision ; extra == 'cv'
-Requires-Dist: transformers >=4.36.0 ; extra == 'cv'
-Provides-Extra: dev
-Requires-Dist: datachain[docs,tests] ; extra == 'dev'
-Requires-Dist: mypy ==1.10.1 ; extra == 'dev'
-Requires-Dist: types-python-dateutil ; extra == 'dev'
-Requires-Dist: types-PyYAML ; extra == 'dev'
-Requires-Dist: types-requests ; extra == 'dev'
-Requires-Dist: types-ujson ; extra == 'dev'
-Provides-Extra: docs
-Requires-Dist: mkdocs >=1.5.2 ; extra == 'docs'
-Requires-Dist: mkdocs-gen-files >=0.5.0 ; extra == 'docs'
-Requires-Dist: mkdocs-material >=9.3.1 ; extra == 'docs'
-Requires-Dist: mkdocs-section-index >=0.3.6 ; extra == 'docs'
-Requires-Dist: mkdocstrings-python >=1.6.3 ; extra == 'docs'
-Requires-Dist: mkdocs-literate-nav >=0.6.1 ; extra == 'docs'
-Provides-Extra: remote
-Requires-Dist: datachain[pandas] ; extra == 'remote'
-Requires-Dist: lz4 ; extra == 'remote'
-Requires-Dist: msgpack <2,>=1.0.4 ; extra == 'remote'
-Requires-Dist: requests >=2.22.0 ; extra == 'remote'
-Provides-Extra: tests
-Requires-Dist: datachain[cv,pandas,remote,vector] ; extra == 'tests'
-Requires-Dist: pytest <9,>=8 ; extra == 'tests'
-Requires-Dist: pytest-sugar >=0.9.6 ; extra == 'tests'
-Requires-Dist: pytest-cov >=4.1.0 ; extra == 'tests'
-Requires-Dist: pytest-mock >=3.12.0 ; extra == 'tests'
-Requires-Dist: pytest-servers[all] >=0.5.4 ; extra == 'tests'
-Requires-Dist: pytest-benchmark[histogram] ; extra == 'tests'
-Requires-Dist: pytest-asyncio >=0.23.2 ; extra == 'tests'
-Requires-Dist: pytest-xdist >=3.3.1 ; extra == 'tests'
-Requires-Dist: virtualenv ; extra == 'tests'
-Requires-Dist: dulwich ; extra == 'tests'
-Requires-Dist: hypothesis ; extra == 'tests'
-Requires-Dist: open-clip-torch ; extra == 'tests'
-Requires-Dist: aiotools >=1.7.0 ; extra == 'tests'
-Requires-Dist: requests-mock ; extra == 'tests'
-Provides-Extra: vector
-Requires-Dist: usearch ; extra == 'vector'
-
-|PyPI| |Python Version| |Codecov| |Tests| |License|
-
-.. |PyPI| image:: https://img.shields.io/pypi/v/datachain.svg
-   :target: https://pypi.org/project/datachain/
-   :alt: PyPI
-.. |Python Version| image:: https://img.shields.io/pypi/pyversions/datachain
-   :target: https://pypi.org/project/datachain
-   :alt: Python Version
-.. |Codecov| image:: https://codecov.io/gh/iterative/dvcx/branch/main/graph/badge.svg?token=VSCP2T9R5X
-   :target: https://app.codecov.io/gh/iterative/dvcx
-   :alt: Codecov
-.. |Tests| image:: https://github.com/iterative/dvcx/workflows/Tests/badge.svg
-   :target: https://github.com/iterative/dvcx/actions?workflow=Tests
-   :alt: Tests
-.. |License| image:: https://img.shields.io/pypi/l/datachain
-   :target: https://opensource.org/licenses/Apache-2.0
-   :alt: License
-
-AI 🔗 DataChain
-----------------
-
-DataChain is an open-source Python data processing library for wrangling unstructured AI data at scale.
-
-It enables batch LLM API calls and local language and vision AI model inferences to run in parallel over many samples as chained operations resolving to table-like datasets. These datasets can be saved, versioned, and sent directly to PyTorch and TensorFlow for training. DataChain employs rigorous `Pydantic`_ data structures, promoting better data processing practices and enabling vectorized analytical operations normally found in databases.
-
-The DataChain fills the gap between dataframe libraries, data warehouses, and Python-based multimodal AI applications. Our primary use cases include massive data curation, LLM analytics and validation, batch image segmentation and pose detection, GenAI data alignment, etc.
-
-.. code:: console
-
-   $ pip install datachain
-
-Basic operation
----------------
-
-DataChain is built by composing wrangling operations.
-
-For example, it can be instructed to read files from the cloud, map them onto a modern AI service returning a Python object, parallelize API calls, save the result as a dataset, and export a column:
-
-.. code:: py
-
-         import os
-         import datachain as dc
-
-         from anthropic.types.message import Message
-         ClaudeModel = dc.pydantic_to_feature(Message)
-         PROMPT = "summarize this book in less than 200 words"
-         service = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
-         source = "gs://datachain-demo/mybooks/"
-
-         chain = dc.DataChain(source)                          \
-                       .filter(File.name.glob("*.txt"))        \
-                       .settings(parallel=4)                   \
-                       .map(                                   \
-         		              claude = lambda file:                                         \
-         				        ClaudeModel(**service.messages.create(                        \
-                                                  model="claude-3-haiku-20240307",         \
-                                                  system=PROMPT,                           \
-                                                  messages=[{"role": "user",               \
-                                                             "content": file.get_value()}] \
-         								),  \
-         							).model_dump()  \
-         							)               \
-         							.save("mydataset")
-
-         dc.DataChain("mydataset").export("./", "claude.response") # export summaries
-
-Dataset persistence
--------------------
-
-In the example above, the chain resolves to a saved dataset  “mydataset”.  DataChain datasets are immutable and versioned. A saved dataset version can be used as a data source:
-
-.. code:: py
-
-   ds = dc.DataChain("mydataset", version = 1)
-
-Note that DataChain represents file samples as pointers into their respective storage locations. This means a newly created dataset version does not duplicate files in storage, and storage remains the single source of truth for the original samples
-
-Vectorized analytics
----------------------
-Since datasets are internally represented as tables, analytical queries can be vectorized:
-
-.. code:: py
-
-         rate = ds.filter(chain.response == "Success").count() / chain.count() # ??
-         print(f"API class success rate: {100*rate:.2f}%")
-         >> 74.68%
-
-         price_input = 0.25
-         price_output = 1.25
-         price=(ds.sum(C.claude.usage.input_tokens)*price_input \
-                + ds.sum(C.claude.usage.output_tokens)*price_output)/1_000_000
-         print(f"Cost of API calls: ${price:.2f}")
-         >> Cost of API calls: $1.42
-
-
-Importing metadata
-------------------------
-
-It is common for AI data to come together with metadata (annotations, classes, etc).
-DataChain understands many metadata formats, and can connect data samples in storage with external metadata (e.g. CSV columns) to form a single dataset:
-
-.. code:: py
-
-         from dc import parse_csv
-
-         files = dc.DataChain("gs://datachain-demo/myimages/")
-         metadata = dc.DataChain("gs://datachain-demo/myimagesmetadata.csv") \
-                        .gen(meta=parse_csv)  # TBD, also dependent on dropping file
-         dataset = chain1.merge(chain2, on = "file.name", right_on="name"])
-
-         print(dataset.select("file.name", "class", "prob").limit(5).to_pandas())
-         ....
-         ....
-         ....
-         ....
-         ....
-
-Nested annotations (like JSON) can be unrolled into rows and columns in the way that best fits the application. For example, the MS COCO dataset includes JSON annotations detailing segmentations. To build a dataset consisting of all segmented objects in all COCO images:
-
-.. code:: py
-
-      image_files = dc.DataChain("gs://datachain-demo/coco/images/")
-      image_meta  = dc.DataChain("gs://datachain-demo/coco.json")  \
-                     .gen(meta=parse_json, key="images")       # list of images
-      images = image_files.merge(image_meta, on = "file.name", right_on="file_name")
-      objects_meta = dc.DataChain("gs://datachain-demo/coco.json") \
-                     .gen(meta=parse_json, key="annotations")  # annotated objects
-
-      objects = image.full_merge(objects_meta, on = "id", right_on = "image_id")
-
-Generating metadata
----------------------
-
-A typical step in data curation is to create features from data samples for future selection. DataChain represents the newly created metadata as columns, which makes it easy to create new features and filter on them:
-
-.. code:: py
-
-      from fashion_clip.fashion_clip import FashionCLIP
-      from sqlalchemy import JSON
-      from tabulate import tabulate
-
-      from datachain.lib.param import Image
-      from datachain.query import C, DatasetQuery, udf
-
-
-      @udf(
-          params=(Image(),),
-          output={"fclip": JSON},
-          method="fashion_clip",
-          batch=10,
-      )
-      class MyFashionClip:
-          def __init__(self):
-              self.fclip = FashionCLIP("fashion-clip")
-
-          def fashion_clip(self, inputs):
-              embeddings = self.fclip.encode_images(
-                  [input[0] for input in inputs], batch_size=1
-              )
-              return [(json.dumps(emb),) for emb in embeddings.tolist()]
-
-      chain = dc.DataChain("gs://datachain-demo/zalando/images/").filter(
-              C.name.glob("*.jpg")
-          ).limit(5).add_signals(MyFashionClip).save("zalando_hd_emb")
-
-      test_image = "cs://datachain-demo/zalando/test/banner.jpg"
-      test_embedding = MyFashionClip.fashion_clip.encode_images(Image(test_image))
-
-      best_matches = chain.filter(similarity_search(test_embeding)).limit(5)
-
-      print best_matches.to_result()
-
-
-Delta updates
--------------
-
-DataChain is capable of “delta updates” – that is, batch-processing only the newly added data samples. For example, let us copy some images into a local folder and run a chain to generate captions with a locally served captioning model from HuggingFace:
-
-.. code:: console
-
-      > mkdir demo-images/
-      > datachain cp gs://datachain-demo/images/ /tmp/demo-images
-
-
-.. code:: py
-
-         import torch
-
-         from datachain.lib.hf_image_to_text import LLaVAdescribe
-         from datachain.query import C, DatasetQuery
-
-         source = "/tmp/demo-images"
-
-         if torch.cuda.is_available():
-             device = "cuda"
-         else:
-             device = "cpu"
-
-         if __name__ == "__main__":
-             results = (
-                 DatasetQuery(
-                     source,
-                     anon=True,
-                 )
-                 .filter(C.name.glob("*.jpg"))
-                 .add_signals(
-                     LLaVAdescribe(
-                         device=device,
-                         model=model,
-                     ),
-                     parallel=False,
-                 )
-                 .save("annotated-images")
-             )
-
-Now let us add few more more images to the same folder:
-
-.. code:: console
-
-         > datachain cp gs://datachain-demo/extra-images/ /tmp/demo-images
-
-and calculate updates only for the delta:
-
-.. code:: py
-
-      processed = dc.DataChain("annotated-images")
-      delta = dc.dataChain("/tmp/demo-images").subtract(processed)
-
-Passing data to training
-------------------------
-
-Datasets can be exported to CSV or webdataset formats. However, a much better way to pass data to training which avoids data copies and re-sharding is  to wrap a DataChain dataset into a PyTorch class, and let the library take care of file downloads and caching under the hood:
-
-.. code:: py
-
-         ds = dc.DataChain("gs://datachain-demo/name-labeled/images/")
-                        .filter(C.name.glob("*.jpg"))
-                        .map(lambda name: (name[:3],), output={"label": str}, parallel=4)
-             )
-
-         train_loader = DataLoader(
-                 ds.to_pytorch(
-                     ImageReader(),
-                     LabelReader("label", classes=CLASSES),
-                     transform=transform,
-                 ),
-                 batch_size=16,
-                 parallel=2,
-             )
-
-💻  More examples
-------------------
-
-* Curating images to train a custom CLIP model without re-sharding the Webdataset files
-* Batch-transforming and indexing images to create a searchable merchandise catalog
-* Evaluating an LLM application at scale
-* Ranking the LLM retrieval strategies
-* Delta updates in batch processing
-
-Contributions
---------------------
-
-Contributions are very welcome.
-To learn more, see the `Contributor Guide`_.
-
-
-License
--------
-
-Distributed under the terms of the `Apache 2.0 license`_,
-*DataChain* is free and open source software.
-
-
-Issues
-------
-
-If you encounter any problems,
-please `file an issue`_ along with a detailed description.
-
-
-.. _Apache 2.0 license: https://opensource.org/licenses/Apache-2.0
-.. _PyPI: https://pypi.org/
-.. _file an issue: https://github.com/iterative/dvcx/issues
-.. _pip: https://pip.pypa.io/
-.. github-only
-.. _Contributor Guide: CONTRIBUTING.rst
-.. _Pydantic: https://github.com/pydantic/pydantic