datachain 0.7.9__py3-none-any.whl → 0.7.10__py3-none-any.whl

datachain/client/fsspec.py

@@ -172,7 +172,7 @@ class Client(ABC):
         return url == cls.PREFIX
 
     @classmethod
-    def get_uri(cls, name) -> "StorageURI":
+    def get_uri(cls, name: str) -> "StorageURI":
         from datachain.dataset import StorageURI
 
         return StorageURI(f"{cls.PREFIX}{name}")
@@ -278,7 +278,9 @@ class Client(ABC):
     ) -> None:
         await self._fetch_nested(start_prefix, result_queue)
 
-    async def _fetch_dir(self, prefix, pbar, result_queue: ResultQueue) -> set[str]:
+    async def _fetch_dir(
+        self, prefix: str, pbar, result_queue: ResultQueue
+    ) -> set[str]:
         path = f"{self.name}/{prefix}"
         infos = await self.ls_dir(path)
         files = []

datachain/client/local.py

@@ -12,6 +12,7 @@ from datachain.lib.file import File
 from .fsspec import Client
 
 if TYPE_CHECKING:
+    from datachain.cache import DataChainCache
     from datachain.dataset import StorageURI
 
 
@@ -21,7 +22,11 @@ class FileClient(Client):
     protocol = "file"
 
     def __init__(
-        self, name: str, fs_kwargs: dict[str, Any], cache, use_symlinks: bool = False
+        self,
+        name: str,
+        fs_kwargs: dict[str, Any],
+        cache: "DataChainCache",
+        use_symlinks: bool = False,
     ) -> None:
         super().__init__(name, fs_kwargs, cache)
         self.use_symlinks = use_symlinks
@@ -30,7 +35,7 @@ class FileClient(Client):
         raise TypeError("Signed urls are not implemented for local file system")
 
     @classmethod
-    def get_uri(cls, name) -> "StorageURI":
+    def get_uri(cls, name: str) -> "StorageURI":
         from datachain.dataset import StorageURI
 
         return StorageURI(f'{cls.PREFIX}/{name.removeprefix("/")}')
@@ -77,7 +82,7 @@ class FileClient(Client):
         return bucket, path
 
     @classmethod
-    def from_name(cls, name: str, cache, kwargs) -> "FileClient":
+    def from_name(cls, name: str, cache: "DataChainCache", kwargs) -> "FileClient":
         use_symlinks = kwargs.pop("use_symlinks", False)
         return cls(name, kwargs, cache, use_symlinks=use_symlinks)
 
@@ -85,7 +90,7 @@ class FileClient(Client):
     def from_source(
         cls,
         uri: str,
-        cache,
+        cache: "DataChainCache",
         use_symlinks: bool = False,
         **kwargs,
     ) -> "FileClient":

datachain/func/__init__.py

@@ -17,8 +17,9 @@ from .aggregate import (
 )
 from .array import cosine_distance, euclidean_distance, length, sip_hash_64
 from .conditional import greatest, least
-from .numeric import bit_and, bit_or, bit_xor, int_hash_64
+from .numeric import bit_and, bit_hamming_distance, bit_or, bit_xor, int_hash_64
 from .random import rand
+from .string import byte_hamming_distance
 from .window import window
 
 __all__ = [
@@ -26,8 +27,10 @@ __all__ = [
     "array",
     "avg",
     "bit_and",
+    "bit_hamming_distance",
     "bit_or",
     "bit_xor",
+    "byte_hamming_distance",
     "case",
     "collect",
     "concat",

datachain/func/numeric.py

@@ -160,3 +160,49 @@ def int_hash_64(col: Union[ColT, int]) -> Func:
     return Func(
         "int_hash_64", inner=numeric.int_hash_64, cols=cols, args=args, result_type=int
     )
+
+
+def bit_hamming_distance(*args: Union[ColT, int]) -> Func:
+    """
+    Computes the Hamming distance between the bit representations of two integer values.
+
+    The Hamming distance is the number of positions at which the corresponding bits
+    are different. This function returns the dissimilarity between the integers,
+    where 0 indicates identical integers and values closer to the number of bits
+    in the integer indicate higher dissimilarity.
+
+    Args:
+        args (str | int): Two integers to compute the Hamming distance between.
+            If a str is provided, it is assumed to be the name of the column.
+            If an int is provided, it is assumed to be an integer literal.
+
+    Returns:
+        Func: A Func object that represents the Hamming distance function.
+
+    Example:
+        ```py
+        dc.mutate(
+            ham_dist=func.bit_hamming_distance("embed1", 123456),
+        )
+        ```
+
+    Notes:
+        - Result column will always be of type int.
+    """
+    cols, func_args = [], []
+    for arg in args:
+        if isinstance(arg, int):
+            func_args.append(arg)
+        else:
+            cols.append(arg)
+
+    if len(cols) + len(func_args) != 2:
+        raise ValueError("bit_hamming_distance() requires exactly two arguments")
+
+    return Func(
+        "bit_hamming_distance",
+        inner=numeric.bit_hamming_distance,
+        cols=cols,
+        args=func_args,
+        result_type=int,
+    )

datachain/func/string.py

@@ -152,3 +152,49 @@ def regexp_replace(col: Union[str, Func], regex: str, replacement: str) -> Func:
         args = None
 
     return Func("regexp_replace", inner=inner, cols=cols, args=args, result_type=str)
+
+
+def byte_hamming_distance(*args: Union[str, Func]) -> Func:
+    """
+    Computes the Hamming distance between two strings.
+
+    The Hamming distance is the number of positions at which the corresponding
+    characters are different. This function returns the dissimilarity between
+    the strings, where 0 indicates identical strings and values closer to the length
+    of the strings indicate higher dissimilarity.
+
+    Args:
+        args (str | literal): Two strings to compute the Hamming distance between.
+            If a str is provided, it is assumed to be the name of the column.
+            If a Literal is provided, it is assumed to be a string literal.
+
+    Returns:
+        Func: A Func object that represents the Hamming distance function.
+
+    Example:
+        ```py
+        dc.mutate(
+            ham_dist=func.byte_hamming_distance("file.phash", literal("hello")),
+        )
+        ```
+
+    Notes:
+        - Result column will always be of type int.
+    """
+    cols, func_args = [], []
+    for arg in args:
+        if get_origin(arg) is literal:
+            func_args.append(arg)
+        else:
+            cols.append(arg)
+
+    if len(cols) + len(func_args) != 2:
+        raise ValueError("byte_hamming_distance() requires exactly two arguments")
+
+    return Func(
+        "byte_hamming_distance",
+        inner=string.byte_hamming_distance,
+        cols=cols,
+        args=func_args,
+        result_type=int,
+    )

datachain/lib/convert/flatten.py

@@ -1,19 +1,21 @@
+from collections.abc import Generator
+
 from pydantic import BaseModel
 
 from datachain.lib.model_store import ModelStore
 
 
-def flatten(obj: BaseModel):
+def flatten(obj: BaseModel) -> tuple:
     return tuple(_flatten_fields_values(obj.model_fields, obj))
 
 
-def flatten_list(obj_list):
+def flatten_list(obj_list: list[BaseModel]) -> tuple:
     return tuple(
         val for obj in obj_list for val in _flatten_fields_values(obj.model_fields, obj)
     )
 
 
-def _flatten_list_field(value: list):
+def _flatten_list_field(value: list) -> list:
     assert isinstance(value, list)
     if value and ModelStore.is_pydantic(type(value[0])):
         return [val.model_dump() for val in value]
@@ -22,7 +24,7 @@ def _flatten_list_field(value: list):
     return value
 
 
-def _flatten_fields_values(fields, obj: BaseModel):
+def _flatten_fields_values(fields: dict, obj: BaseModel) -> Generator:
     for name, f_info in fields.items():
         anno = f_info.annotation
         # Optimization: Access attributes directly to skip the model_dump() call.
@@ -40,5 +42,5 @@ def _flatten_fields_values(fields, obj: BaseModel):
             yield value
 
 
-def _flatten(obj):
+def _flatten(obj: BaseModel) -> tuple:
     return tuple(_flatten_fields_values(obj.model_fields, obj))

datachain/lib/convert/unflatten.py

@@ -9,12 +9,12 @@ from pydantic import BaseModel
 from datachain.query.schema import DEFAULT_DELIMITER
 
 
-def unflatten_to_json(model: type[BaseModel], row: Sequence[Any], pos=0) -> dict:
+def unflatten_to_json(model: type[BaseModel], row: Sequence[Any], pos: int = 0) -> dict:
     return unflatten_to_json_pos(model, row, pos)[0]
 
 
 def unflatten_to_json_pos(
-    model: type[BaseModel], row: Sequence[Any], pos=0
+    model: type[BaseModel], row: Sequence[Any], pos: int = 0
 ) -> tuple[dict, int]:
     res = {}
     for name, f_info in model.model_fields.items():

datachain/lib/convert/values_to_tuples.py

@@ -11,7 +11,7 @@ from datachain.lib.utils import DataChainParamsError
 
 
 class ValuesToTupleError(DataChainParamsError):
-    def __init__(self, ds_name, msg):
+    def __init__(self, ds_name: str, msg: str):
         if ds_name:
             ds_name = f"' {ds_name}'"
         super().__init__(f"Cannot convert signals for dataset{ds_name}: {msg}")

datachain/lib/utils.py

@@ -28,7 +28,7 @@ class DataChainParamsError(DataChainError):
 
 
 class DataChainColumnError(DataChainParamsError):
-    def __init__(self, col_name, msg):
+    def __init__(self, col_name: str, msg: str):
         super().__init__(f"Error for column {col_name}: {msg}")
 
 

datachain/query/dataset.py

@@ -215,7 +215,7 @@ class DatasetDiffOperation(Step):
         Should return select query that calculates desired diff between dataset queries
         """
 
-    def apply(self, query_generator, temp_tables: list[str]):
+    def apply(self, query_generator, temp_tables: list[str]) -> "StepResult":
         source_query = query_generator.exclude(("sys__id",))
         target_query = self.dq.apply_steps().select()
         temp_tables.extend(self.dq.temp_table_names)

datachain/sql/functions/numeric.py

@@ -35,9 +35,21 @@ class int_hash_64(GenericFunction):  # noqa: N801
     inherit_cache = True
 
 
+class bit_hamming_distance(GenericFunction):  # noqa: N801
+    """
+    Returns the Hamming distance between two integers.
+    """
+
+    type = Int64()
+    package = "numeric"
+    name = "hamming_distance"
+    inherit_cache = True
+
+
 compiler_not_implemented(bit_and)
 compiler_not_implemented(bit_or)
 compiler_not_implemented(bit_xor)
 compiler_not_implemented(bit_rshift)
 compiler_not_implemented(bit_lshift)
 compiler_not_implemented(int_hash_64)
+compiler_not_implemented(bit_hamming_distance)

datachain/sql/functions/string.py

@@ -48,7 +48,19 @@ class replace(GenericFunction):  # noqa: N801
     inherit_cache = True
 
 
+class byte_hamming_distance(GenericFunction):  # noqa: N801
+    """
+    Returns the Hamming distance between two strings.
+    """
+
+    type = Int64()
+    package = "string"
+    name = "hamming_distance"
+    inherit_cache = True
+
+
 compiler_not_implemented(length)
 compiler_not_implemented(split)
 compiler_not_implemented(regexp_replace)
 compiler_not_implemented(replace)
+compiler_not_implemented(byte_hamming_distance)

datachain/sql/sqlite/base.py

@@ -90,6 +90,7 @@ def setup():
     compiles(string.split, "sqlite")(compile_string_split)
     compiles(string.regexp_replace, "sqlite")(compile_string_regexp_replace)
     compiles(string.replace, "sqlite")(compile_string_replace)
+    compiles(string.byte_hamming_distance, "sqlite")(compile_byte_hamming_distance)
     compiles(conditional.greatest, "sqlite")(compile_greatest)
     compiles(conditional.least, "sqlite")(compile_least)
     compiles(Values, "sqlite")(compile_values)
@@ -104,6 +105,7 @@ def setup():
     compiles(numeric.bit_rshift, "sqlite")(compile_bitwise_rshift)
     compiles(numeric.bit_lshift, "sqlite")(compile_bitwise_lshift)
     compiles(numeric.int_hash_64, "sqlite")(compile_int_hash_64)
+    compiles(numeric.bit_hamming_distance, "sqlite")(compile_bit_hamming_distance)
 
     if load_usearch_extension(sqlite3.connect(":memory:")):
         compiles(array.cosine_distance, "sqlite")(compile_cosine_distance_ext)
@@ -191,6 +193,26 @@ def sqlite_int_hash_64(x: int) -> int:
     return x if x < 1 << 63 else (x & MAX_INT64) - (1 << 64)
 
 
+def sqlite_bit_hamming_distance(a: int, b: int) -> int:
+    """Calculate the Hamming distance between two integers."""
+    diff = (a & MAX_INT64) ^ (b & MAX_INT64)
+    if hasattr(diff, "bit_count"):
+        return diff.bit_count()
+    return bin(diff).count("1")
+
+
+def sqlite_byte_hamming_distance(a: str, b: str) -> int:
+    """Calculate the Hamming distance between two strings."""
+    diff = 0
+    if len(a) < len(b):
+        diff = len(b) - len(a)
+        b = b[: len(a)]
+    elif len(b) < len(a):
+        diff = len(a) - len(b)
+        a = a[: len(b)]
+    return diff + sum(c1 != c2 for c1, c2 in zip(a, b))
+
+
 def register_user_defined_sql_functions() -> None:
     # Register optional functions if we have the necessary dependencies
     # and otherwise register functions that will raise an exception with
@@ -225,6 +247,9 @@ def register_user_defined_sql_functions() -> None:
             "bitwise_lshift", 2, lambda a, b: a << b, deterministic=True
         )
         conn.create_function("int_hash_64", 1, sqlite_int_hash_64, deterministic=True)
+        conn.create_function(
+            "bit_hamming_distance", 2, sqlite_bit_hamming_distance, deterministic=True
+        )
 
     _registered_function_creators["numeric_functions"] = create_numeric_functions
 
@@ -237,6 +262,9 @@ def register_user_defined_sql_functions() -> None:
         conn.create_function(
             "regexp_replace", 3, sqlite_regexp_replace, deterministic=True
         )
+        conn.create_function(
+            "byte_hamming_distance", 2, sqlite_byte_hamming_distance, deterministic=True
+        )
 
     _registered_function_creators["string_functions"] = create_string_functions
 
@@ -383,6 +411,18 @@ def compile_int_hash_64(element, compiler, **kwargs):
     return compiler.process(func.int_hash_64(*element.clauses.clauses), **kwargs)
 
 
+def compile_bit_hamming_distance(element, compiler, **kwargs):
+    return compiler.process(
+        func.bit_hamming_distance(*element.clauses.clauses), **kwargs
+    )
+
+
+def compile_byte_hamming_distance(element, compiler, **kwargs):
+    return compiler.process(
+        func.byte_hamming_distance(*element.clauses.clauses), **kwargs
+    )
+
+
 def py_json_array_length(arr):
     return len(orjson.loads(arr))
 

datachain-0.7.10.dist-info/METADATA

@@ -0,0 +1,207 @@
+Metadata-Version: 2.1
+Name: datachain
+Version: 0.7.10
+Summary: Wrangle unstructured AI data at scale
+Author-email: Dmitry Petrov <support@dvc.org>
+License: Apache-2.0
+Project-URL: Documentation, https://datachain.dvc.ai
+Project-URL: Issues, https://github.com/iterative/datachain/issues
+Project-URL: Source, https://github.com/iterative/datachain
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Development Status :: 2 - Pre-Alpha
+Requires-Python: >=3.9
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: pyyaml
+Requires-Dist: tomlkit
+Requires-Dist: tqdm
+Requires-Dist: numpy<3,>=1
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow
+Requires-Dist: typing-extensions
+Requires-Dist: python-dateutil>=2
+Requires-Dist: attrs>=21.3.0
+Requires-Dist: s3fs>=2024.2.0
+Requires-Dist: gcsfs>=2024.2.0
+Requires-Dist: adlfs>=2024.2.0
+Requires-Dist: dvc-data<4,>=3.10
+Requires-Dist: dvc-objects<6,>=4
+Requires-Dist: shtab<2,>=1.3.4
+Requires-Dist: sqlalchemy>=2
+Requires-Dist: multiprocess==0.70.16
+Requires-Dist: cloudpickle
+Requires-Dist: orjson>=3.10.5
+Requires-Dist: pydantic<3,>=2
+Requires-Dist: jmespath>=1.0
+Requires-Dist: datamodel-code-generator>=0.25
+Requires-Dist: Pillow<12,>=10.0.0
+Requires-Dist: msgpack<2,>=1.0.4
+Requires-Dist: psutil
+Requires-Dist: huggingface_hub
+Requires-Dist: iterative-telemetry>=0.0.9
+Requires-Dist: platformdirs
+Requires-Dist: dvc-studio-client<1,>=0.21
+Requires-Dist: tabulate
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.5.2; extra == "docs"
+Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
+Requires-Dist: mkdocs-material>=9.3.1; extra == "docs"
+Requires-Dist: mkdocs-section-index>=0.3.6; extra == "docs"
+Requires-Dist: mkdocstrings-python>=1.6.3; extra == "docs"
+Requires-Dist: mkdocs-literate-nav>=0.6.1; extra == "docs"
+Provides-Extra: torch
+Requires-Dist: torch>=2.1.0; extra == "torch"
+Requires-Dist: torchvision; extra == "torch"
+Requires-Dist: transformers>=4.36.0; extra == "torch"
+Provides-Extra: remote
+Requires-Dist: lz4; extra == "remote"
+Requires-Dist: requests>=2.22.0; extra == "remote"
+Provides-Extra: vector
+Requires-Dist: usearch; extra == "vector"
+Provides-Extra: hf
+Requires-Dist: numba>=0.60.0; extra == "hf"
+Requires-Dist: datasets[audio,vision]>=2.21.0; extra == "hf"
+Provides-Extra: tests
+Requires-Dist: datachain[hf,remote,torch,vector]; extra == "tests"
+Requires-Dist: pytest<9,>=8; extra == "tests"
+Requires-Dist: pytest-sugar>=0.9.6; extra == "tests"
+Requires-Dist: pytest-cov>=4.1.0; extra == "tests"
+Requires-Dist: pytest-mock>=3.12.0; extra == "tests"
+Requires-Dist: pytest-servers[all]>=0.5.8; extra == "tests"
+Requires-Dist: pytest-benchmark[histogram]; extra == "tests"
+Requires-Dist: pytest-xdist>=3.3.1; extra == "tests"
+Requires-Dist: virtualenv; extra == "tests"
+Requires-Dist: dulwich; extra == "tests"
+Requires-Dist: hypothesis; extra == "tests"
+Requires-Dist: open_clip_torch; extra == "tests"
+Requires-Dist: aiotools>=1.7.0; extra == "tests"
+Requires-Dist: requests-mock; extra == "tests"
+Requires-Dist: scipy; extra == "tests"
+Provides-Extra: dev
+Requires-Dist: datachain[docs,tests]; extra == "dev"
+Requires-Dist: mypy==1.13.0; extra == "dev"
+Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: types-pytz; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: types-tabulate; extra == "dev"
+Provides-Extra: examples
+Requires-Dist: datachain[tests]; extra == "examples"
+Requires-Dist: numpy<2,>=1; extra == "examples"
+Requires-Dist: defusedxml; extra == "examples"
+Requires-Dist: accelerate; extra == "examples"
+Requires-Dist: unstructured[embed-huggingface,pdf]<0.16.0; extra == "examples"
+Requires-Dist: pdfplumber==0.11.4; extra == "examples"
+Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
+Requires-Dist: onnx==1.16.1; extra == "examples"
+Requires-Dist: ultralytics==8.3.37; extra == "examples"
+
+================
+|logo| DataChain
+================
+
+|PyPI| |Python Version| |Codecov| |Tests|
+
+.. |logo| image:: docs/assets/datachain.svg
+   :height: 24
+.. |PyPI| image:: https://img.shields.io/pypi/v/datachain.svg
+   :target: https://pypi.org/project/datachain/
+   :alt: PyPI
+.. |Python Version| image:: https://img.shields.io/pypi/pyversions/datachain
+   :target: https://pypi.org/project/datachain
+   :alt: Python Version
+.. |Codecov| image:: https://codecov.io/gh/iterative/datachain/graph/badge.svg?token=byliXGGyGB
+   :target: https://codecov.io/gh/iterative/datachain
+   :alt: Codecov
+.. |Tests| image:: https://github.com/iterative/datachain/actions/workflows/tests.yml/badge.svg
+   :target: https://github.com/iterative/datachain/actions/workflows/tests.yml
+   :alt: Tests
+
+DataChain is a Python-based AI-data warehouse for transforming and analyzing unstructured
+data like images, audio, videos, text and PDFs. It integrates with external storage
+(e.g. S3) to process data efficiently without data duplication and manages metadata
+in an internal database for easy and efficient querying.
+
+
+Use Cases
+=========
+
+1. **ETL.** Pythonic framework for describing and running unstructured data transformations
+   and enrichments, applying models to data, including LLMs.
+2. **Analytics.** DataChain dataset is a table that combines all the information about data
+   objects in one place + it provides dataframe-like API and vecrorized engine to do analytics
+   on these tables at scale.
+3. **Versioning.** DataChain doesn't store, require moving or copying data (unlike DVC).
+   Perfect use case is a bucket with thousands or millions of images, videos, audio, PDFs.
+
+
+Key Features
+============
+
+📂 **Multimodal Dataset Versioning.**
+   - Version unstructured data without moving or creating data copies, by supporting
+     references to S3, GCP, Azure, and local file systems.
+   - Multimodal data support: images, video, text, PDFs, JSONs, CSVs, parquet, etc.
+   - Unite files and metadata together into persistent, versioned, columnar datasets.
+
+🐍 **Python-friendly.**
+   - Operate on Python objects and object fields: float scores, strings, matrixes,
+     LLM response objects.
+   - Run Python code in a high-scale, terabytes size datasets, with built-in
+     parallelization and memory-efficient computing — no SQL or Spark required.
+
+🧠 **Data Enrichment and Processing.**
+   - Generate metadata using local AI models and LLM APIs.
+   - Filter, join, and group datasets by metadata. Search by vector embeddings.
+   - High-performance vectorized operations on Python objects: sum, count, avg, etc.
+   - Pass datasets to Pytorch and Tensorflow, or export them back into storage.
+
+
+Getting Started
+===============
+
+Visit `Quick Start <https://docs.datachain.ai/quick-start>`_ to get started with `DataChain` and learn more.
+
+
+Contributing
+============
+
+Contributions are very welcome. To learn more, see the `Contributor Guide`_.
+
+
+Community and Support
+=====================
+
+* `Docs <https://docs.datachain.ai/>`_
+* `File an issue`_ if you encounter any problems
+* `Discord Chat <https://dvc.org/chat>`_
+* `Email <mailto:support@dvc.org>`_
+* `Twitter <https://twitter.com/DVCorg>`_
+
+
+DataChain Studio Platform
+=========================
+
+`DataChain Studio`_ is a proprietary solution for teams that offers:
+
+- **Centralized dataset registry** to manage data, code and dependency
+  dependencies in one place.
+- **Data Lineage** for data sources as well as derivative dataset.
+- **UI for Multimodal Data** like images, videos, and PDFs.
+- **Scalable Compute** to handle large datasets (100M+ files) and in-house
+  AI model inference.
+- **Access control** including SSO and team based collaboration.
+
+.. _PyPI: https://pypi.org/
+.. _file an issue: https://github.com/iterative/datachain/issues
+.. github-only
+.. _Contributor Guide: https://docs.datachain.ai/contributing
+.. _Pydantic: https://github.com/pydantic/pydantic
+.. _publicly available: https://radar.kit.edu/radar/en/dataset/FdJmclKpjHzLfExE.ExpBot%2B-%2BA%2Bdataset%2Bof%2B79%2Bdialogs%2Bwith%2Ban%2Bexperimental%2Bcustomer%2Bservice%2Bchatbot
+.. _SQLite: https://www.sqlite.org/
+.. _Getting Started: https://docs.datachain.ai/
+.. _DataChain Studio: https://studio.datachain.ai/

{datachain-0.7.9.dist-info → datachain-0.7.10.dist-info}/RECORD

@@ -24,10 +24,10 @@ datachain/catalog/loader.py,sha256=HA_mBC7q_My8j2WnSvIjUGuJpl6SIdg5vvy_lagxJlA,5
 datachain/client/__init__.py,sha256=T4wiYL9KIM0ZZ_UqIyzV8_ufzYlewmizlV4iymHNluE,86
 datachain/client/azure.py,sha256=ffxs26zm6KLAL1aUWJm-vtzuZP3LSNha7UDGXynMBKo,2234
 datachain/client/fileslice.py,sha256=bT7TYco1Qe3bqoc8aUkUZcPdPofJDHlryL5BsTn9xsY,3021
-datachain/client/fsspec.py,sha256=KDGLhJMnive73hI8GABeP_aQZv1w5M_6rxz6KRRxaHI,12712
+datachain/client/fsspec.py,sha256=kf1blSGNcEXJ0tra3y5i35jc1aAy-67wMHXkqjlRMXg,12736
 datachain/client/gcs.py,sha256=cnTIr5GS6dbYOEYfqehhyQu3dr6XNjPHSg5U3FkivUk,4124
 datachain/client/hf.py,sha256=XeVJVbiNViZCpn3sfb90Fr8SYO3BdLmfE3hOWMoqInE,951
-datachain/client/local.py,sha256=vwbgCwZ7IqY2voj2l7tLJjgov7Dp--fEUvUwUBsMbls,4457
+datachain/client/local.py,sha256=f2HBqWH8SQM5CyiJ0ljfePVROg2FszWaAn6E2c8RiLE,4596
 datachain/client/s3.py,sha256=CVHBUZ1Ic2Q3370nl-Bbe69phuWjFlrVv9dTJKBpRT0,6019
 datachain/data_storage/__init__.py,sha256=9Wit-oe5P46V7CJQTD0BJ5MhOa2Y9h3ddJ4VWTe-Lec,273
 datachain/data_storage/db_engine.py,sha256=81Ol1of9TTTzD97ORajCnP366Xz2mEJt6C-kTUCaru4,3406
@@ -37,16 +37,16 @@ datachain/data_storage/schema.py,sha256=-QVlRvD0dfu-ZFUxylEoSnLJLnleMEjVlcAb2OGu
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
 datachain/data_storage/sqlite.py,sha256=D_ZQ0PHmZzHO2dinv4naVJocUDIZUwV4WAz692C1cyk,22521
 datachain/data_storage/warehouse.py,sha256=tjIkU-5JywBR0apCyqTcwSyaRtGxhu2L7IVjrz-55uc,30802
-datachain/func/__init__.py,sha256=oz-GbCcp5jnN82u6cghWTGzmU9IQvtvllOof73wE52g,934
+datachain/func/__init__.py,sha256=TG6JHFKtLi06Nd5iLszXIflEq-VKZcKMdgo_KiQ8SGQ,1055
 datachain/func/aggregate.py,sha256=7_IPrIwb2XSs3zG4iOr1eTvzn6kNVe2mkzvNzjusDHk,10942
 datachain/func/array.py,sha256=zHDNWuWLA7HVa9FEvQeHhVi00_xqenyleTqcLwkXWBI,5477
 datachain/func/base.py,sha256=wA0sBQAVyN9LPxoo7Ox83peS0zUVnyuKxukwAcjGLfY,534
 datachain/func/conditional.py,sha256=mQroxsoExpBW84Zm5dAYP4OpBblWmzfnF2qJq9rba54,2223
 datachain/func/func.py,sha256=mJ_rOXMpoqnK4-d5eF9boSMx5hWzgKoMLPGpZQqLAfw,15222
-datachain/func/numeric.py,sha256=GcUX6ijZvzfac8CZrHE0gRc9WCPiutcMLKqNXtbn-Yo,4186
+datachain/func/numeric.py,sha256=gMe1Ks0dqQKHkjcpvj7I5S-neECzQ_gltPQLNoaWOyo,5632
 datachain/func/path.py,sha256=mqN_mfkwv44z2II7DMTp_fGGw95hmTCNls_TOFNpr4k,3155
 datachain/func/random.py,sha256=pENOLj9rSmWfGCnOsUIaCsVC5486zQb66qfQvXaz9Z4,452
-datachain/func/string.py,sha256=NQzaXXYu7yb72HPADy4WrFlcgvTS77L9x7-qvCKJtnk,4522
+datachain/func/string.py,sha256=8az3BTeezlaZt6NW-54GWX7WSosAOVMbTr6bXIYyJq4,5958
 datachain/func/window.py,sha256=0MB1yjpVbwOrl_WNLZ8V3jkJz3o0XlYinpAcZQJuxiA,1688
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/arrow.py,sha256=b5efxAUaNNYVwtXVJqj07D3zf5KC-BPlLCxKEZbEG6w,9429
@@ -68,16 +68,16 @@ datachain/lib/tar.py,sha256=3WIzao6yD5fbLqXLTt9GhPGNonbFIs_fDRu-9vgLgsA,1038
 datachain/lib/text.py,sha256=UNHm8fhidk7wdrWqacEWaA6I9ykfYqarQ2URby7jc7M,1261
 datachain/lib/udf.py,sha256=-j0krjNAELTqRI0dB1N65AmawtcIY5vN---AuUcW8Us,13637
 datachain/lib/udf_signature.py,sha256=GXw24A-Olna6DWCdgy2bC-gZh_gLGPQ-KvjuI6pUjC0,7281
-datachain/lib/utils.py,sha256=om-MCiyYwvPHtFq3V2rBKrRDNkio9XXofj7RsUIlHKU,1586
+datachain/lib/utils.py,sha256=QrjVs_oLRXEotOPUYurBJypBFi_ReTJmxcnJeH4j2Uk,1596
 datachain/lib/vfile.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/webdataset.py,sha256=o7SHk5HOUWsZ5Ln04xOM04eQqiBHiJNO7xLgyVBrwo8,6924
 datachain/lib/webdataset_laion.py,sha256=xvT6m_r5y0KbOx14BUe7UC5mOgrktJq53Mh-H0EVlUE,2525
 datachain/lib/convert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/lib/convert/flatten.py,sha256=Uebc5CeqCsacp-nr6IG9i6OGuUavXqdqnoGctZBk3RQ,1384
+datachain/lib/convert/flatten.py,sha256=IZFiUYbgXSxXhPSG5Cqf5IjnJ4ZDZKXMr4o_yCR1NY4,1505
 datachain/lib/convert/python_to_sql.py,sha256=40SAOdoOgikZRhn8iomCPDRoxC3RFxjJLivEAA9MHDU,2880
 datachain/lib/convert/sql_to_python.py,sha256=XXCBYDQFUXJIBNWkjEP944cnCfJ8GF2Tji0DLF3A_zQ,315
-datachain/lib/convert/unflatten.py,sha256=Ogvh_5wg2f38_At_1lN0D_e2uZOOpYEvwvB2xdq56Tw,2012
-datachain/lib/convert/values_to_tuples.py,sha256=varRCnSMT_pZmHznrd2Yi05qXLLz_v9YH_pOCpHSkdc,3921
+datachain/lib/convert/unflatten.py,sha256=5RLIEB7utQFcXlyUIRGqu6VtmAN4N4whlslpO7xMQyI,2026
+datachain/lib/convert/values_to_tuples.py,sha256=EFfIGBiVVltJQG8blzsQ1dGXneh4D3wdLfSUeoK10OI,3931
 datachain/model/__init__.py,sha256=R9faX5OHV1xh2EW-g2MPedwbtEqt3LodJRyluB-QylI,189
 datachain/model/bbox.py,sha256=1Li1G3RdiQwLOAc2Mak2nQU0bcvdH-lXmXtA984CUWM,3154
 datachain/model/pose.py,sha256=q9NgB8h66aKnYnLi7Pyf9bU-F_90W4cbvtSO3-_hkdk,3078
@@ -88,7 +88,7 @@ datachain/model/ultralytics/pose.py,sha256=71KBTcoST2wcEtsyGXqLVpvUtqbp9gwZGA15p
 datachain/model/ultralytics/segment.py,sha256=Z1ab0tZRJubSYNH4KkFlzhYeGNTfAyC71KmkQcToHDQ,2760
 datachain/query/__init__.py,sha256=7DhEIjAA8uZJfejruAVMZVcGFmvUpffuZJwgRqNwe-c,263
 datachain/query/batch.py,sha256=5fEhORFe7li12SdYddaSK3LyqksMfCHhwN1_A6TfsA4,3485
-datachain/query/dataset.py,sha256=J6SbCLnFlZgCxRchc3tVk5tcC7xo1Hp616JGlEZXCDo,54547
+datachain/query/dataset.py,sha256=eXr9fJz2grX2evmkmsiH0Xeqajd8gFnujmt_USMxy0c,54563
 datachain/query/dispatch.py,sha256=fZ0TgGFRcsrYh1iXQoZVjkUl4Xetom9PSHoeDes3IRs,11606
 datachain/query/metrics.py,sha256=r5b0ygYhokbXp8Mg3kCH8iFSRw0jxzyeBe-C-J_bKFc,938
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
@@ -107,20 +107,20 @@ datachain/sql/functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
 datachain/sql/functions/aggregate.py,sha256=3AQdA8YHPFdtCEfwZKQXTT8SlQWdG9gD5PBtGN3Odqs,944
 datachain/sql/functions/array.py,sha256=Zq59CaMHf_hFapU4kxvy2mwteH344k5Wksxja4MfBks,1204
 datachain/sql/functions/conditional.py,sha256=q7YUKfunXeEldXaxgT-p5pUTcOEVU_tcQ2BJlquTRPs,207
-datachain/sql/functions/numeric.py,sha256=DFTTEWsvBBXwbaaC4zdxhAoqUYwI6nbymG-nzbzdPv8,972
+datachain/sql/functions/numeric.py,sha256=BK2KCiPSgM2IveCq-9M_PG3CtPBlztaS9TTn1LGzyLs,1250
 datachain/sql/functions/path.py,sha256=zixpERotTFP6LZ7I4TiGtyRA8kXOoZmH1yzH9oRW0mg,1294
 datachain/sql/functions/random.py,sha256=vBwEEj98VH4LjWixUCygQ5Bz1mv1nohsCG0-ZTELlVg,271
-datachain/sql/functions/string.py,sha256=DYgiw8XSk7ge7GXvyRI1zbaMruIizNeI-puOjriQGZQ,1148
+datachain/sql/functions/string.py,sha256=E-T9OIzUR-GKaLgjZsEtg5CJrY_sLf1lt1awTvY7w2w,1426
 datachain/sql/sqlite/__init__.py,sha256=TAdJX0Bg28XdqPO-QwUVKy8rg78cgMileHvMNot7d04,166
-datachain/sql/sqlite/base.py,sha256=eQv2U32jChG9tnYSFE4SS2Mvfb7-W3Ok3Ffhew9qkKI,17254
+datachain/sql/sqlite/base.py,sha256=E2PK3hoGlHey1eEjcReXRrI-c_ASr3AmAXaNYKDY_o8,18634
 datachain/sql/sqlite/types.py,sha256=lPXS1XbkmUtlkkiRxy_A_UzsgpPv2VSkXYOD4zIHM4w,1734
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ZgDcrNiKiPXZmKD591_1z9qRIXitu5zwAsoVPB7ykiU,2508
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.7.9.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.7.9.dist-info/METADATA,sha256=iu58cwfGQVYTwn53symALXVpe9292EWXdOly2MWuPZY,18006
-datachain-0.7.9.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-datachain-0.7.9.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.7.9.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.7.9.dist-info/RECORD,,
+datachain-0.7.10.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.7.10.dist-info/METADATA,sha256=qtw_rToRdmR9-CO6MFCAGv6NWJJ87C95iQaDEnDE4H8,8371
+datachain-0.7.10.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+datachain-0.7.10.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.7.10.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.7.10.dist-info/RECORD,,

datachain-0.7.9.dist-info/METADATA

@@ -1,488 +0,0 @@
-Metadata-Version: 2.1
-Name: datachain
-Version: 0.7.9
-Summary: Wrangle unstructured AI data at scale
-Author-email: Dmitry Petrov <support@dvc.org>
-License: Apache-2.0
-Project-URL: Documentation, https://datachain.dvc.ai
-Project-URL: Issues, https://github.com/iterative/datachain/issues
-Project-URL: Source, https://github.com/iterative/datachain
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Development Status :: 2 - Pre-Alpha
-Requires-Python: >=3.9
-Description-Content-Type: text/x-rst
-License-File: LICENSE
-Requires-Dist: pyyaml
-Requires-Dist: tomlkit
-Requires-Dist: tqdm
-Requires-Dist: numpy<3,>=1
-Requires-Dist: pandas>=2.0.0
-Requires-Dist: pyarrow
-Requires-Dist: typing-extensions
-Requires-Dist: python-dateutil>=2
-Requires-Dist: attrs>=21.3.0
-Requires-Dist: s3fs>=2024.2.0
-Requires-Dist: gcsfs>=2024.2.0
-Requires-Dist: adlfs>=2024.2.0
-Requires-Dist: dvc-data<4,>=3.10
-Requires-Dist: dvc-objects<6,>=4
-Requires-Dist: shtab<2,>=1.3.4
-Requires-Dist: sqlalchemy>=2
-Requires-Dist: multiprocess==0.70.16
-Requires-Dist: cloudpickle
-Requires-Dist: orjson>=3.10.5
-Requires-Dist: pydantic<3,>=2
-Requires-Dist: jmespath>=1.0
-Requires-Dist: datamodel-code-generator>=0.25
-Requires-Dist: Pillow<12,>=10.0.0
-Requires-Dist: msgpack<2,>=1.0.4
-Requires-Dist: psutil
-Requires-Dist: huggingface_hub
-Requires-Dist: iterative-telemetry>=0.0.9
-Requires-Dist: platformdirs
-Requires-Dist: dvc-studio-client<1,>=0.21
-Requires-Dist: tabulate
-Provides-Extra: docs
-Requires-Dist: mkdocs>=1.5.2; extra == "docs"
-Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
-Requires-Dist: mkdocs-material>=9.3.1; extra == "docs"
-Requires-Dist: mkdocs-section-index>=0.3.6; extra == "docs"
-Requires-Dist: mkdocstrings-python>=1.6.3; extra == "docs"
-Requires-Dist: mkdocs-literate-nav>=0.6.1; extra == "docs"
-Provides-Extra: torch
-Requires-Dist: torch>=2.1.0; extra == "torch"
-Requires-Dist: torchvision; extra == "torch"
-Requires-Dist: transformers>=4.36.0; extra == "torch"
-Provides-Extra: remote
-Requires-Dist: lz4; extra == "remote"
-Requires-Dist: requests>=2.22.0; extra == "remote"
-Provides-Extra: vector
-Requires-Dist: usearch; extra == "vector"
-Provides-Extra: hf
-Requires-Dist: numba>=0.60.0; extra == "hf"
-Requires-Dist: datasets[audio,vision]>=2.21.0; extra == "hf"
-Provides-Extra: tests
-Requires-Dist: datachain[hf,remote,torch,vector]; extra == "tests"
-Requires-Dist: pytest<9,>=8; extra == "tests"
-Requires-Dist: pytest-sugar>=0.9.6; extra == "tests"
-Requires-Dist: pytest-cov>=4.1.0; extra == "tests"
-Requires-Dist: pytest-mock>=3.12.0; extra == "tests"
-Requires-Dist: pytest-servers[all]>=0.5.8; extra == "tests"
-Requires-Dist: pytest-benchmark[histogram]; extra == "tests"
-Requires-Dist: pytest-xdist>=3.3.1; extra == "tests"
-Requires-Dist: virtualenv; extra == "tests"
-Requires-Dist: dulwich; extra == "tests"
-Requires-Dist: hypothesis; extra == "tests"
-Requires-Dist: open_clip_torch; extra == "tests"
-Requires-Dist: aiotools>=1.7.0; extra == "tests"
-Requires-Dist: requests-mock; extra == "tests"
-Requires-Dist: scipy; extra == "tests"
-Provides-Extra: dev
-Requires-Dist: datachain[docs,tests]; extra == "dev"
-Requires-Dist: mypy==1.13.0; extra == "dev"
-Requires-Dist: types-python-dateutil; extra == "dev"
-Requires-Dist: types-pytz; extra == "dev"
-Requires-Dist: types-PyYAML; extra == "dev"
-Requires-Dist: types-requests; extra == "dev"
-Requires-Dist: types-tabulate; extra == "dev"
-Provides-Extra: examples
-Requires-Dist: datachain[tests]; extra == "examples"
-Requires-Dist: numpy<2,>=1; extra == "examples"
-Requires-Dist: defusedxml; extra == "examples"
-Requires-Dist: accelerate; extra == "examples"
-Requires-Dist: unstructured[embed-huggingface,pdf]<0.16.0; extra == "examples"
-Requires-Dist: pdfplumber==0.11.4; extra == "examples"
-Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
-Requires-Dist: onnx==1.16.1; extra == "examples"
-Requires-Dist: ultralytics==8.3.37; extra == "examples"
-
-================
-|logo| DataChain
-================
-
-|PyPI| |Python Version| |Codecov| |Tests|
-
-.. |logo| image:: docs/assets/datachain.svg
-   :height: 24
-.. |PyPI| image:: https://img.shields.io/pypi/v/datachain.svg
-   :target: https://pypi.org/project/datachain/
-   :alt: PyPI
-.. |Python Version| image:: https://img.shields.io/pypi/pyversions/datachain
-   :target: https://pypi.org/project/datachain
-   :alt: Python Version
-.. |Codecov| image:: https://codecov.io/gh/iterative/datachain/graph/badge.svg?token=byliXGGyGB
-   :target: https://codecov.io/gh/iterative/datachain
-   :alt: Codecov
-.. |Tests| image:: https://github.com/iterative/datachain/actions/workflows/tests.yml/badge.svg
-   :target: https://github.com/iterative/datachain/actions/workflows/tests.yml
-   :alt: Tests
-
-DataChain is a Python-based AI-data warehouse for transforming and analyzing unstructured
-data like images, audio, videos, text and PDFs. It integrates with external storage
-(e.g., S3) to process data efficiently without data duplication and manages metadata
-in an internal database for easy and efficient querying.
-
-
-Use Cases
-=========
-
-1. **Multimodal Dataset Preparation and Curation**: ideal for organizing and
-   refining data in pre-training, finetuning or LLM evaluating stages.
-2. **GenAI Data Analytics**: Enables advanced analytics for multimodal data and
-   ad-hoc analytics using LLMs.
-
-Key Features
-============
-
-📂 **Multimodal Dataset Versioning.**
-   - Version unstructured data without redundant data copies, by supporting
-     references to S3, GCP, Azure, and local file systems.
-   - Multimodal data support: images, video, text, PDFs, JSONs, CSVs, parquet, etc.
-   - Unite files and metadata together into persistent, versioned, columnar datasets.
-
-🐍 **Python-friendly.**
-   - Operate on Python objects and object fields: float scores, strings, matrixes,
-     LLM response objects.
-   - Run Python code in a high-scale, terabytes size datasets, with built-in
-     parallelization and memory-efficient computing — no SQL or Spark required.
-
-🧠 **Data Enrichment and Processing.**
-   - Generate metadata using local AI models and LLM APIs.
-   - Filter, join, and group datasets by metadata. Search by vector embeddings.
-   - High-performance vectorized operations on Python objects: sum, count, avg, etc.
-   - Pass datasets to Pytorch and Tensorflow, or export them back into storage.
-
-
-Quick Start
------------
-
-.. code:: console
-
-   $ pip install datachain
-
-
-Selecting files using JSON metadata
-======================================
-
-A storage consists of images of cats and dogs (`dog.1048.jpg`, `cat.1009.jpg`),
-annotated with ground truth and model inferences in the 'json-pairs' format,
-where each image has a matching JSON file like `cat.1009.json`:
-
-.. code:: json
-
-    {
-        "class": "cat", "id": "1009", "num_annotators": 8,
-        "inference": {"class": "dog", "confidence": 0.68}
-    }
-
-Example of downloading only "high-confidence cat" inferred images using JSON metadata:
-
-
-.. code:: py
-
-    from datachain import Column, DataChain
-
-    meta = DataChain.from_json("gs://datachain-demo/dogs-and-cats/*json", object_name="meta")
-    images = DataChain.from_storage("gs://datachain-demo/dogs-and-cats/*jpg")
-
-    images_id = images.map(id=lambda file: file.path.split('.')[-2])
-    annotated = images_id.merge(meta, on="id", right_on="meta.id")
-
-    likely_cats = annotated.filter((Column("meta.inference.confidence") > 0.93) \
-                                   & (Column("meta.inference.class_") == "cat"))
-    likely_cats.export_files("high-confidence-cats/", signal="file")
-
-
-Data curation with a local AI model
-===================================
-Batch inference with a simple sentiment model using the `transformers` library:
-
-.. code:: shell
-
-    pip install transformers
-
-The code below downloads files from the cloud, and applies a user-defined function
-to each one of them. All files with a positive sentiment
-detected are then copied to the local directory.
-
-.. code:: py
-
-    from transformers import pipeline
-    from datachain import DataChain, Column
-
-    classifier = pipeline("sentiment-analysis", device="cpu",
-                    model="distilbert/distilbert-base-uncased-finetuned-sst-2-english")
-
-    def is_positive_dialogue_ending(file) -> bool:
-        dialogue_ending = file.read()[-512:]
-        return classifier(dialogue_ending)[0]["label"] == "POSITIVE"
-
-    chain = (
-       DataChain.from_storage("gs://datachain-demo/chatbot-KiT/",
-                              object_name="file", type="text")
-       .settings(parallel=8, cache=True)
-       .map(is_positive=is_positive_dialogue_ending)
-       .save("file_response")
-    )
-
-    positive_chain = chain.filter(Column("is_positive") == True)
-    positive_chain.export_files("./output")
-
-    print(f"{positive_chain.count()} files were exported")
-
-
-
-13 files were exported
-
-.. code:: shell
-
-    $ ls output/datachain-demo/chatbot-KiT/
-    15.txt 20.txt 24.txt 27.txt 28.txt 29.txt 33.txt 37.txt 38.txt 43.txt ...
-    $ ls output/datachain-demo/chatbot-KiT/ | wc -l
-    13
-
-
-LLM judging chatbots
-=============================
-
-LLMs can work as universal classifiers. In the example below,
-we employ a free API from Mistral to judge the `publicly available`_ chatbot dialogs. Please get a free
-Mistral API key at https://console.mistral.ai
-
-
-.. code:: shell
-
-    $ pip install mistralai (Requires version >=1.0.0)
-    $ export MISTRAL_API_KEY=_your_key_
-
-DataChain can parallelize API calls; the free Mistral tier supports up to 4 requests at the same time.
-
-.. code:: py
-
-    from mistralai import Mistral
-    from datachain import File, DataChain, Column
-
-    PROMPT = "Was this dialog successful? Answer in a single word: Success or Failure."
-
-    def eval_dialogue(file: File) -> bool:
-         client = Mistral()
-         response = client.chat.complete(
-             model="open-mixtral-8x22b",
-             messages=[{"role": "system", "content": PROMPT},
-                       {"role": "user", "content": file.read()}])
-         result = response.choices[0].message.content
-         return result.lower().startswith("success")
-
-    chain = (
-       DataChain.from_storage("gs://datachain-demo/chatbot-KiT/", object_name="file")
-       .settings(parallel=4, cache=True)
-       .map(is_success=eval_dialogue)
-       .save("mistral_files")
-    )
-
-    successful_chain = chain.filter(Column("is_success") == True)
-    successful_chain.export_files("./output_mistral")
-
-    print(f"{successful_chain.count()} files were exported")
-
-
-With the instruction above, the Mistral model considers 31/50 files to hold the successful dialogues:
-
-.. code:: shell
-
-    $ ls output_mistral/datachain-demo/chatbot-KiT/
-    1.txt  15.txt 18.txt 2.txt  22.txt 25.txt 28.txt 33.txt 37.txt 4.txt  41.txt ...
-    $ ls output_mistral/datachain-demo/chatbot-KiT/ | wc -l
-    31
-
-
-
-Serializing Python-objects
-==========================
-
-LLM responses may contain valuable information for analytics – such as the number of tokens used, or the
-model performance parameters.
-
-Instead of extracting this information from the Mistral response data structure (class
-`ChatCompletionResponse`), DataChain can serialize the entire LLM response to the internal DB:
-
-
-.. code:: py
-
-    from mistralai import Mistral
-    from mistralai.models import ChatCompletionResponse
-    from datachain import File, DataChain, Column
-
-    PROMPT = "Was this dialog successful? Answer in a single word: Success or Failure."
-
-    def eval_dialog(file: File) -> ChatCompletionResponse:
-         client = MistralClient()
-         return client.chat(
-             model="open-mixtral-8x22b",
-             messages=[{"role": "system", "content": PROMPT},
-                       {"role": "user", "content": file.read()}])
-
-    chain = (
-       DataChain.from_storage("gs://datachain-demo/chatbot-KiT/", object_name="file")
-       .settings(parallel=4, cache=True)
-       .map(response=eval_dialog)
-       .map(status=lambda response: response.choices[0].message.content.lower()[:7])
-       .save("response")
-    )
-
-    chain.select("file.name", "status", "response.usage").show(5)
-
-    success_rate = chain.filter(Column("status") == "success").count() / chain.count()
-    print(f"{100*success_rate:.1f}% dialogs were successful")
-
-Output:
-
-.. code:: shell
-
-         file   status      response     response          response
-         name                  usage        usage             usage
-                       prompt_tokens total_tokens completion_tokens
-    0   1.txt  success           547          548                 1
-    1  10.txt  failure          3576         3578                 2
-    2  11.txt  failure           626          628                 2
-    3  12.txt  failure          1144         1182                38
-    4  13.txt  success          1100         1101                 1
-
-    [Limited by 5 rows]
-    64.0% dialogs were successful
-
-
-Iterating over Python data structures
-=============================================
-
-In the previous examples, datasets were saved in the embedded database
-(`SQLite`_ in folder `.datachain` of the working directory).
-These datasets were automatically versioned, and can be accessed using
-`DataChain.from_dataset("dataset_name")`.
-
-Here is how to retrieve a saved dataset and iterate over the objects:
-
-.. code:: py
-
-    chain = DataChain.from_dataset("response")
-
-    # Iterating one-by-one: support out-of-memory workflow
-    for file, response in chain.limit(5).collect("file", "response"):
-        # verify the collected Python objects
-        assert isinstance(response, ChatCompletionResponse)
-
-        status = response.choices[0].message.content[:7]
-        tokens = response.usage.total_tokens
-        print(f"{file.get_uri()}: {status}, file size: {file.size}, tokens: {tokens}")
-
-Output:
-
-.. code:: shell
-
-    gs://datachain-demo/chatbot-KiT/1.txt: Success, file size: 1776, tokens: 548
-    gs://datachain-demo/chatbot-KiT/10.txt: Failure, file size: 11576, tokens: 3578
-    gs://datachain-demo/chatbot-KiT/11.txt: Failure, file size: 2045, tokens: 628
-    gs://datachain-demo/chatbot-KiT/12.txt: Failure, file size: 3833, tokens: 1207
-    gs://datachain-demo/chatbot-KiT/13.txt: Success, file size: 3657, tokens: 1101
-
-
-Vectorized analytics over Python objects
-========================================
-
-Some operations can run inside the DB without deserialization.
-For instance, let's calculate the total cost of using the LLM APIs, assuming the Mixtral call costs $2 per 1M input tokens and $6 per 1M output tokens:
-
-.. code:: py
-
-    chain = DataChain.from_dataset("mistral_dataset")
-
-    cost = chain.sum("response.usage.prompt_tokens")*0.000002 \
-               + chain.sum("response.usage.completion_tokens")*0.000006
-    print(f"Spent ${cost:.2f} on {chain.count()} calls")
-
-Output:
-
-.. code:: shell
-
-    Spent $0.08 on 50 calls
-
-
-PyTorch data loader
-===================
-
-Chain results can be exported or passed directly to PyTorch dataloader.
-For example, if we are interested in passing image and a label based on file
-name suffix, the following code will do it:
-
-.. code:: py
-
-    from torch.utils.data import DataLoader
-    from transformers import CLIPProcessor
-
-    from datachain import C, DataChain
-
-    processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
-
-    chain = (
-        DataChain.from_storage("gs://datachain-demo/dogs-and-cats/", type="image")
-        .map(label=lambda name: name.split(".")[0], params=["file.name"])
-        .select("file", "label").to_pytorch(
-            transform=processor.image_processor,
-            tokenizer=processor.tokenizer,
-        )
-    )
-    loader = DataLoader(chain, batch_size=1)
-
-
-DataChain Studio Platform
--------------------------
-
-`DataChain Studio`_ is a proprietary solution for teams that offers:
-
-- **Centralized dataset registry** to manage data, code and dependency
-  dependencies in one place.
-- **Data Lineage** for data sources as well as direvative dataset.
-- **UI for Multimodal Data** like images, videos, and PDFs.
-- **Scalable Compute** to handle large datasets (100M+ files) and in-house
-  AI model inference.
-- **Access control** including SSO and team based collaboration.
-
-Tutorials
----------
-
-* `Getting Started`_
-* `Multimodal <https://github.com/iterative/datachain-examples/blob/main/multimodal/clip_fine_tuning.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain-examples/blob/main/multimodal/clip_fine_tuning.ipynb>`__)
-* `LLM evaluations <https://github.com/iterative/datachain-examples/blob/main/llm/llm_chatbot_evaluation.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain-examples/blob/main/llm/llm_chatbot_evaluation.ipynb>`__)
-* `Reading JSON metadata <https://github.com/iterative/datachain-examples/blob/main/formats/json-metadata-tutorial.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/datachain-examples/blob/main/formats/json-metadata-tutorial.ipynb>`__)
-
-
-Contributions
--------------
-
-Contributions are very welcome.
-To learn more, see the `Contributor Guide`_.
-
-
-Community and Support
----------------------
-
-* `Docs <https://datachain.dvc.ai/>`_
-* `File an issue`_ if you encounter any problems
-* `Discord Chat <https://dvc.org/chat>`_
-* `Email <mailto:support@dvc.org>`_
-* `Twitter <https://twitter.com/DVCorg>`_
-
-
-.. _PyPI: https://pypi.org/
-.. _file an issue: https://github.com/iterative/datachain/issues
-.. github-only
-.. _Contributor Guide: CONTRIBUTING.rst
-.. _Pydantic: https://github.com/pydantic/pydantic
-.. _publicly available: https://radar.kit.edu/radar/en/dataset/FdJmclKpjHzLfExE.ExpBot%2B-%2BA%2Bdataset%2Bof%2B79%2Bdialogs%2Bwith%2Ban%2Bexperimental%2Bcustomer%2Bservice%2Bchatbot
-.. _SQLite: https://www.sqlite.org/
-.. _Getting Started: https://docs.datachain.ai/
-.. _DataChain Studio: https://studio.datachain.ai/