datachain 0.3.17__tar.gz → 0.3.19__tar.gz

{datachain-0.3.17 → datachain-0.3.19}/.github/workflows/benchmarks.yml

@@ -23,15 +23,18 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
-          cache: 'pip'
 
-      - name: Upgrade nox and uv
-        run: |
-          python -m pip install --upgrade 'nox[uv]'
-          nox --version
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-suffix: benchmarks
+          cache-dependency-glob: pyproject.toml
+
+      - name: Install nox and dvc
+        run: uv pip install dvc[gs] nox --system
 
-      - run: uv pip install dvc[gs] --system
-      - run: dvc --cd tests/benchmarks/datasets pull
+      - name: Pull dataset
+        run: dvc --cd tests/benchmarks/datasets pull
       - name: Run benchmarks
         run: nox -s bench

{datachain-0.3.17 → datachain-0.3.19}/.github/workflows/release.yml

@@ -21,17 +21,16 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12'
 
-      - name: Upgrade nox and uv
-        run: |
-          python -m pip install --upgrade 'nox[uv]'
-          nox --version
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
 
+      - name: Install nox
+        run: uv pip install nox --system
       - name: Build package
         run: nox -s build
 

{datachain-0.3.17 → datachain-0.3.19}/.github/workflows/tests-studio.yml

@@ -82,12 +82,15 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.pyv }}
-          cache: 'pip'
 
-      - name: Install uv
-        run: |
-          python -m pip install --upgrade uv
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-suffix: studio
+          cache-dependency-glob: |
+            backend/datachain_server/pyproject.toml
+            backend/datachain/pyproject.toml
 
       - name: Install dependencies
         run: uv pip install --system ./backend/datachain_server[tests] ./backend/datachain[tests]

{datachain-0.3.17 → datachain-0.3.19}/.github/workflows/tests.yml

@@ -26,13 +26,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.9'
-          cache: 'pip'
 
-      - name: Upgrade nox and uv
-        run: |
-          python -m pip install --upgrade 'nox[uv]'
-          nox --version
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-suffix: lint
+          cache-dependency-glob: pyproject.toml
+
+      - name: Install nox
+        run: uv pip install nox --system
 
       - name: Cache mypy
         uses: actions/cache@v4
@@ -77,13 +80,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.pyv }}
-          cache: 'pip'
 
-      - name: Upgrade nox and uv
-        run: |
-          python -m pip install --upgrade 'nox[uv]'
-          nox --version
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-suffix: tests-${{ matrix.pyv }}
+          cache-dependency-glob: pyproject.toml
+
+      - name: Install nox
+        run: uv pip install nox --system
 
       - name: Skip flaky azure, gs remotes on macOS
         if: runner.os == 'macOS'
@@ -134,13 +140,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.pyv }}
-          cache: 'pip'
 
-      - name: Upgrade nox and uv
-        run: |
-          python -m pip install --upgrade 'nox[uv]'
-          nox --version
-          uv --version
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-suffix: examples-${{ matrix.pyv }}
+          cache-dependency-glob: pyproject.toml
+
+      - name: Install nox
+        run: uv pip install nox --system
 
       - name: Run examples
         run: nox -s examples -p ${{ matrix.pyv }} -- -m "${{ matrix.group }}"

{datachain-0.3.17/src/datachain.egg-info → datachain-0.3.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.17
+Version: 0.3.19
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -34,7 +34,6 @@ Requires-Dist: dvc-objects<6,>=4
 Requires-Dist: shtab<2,>=1.3.4
 Requires-Dist: sqlalchemy>=2
 Requires-Dist: multiprocess==0.70.16
-Requires-Dist: dill==0.3.8
 Requires-Dist: cloudpickle
 Requires-Dist: orjson>=3.10.5
 Requires-Dist: pydantic<3,>=2
@@ -44,6 +43,7 @@ Requires-Dist: Pillow<11,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
 Requires-Dist: huggingface_hub
+Requires-Dist: iterative-telemetry>=0.0.9
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
@@ -69,7 +69,7 @@ Requires-Dist: pytest<9,>=8; extra == "tests"
 Requires-Dist: pytest-sugar>=0.9.6; extra == "tests"
 Requires-Dist: pytest-cov>=4.1.0; extra == "tests"
 Requires-Dist: pytest-mock>=3.12.0; extra == "tests"
-Requires-Dist: pytest-servers[all]>=0.5.5; extra == "tests"
+Requires-Dist: pytest-servers[all]>=0.5.7; extra == "tests"
 Requires-Dist: pytest-benchmark[histogram]; extra == "tests"
 Requires-Dist: pytest-xdist>=3.3.1; extra == "tests"
 Requires-Dist: virtualenv; extra == "tests"
@@ -91,9 +91,10 @@ Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: numpy<2,>=1; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
-Requires-Dist: unstructured[pdf]; extra == "examples"
+Requires-Dist: unstructured[embed-huggingface,pdf]; extra == "examples"
 Requires-Dist: pdfplumber==0.11.4; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
+Requires-Dist: onnx==1.16.1; extra == "examples"
 
 ================
 |logo| DataChain

{datachain-0.3.17 → datachain-0.3.19}/docs/references/file.md

@@ -7,6 +7,8 @@ automatically when creating a `DataChain` from files, like in
 classes include various metadata fields about the underlying file as well as methods to
 read from the files and otherwise work with the file contents.
 
+::: datachain.lib.file.ArrowRow
+
 ::: datachain.lib.file.ExportPlacement
 
 ::: datachain.lib.file.File
@@ -15,8 +17,6 @@ read from the files and otherwise work with the file contents.
 
 ::: datachain.lib.file.ImageFile
 
-::: datachain.lib.file.IndexedFile
-
 ::: datachain.lib.file.TarVFile
 
 ::: datachain.lib.file.TextFile

datachain-0.3.19/examples/llm_and_nlp/unstructured-embeddings-gen.py

@@ -0,0 +1,76 @@
+"""
+To install the required dependencies:
+
+  pip install datachain[examples]
+
+"""
+
+from collections.abc import Iterator
+
+from unstructured.cleaners.core import (
+    clean,
+    group_broken_paragraphs,
+    replace_unicode_quotes,
+)
+from unstructured.embed.huggingface import (
+    HuggingFaceEmbeddingConfig,
+    HuggingFaceEmbeddingEncoder,
+)
+from unstructured.partition.pdf import partition_pdf
+
+from datachain import C, DataChain, DataModel, File
+
+source = "gs://datachain-demo/neurips/1987/"
+
+
+# Define the output as a DataModel class
+class Chunk(DataModel):
+    key: str
+    text: str
+    embeddings: list[float]
+
+
+# Define embedding encoder
+
+embedding_encoder = HuggingFaceEmbeddingEncoder(config=HuggingFaceEmbeddingConfig())
+
+
+# Use signatures to define UDF input/output
+# these can be pydantic model or regular Python types
+def process_pdf(file: File) -> Iterator[Chunk]:
+    # Ingest the file
+    with file.open() as f:
+        chunks = partition_pdf(file=f, chunking_strategy="by_title", strategy="fast")
+
+    # Clean the chunks and add new columns
+    for chunk in chunks:
+        chunk.apply(
+            lambda text: clean(
+                text, bullets=True, extra_whitespace=True, trailing_punctuation=True
+            )
+        )
+        chunk.apply(replace_unicode_quotes)
+        chunk.apply(group_broken_paragraphs)
+
+    # create embeddings
+    chunks_embedded = embedding_encoder.embed_documents(chunks)
+
+    # Add new rows to DataChain
+    for chunk in chunks_embedded:
+        yield Chunk(
+            key=file.path,
+            text=chunk.text,
+            embeddings=chunk.embeddings,
+        )
+
+
+dc = (
+    DataChain.from_storage(source)
+    .settings(parallel=-1)
+    .filter(C.file.path.glob("*.pdf"))
+    .gen(document=process_pdf)
+)
+
+dc.save("embedded-documents")
+
+DataChain.from_dataset("embedded-documents").show()

datachain-0.3.17/examples/llm_and_nlp/unstructured-text.py → datachain-0.3.19/examples/llm_and_nlp/unstructured-summary-map.py

@@ -1,6 +1,10 @@
-#
-# pip install unstructured[pdf] huggingface_hub[hf_transfer]
-#
+"""
+To install the required dependencies:
+
+  pip install datachain[examples]
+
+"""
+
 import os
 
 os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"

{datachain-0.3.17 → datachain-0.3.19}/examples/multimodal/hf_pipeline.py

@@ -1,4 +1,10 @@
-# pip install scipy torch transformers huggingface_hub[hf_transfer]
+"""
+To install the required dependencies:
+
+  pip install datachain[examples]
+
+"""
+
 # NOTE: also need to install ffmpeg binary
 import json
 import os

{datachain-0.3.17 → datachain-0.3.19}/examples/multimodal/openai_image_desc_lib.py

@@ -1,5 +1,3 @@
-# pip install Pillow
-
 import base64
 import os
 

{datachain-0.3.17 → datachain-0.3.19}/noxfile.py

@@ -57,8 +57,8 @@ def lint(session: nox.Session) -> None:
 
 @nox.session
 def build(session: nox.Session) -> None:
-    session.install("build", "twine", "uv")
-    session.run("python", "-m", "build", "--installer", "uv")
+    session.install("twine", "uv")
+    session.run("uv", "build")
     dists = glob.glob("dist/*")
     session.run("twine", "check", *dists, silent=True)
 

{datachain-0.3.17 → datachain-0.3.19}/pyproject.toml

@@ -37,7 +37,6 @@ dependencies = [
   "shtab>=1.3.4,<2",
   "sqlalchemy>=2",
   "multiprocess==0.70.16",
-  "dill==0.3.8",
   "cloudpickle",
   "orjson>=3.10.5",
   "pydantic>=2,<3",
@@ -46,7 +45,8 @@ dependencies = [
   "Pillow>=10.0.0,<11",
   "msgpack>=1.0.4,<2",
   "psutil",
-  "huggingface_hub"
+  "huggingface_hub",
+  "iterative-telemetry>=0.0.9"
 ]
 
 [project.optional-dependencies]
@@ -80,7 +80,7 @@ tests = [
   "pytest-sugar>=0.9.6",
   "pytest-cov>=4.1.0",
   "pytest-mock>=3.12.0",
-  "pytest-servers[all]>=0.5.5",
+  "pytest-servers[all]>=0.5.7",
   "pytest-benchmark[histogram]",
   "pytest-xdist>=3.3.1",
   "virtualenv",
@@ -104,9 +104,10 @@ examples = [
   "numpy>=1,<2",
   "defusedxml",
   "accelerate",
-  "unstructured[pdf]",
+  "unstructured[pdf, embed-huggingface]",
   "pdfplumber==0.11.4",
-  "huggingface_hub[hf_transfer]"
+  "huggingface_hub[hf_transfer]",
+  "onnx==1.16.1"
 ]
 
 [project.urls]

{datachain-0.3.17 → datachain-0.3.19}/src/datachain/__init__.py

@@ -1,21 +1,23 @@
 from datachain.lib.data_model import DataModel, DataType, is_chain_type
 from datachain.lib.dc import C, Column, DataChain, Sys
 from datachain.lib.file import (
+    ArrowRow,
     File,
     FileError,
     ImageFile,
-    IndexedFile,
     TarVFile,
     TextFile,
 )
 from datachain.lib.model_store import ModelStore
 from datachain.lib.udf import Aggregator, Generator, Mapper
 from datachain.lib.utils import AbstractUDF, DataChainError
+from datachain.query import metrics, param
 from datachain.query.session import Session
 
 __all__ = [
     "AbstractUDF",
     "Aggregator",
+    "ArrowRow",
     "C",
     "Column",
     "DataChain",
@@ -26,7 +28,6 @@ __all__ = [
     "FileError",
     "Generator",
     "ImageFile",
-    "IndexedFile",
     "Mapper",
     "ModelStore",
     "Session",
@@ -34,4 +35,6 @@ __all__ = [
     "TarVFile",
     "TextFile",
     "is_chain_type",
+    "metrics",
+    "param",
 ]

{datachain-0.3.17 → datachain-0.3.19}/src/datachain/cache.py

@@ -1,56 +1,15 @@
-import hashlib
-import json
 import os
-from datetime import datetime
-from functools import partial
 from typing import TYPE_CHECKING, Optional
 
-import attrs
 from dvc_data.hashfile.db.local import LocalHashFileDB
 from dvc_objects.fs.local import LocalFileSystem
 from fsspec.callbacks import Callback, TqdmCallback
 
-from datachain.utils import TIME_ZERO
-
 from .progress import Tqdm
 
 if TYPE_CHECKING:
     from datachain.client import Client
-    from datachain.storage import StorageURI
-
-sha256 = partial(hashlib.sha256, usedforsecurity=False)
-
-
-@attrs.frozen
-class UniqueId:
-    storage: "StorageURI"
-    path: str
-    size: int
-    etag: str
-    version: str = ""
-    is_latest: bool = True
-    location: Optional[str] = None
-    last_modified: datetime = TIME_ZERO
-
-    def get_parsed_location(self) -> Optional[dict]:
-        if not self.location:
-            return None
-
-        loc_stack = (
-            json.loads(self.location)
-            if isinstance(self.location, str)
-            else self.location
-        )
-        if len(loc_stack) > 1:
-            raise NotImplementedError("Nested v-objects are not supported yet.")
-
-        return loc_stack[0]
-
-    def get_hash(self) -> str:
-        fingerprint = f"{self.storage}/{self.path}/{self.version}/{self.etag}"
-        if self.location:
-            fingerprint += f"/{self.location}"
-        return sha256(fingerprint.encode()).hexdigest()
+    from datachain.lib.file import File
 
 
 def try_scandir(path):
@@ -77,30 +36,30 @@ class DataChainCache:
     def tmp_dir(self):
         return self.odb.tmp_dir
 
-    def get_path(self, uid: UniqueId) -> Optional[str]:
-        if self.contains(uid):
-            return self.path_from_checksum(uid.get_hash())
+    def get_path(self, file: "File") -> Optional[str]:
+        if self.contains(file):
+            return self.path_from_checksum(file.get_hash())
         return None
 
-    def contains(self, uid: UniqueId) -> bool:
-        return self.odb.exists(uid.get_hash())
+    def contains(self, file: "File") -> bool:
+        return self.odb.exists(file.get_hash())
 
     def path_from_checksum(self, checksum: str) -> str:
         assert checksum
         return self.odb.oid_to_path(checksum)
 
-    def remove(self, uid: UniqueId) -> None:
-        self.odb.delete(uid.get_hash())
+    def remove(self, file: "File") -> None:
+        self.odb.delete(file.get_hash())
 
     async def download(
-        self, uid: UniqueId, client: "Client", callback: Optional[Callback] = None
+        self, file: "File", client: "Client", callback: Optional[Callback] = None
     ) -> None:
-        from_path = f"{uid.storage}/{uid.path}"
+        from_path = f"{file.source}/{file.path}"
         from dvc_objects.fs.utils import tmp_fname
 
         odb_fs = self.odb.fs
         tmp_info = odb_fs.join(self.odb.tmp_dir, tmp_fname())  # type: ignore[arg-type]
-        size = uid.size
+        size = file.size
         if size < 0:
             size = await client.get_size(from_path)
         cb = callback or TqdmCallback(
@@ -115,13 +74,13 @@ class DataChainCache:
                 cb.close()
 
         try:
-            oid = uid.get_hash()
+            oid = file.get_hash()
             self.odb.add(tmp_info, self.odb.fs, oid)
         finally:
             os.unlink(tmp_info)
 
-    def store_data(self, uid: UniqueId, contents: bytes) -> None:
-        checksum = uid.get_hash()
+    def store_data(self, file: "File", contents: bytes) -> None:
+        checksum = file.get_hash()
         dst = self.path_from_checksum(checksum)
         if not os.path.exists(dst):
             # Create the file only if it's not already in cache