datachain 0.8.3__tar.gz → 0.8.5__tar.gz

{datachain-0.8.3 → datachain-0.8.5}/.github/workflows/tests-studio.yml

@@ -32,7 +32,7 @@ jobs:
           POSTGRES_DB: database
           POSTGRES_HOST_AUTH_METHOD: trust
       clickhouse:
-        image: clickhouse/clickhouse-server:24.6
+        image: clickhouse/clickhouse-server:24.8
         ports:
           - 8123:8123
           - 9010:9000

{datachain-0.8.3 → datachain-0.8.5}/.github/workflows/tests.yml

@@ -138,7 +138,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest]
         pyv: ['3.9', '3.12']
-        group: ['get_started', 'llm_and_nlp or computer_vision', 'multimodal']
+        group: ['get_started', 'computer_vision', 'llm_and_nlp', 'multimodal']
         exclude:
           - {os: ubuntu-latest, pyv: '3.9', group: 'multimodal'}
           - {os: ubuntu-latest, pyv: '3.12', group: 'multimodal'}

{datachain-0.8.3 → datachain-0.8.5}/.gitignore

@@ -140,3 +140,6 @@ cython_debug/
 .vscode/
 .datachain/
 .dvcx/
+
+# pt files produced by ultralytics examples
+*.pt

{datachain-0.8.3 → datachain-0.8.5}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.8.4'
+    rev: 'v0.8.6'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.8.3 → datachain-0.8.5}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.3
+Version: 0.8.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -50,7 +50,7 @@ Requires-Dist: websockets
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
-Requires-Dist: mkdocs-material>=9.3.1; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.22; extra == "docs"
 Requires-Dist: mkdocs-section-index>=0.3.6; extra == "docs"
 Requires-Dist: mkdocstrings-python>=1.6.3; extra == "docs"
 Requires-Dist: mkdocs-literate-nav>=0.6.1; extra == "docs"
@@ -84,7 +84,7 @@ Requires-Dist: requests-mock; extra == "tests"
 Requires-Dist: scipy; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
-Requires-Dist: mypy==1.14.0; extra == "dev"
+Requires-Dist: mypy==1.14.1; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
@@ -95,11 +95,11 @@ Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
 Requires-Dist: unstructured_ingest[embed-huggingface]; extra == "examples"
-Requires-Dist: unstructured[pdf]; extra == "examples"
-Requires-Dist: pdfplumber==0.11.4; extra == "examples"
+Requires-Dist: unstructured[pdf]<0.16.12; extra == "examples"
+Requires-Dist: pdfplumber==0.11.5; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
 Requires-Dist: onnx==1.16.1; extra == "examples"
-Requires-Dist: ultralytics==8.3.53; extra == "examples"
+Requires-Dist: ultralytics==8.3.58; extra == "examples"
 
 ================
 |logo| DataChain
@@ -134,7 +134,7 @@ Use Cases
 1. **ETL.** Pythonic framework for describing and running unstructured data transformations
    and enrichments, applying models to data, including LLMs.
 2. **Analytics.** DataChain dataset is a table that combines all the information about data
-   objects in one place + it provides dataframe-like API and vecrorized engine to do analytics
+   objects in one place + it provides dataframe-like API and vectorized engine to do analytics
    on these tables at scale.
 3. **Versioning.** DataChain doesn't store, require moving or copying data (unlike DVC).
    Perfect use case is a bucket with thousands or millions of images, videos, audio, PDFs.
@@ -270,7 +270,7 @@ DataChain Studio Platform
 
 `DataChain Studio`_ is a proprietary solution for teams that offers:
 
-- **Centralized dataset registry** to manage data, code and dependency
+- **Centralized dataset registry** to manage data, code and
   dependencies in one place.
 - **Data Lineage** for data sources as well as derivative dataset.
 - **UI for Multimodal Data** like images, videos, and PDFs.

{datachain-0.8.3 → datachain-0.8.5}/README.rst

@@ -31,7 +31,7 @@ Use Cases
 1. **ETL.** Pythonic framework for describing and running unstructured data transformations
    and enrichments, applying models to data, including LLMs.
 2. **Analytics.** DataChain dataset is a table that combines all the information about data
-   objects in one place + it provides dataframe-like API and vecrorized engine to do analytics
+   objects in one place + it provides dataframe-like API and vectorized engine to do analytics
    on these tables at scale.
 3. **Versioning.** DataChain doesn't store, require moving or copying data (unlike DVC).
    Perfect use case is a bucket with thousands or millions of images, videos, audio, PDFs.
@@ -167,7 +167,7 @@ DataChain Studio Platform
 
 `DataChain Studio`_ is a proprietary solution for teams that offers:
 
-- **Centralized dataset registry** to manage data, code and dependency
+- **Centralized dataset registry** to manage data, code and
   dependencies in one place.
 - **Data Lineage** for data sources as well as derivative dataset.
 - **UI for Multimodal Data** like images, videos, and PDFs.

{datachain-0.8.3 → datachain-0.8.5}/docs/index.md

@@ -42,7 +42,7 @@ database for easy and efficient querying.
     including LLMs.
 2.  **Analytics.** DataChain dataset is a table that combines all the
     information about data objects in one place + it provides
-    dataframe-like API and vecrorized engine to do analytics on these
+    dataframe-like API and vectorized engine to do analytics on these
     tables at scale.
 3.  **Versioning.** DataChain doesn't store, require moving or copying
     data (unlike DVC). Perfect use case is a bucket with thousands or

{datachain-0.8.3 → datachain-0.8.5}/examples/computer_vision/ultralytics-bbox.py

@@ -1,3 +1,8 @@
+import os
+
+os.environ["YOLO_VERBOSE"] = "false"
+
+
 from io import BytesIO
 
 from PIL import Image

{datachain-0.8.3 → datachain-0.8.5}/examples/computer_vision/ultralytics-pose.py

@@ -1,3 +1,8 @@
+import os
+
+os.environ["YOLO_VERBOSE"] = "false"
+
+
 from io import BytesIO
 
 from PIL import Image

{datachain-0.8.3 → datachain-0.8.5}/examples/computer_vision/ultralytics-segment.py

@@ -1,3 +1,8 @@
+import os
+
+os.environ["YOLO_VERBOSE"] = "false"
+
+
 from io import BytesIO
 
 from PIL import Image

{datachain-0.8.3 → datachain-0.8.5}/examples/get_started/torch-loader.py

@@ -56,7 +56,7 @@ class CNN(nn.Module):
 if __name__ == "__main__":
     ds = (
         DataChain.from_storage(STORAGE, type="image")
-        .settings(cache=True, prefetch=25)
+        .settings(prefetch=25)
         .filter(C("file.path").glob("*.jpg"))
         .map(
             label=lambda path: label_to_int(basename(path)[:3], CLASSES),
@@ -68,7 +68,7 @@ if __name__ == "__main__":
     train_loader = DataLoader(
         ds.to_pytorch(transform=transform),
         batch_size=25,
-        num_workers=max(4, os.cpu_count() or 2),
+        num_workers=min(4, os.cpu_count() or 2),
         persistent_workers=True,
         multiprocessing_context=multiprocessing.get_context("spawn"),
     )

{datachain-0.8.3 → datachain-0.8.5}/mkdocs.yml

@@ -136,6 +136,7 @@ plugins:
             show_root_heading: true
             show_signature_annotations: true
             show_symbol_type_heading: true
+            show_symbol_type_toc: true
             signature_crossrefs: true
           import:
             - https://docs.python.org/3/objects.inv

{datachain-0.8.3 → datachain-0.8.5}/pyproject.toml

@@ -56,7 +56,7 @@ dependencies = [
 docs = [
   "mkdocs>=1.5.2",
   "mkdocs-gen-files>=0.5.0",
-  "mkdocs-material>=9.3.1",
+  "mkdocs-material==9.5.22",
   "mkdocs-section-index>=0.3.6",
   "mkdocstrings-python>=1.6.3",
   "mkdocs-literate-nav>=0.6.1"
@@ -96,7 +96,7 @@ tests = [
 ]
 dev = [
   "datachain[docs,tests]",
-  "mypy==1.14.0",
+  "mypy==1.14.1",
   "types-python-dateutil",
   "types-pytz",
   "types-PyYAML",
@@ -108,11 +108,11 @@ examples = [
   "defusedxml",
   "accelerate",
   "unstructured_ingest[embed-huggingface]",
-  "unstructured[pdf]",
-  "pdfplumber==0.11.4",
+  "unstructured[pdf]<0.16.12",
+  "pdfplumber==0.11.5",
   "huggingface_hub[hf_transfer]",
   "onnx==1.16.1",
-  "ultralytics==8.3.53"
+  "ultralytics==8.3.58"
 ]
 
 [project.urls]

{datachain-0.8.3 → datachain-0.8.5}/src/datachain/asyn.py

@@ -8,12 +8,14 @@ from collections.abc import (
     Iterable,
     Iterator,
 )
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, wait
 from heapq import heappop, heappush
 from typing import Any, Callable, Generic, Optional, TypeVar
 
 from fsspec.asyn import get_loop
 
+from datachain.utils import safe_closing
+
 ASYNC_WORKERS = 20
 
 InputT = TypeVar("InputT", contravariant=True)  # noqa: PLC0105
@@ -56,6 +58,7 @@ class AsyncMapper(Generic[InputT, ResultT]):
         self.pool = ThreadPoolExecutor(workers)
         self._tasks: set[asyncio.Task] = set()
         self._shutdown_producer = threading.Event()
+        self._producer_is_shutdown = threading.Event()
 
     def start_task(self, coro: Coroutine) -> asyncio.Task:
         task = self.loop.create_task(coro)
@@ -64,11 +67,16 @@ class AsyncMapper(Generic[InputT, ResultT]):
         return task
 
     def _produce(self) -> None:
-        for item in self.iterable:
-            if self._shutdown_producer.is_set():
-                return
-            fut = asyncio.run_coroutine_threadsafe(self.work_queue.put(item), self.loop)
-            fut.result()  # wait until the item is in the queue
+        try:
+            with safe_closing(self.iterable):
+                for item in self.iterable:
+                    if self._shutdown_producer.is_set():
+                        return
+                    coro = self.work_queue.put(item)
+                    fut = asyncio.run_coroutine_threadsafe(coro, self.loop)
+                    fut.result()  # wait until the item is in the queue
+        finally:
+            self._producer_is_shutdown.set()
 
     async def produce(self) -> None:
         await self.to_thread(self._produce)
@@ -179,6 +187,8 @@ class AsyncMapper(Generic[InputT, ResultT]):
             self.shutdown_producer()
             if not async_run.done():
                 async_run.cancel()
+                wait([async_run])
+            self._producer_is_shutdown.wait()
 
     def __iter__(self):
         return self.iterate()

{datachain-0.8.3 → datachain-0.8.5}/src/datachain/cache.py

@@ -1,8 +1,12 @@
 import os
+from collections.abc import Iterator
+from contextlib import contextmanager
+from tempfile import mkdtemp
 from typing import TYPE_CHECKING, Optional
 
 from dvc_data.hashfile.db.local import LocalHashFileDB
 from dvc_objects.fs.local import LocalFileSystem
+from dvc_objects.fs.utils import remove
 from fsspec.callbacks import Callback, TqdmCallback
 
 from .progress import Tqdm
@@ -20,6 +24,23 @@ def try_scandir(path):
         pass
 
 
+def get_temp_cache(tmp_dir: str, prefix: Optional[str] = None) -> "DataChainCache":
+    cache_dir = mkdtemp(prefix=prefix, dir=tmp_dir)
+    return DataChainCache(cache_dir, tmp_dir=tmp_dir)
+
+
+@contextmanager
+def temporary_cache(
+    tmp_dir: str, prefix: Optional[str] = None, delete: bool = True
+) -> Iterator["DataChainCache"]:
+    cache = get_temp_cache(tmp_dir, prefix=prefix)
+    try:
+        yield cache
+    finally:
+        if delete:
+            cache.destroy()
+
+
 class DataChainCache:
     def __init__(self, cache_dir: str, tmp_dir: str):
         self.odb = LocalHashFileDB(
@@ -28,6 +49,9 @@ class DataChainCache:
             tmp_dir=tmp_dir,
         )
 
+    def __eq__(self, other) -> bool:
+        return self.odb == other.odb
+
     @property
     def cache_dir(self):
         return self.odb.path
@@ -63,7 +87,7 @@ class DataChainCache:
         if size < 0:
             size = await client.get_size(from_path, version_id=file.version)
         cb = callback or TqdmCallback(
-            tqdm_kwargs={"desc": odb_fs.name(from_path), "bytes": True},
+            tqdm_kwargs={"desc": odb_fs.name(from_path), "bytes": True, "leave": False},
             tqdm_cls=Tqdm,
             size=size,
         )
@@ -82,20 +106,18 @@ class DataChainCache:
             os.unlink(tmp_info)
 
     def store_data(self, file: "File", contents: bytes) -> None:
-        checksum = file.get_hash()
-        dst = self.path_from_checksum(checksum)
-        if not os.path.exists(dst):
-            # Create the file only if it's not already in cache
-            os.makedirs(os.path.dirname(dst), exist_ok=True)
-            with open(dst, mode="wb") as f:
-                f.write(contents)
-
-    def clear(self):
+        self.odb.add_bytes(file.get_hash(), contents)
+
+    def clear(self) -> None:
         """
         Completely clear the cache.
         """
         self.odb.clear()
 
+    def destroy(self) -> None:
+        # `clear` leaves the prefix directory structure intact.
+        remove(self.cache_dir)
+
     def get_total_size(self) -> int:
         total = 0
         for subdir in try_scandir(self.odb.path):

{datachain-0.8.3 → datachain-0.8.5}/src/datachain/catalog/catalog.py

@@ -405,6 +405,7 @@ def get_download_bar(bar_format: str, total_size: int):
         unit_scale=True,
         unit_divisor=1000,
         total=total_size,
+        leave=False,
     )
 
 
@@ -429,6 +430,7 @@ def instantiate_node_groups(
             unit_scale=True,
             unit_divisor=1000,
             total=total_files,
+            leave=False,
         )
     )
 
@@ -534,6 +536,12 @@ def find_column_to_str(  # noqa: PLR0911
     return ""
 
 
+def clone_catalog_with_cache(catalog: "Catalog", cache: "DataChainCache") -> "Catalog":
+    clone = catalog.copy()
+    clone.cache = cache
+    return clone
+
+
 class Catalog:
     def __init__(
         self,
@@ -1242,10 +1250,17 @@ class Catalog:
         path: str,
         version_id: Optional[str] = None,
         client_config=None,
+        content_disposition: Optional[str] = None,
+        **kwargs,
     ) -> str:
         client_config = client_config or self.client_config
         client = Client.get_client(source, self.cache, **client_config)
-        return client.url(path, version_id=version_id)
+        return client.url(
+            path,
+            version_id=version_id,
+            content_disposition=content_disposition,
+            **kwargs,
+        )
 
     def export_dataset_table(
         self,
@@ -1437,6 +1452,7 @@ class Catalog:
             unit_scale=True,
             unit_divisor=1000,
             total=ds_stats.num_objects,  # type: ignore [union-attr]
+            leave=False,
         )
 
         schema = DatasetRecord.parse_schema(remote_ds_version.schema)