datachain 0.8.13__tar.gz → 0.9.0__tar.gz

{datachain-0.8.13 → datachain-0.9.0}/.github/workflows/tests-studio.yml

@@ -75,6 +75,9 @@ jobs:
           path: './backend/datachain'
           fetch-depth: 0
 
+      - name: Set up FFmpeg
+        uses: AnimMouse/setup-ffmpeg@v1
+
       - name: Set up Python ${{ matrix.pyv }}
         uses: actions/setup-python@v5
         with:

{datachain-0.8.13 → datachain-0.9.0}/.github/workflows/tests.yml

@@ -78,6 +78,9 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
 
+      - name: Set up FFmpeg
+        uses: AnimMouse/setup-ffmpeg@v1
+
       - name: Set up Python ${{ matrix.pyv }}
         uses: actions/setup-python@v5
         with:

{datachain-0.8.13 → datachain-0.9.0}/.pre-commit-config.yaml

@@ -24,13 +24,13 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.9.3'
+    rev: 'v0.9.6'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]

{datachain-0.8.13 → datachain-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: datachain
-Version: 0.8.13
+Version: 0.9.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -21,10 +21,12 @@ Requires-Dist: tomlkit
 Requires-Dist: tqdm
 Requires-Dist: numpy<3,>=1
 Requires-Dist: pandas>=2.0.0
+Requires-Dist: packaging
 Requires-Dist: pyarrow
 Requires-Dist: typing-extensions
 Requires-Dist: python-dateutil>=2
 Requires-Dist: attrs>=21.3.0
+Requires-Dist: fsspec>=2024.2.0
 Requires-Dist: s3fs>=2024.2.0
 Requires-Dist: gcsfs>=2024.2.0
 Requires-Dist: adlfs>=2024.2.0
@@ -42,7 +44,7 @@ Requires-Dist: Pillow<12,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
 Requires-Dist: huggingface_hub
-Requires-Dist: iterative-telemetry>=0.0.9
+Requires-Dist: iterative-telemetry>=0.0.10
 Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21
 Requires-Dist: tabulate
@@ -54,6 +56,7 @@ Requires-Dist: mkdocs-material==9.5.22; extra == "docs"
 Requires-Dist: mkdocs-section-index>=0.3.6; extra == "docs"
 Requires-Dist: mkdocstrings-python>=1.6.3; extra == "docs"
 Requires-Dist: mkdocs-literate-nav>=0.6.1; extra == "docs"
+Requires-Dist: eval-type-backport; extra == "docs"
 Provides-Extra: torch
 Requires-Dist: torch>=2.1.0; extra == "torch"
 Requires-Dist: torchvision; extra == "torch"
@@ -66,8 +69,13 @@ Requires-Dist: usearch; extra == "vector"
 Provides-Extra: hf
 Requires-Dist: numba>=0.60.0; extra == "hf"
 Requires-Dist: datasets[audio,vision]>=2.21.0; extra == "hf"
+Provides-Extra: video
+Requires-Dist: av<14; extra == "video"
+Requires-Dist: ffmpeg-python; extra == "video"
+Requires-Dist: imageio[ffmpeg]; extra == "video"
+Requires-Dist: opencv-python; extra == "video"
 Provides-Extra: tests
-Requires-Dist: datachain[hf,remote,torch,vector]; extra == "tests"
+Requires-Dist: datachain[hf,remote,torch,vector,video]; extra == "tests"
 Requires-Dist: pytest<9,>=8; extra == "tests"
 Requires-Dist: pytest-sugar>=0.9.6; extra == "tests"
 Requires-Dist: pytest-cov>=4.1.0; extra == "tests"
@@ -83,7 +91,7 @@ Requires-Dist: requests-mock; extra == "tests"
 Requires-Dist: scipy; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
-Requires-Dist: mypy==1.14.1; extra == "dev"
+Requires-Dist: mypy==1.15.0; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
@@ -94,7 +102,7 @@ Requires-Dist: datachain[tests]; extra == "examples"
 Requires-Dist: defusedxml; extra == "examples"
 Requires-Dist: accelerate; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
-Requires-Dist: ultralytics==8.3.68; extra == "examples"
+Requires-Dist: ultralytics==8.3.74; extra == "examples"
 Requires-Dist: open_clip_torch; extra == "examples"
 
 ================

datachain-0.9.0/docs/references/data-types/arrowrow.md

@@ -0,0 +1,3 @@
+# ArrowRow
+
+::: datachain.lib.file.ArrowRow

datachain-0.9.0/docs/references/data-types/bbox.md

@@ -0,0 +1,5 @@
+# Bounding Box
+
+::: datachain.model.bbox.BBox
+
+::: datachain.model.bbox.OBBox

datachain-0.9.0/docs/references/data-types/file.md

@@ -0,0 +1,35 @@
+# File
+
+`File` is a special [`DataModel`](index.md#datachain.lib.data_model.DataModel),
+which is automatically generated when a `DataChain` is created from files,
+such as in [`DataChain.from_storage`](../datachain.md#datachain.lib.dc.DataChain.from_storage):
+
+```python
+from datachain import DataChain
+
+dc = DataChain.from_storage("gs://datachain-demo/dogs-and-cats")
+dc.print_schema()
+```
+
+Output:
+
+```
+file: File@v1
+    source: str
+    path: str
+    size: int
+    version: str
+    etag: str
+    is_latest: bool
+    last_modified: datetime
+    location: Union[dict, list[dict], NoneType]
+```
+
+`File` classes include various metadata fields describing the underlying file,
+along with methods to read and manipulate file contents.
+
+::: datachain.lib.file.File
+
+::: datachain.lib.file.FileError
+
+::: datachain.lib.file.TarVFile

datachain-0.9.0/docs/references/data-types/imagefile.md

@@ -0,0 +1,15 @@
+# ImageFile
+
+`ImageFile` is inherited from [`File`](file.md) with additional methods for working with image files.
+
+`ImageFile` is generated when a `DataChain` is created [from storage](../datachain.md#datachain.lib.dc.DataChain.from_storage), using `type="image"` param:
+
+```python
+from datachain import DataChain
+
+dc = DataChain.from_storage("s3://bucket-name/", type="image")
+```
+
+::: datachain.lib.file.ImageFile
+
+::: datachain.lib.file.Image

datachain-0.8.13/docs/references/datatype.md → datachain-0.9.0/docs/references/data-types/index.md

@@ -1,4 +1,4 @@
-# DataType
+# Data Types
 
 Data types supported by `DataChain` must be of type
 [`DataType`](#datachain.lib.data_model.DataType). `DataType` includes most Python types

datachain-0.9.0/docs/references/data-types/pose.md

@@ -0,0 +1,5 @@
+# Pose
+
+::: datachain.model.pose.Pose
+
+::: datachain.model.pose.Pose3D

datachain-0.9.0/docs/references/data-types/segment.md

@@ -0,0 +1,3 @@
+# Segment
+
+::: datachain.model.segment.Segment

datachain-0.9.0/docs/references/data-types/tarvfile.md

@@ -0,0 +1,3 @@
+# TarVFile
+
+::: datachain.lib.file.TarVFile

datachain-0.9.0/docs/references/data-types/textfile.md

@@ -0,0 +1,13 @@
+# TextFile
+
+`TextFile` is inherited from [`File`](file.md) with additional methods for working with text files.
+
+`TextFile` is generated when a `DataChain` is created [from storage](../datachain.md#datachain.lib.dc.DataChain.from_storage), using `type="text"` param:
+
+```python
+from datachain import DataChain
+
+dc = DataChain.from_storage("s3://bucket-name/", type="text")
+```
+
+::: datachain.lib.file.TextFile

datachain-0.9.0/docs/references/data-types/videofile.md

@@ -0,0 +1,29 @@
+# VideoFile
+
+`VideoFile` extends [`File`](file.md) and provides additional methods for working with video files.
+
+`VideoFile` instances are created when a `DataChain` is initialized [from storage](../datachain.md#datachain.lib.dc.DataChain.from_storage) with the `type="video"` parameter:
+
+```python
+from datachain import DataChain
+
+dc = DataChain.from_storage("s3://bucket-name/", type="video")
+```
+
+There are additional models for working with video files:
+
+- `VideoFrame` - represents a single frame of a video file.
+- `VideoFragment` - represents a fragment of a video file.
+
+These are virtual models that do not create physical files.
+Instead, they are used to represent the data in the `VideoFile` these models are referring to.
+If you need to save the data, you can use the `save` method of these models,
+allowing you to save data locally or upload it to a storage service.
+
+::: datachain.lib.file.VideoFile
+
+::: datachain.lib.file.VideoFrame
+
+::: datachain.lib.file.VideoFragment
+
+::: datachain.lib.file.Video

datachain-0.9.0/docs/references/index.md

@@ -0,0 +1,23 @@
+---
+title: API Reference
+---
+
+# API Reference
+
+DataChain's API is organized into several modules:
+
+- [DataChain](./datachain.md) - Core chain operations and dataset management
+- [Data Types](./data-types/index.md) - Supported data types and schema definitions
+    - [File](./data-types/file.md) - File handling and storage operations
+    - [TextFile](./data-types/textfile.md) - Text file
+    - [ImageFile](./data-types/imagefile.md) - Image file
+    - [VideoFile](./data-types/imagefile.md) - Video file
+    - [TarVFile](./data-types/tarvfile.md) - Virtual file model for files extracted from tar archives
+    - [ArrowRow](./data-types/arrowrow.md) - Working with Arrow-supported file
+    - [BBox](./data-types/bbox.md) - Bounding box data type
+    - [Pose](./data-types/pose.md) - Pose data type
+    - [Segment](./data-types/segment.md) - Segment data type
+- [UDF](./udf.md) - User-defined functions and transformations
+- [Functions](./func.md) - Built-in functions for data manipulation and analysis
+- [Torch](./torch.md) - PyTorch data loading utilities
+- [Toolkit](./toolkit.md) - Functions for common DS/ML operations

datachain-0.9.0/docs/references/toolkit.md

@@ -0,0 +1,5 @@
+# Toolkit
+
+Here you can find the toolkit functions on top of DataChain for common DS/ML operations (e.g. train/test split). Import these functions from `datachain.toolkit`.
+
+::: datachain.toolkit

datachain-0.9.0/examples/get_started/common_sql_functions.py

@@ -0,0 +1,54 @@
+from datachain import C, DataChain
+from datachain.func import array, greatest, least, path, string
+
+
+def num_chars_udf(file):
+    parts = file.name.split(".")
+    if len(parts) > 1:
+        return (list(parts[1]),)
+    return ([],)
+
+
+dc = DataChain.from_storage("gs://datachain-demo/dogs-and-cats/", anon=True)
+dc.map(num_chars_udf, params=["file"], output={"num_chars": list[str]}).select(
+    "file.path", "num_chars"
+).show(5)
+
+(
+    dc.mutate(
+        length=string.length(path.name(C("file.path"))),
+        parts=string.split(path.name(C("file.path")), "."),
+    )
+    .select("file.path", "length", "parts")
+    .show(5)
+)
+
+(
+    dc.mutate(
+        stem=path.file_stem(C("file.path")),
+        ext=path.file_ext(C("file.path")),
+    )
+    .select("file.path", "stem", "ext")
+    .show(5)
+)
+
+parts = string.split(path.name(C("file.path")), ".")
+chain = dc.mutate(
+    isdog=array.contains(parts, "dog"),
+    iscat=array.contains(parts, "cat"),
+)
+chain.select("file.path", "isdog", "iscat").show(5)
+
+chain = dc.mutate(
+    a=array.length(string.split("file.path", "/")),
+    b=array.length(string.split(path.name("file.path"), "0")),
+)
+
+(
+    chain.mutate(
+        greatest=greatest(chain.column("a"), C("b")),
+        least=least(chain.column("a"), C("b")),
+    )
+    .select("a", "b", "greatest", "least")
+    .show(10)
+)

{datachain-0.8.13 → datachain-0.9.0}/mkdocs.yml

@@ -69,11 +69,21 @@ nav:
       - 🐍 API Reference:
           - Overview: references/index.md
           - DataChain: references/datachain.md
-          - DataType: references/datatype.md
-          - File: references/file.md
+          - Data Types:
+              - Overview: references/data-types/index.md
+              - File: references/data-types/file.md
+              - TextFile: references/data-types/textfile.md
+              - ImageFile: references/data-types/imagefile.md
+              - VideoFile: references/data-types/videofile.md
+              - TarVFile: references/data-types/tarvfile.md
+              - ArrowRow: references/data-types/arrowrow.md
+              - BBox: references/data-types/bbox.md
+              - Pose: references/data-types/pose.md
+              - Segment: references/data-types/segment.md
           - UDF: references/udf.md
           - Torch: references/torch.md
           - Functions: references/func.md
+          - Toolkit: references/toolkit.md
       - 🤝 Contributing: contributing.md
 
   - DataChain Website ↗: https://datachain.ai" target="_blank"

{datachain-0.8.13 → datachain-0.9.0}/noxfile.py

@@ -32,6 +32,12 @@ def bench(session: nox.Session) -> None:
 @nox.session(python=["3.9", "3.10", "3.11", "3.12", "pypy3.9", "pypy3.10"])
 def tests(session: nox.Session) -> None:
     session.install(".[tests]")
+    env = {"COVERAGE_FILE": f".coverage.{session.python}"}
+    if session.python == "3.12":
+        # improve performance of tests in Python 3.12 when used with coverage
+        # https://github.com/nedbat/coveragepy/issues/1665
+        # https://github.com/python/cpython/issues/107674
+        env["COVERAGE_CORE"] = "sysmon"
     session.run(
         "pytest",
         "--cov",
@@ -41,7 +47,7 @@ def tests(session: nox.Session) -> None:
         "--numprocesses=logical",
         "--dist=loadgroup",
         *session.posargs,
-        env={"COVERAGE_FILE": f".coverage.{session.python}"},
+        env=env,
     )
 
 

{datachain-0.8.13 → datachain-0.9.0}/pyproject.toml

@@ -24,10 +24,12 @@ dependencies = [
   "tqdm",
   "numpy>=1,<3",
   "pandas>=2.0.0",
+  "packaging",
   "pyarrow",
   "typing-extensions",
   "python-dateutil>=2",
   "attrs>=21.3.0",
+  "fsspec>=2024.2.0",
   "s3fs>=2024.2.0",
   "gcsfs>=2024.2.0",
   "adlfs>=2024.2.0",
@@ -45,7 +47,7 @@ dependencies = [
   "msgpack>=1.0.4,<2",
   "psutil",
   "huggingface_hub",
-  "iterative-telemetry>=0.0.9",
+  "iterative-telemetry>=0.0.10",
   "platformdirs",
   "dvc-studio-client>=0.21,<1",
   "tabulate",
@@ -59,7 +61,8 @@ docs = [
   "mkdocs-material==9.5.22",
   "mkdocs-section-index>=0.3.6",
   "mkdocstrings-python>=1.6.3",
-  "mkdocs-literate-nav>=0.6.1"
+  "mkdocs-literate-nav>=0.6.1",
+  "eval-type-backport"
 ]
 torch = [
   "torch>=2.1.0",
@@ -77,8 +80,16 @@ hf = [
   "numba>=0.60.0",
   "datasets[audio,vision]>=2.21.0"
 ]
+video = [
+  # Use 'av<14' because of incompatibility with imageio
+  # See https://github.com/PyAV-Org/PyAV/discussions/1700
+  "av<14",
+  "ffmpeg-python",
+  "imageio[ffmpeg]",
+  "opencv-python"
+]
 tests = [
-  "datachain[torch,remote,vector,hf]",
+  "datachain[torch,remote,vector,hf,video]",
   "pytest>=8,<9",
   "pytest-sugar>=0.9.6",
   "pytest-cov>=4.1.0",
@@ -95,7 +106,7 @@ tests = [
 ]
 dev = [
   "datachain[docs,tests]",
-  "mypy==1.14.1",
+  "mypy==1.15.0",
   "types-python-dateutil",
   "types-pytz",
   "types-PyYAML",
@@ -107,7 +118,7 @@ examples = [
   "defusedxml",
   "accelerate",
   "huggingface_hub[hf_transfer]",
-  "ultralytics==8.3.68",
+  "ultralytics==8.3.74",
   "open_clip_torch"
 ]
 

{datachain-0.8.13 → datachain-0.9.0}/src/datachain/__init__.py

@@ -4,9 +4,14 @@ from datachain.lib.file import (
     ArrowRow,
     File,
     FileError,
+    Image,
     ImageFile,
     TarVFile,
     TextFile,
+    Video,
+    VideoFile,
+    VideoFragment,
+    VideoFrame,
 )
 from datachain.lib.model_store import ModelStore
 from datachain.lib.udf import Aggregator, Generator, Mapper
@@ -27,6 +32,7 @@ __all__ = [
     "File",
     "FileError",
     "Generator",
+    "Image",
     "ImageFile",
     "Mapper",
     "ModelStore",
@@ -34,6 +40,10 @@ __all__ = [
     "Sys",
     "TarVFile",
     "TextFile",
+    "Video",
+    "VideoFile",
+    "VideoFragment",
+    "VideoFrame",
     "is_chain_type",
     "metrics",
     "param",

{datachain-0.8.13 → datachain-0.9.0}/src/datachain/catalog/catalog.py

@@ -89,10 +89,6 @@ PULL_DATASET_SLEEP_INTERVAL = 0.1  # sleep time while waiting for chunk to be av
 PULL_DATASET_CHECK_STATUS_INTERVAL = 20  # interval to check export status in Studio
 
 
-def raise_remote_error(error_message: str) -> NoReturn:
-    raise DataChainError(f"Error from server: {error_message}")
-
-
 def noop(_: str):
     pass
 
@@ -211,14 +207,14 @@ class DatasetRowsFetcher(NodesThreadPool):
             self.remote_ds_name, self.remote_ds_version
         )
         if not export_status_response.ok:
-            raise_remote_error(export_status_response.message)
+            raise DataChainError(export_status_response.message)
 
         export_status = export_status_response.data["status"]  # type: ignore [index]
 
         if export_status == "failed":
-            raise_remote_error("Dataset export failed in Studio")
+            raise DataChainError("Dataset export failed in Studio")
         if export_status == "removed":
-            raise_remote_error("Dataset export removed in Studio")
+            raise DataChainError("Dataset export removed in Studio")
 
         self.last_status_check = time.time()
 
@@ -1101,6 +1097,31 @@ class Catalog:
     def get_dataset(self, name: str) -> DatasetRecord:
         return self.metastore.get_dataset(name)
 
+    def get_dataset_with_remote_fallback(
+        self, name: str, version: Optional[int] = None
+    ) -> DatasetRecord:
+        try:
+            ds = self.get_dataset(name)
+            if version and not ds.has_version(version):
+                raise DatasetVersionNotFoundError(
+                    f"Dataset {name} does not have version {version}"
+                )
+            return ds
+
+        except (DatasetNotFoundError, DatasetVersionNotFoundError):
+            print("Dataset not found in local catalog, trying to get from studio")
+
+            remote_ds_uri = f"{DATASET_PREFIX}{name}"
+            if version:
+                remote_ds_uri += f"@v{version}"
+
+            self.pull_dataset(
+                remote_ds_uri=remote_ds_uri,
+                local_ds_name=name,
+                local_ds_version=version,
+            )
+            return self.get_dataset(name)
+
     def get_dataset_with_version_uuid(self, uuid: str) -> DatasetRecord:
         """Returns dataset that contains version with specific uuid"""
         for dataset in self.ls_datasets():
@@ -1113,7 +1134,7 @@ class Catalog:
 
         info_response = studio_client.dataset_info(name)
         if not info_response.ok:
-            raise_remote_error(info_response.message)
+            raise DataChainError(info_response.message)
 
         dataset_info = info_response.data
         assert isinstance(dataset_info, dict)
@@ -1209,6 +1230,8 @@ class Catalog:
         **kwargs,
     ) -> str:
         client_config = client_config or self.client_config
+        if client_config.get("anon"):
+            content_disposition = None
         client = Client.get_client(source, self.cache, **client_config)
         return client.url(
             path,
@@ -1407,7 +1430,7 @@ class Catalog:
             remote_ds_name, remote_ds_version.version
         )
         if not export_response.ok:
-            raise_remote_error(export_response.message)
+            raise DataChainError(export_response.message)
 
         signed_urls = export_response.data
 

{datachain-0.8.13 → datachain-0.9.0}/src/datachain/cli/__init__.py

@@ -160,6 +160,8 @@ def handle_dataset_command(args, catalog):
             local=args.local,
             all=args.all,
             team=args.team,
+            latest_only=not args.versions,
+            name=args.name,
         ),
         "rm": lambda: rm_dataset(
             catalog,