datachain 0.25.0__tar.gz → 0.25.2__tar.gz

{datachain-0.25.0 → datachain-0.25.2}/.github/workflows/benchmarks.yml

@@ -30,9 +30,6 @@ jobs:
           enable-cache: true
           cache-suffix: benchmarks
           cache-dependency-glob: pyproject.toml
-          # revert after this is fixed
-          # https://github.com/wntrblm/nox/issues/953
-          version: ">=0.6,<0.7"
 
       - name: Install nox and dvc
         run: uv pip install dvc[gs] nox --system

{datachain-0.25.0 → datachain-0.25.2}/.github/workflows/tests-studio.yml

@@ -75,8 +75,21 @@ jobs:
           path: './backend/datachain'
           fetch-depth: 0
 
-      - name: Set up FFmpeg
-        uses: AnimMouse/setup-ffmpeg@v1
+      - name: Install FFmpeg on Windows
+        if: runner.os == 'Windows'
+        run: choco install ffmpeg
+
+      - name: Install FFmpeg on macOS
+        if: runner.os == 'macOS'
+        run: |
+          brew install ffmpeg
+          echo 'DYLD_FALLBACK_LIBRARY_PATH=/opt/homebrew/lib' >> "$GITHUB_ENV"
+
+      - name: Install FFmpeg on Ubuntu
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg
 
       - name: Set up Python ${{ matrix.pyv }}
         uses: actions/setup-python@v5

{datachain-0.25.0 → datachain-0.25.2}/.github/workflows/tests.yml

@@ -34,9 +34,6 @@ jobs:
           enable-cache: true
           cache-suffix: lint
           cache-dependency-glob: pyproject.toml
-          # revert after this is fixed
-          # https://github.com/wntrblm/nox/issues/953
-          version: ">=0.6,<0.7"
 
       - name: Install nox
         run: uv pip install nox --system
@@ -81,9 +78,6 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
 
-      - name: Set up FFmpeg
-        uses: AnimMouse/setup-ffmpeg@v1
-
       - name: Set up Python ${{ matrix.pyv }}
         uses: actions/setup-python@v5
         with:
@@ -95,9 +89,22 @@ jobs:
           enable-cache: true
           cache-suffix: tests-${{ matrix.pyv }}
           cache-dependency-glob: pyproject.toml
-          # revert after this is fixed
-          # https://github.com/wntrblm/nox/issues/953
-          version: ">=0.6,<0.7"
+
+      - name: Install FFmpeg on Windows
+        if: runner.os == 'Windows'
+        run: choco install ffmpeg
+
+      - name: Install FFmpeg on macOS
+        if: runner.os == 'macOS'
+        run: |
+          brew install ffmpeg
+          echo 'DYLD_FALLBACK_LIBRARY_PATH=/opt/homebrew/lib' >> "$GITHUB_ENV"
+
+      - name: Install FFmpeg on Ubuntu
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg
 
       - name: Install nox
         run: uv pip install nox --system
@@ -165,9 +172,6 @@ jobs:
           enable-cache: true
           cache-suffix: examples-${{ matrix.pyv }}
           cache-dependency-glob: pyproject.toml
-          # revert after this is fixed
-          # https://github.com/wntrblm/nox/issues/953
-          version: ">=0.6,<0.7"
 
       - name: Install nox
         run: uv pip install nox --system

{datachain-0.25.0 → datachain-0.25.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.25.0
+Version: 0.25.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -70,7 +70,8 @@ Provides-Extra: vector
 Requires-Dist: usearch; extra == "vector"
 Provides-Extra: hf
 Requires-Dist: numba>=0.60.0; extra == "hf"
-Requires-Dist: datasets[audio,vision]>=2.21.0; extra == "hf"
+Requires-Dist: datasets[vision]>=4.0.0; extra == "hf"
+Requires-Dist: datasets[audio]>=4.0.0; (sys_platform == "linux" or sys_platform == "darwin") and extra == "hf"
 Requires-Dist: fsspec>=2024.12.0; extra == "hf"
 Provides-Extra: video
 Requires-Dist: ffmpeg-python; extra == "video"

{datachain-0.25.0 → datachain-0.25.2}/docs/references/datachain.md

@@ -15,6 +15,10 @@ for examples of how to create a chain.
 
 ::: datachain.lib.dc.datasets.datasets
 
+::: datachain.lib.dc.datasets.delete_dataset
+
+::: datachain.lib.dc.datasets.move_dataset
+
 ::: datachain.lib.dc.hf.read_hf
 
 ::: datachain.lib.dc.json.read_json

{datachain-0.25.0 → datachain-0.25.2}/pyproject.toml

@@ -81,7 +81,9 @@ vector = [
 ]
 hf = [
   "numba>=0.60.0",
-  "datasets[audio,vision]>=2.21.0",
+  "datasets[vision]>=4.0.0",
+  # https://github.com/pytorch/torchcodec/issues/640
+  "datasets[audio]>=4.0.0 ; (sys_platform == 'linux' or sys_platform == 'darwin')",
   "fsspec>=2024.12.0"
 ]
 video = [

{datachain-0.25.0 → datachain-0.25.2}/src/datachain/lib/arrow.py

@@ -126,7 +126,16 @@ class ArrowGenerator(Generator):
             if isinstance(kwargs.get("format"), CsvFileFormat):
                 kwargs["format"] = "csv"
             arrow_file = ArrowRow(file=file, index=index, kwargs=kwargs)
+
+            if self.output_schema and hasattr(vals[0], "source"):
+                # if we are reading parquet file written by datachain it might have
+                # source inside of it already, so we should not duplicate it, instead
+                # we are re-creating it of the self.source flag
+                vals[0].source = arrow_file  # type: ignore[attr-defined]
+
+                return vals
             return [arrow_file, *vals]
+
         return vals
 
     def _process_non_datachain_record(

{datachain-0.25.0 → datachain-0.25.2}/src/datachain/lib/dc/datasets.py

@@ -376,7 +376,7 @@ def move_dataset(
             the namespace and project, or a regular name. If a regular name is used,
             default values will be applied. The source dataset will no longer exist
             after the move.
-        dst: The destination dataset name. This can also be a fully qualified
+        dest: The destination dataset name. This can also be a fully qualified
             name with a namespace and project, or just a regular name (default values
             will be used in that case). The original dataset will be moved here.
         session: An optional session instance. If not provided, the default session

{datachain-0.25.0 → datachain-0.25.2}/src/datachain/lib/hf.py

@@ -11,7 +11,7 @@ try:
         Image,
         IterableDataset,
         IterableDatasetDict,
-        Sequence,
+        List,
         Value,
         load_dataset,
     )
@@ -59,7 +59,6 @@ class HFImage(DataModel):
 
 
 class HFAudio(DataModel):
-    path: str
     array: list[float]
     sampling_rate: int
 
@@ -116,26 +115,24 @@ def stream_splits(ds: Union[str, HFDatasetType], *args, **kwargs):
     return {"": ds}
 
 
-def convert_feature(val: Any, feat: Any, anno: Any) -> Any:  # noqa: PLR0911
-    if isinstance(feat, (Value, Array2D, Array3D, Array4D, Array5D)):
+def convert_feature(val: Any, feat: Any, anno: Any) -> Any:
+    if isinstance(feat, (Value, Array2D, Array3D, Array4D, Array5D, List)):
         return val
     if isinstance(feat, ClassLabel):
         return HFClassLabel(string=feat.names[val], integer=val)
-    if isinstance(feat, Sequence):
-        if isinstance(feat.feature, dict):
-            sdict = {}
-            for sname in val:
-                sfeat = feat.feature[sname]
-                sanno = anno.model_fields[sname].annotation
-                sdict[sname] = [convert_feature(v, sfeat, sanno) for v in val[sname]]
-            return anno(**sdict)
-        return val
+    if isinstance(feat, dict):
+        sdict = {}
+        for sname in val:
+            sfeat = feat[sname]
+            sanno = anno.model_fields[sname].annotation
+            sdict[sname] = [convert_feature(v, sfeat, sanno) for v in val[sname]]
+        return anno(**sdict)
     if isinstance(feat, Image):
         if isinstance(val, dict):
             return HFImage(img=val["bytes"])
         return HFImage(img=image_to_bytes(val))
     if isinstance(feat, Audio):
-        return HFAudio(**val)
+        return HFAudio(array=val["array"], sampling_rate=val["sampling_rate"])
 
 
 def get_output_schema(features: Features) -> dict[str, DataType]:
@@ -151,13 +148,13 @@ def _feature_to_chain_type(name: str, val: Any) -> DataType:  # noqa: PLR0911
         return arrow_type_mapper(val.pa_type)
     if isinstance(val, ClassLabel):
         return HFClassLabel
-    if isinstance(val, Sequence):
-        if isinstance(val.feature, dict):
-            sequence_dict = {}
-            for sname, sval in val.feature.items():
-                dtype = _feature_to_chain_type(sname, sval)
-                sequence_dict[sname] = list[dtype]  # type: ignore[valid-type]
-            return dict_to_data_model(name, sequence_dict)  # type: ignore[arg-type]
+    if isinstance(val, dict):
+        sequence_dict = {}
+        for sname, sval in val.items():
+            dtype = _feature_to_chain_type(sname, sval)
+            sequence_dict[sname] = dtype  # type: ignore[valid-type]
+        return dict_to_data_model(name, sequence_dict)  # type: ignore[arg-type]
+    if isinstance(val, List):
         return list[_feature_to_chain_type(name, val.feature)]  # type: ignore[arg-type,misc,return-value]
     if isinstance(val, Array2D):
         dtype = arrow_type_mapper(string_to_arrow(val.dtype))

{datachain-0.25.0 → datachain-0.25.2}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.25.0
+Version: 0.25.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -70,7 +70,8 @@ Provides-Extra: vector
 Requires-Dist: usearch; extra == "vector"
 Provides-Extra: hf
 Requires-Dist: numba>=0.60.0; extra == "hf"
-Requires-Dist: datasets[audio,vision]>=2.21.0; extra == "hf"
+Requires-Dist: datasets[vision]>=4.0.0; extra == "hf"
+Requires-Dist: datasets[audio]>=4.0.0; (sys_platform == "linux" or sys_platform == "darwin") and extra == "hf"
 Requires-Dist: fsspec>=2024.12.0; extra == "hf"
 Provides-Extra: video
 Requires-Dist: ffmpeg-python; extra == "video"

{datachain-0.25.0 → datachain-0.25.2}/src/datachain.egg-info/requires.txt

@@ -63,9 +63,12 @@ open_clip_torch
 
 [hf]
 numba>=0.60.0
-datasets[audio,vision]>=2.21.0
+datasets[vision]>=4.0.0
 fsspec>=2024.12.0
 
+[hf:sys_platform == "linux" or sys_platform == "darwin"]
+datasets[audio]>=4.0.0
+
 [remote]
 lz4
 requests>=2.22.0

{datachain-0.25.0 → datachain-0.25.2}/tests/func/test_hf.py

@@ -1,4 +1,7 @@
+import importlib
+
 import numpy as np
+import pytest
 from datasets import load_dataset
 from datasets.features.image import image_to_bytes
 from PIL import Image
@@ -12,6 +15,18 @@ from datachain.lib.hf import (
 )
 
 
+def require_torchcodec(test_case):
+    """
+    Decorator marking a test that requires torchcodec (not available on Windows).
+    These tests are skipped when torchcodec isn't installed.
+    """
+    if not importlib.util.find_spec("torchcodec"):
+        test_case = pytest.mark.skip(
+            "test requires torchcoded, not available on Windows yet"
+        )(test_case)
+    return test_case
+
+
 def test_hf_image(tmp_path):
     train_dir = tmp_path / "train"
     train_dir.mkdir()
@@ -28,6 +43,7 @@ def test_hf_image(tmp_path):
     assert row.image.img == image_to_bytes(img)
 
 
+@require_torchcodec
 def test_hf_audio(tmp_path):
     # See https://stackoverflow.com/questions/66191480/how-to-convert-a-numpy-array-to-a-mp3-file
     samplerate = 44100
@@ -45,6 +61,5 @@ def test_hf_audio(tmp_path):
     gen = HFGenerator(ds, dict_to_data_model("", schema))
     gen.setup()
     row = next(iter(gen.process("train")))
-    assert row.audio.path == str(train_dir / "example.wav")
     assert np.allclose(row.audio.array, data / amplitude, atol=1e-4)
     assert row.audio.sampling_rate == samplerate

{datachain-0.25.0 → datachain-0.25.2}/tests/unit/lib/test_datachain.py

@@ -1760,6 +1760,22 @@ def test_read_parquet(tmp_dir, test_session):
     assert df_equal(df1, df)
 
 
+def test_read_parquet_exported_with_source(test_session, tmp_dir):
+    path = tmp_dir / "df.parquet"
+    path2 = tmp_dir / "df2.parquet"
+    df = pd.DataFrame(DF_DATA)
+
+    df.to_parquet(path)
+    dc.read_parquet(path, source=True).to_parquet(path2)
+    df1 = (
+        dc.read_parquet(path2, source=True)
+        .select("first_name", "age", "city")
+        .to_pandas()
+    )
+
+    assert df_equal(df1, df)
+
+
 @skip_if_not_sqlite
 def test_read_parquet_in_memory(tmp_dir):
     df = pd.DataFrame(DF_DATA)