datachain 0.30.2__tar.gz → 0.30.3__tar.gz

{datachain-0.30.2 → datachain-0.30.3}/.github/workflows/benchmarks.yml

@@ -18,7 +18,7 @@ jobs:
   run:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Set up Python 3.13
         uses: actions/setup-python@v5
         with:

{datachain-0.30.2 → datachain-0.30.3}/.github/workflows/release.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
 

{datachain-0.30.2 → datachain-0.30.3}/.github/workflows/tests-studio.yml

@@ -62,7 +62,7 @@ jobs:
           echo "Studio branch: $STUDIO_BRANCH"
 
       - name: Check out Studio
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           repository: iterative/studio
@@ -70,7 +70,7 @@ jobs:
           token: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
 
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           path: './backend/datachain'
           fetch-depth: 0

{datachain-0.30.2 → datachain-0.30.3}/.github/workflows/tests.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
@@ -73,7 +73,7 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
@@ -175,7 +175,7 @@ jobs:
           - {os: ubuntu-latest-4-cores, pyv: "3.13", group: multimodal}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
 
@@ -194,6 +194,22 @@ jobs:
       - name: Install nox
         run: uv pip install nox --system
 
+      - name: Install FFmpeg on Windows
+        if: runner.os == 'Windows'
+        run: choco install ffmpeg
+
+      - name: Install FFmpeg on macOS
+        if: runner.os == 'macOS'
+        run: |
+          brew install ffmpeg
+          echo 'DYLD_FALLBACK_LIBRARY_PATH=/opt/homebrew/lib' >> "$GITHUB_ENV"
+
+      - name: Install FFmpeg on Ubuntu
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg
+
       - name: Set hf token
         if: matrix.group == 'llm_and_nlp'
         run: echo 'HF_TOKEN=${{ secrets.HF_TOKEN }}' >> "$GITHUB_ENV"

{datachain-0.30.2 → datachain-0.30.3}/.github/workflows/update-template.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Update template
         uses: iterative/py-template@main

{datachain-0.30.2 → datachain-0.30.3}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.12.8'
+    rev: 'v0.12.9'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.30.2 → datachain-0.30.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.30.2
+Version: 0.30.3
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -45,7 +45,7 @@ Requires-Dist: datamodel-code-generator>=0.25
 Requires-Dist: Pillow<12,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
-Requires-Dist: huggingface_hub<0.34.0
+Requires-Dist: huggingface_hub
 Requires-Dist: iterative-telemetry>=0.0.10
 Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21

datachain-0.30.3/docs/references/func.md

@@ -0,0 +1,38 @@
+# Functions
+
+Use built-in functions for data manipulation and analysis to operate on the underlying database storing the chain data. These functions are useful for operations like [`DataChain.filter`](datachain.md#datachain.lib.dc.DataChain.filter) and [`DataChain.mutate`](datachain.md#datachain.lib.dc.DataChain.mutate).
+
+Functions are organized by category and accessed through their respective modules. For example, string functions are accessed via `func.string.length()`, array functions via `func.array.contains()`, etc.
+
+!!! note "Global Function Access"
+    Only a subset of functions are available directly from `datachain.func` (e.g., `func.length`). Most functions should be accessed through their specific module namespace (e.g., `func.string.length`) to avoid naming conflicts.
+
+## Function Categories
+
+DataChain provides several categories of functions for different types of operations:
+
+- **[Aggregate Functions](functions/aggregate.md)** - Functions for aggregating data like `sum`, `count`, `avg`, etc.
+- **[Array Functions](functions/array.md)** - Functions for working with arrays and lists
+- **[Conditional Functions](functions/conditional.md)** - Functions for conditional logic like `ifelse`, `case`, etc.
+- **[Numeric Functions](functions/numeric.md)** - Functions for numeric operations and computations
+- **[Path Functions](functions/path.md)** - Functions for working with file paths
+- **[Random Functions](functions/random.md)** - Functions for generating random values
+- **[String Functions](functions/string.md)** - Functions for string manipulation and processing
+- **[Window Functions](functions/window.md)** - Functions for window operations
+
+## Usage
+
+```python
+from datachain.func import aggregate, array, conditional, numeric, path, random, string, window
+
+# Access functions through their module namespaces
+dc.mutate(
+    text_length=string.length("text_column"),
+    contains_item=array.contains("array_column", "value"),
+    file_extension=path.file_ext("file_path")
+)
+
+# Some commonly used functions are also available directly
+from datachain.func import sum, count, length, ifelse
+dc.mutate(total=sum("amount"))
+```

datachain-0.30.3/docs/references/functions/aggregate.md

@@ -0,0 +1,5 @@
+# Aggregate Functions
+
+Aggregate functions perform calculations on sets of values and return a single result.
+
+::: datachain.func.aggregate

datachain-0.30.3/docs/references/functions/array.md

@@ -0,0 +1,5 @@
+# Array Functions
+
+Functions for working with arrays and lists, including operations like distance calculations and array manipulation.
+
+::: datachain.func.array

datachain-0.30.3/docs/references/functions/conditional.md

@@ -0,0 +1,5 @@
+# Conditional Functions
+
+Functions for conditional logic and control flow in data processing.
+
+::: datachain.func.conditional

datachain-0.30.3/docs/references/functions/numeric.md

@@ -0,0 +1,5 @@
+# Numeric Functions
+
+Functions for numeric operations, bit manipulation, and mathematical computations.
+
+::: datachain.func.numeric

datachain-0.30.3/docs/references/functions/path.md

@@ -0,0 +1,5 @@
+# Path Functions
+
+Functions for working with file paths and extracting path components.
+
+::: datachain.func.path

datachain-0.30.3/docs/references/functions/random.md

@@ -0,0 +1,5 @@
+# Random Functions
+
+Functions for generating random values and sampling.
+
+::: datachain.func.random

datachain-0.30.3/docs/references/functions/string.md

@@ -0,0 +1,22 @@
+# String Functions
+
+Functions for string manipulation, text processing, and string analysis.
+
+## Usage
+
+String functions are available under the `func.string` namespace to avoid name collisions with other functions:
+
+```python
+from datachain.func import string
+
+# Use string functions with the string namespace
+dc.mutate(
+    str_len=string.length("text_column"),
+    parts=string.split("text_column", ","),
+    cleaned=string.replace("text_column", "old", "new"),
+    regex_cleaned=string.regexp_replace("text_column", r"\d+", "X"),
+    distance=string.byte_hamming_distance("col1", "col2")
+)
+```
+
+::: datachain.func.string

datachain-0.30.3/docs/references/functions/window.md

@@ -0,0 +1,5 @@
+# Window Functions
+
+Functions for window operations and analytical processing.
+
+::: datachain.func.window

{datachain-0.30.2 → datachain-0.30.3}/examples/multimodal/audio-to-text.py

@@ -35,7 +35,10 @@ def process(fragment: AudioFragment, pipeline: Pipeline) -> str:
         audio_array = audio_array.mean(axis=1)
 
     # Pass the numpy array with exact sampling rate from fragment
-    result = pipeline({"raw": audio_array, "sampling_rate": sample_rate})
+    result = pipeline(
+        {"raw": audio_array, "sampling_rate": sample_rate},
+        generate_kwargs={"language": "en"},
+    )
     return str(result["text"])
 
 

{datachain-0.30.2 → datachain-0.30.3}/mkdocs.yml

@@ -82,7 +82,16 @@ nav:
               - Segment: references/data-types/segment.md
           - UDF: references/udf.md
           - Torch: references/torch.md
-          - Functions: references/func.md
+          - Functions:
+              - Overview: references/func.md
+              - Aggregate: references/functions/aggregate.md
+              - Array: references/functions/array.md
+              - Conditional: references/functions/conditional.md
+              - Numeric: references/functions/numeric.md
+              - Path: references/functions/path.md
+              - Random: references/functions/random.md
+              - String: references/functions/string.md
+              - Window: references/functions/window.md
           - Toolkit: references/toolkit.md
       - 📖 CLI Reference:
           - Overview: commands/index.md
@@ -177,7 +186,7 @@ plugins:
             - https://numpy.org/doc/stable/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
             - https://arrow.apache.org/docs/objects.inv
-            # - https://docs.sqlalchemy.org/objects.inv  # SSL certificate issue
+            - https://docs.sqlalchemy.org/objects.inv  # SSL certificate issue
             - https://docs.pydantic.dev/latest/objects.inv
 
 watch:

{datachain-0.30.2 → datachain-0.30.3}/pyproject.toml

@@ -49,7 +49,7 @@ dependencies = [
   "Pillow>=10.0.0,<12",
   "msgpack>=1.0.4,<2",
   "psutil",
-  "huggingface_hub<0.34.0",
+  "huggingface_hub",
   "iterative-telemetry>=0.0.10",
   "platformdirs",
   "dvc-studio-client>=0.21,<1",

{datachain-0.30.2 → datachain-0.30.3}/src/datachain/catalog/catalog.py

@@ -680,8 +680,9 @@ class Catalog:
                 ds_namespace, ds_project, ds_name = parse_dataset_name(ds_name)
                 assert ds_namespace
                 assert ds_project
-                project = self.metastore.get_project(ds_project, ds_namespace)
-                dataset = self.get_dataset(ds_name, project)
+                dataset = self.get_dataset(
+                    ds_name, namespace_name=ds_namespace, project_name=ds_project
+                )
                 if not ds_version:
                     ds_version = dataset.latest_version
                 dataset_sources = self.warehouse.get_dataset_sources(
@@ -807,7 +808,11 @@ class Catalog:
             )
         default_version = DEFAULT_DATASET_VERSION
         try:
-            dataset = self.get_dataset(name, project)
+            dataset = self.get_dataset(
+                name,
+                namespace_name=project.namespace.name if project else None,
+                project_name=project.name if project else None,
+            )
             default_version = dataset.next_version_patch
             if update_version == "major":
                 default_version = dataset.next_version_major
@@ -1016,7 +1021,11 @@ class Catalog:
             dc.save(name)
         except Exception as e:  # noqa: BLE001
             try:
-                ds = self.get_dataset(name, project)
+                ds = self.get_dataset(
+                    name,
+                    namespace_name=project.namespace.name,
+                    project_name=project.name,
+                )
                 self.metastore.update_dataset_status(
                     ds,
                     DatasetStatus.FAILED,
@@ -1033,7 +1042,11 @@ class Catalog:
             except DatasetNotFoundError:
                 raise e from None
 
-        ds = self.get_dataset(name, project)
+        ds = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name,
+            project_name=project.name,
+        )
 
         self.update_dataset_version_with_warehouse_info(
             ds,
@@ -1041,7 +1054,11 @@ class Catalog:
             sources="\n".join(sources),
         )
 
-        return self.get_dataset(name, project)
+        return self.get_dataset(
+            name,
+            namespace_name=project.namespace.name,
+            project_name=project.name,
+        )
 
     def get_full_dataset_name(
         self,
@@ -1077,22 +1094,23 @@ class Catalog:
         return namespace_name, project_name, name
 
     def get_dataset(
-        self, name: str, project: Optional[Project] = None
+        self,
+        name: str,
+        namespace_name: Optional[str] = None,
+        project_name: Optional[str] = None,
     ) -> DatasetRecord:
         from datachain.lib.listing import is_listing_dataset
 
-        project = project or self.metastore.default_project
+        namespace_name = namespace_name or self.metastore.default_namespace_name
+        project_name = project_name or self.metastore.default_project_name
 
         if is_listing_dataset(name):
-            project = self.metastore.listing_project
+            namespace_name = self.metastore.system_namespace_name
+            project_name = self.metastore.listing_project_name
 
-        try:
-            return self.metastore.get_dataset(name, project.id if project else None)
-        except DatasetNotFoundError:
-            raise DatasetNotFoundError(
-                f"Dataset {name} not found in namespace {project.namespace.name}"
-                f" and project {project.name}"
-            ) from None
+        return self.metastore.get_dataset(
+            name, namespace_name=namespace_name, project_name=project_name
+        )
 
     def get_dataset_with_remote_fallback(
         self,
@@ -1113,8 +1131,11 @@ class Catalog:
 
         if self.metastore.is_local_dataset(namespace_name) or not update:
             try:
-                project = self.metastore.get_project(project_name, namespace_name)
-                ds = self.get_dataset(name, project)
+                ds = self.get_dataset(
+                    name,
+                    namespace_name=namespace_name,
+                    project_name=project_name,
+                )
                 if not version or ds.has_version(version):
                     return ds
             except (NamespaceNotFoundError, ProjectNotFoundError, DatasetNotFoundError):
@@ -1139,7 +1160,9 @@ class Catalog:
                 local_ds_version=version,
             )
             return self.get_dataset(
-                name, self.metastore.get_project(project_name, namespace_name)
+                name,
+                namespace_name=namespace_name,
+                project_name=project_name,
             )
 
         return self.get_remote_dataset(namespace_name, project_name, name)
@@ -1148,7 +1171,11 @@ class Catalog:
         """Returns dataset that contains version with specific uuid"""
         for dataset in self.ls_datasets():
             if dataset.has_version_with_uuid(uuid):
-                return self.get_dataset(dataset.name, dataset.project)
+                return self.get_dataset(
+                    dataset.name,
+                    namespace_name=dataset.project.namespace.name,
+                    project_name=dataset.project.name,
+                )
         raise DatasetNotFoundError(f"Dataset with version uuid {uuid} not found.")
 
     def get_remote_dataset(
@@ -1171,9 +1198,18 @@ class Catalog:
         return DatasetRecord.from_dict(dataset_info)
 
     def get_dataset_dependencies(
-        self, name: str, version: str, project: Optional[Project] = None, indirect=False
+        self,
+        name: str,
+        version: str,
+        namespace_name: Optional[str] = None,
+        project_name: Optional[str] = None,
+        indirect=False,
     ) -> list[Optional[DatasetDependency]]:
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=namespace_name,
+            project_name=project_name,
+        )
 
         direct_dependencies = self.metastore.get_direct_dataset_dependencies(
             dataset, version
@@ -1187,10 +1223,13 @@ class Catalog:
                 # dependency has been removed
                 continue
             if d.is_dataset:
-                project = self.metastore.get_project(d.project, d.namespace)
                 # only datasets can have dependencies
                 d.dependencies = self.get_dataset_dependencies(
-                    d.name, d.version, project, indirect=indirect
+                    d.name,
+                    d.version,
+                    namespace_name=d.namespace,
+                    project_name=d.project,
+                    indirect=indirect,
                 )
 
         return direct_dependencies
@@ -1340,7 +1379,11 @@ class Catalog:
         project: Optional[Project] = None,
         client_config=None,
     ) -> list[str]:
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
 
         return self.warehouse.export_dataset_table(
             bucket_uri, dataset, version, client_config
@@ -1349,7 +1392,11 @@ class Catalog:
     def dataset_table_export_file_names(
         self, name: str, version: str, project: Optional[Project] = None
     ) -> list[str]:
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
         return self.warehouse.dataset_table_export_file_names(dataset, version)
 
     def remove_dataset(
@@ -1359,7 +1406,11 @@ class Catalog:
         version: Optional[str] = None,
         force: Optional[bool] = False,
     ):
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
         if not version and not force:
             raise ValueError(f"Missing dataset version from input for dataset {name}")
         if version and not dataset.has_version(version):
@@ -1395,7 +1446,11 @@ class Catalog:
         if attrs is not None:
             update_data["attrs"] = attrs  # type: ignore[assignment]
 
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
         return self.update_dataset(dataset, **update_data)
 
     def ls(
@@ -1549,7 +1604,9 @@ class Catalog:
         )
 
         try:
-            local_dataset = self.get_dataset(local_ds_name, project=project)
+            local_dataset = self.get_dataset(
+                local_ds_name, namespace_name=namespace.name, project_name=project.name
+            )
             if local_dataset and local_dataset.has_version(local_ds_version):
                 raise DataChainError(
                     f"Local dataset {local_ds_uri} already exists with different uuid,"

{datachain-0.30.2 → datachain-0.30.3}/src/datachain/cli/commands/datasets.py

@@ -107,8 +107,9 @@ def list_datasets_local(catalog: "Catalog", name: Optional[str] = None):
 def list_datasets_local_versions(catalog: "Catalog", name: str):
     namespace_name, project_name, name = catalog.get_full_dataset_name(name)
 
-    project = catalog.metastore.get_project(project_name, namespace_name)
-    ds = catalog.get_dataset(name, project)
+    ds = catalog.get_dataset(
+        name, namespace_name=namespace_name, project_name=project_name
+    )
     for v in ds.versions:
         yield (name, v.version)
 

{datachain-0.30.2 → datachain-0.30.3}/src/datachain/data_storage/metastore.py

@@ -301,7 +301,13 @@ class AbstractMetastore(ABC, Serializable):
         """
 
     @abstractmethod
-    def get_dataset(self, name: str, project_id: Optional[int] = None) -> DatasetRecord:
+    def get_dataset(
+        self,
+        name: str,  # normal, not full dataset name
+        namespace_name: Optional[str] = None,
+        project_name: Optional[str] = None,
+        conn=None,
+    ) -> DatasetRecord:
         """Gets a single dataset by name."""
 
     @abstractmethod
@@ -912,11 +918,14 @@ class AbstractDBMetastore(AbstractMetastore):
         **kwargs,  # TODO registered = True / False
     ) -> DatasetRecord:
         """Creates new dataset."""
-        project_id = project_id or self.default_project.id
+        if not project_id:
+            project = self.default_project
+        else:
+            project = self.get_project_by_id(project_id)
 
         query = self._datasets_insert().values(
             name=name,
-            project_id=project_id,
+            project_id=project.id,
             status=status,
             feature_schema=json.dumps(feature_schema or {}),
             created_at=datetime.now(timezone.utc),
@@ -935,7 +944,9 @@ class AbstractDBMetastore(AbstractMetastore):
             query = query.on_conflict_do_nothing(index_elements=["project_id", "name"])
         self.db.execute(query)
 
-        return self.get_dataset(name, project_id)
+        return self.get_dataset(
+            name, namespace_name=project.namespace.name, project_name=project.name
+        )
 
     def create_dataset_version(  # noqa: PLR0913
         self,
@@ -992,7 +1003,12 @@ class AbstractDBMetastore(AbstractMetastore):
             )
         self.db.execute(query, conn=conn)
 
-        return self.get_dataset(dataset.name, dataset.project.id, conn=conn)
+        return self.get_dataset(
+            dataset.name,
+            namespace_name=dataset.project.namespace.name,
+            project_name=dataset.project.name,
+            conn=conn,
+        )
 
     def remove_dataset(self, dataset: DatasetRecord) -> None:
         """Removes dataset."""
@@ -1216,21 +1232,30 @@ class AbstractDBMetastore(AbstractMetastore):
     def get_dataset(
         self,
         name: str,  # normal, not full dataset name
-        project_id: Optional[int] = None,
+        namespace_name: Optional[str] = None,
+        project_name: Optional[str] = None,
         conn=None,
     ) -> DatasetRecord:
         """
         Gets a single dataset in project by dataset name.
         """
-        project_id = project_id or self.default_project.id
+        namespace_name = namespace_name or self.default_namespace_name
+        project_name = project_name or self.default_project_name
 
         d = self._datasets
+        n = self._namespaces
+        p = self._projects
         query = self._base_dataset_query()
-        query = query.where(d.c.name == name, d.c.project_id == project_id)  # type: ignore [attr-defined]
+        query = query.where(
+            d.c.name == name,
+            n.c.name == namespace_name,
+            p.c.name == project_name,
+        )  # type: ignore [attr-defined]
         ds = self._parse_dataset(self.db.execute(query, conn=conn))
         if not ds:
             raise DatasetNotFoundError(
-                f"Dataset {name} not found in project with id {project_id}"
+                f"Dataset {name} not found in namespace {namespace_name}"
+                f" and project {project_name}"
             )
 
         return ds