datachain 0.30.2__tar.gz → 0.30.4__tar.gz

{datachain-0.30.2 → datachain-0.30.4}/.github/workflows/benchmarks.yml

@@ -18,7 +18,7 @@ jobs:
   run:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Set up Python 3.13
         uses: actions/setup-python@v5
         with:

{datachain-0.30.2 → datachain-0.30.4}/.github/workflows/release.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
 

{datachain-0.30.2 → datachain-0.30.4}/.github/workflows/tests-studio.yml

@@ -62,7 +62,7 @@ jobs:
           echo "Studio branch: $STUDIO_BRANCH"
 
       - name: Check out Studio
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           repository: iterative/studio
@@ -70,7 +70,7 @@ jobs:
           token: ${{ secrets.ITERATIVE_STUDIO_READ_ACCESS_TOKEN }}
 
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           path: './backend/datachain'
           fetch-depth: 0

{datachain-0.30.2 → datachain-0.30.4}/.github/workflows/tests.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
@@ -73,7 +73,7 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
@@ -175,7 +175,7 @@ jobs:
           - {os: ubuntu-latest-4-cores, pyv: "3.13", group: multimodal}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
 
@@ -194,6 +194,22 @@ jobs:
       - name: Install nox
         run: uv pip install nox --system
 
+      - name: Install FFmpeg on Windows
+        if: runner.os == 'Windows'
+        run: choco install ffmpeg
+
+      - name: Install FFmpeg on macOS
+        if: runner.os == 'macOS'
+        run: |
+          brew install ffmpeg
+          echo 'DYLD_FALLBACK_LIBRARY_PATH=/opt/homebrew/lib' >> "$GITHUB_ENV"
+
+      - name: Install FFmpeg on Ubuntu
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg
+
       - name: Set hf token
         if: matrix.group == 'llm_and_nlp'
         run: echo 'HF_TOKEN=${{ secrets.HF_TOKEN }}' >> "$GITHUB_ENV"

{datachain-0.30.2 → datachain-0.30.4}/.github/workflows/update-template.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Update template
         uses: iterative/py-template@main

{datachain-0.30.2 → datachain-0.30.4}/.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: trailing-whitespace
         exclude: '^LICENSES/'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.12.8'
+    rev: 'v0.12.10'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{datachain-0.30.2 → datachain-0.30.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.30.2
+Version: 0.30.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -45,7 +45,7 @@ Requires-Dist: datamodel-code-generator>=0.25
 Requires-Dist: Pillow<12,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
-Requires-Dist: huggingface_hub<0.34.0
+Requires-Dist: huggingface_hub
 Requires-Dist: iterative-telemetry>=0.0.10
 Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21

datachain-0.30.4/docs/references/func.md

@@ -0,0 +1,38 @@
+# Functions
+
+Use built-in functions for data manipulation and analysis to operate on the underlying database storing the chain data. These functions are useful for operations like [`DataChain.filter`](datachain.md#datachain.lib.dc.DataChain.filter) and [`DataChain.mutate`](datachain.md#datachain.lib.dc.DataChain.mutate).
+
+Functions are organized by category and accessed through their respective modules. For example, string functions are accessed via `func.string.length()`, array functions via `func.array.contains()`, etc.
+
+!!! note "Global Function Access"
+    Only a subset of functions are available directly from `datachain.func` (e.g., `func.length`). Most functions should be accessed through their specific module namespace (e.g., `func.string.length`) to avoid naming conflicts.
+
+## Function Categories
+
+DataChain provides several categories of functions for different types of operations:
+
+- **[Aggregate Functions](functions/aggregate.md)** - Functions for aggregating data like `sum`, `count`, `avg`, etc.
+- **[Array Functions](functions/array.md)** - Functions for working with arrays and lists
+- **[Conditional Functions](functions/conditional.md)** - Functions for conditional logic like `ifelse`, `case`, etc.
+- **[Numeric Functions](functions/numeric.md)** - Functions for numeric operations and computations
+- **[Path Functions](functions/path.md)** - Functions for working with file paths
+- **[Random Functions](functions/random.md)** - Functions for generating random values
+- **[String Functions](functions/string.md)** - Functions for string manipulation and processing
+- **[Window Functions](functions/window.md)** - Functions for window operations
+
+## Usage
+
+```python
+from datachain.func import aggregate, array, conditional, numeric, path, random, string, window
+
+# Access functions through their module namespaces
+dc.mutate(
+    text_length=string.length("text_column"),
+    contains_item=array.contains("array_column", "value"),
+    file_extension=path.file_ext("file_path")
+)
+
+# Some commonly used functions are also available directly
+from datachain.func import sum, count, length, ifelse
+dc.mutate(total=sum("amount"))
+```

datachain-0.30.4/docs/references/functions/aggregate.md

@@ -0,0 +1,5 @@
+# Aggregate Functions
+
+Aggregate functions perform calculations on sets of values and return a single result.
+
+::: datachain.func.aggregate

datachain-0.30.4/docs/references/functions/array.md

@@ -0,0 +1,5 @@
+# Array Functions
+
+Functions for working with arrays and lists, including operations like distance calculations and array manipulation.
+
+::: datachain.func.array

datachain-0.30.4/docs/references/functions/conditional.md

@@ -0,0 +1,5 @@
+# Conditional Functions
+
+Functions for conditional logic and control flow in data processing.
+
+::: datachain.func.conditional

datachain-0.30.4/docs/references/functions/numeric.md

@@ -0,0 +1,5 @@
+# Numeric Functions
+
+Functions for numeric operations, bit manipulation, and mathematical computations.
+
+::: datachain.func.numeric

datachain-0.30.4/docs/references/functions/path.md

@@ -0,0 +1,5 @@
+# Path Functions
+
+Functions for working with file paths and extracting path components.
+
+::: datachain.func.path

datachain-0.30.4/docs/references/functions/random.md

@@ -0,0 +1,5 @@
+# Random Functions
+
+Functions for generating random values and sampling.
+
+::: datachain.func.random

datachain-0.30.4/docs/references/functions/string.md

@@ -0,0 +1,22 @@
+# String Functions
+
+Functions for string manipulation, text processing, and string analysis.
+
+## Usage
+
+String functions are available under the `func.string` namespace to avoid name collisions with other functions:
+
+```python
+from datachain.func import string
+
+# Use string functions with the string namespace
+dc.mutate(
+    str_len=string.length("text_column"),
+    parts=string.split("text_column", ","),
+    cleaned=string.replace("text_column", "old", "new"),
+    regex_cleaned=string.regexp_replace("text_column", r"\d+", "X"),
+    distance=string.byte_hamming_distance("col1", "col2")
+)
+```
+
+::: datachain.func.string

datachain-0.30.4/docs/references/functions/window.md

@@ -0,0 +1,5 @@
+# Window Functions
+
+Functions for window operations and analytical processing.
+
+::: datachain.func.window

{datachain-0.30.2 → datachain-0.30.4}/examples/multimodal/audio-to-text.py

@@ -35,7 +35,10 @@ def process(fragment: AudioFragment, pipeline: Pipeline) -> str:
         audio_array = audio_array.mean(axis=1)
 
     # Pass the numpy array with exact sampling rate from fragment
-    result = pipeline({"raw": audio_array, "sampling_rate": sample_rate})
+    result = pipeline(
+        {"raw": audio_array, "sampling_rate": sample_rate},
+        generate_kwargs={"language": "en"},
+    )
     return str(result["text"])
 
 

{datachain-0.30.2 → datachain-0.30.4}/mkdocs.yml

@@ -82,7 +82,16 @@ nav:
               - Segment: references/data-types/segment.md
           - UDF: references/udf.md
           - Torch: references/torch.md
-          - Functions: references/func.md
+          - Functions:
+              - Overview: references/func.md
+              - Aggregate: references/functions/aggregate.md
+              - Array: references/functions/array.md
+              - Conditional: references/functions/conditional.md
+              - Numeric: references/functions/numeric.md
+              - Path: references/functions/path.md
+              - Random: references/functions/random.md
+              - String: references/functions/string.md
+              - Window: references/functions/window.md
           - Toolkit: references/toolkit.md
       - 📖 CLI Reference:
           - Overview: commands/index.md
@@ -177,7 +186,7 @@ plugins:
             - https://numpy.org/doc/stable/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
             - https://arrow.apache.org/docs/objects.inv
-            # - https://docs.sqlalchemy.org/objects.inv  # SSL certificate issue
+            - https://docs.sqlalchemy.org/objects.inv  # SSL certificate issue
             - https://docs.pydantic.dev/latest/objects.inv
 
 watch:

{datachain-0.30.2 → datachain-0.30.4}/pyproject.toml

@@ -49,7 +49,7 @@ dependencies = [
   "Pillow>=10.0.0,<12",
   "msgpack>=1.0.4,<2",
   "psutil",
-  "huggingface_hub<0.34.0",
+  "huggingface_hub",
   "iterative-telemetry>=0.0.10",
   "platformdirs",
   "dvc-studio-client>=0.21,<1",

{datachain-0.30.2 → datachain-0.30.4}/src/datachain/__init__.py

@@ -6,6 +6,7 @@ from datachain.lib.dc import (
     Sys,
     datasets,
     delete_dataset,
+    is_studio,
     listings,
     move_dataset,
     read_csv,
@@ -74,6 +75,7 @@ __all__ = [
     "datasets",
     "delete_dataset",
     "is_chain_type",
+    "is_studio",
     "listings",
     "metrics",
     "move_dataset",

{datachain-0.30.2 → datachain-0.30.4}/src/datachain/catalog/__init__.py

@@ -3,6 +3,7 @@ from .catalog import (
     QUERY_SCRIPT_CANCELED_EXIT_CODE,
     QUERY_SCRIPT_INVALID_LAST_STATEMENT_EXIT_CODE,
     Catalog,
+    is_namespace_local,
 )
 from .loader import get_catalog
 
@@ -12,4 +13,5 @@ __all__ = [
     "QUERY_SCRIPT_INVALID_LAST_STATEMENT_EXIT_CODE",
     "Catalog",
     "get_catalog",
+    "is_namespace_local",
 ]

{datachain-0.30.2 → datachain-0.30.4}/src/datachain/catalog/catalog.py

@@ -113,6 +113,11 @@ else:
     SIGINT = signal.SIGINT
 
 
+def is_namespace_local(namespace_name) -> bool:
+    """Checks if namespace is from local environment, i.e. is `local`"""
+    return namespace_name == "local"
+
+
 def shutdown_process(
     proc: subprocess.Popen,
     interrupt_timeout: Optional[int] = None,
@@ -680,8 +685,9 @@ class Catalog:
                 ds_namespace, ds_project, ds_name = parse_dataset_name(ds_name)
                 assert ds_namespace
                 assert ds_project
-                project = self.metastore.get_project(ds_project, ds_namespace)
-                dataset = self.get_dataset(ds_name, project)
+                dataset = self.get_dataset(
+                    ds_name, namespace_name=ds_namespace, project_name=ds_project
+                )
                 if not ds_version:
                     ds_version = dataset.latest_version
                 dataset_sources = self.warehouse.get_dataset_sources(
@@ -807,7 +813,11 @@ class Catalog:
             )
         default_version = DEFAULT_DATASET_VERSION
         try:
-            dataset = self.get_dataset(name, project)
+            dataset = self.get_dataset(
+                name,
+                namespace_name=project.namespace.name if project else None,
+                project_name=project.name if project else None,
+            )
             default_version = dataset.next_version_patch
             if update_version == "major":
                 default_version = dataset.next_version_major
@@ -1016,7 +1026,11 @@ class Catalog:
             dc.save(name)
         except Exception as e:  # noqa: BLE001
             try:
-                ds = self.get_dataset(name, project)
+                ds = self.get_dataset(
+                    name,
+                    namespace_name=project.namespace.name,
+                    project_name=project.name,
+                )
                 self.metastore.update_dataset_status(
                     ds,
                     DatasetStatus.FAILED,
@@ -1033,7 +1047,11 @@ class Catalog:
             except DatasetNotFoundError:
                 raise e from None
 
-        ds = self.get_dataset(name, project)
+        ds = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name,
+            project_name=project.name,
+        )
 
         self.update_dataset_version_with_warehouse_info(
             ds,
@@ -1041,7 +1059,11 @@ class Catalog:
             sources="\n".join(sources),
         )
 
-        return self.get_dataset(name, project)
+        return self.get_dataset(
+            name,
+            namespace_name=project.namespace.name,
+            project_name=project.name,
+        )
 
     def get_full_dataset_name(
         self,
@@ -1077,22 +1099,23 @@ class Catalog:
         return namespace_name, project_name, name
 
     def get_dataset(
-        self, name: str, project: Optional[Project] = None
+        self,
+        name: str,
+        namespace_name: Optional[str] = None,
+        project_name: Optional[str] = None,
     ) -> DatasetRecord:
         from datachain.lib.listing import is_listing_dataset
 
-        project = project or self.metastore.default_project
+        namespace_name = namespace_name or self.metastore.default_namespace_name
+        project_name = project_name or self.metastore.default_project_name
 
         if is_listing_dataset(name):
-            project = self.metastore.listing_project
+            namespace_name = self.metastore.system_namespace_name
+            project_name = self.metastore.listing_project_name
 
-        try:
-            return self.metastore.get_dataset(name, project.id if project else None)
-        except DatasetNotFoundError:
-            raise DatasetNotFoundError(
-                f"Dataset {name} not found in namespace {project.namespace.name}"
-                f" and project {project.name}"
-            ) from None
+        return self.metastore.get_dataset(
+            name, namespace_name=namespace_name, project_name=project_name
+        )
 
     def get_dataset_with_remote_fallback(
         self,
@@ -1103,6 +1126,8 @@ class Catalog:
         pull_dataset: bool = False,
         update: bool = False,
     ) -> DatasetRecord:
+        from datachain.lib.dc.utils import is_studio
+
         # Intentionally ignore update flag is version is provided. Here only exact
         # version can be provided and update then doesn't make sense.
         # It corresponds to a query like this for example:
@@ -1111,16 +1136,24 @@ class Catalog:
         if version:
             update = False
 
-        if self.metastore.is_local_dataset(namespace_name) or not update:
+        # we don't do Studio fallback is script is already ran in Studio, or if we try
+        # to fetch dataset with local namespace as that one cannot
+        # exist in Studio in the first place
+        no_fallback = is_studio() or is_namespace_local(namespace_name)
+
+        if no_fallback or not update:
             try:
-                project = self.metastore.get_project(project_name, namespace_name)
-                ds = self.get_dataset(name, project)
+                ds = self.get_dataset(
+                    name,
+                    namespace_name=namespace_name,
+                    project_name=project_name,
+                )
                 if not version or ds.has_version(version):
                     return ds
             except (NamespaceNotFoundError, ProjectNotFoundError, DatasetNotFoundError):
                 pass
 
-        if self.metastore.is_local_dataset(namespace_name):
+        if no_fallback:
             raise DatasetNotFoundError(
                 f"Dataset {name}"
                 + (f" version {version} " if version else " ")
@@ -1139,7 +1172,9 @@ class Catalog:
                 local_ds_version=version,
             )
             return self.get_dataset(
-                name, self.metastore.get_project(project_name, namespace_name)
+                name,
+                namespace_name=namespace_name,
+                project_name=project_name,
             )
 
         return self.get_remote_dataset(namespace_name, project_name, name)
@@ -1148,7 +1183,11 @@ class Catalog:
         """Returns dataset that contains version with specific uuid"""
         for dataset in self.ls_datasets():
             if dataset.has_version_with_uuid(uuid):
-                return self.get_dataset(dataset.name, dataset.project)
+                return self.get_dataset(
+                    dataset.name,
+                    namespace_name=dataset.project.namespace.name,
+                    project_name=dataset.project.name,
+                )
         raise DatasetNotFoundError(f"Dataset with version uuid {uuid} not found.")
 
     def get_remote_dataset(
@@ -1171,9 +1210,18 @@ class Catalog:
         return DatasetRecord.from_dict(dataset_info)
 
     def get_dataset_dependencies(
-        self, name: str, version: str, project: Optional[Project] = None, indirect=False
+        self,
+        name: str,
+        version: str,
+        namespace_name: Optional[str] = None,
+        project_name: Optional[str] = None,
+        indirect=False,
     ) -> list[Optional[DatasetDependency]]:
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=namespace_name,
+            project_name=project_name,
+        )
 
         direct_dependencies = self.metastore.get_direct_dataset_dependencies(
             dataset, version
@@ -1187,10 +1235,13 @@ class Catalog:
                 # dependency has been removed
                 continue
             if d.is_dataset:
-                project = self.metastore.get_project(d.project, d.namespace)
                 # only datasets can have dependencies
                 d.dependencies = self.get_dataset_dependencies(
-                    d.name, d.version, project, indirect=indirect
+                    d.name,
+                    d.version,
+                    namespace_name=d.namespace,
+                    project_name=d.project,
+                    indirect=indirect,
                 )
 
         return direct_dependencies
@@ -1340,7 +1391,11 @@ class Catalog:
         project: Optional[Project] = None,
         client_config=None,
     ) -> list[str]:
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
 
         return self.warehouse.export_dataset_table(
             bucket_uri, dataset, version, client_config
@@ -1349,7 +1404,11 @@ class Catalog:
     def dataset_table_export_file_names(
         self, name: str, version: str, project: Optional[Project] = None
     ) -> list[str]:
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
         return self.warehouse.dataset_table_export_file_names(dataset, version)
 
     def remove_dataset(
@@ -1359,7 +1418,11 @@ class Catalog:
         version: Optional[str] = None,
         force: Optional[bool] = False,
     ):
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
         if not version and not force:
             raise ValueError(f"Missing dataset version from input for dataset {name}")
         if version and not dataset.has_version(version):
@@ -1395,7 +1458,11 @@ class Catalog:
         if attrs is not None:
             update_data["attrs"] = attrs  # type: ignore[assignment]
 
-        dataset = self.get_dataset(name, project)
+        dataset = self.get_dataset(
+            name,
+            namespace_name=project.namespace.name if project else None,
+            project_name=project.name if project else None,
+        )
         return self.update_dataset(dataset, **update_data)
 
     def ls(
@@ -1549,7 +1616,9 @@ class Catalog:
         )
 
         try:
-            local_dataset = self.get_dataset(local_ds_name, project=project)
+            local_dataset = self.get_dataset(
+                local_ds_name, namespace_name=namespace.name, project_name=project.name
+            )
             if local_dataset and local_dataset.has_version(local_ds_version):
                 raise DataChainError(
                     f"Local dataset {local_ds_uri} already exists with different uuid,"

{datachain-0.30.2 → datachain-0.30.4}/src/datachain/catalog/loader.py

@@ -127,7 +127,8 @@ def get_udf_distributor_class() -> Optional[type["AbstractUDFDistributor"]]:
 
 
 def get_catalog(
-    client_config: Optional[dict[str, Any]] = None, in_memory: bool = False
+    client_config: Optional[dict[str, Any]] = None,
+    in_memory: bool = False,
 ) -> "Catalog":
     """
     Function that creates Catalog instance with appropriate metastore
@@ -142,8 +143,9 @@ def get_catalog(
     """
     from datachain.catalog import Catalog
 
+    metastore = get_metastore(in_memory=in_memory)
     return Catalog(
-        metastore=get_metastore(in_memory=in_memory),
+        metastore=metastore,
         warehouse=get_warehouse(in_memory=in_memory),
         client_config=client_config,
         in_memory=in_memory,

{datachain-0.30.2 → datachain-0.30.4}/src/datachain/cli/__init__.py

@@ -6,6 +6,7 @@ from multiprocessing import freeze_support
 from typing import Optional
 
 from datachain.cli.utils import get_logging_level
+from datachain.error import DataChainError as DataChainError
 
 from .commands import (
     clear_cache,