datachain 0.23.0__tar.gz → 0.24.0__tar.gz

{datachain-0.23.0 → datachain-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.23.0
+Version: 0.24.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.23.0 → datachain-0.24.0}/src/datachain/catalog/catalog.py

@@ -49,6 +49,7 @@ from datachain.error import (
     DatasetInvalidVersionError,
     DatasetNotFoundError,
     DatasetVersionNotFoundError,
+    NamespaceNotFoundError,
     ProjectNotFoundError,
     QueryScriptCancelError,
     QueryScriptRunError,
@@ -1107,21 +1108,26 @@ class Catalog:
         namespace_name: str,
         project_name: str,
         version: Optional[str] = None,
+        pull_dataset: bool = False,
+        update: bool = False,
     ) -> DatasetRecord:
-        try:
-            project = self.metastore.get_project(project_name, namespace_name)
-            ds = self.get_dataset(name, project)
-            if version and not ds.has_version(version):
-                raise DatasetVersionNotFoundError(
-                    f"Dataset {name} does not have version {version}"
-                )
-            return ds
+        if self.metastore.is_local_dataset(namespace_name) or not update:
+            try:
+                project = self.metastore.get_project(project_name, namespace_name)
+                ds = self.get_dataset(name, project)
+                if not version or ds.has_version(version):
+                    return ds
+            except (NamespaceNotFoundError, ProjectNotFoundError, DatasetNotFoundError):
+                pass
+
+        if self.metastore.is_local_dataset(namespace_name):
+            raise DatasetNotFoundError(
+                f"Dataset {name}"
+                + (f" version {version} " if version else " ")
+                + "not found"
+            )
 
-        except (
-            ProjectNotFoundError,
-            DatasetNotFoundError,
-            DatasetVersionNotFoundError,
-        ):
+        if pull_dataset:
             print("Dataset not found in local catalog, trying to get from studio")
             remote_ds_uri = create_dataset_uri(
                 name, namespace_name, project_name, version
@@ -1136,6 +1142,8 @@ class Catalog:
                 name, self.metastore.get_project(project_name, namespace_name)
             )
 
+        return self.get_remote_dataset(namespace_name, project_name, name)
+
     def get_dataset_with_version_uuid(self, uuid: str) -> DatasetRecord:
         """Returns dataset that contains version with specific uuid"""
         for dataset in self.ls_datasets():
@@ -1152,6 +1160,10 @@ class Catalog:
 
         info_response = studio_client.dataset_info(namespace, project, name)
         if not info_response.ok:
+            if info_response.status == 404:
+                raise DatasetNotFoundError(
+                    f"Dataset {namespace}.{project}.{name} not found"
+                )
             raise DataChainError(info_response.message)
 
         dataset_info = info_response.data

{datachain-0.23.0 → datachain-0.24.0}/src/datachain/dataset.py

@@ -12,6 +12,9 @@ from typing import (
 )
 from urllib.parse import urlparse
 
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
+
 from datachain import semver
 from datachain.error import DatasetVersionNotFoundError, InvalidDatasetNameError
 from datachain.namespace import Namespace
@@ -661,13 +664,39 @@ class DatasetRecord:
             return None
         return max(versions).version
 
-    @property
-    def prev_version(self) -> Optional[str]:
-        """Returns previous version of a dataset"""
-        if len(self.versions) == 1:
+    def latest_compatible_version(self, version_spec: str) -> Optional[str]:
+        """
+        Returns the latest version that matches the given version specifier.
+
+        Supports Python version specifiers like:
+        - ">=1.0.0,<2.0.0" (compatible release range)
+        - "~=1.4.2" (compatible release clause)
+        - "==1.2.*" (prefix matching)
+        - ">1.0.0" (exclusive ordered comparison)
+        - ">=1.0.0" (inclusive ordered comparison)
+        - "!=1.3.0" (version exclusion)
+
+        Args:
+            version_spec: Version specifier string following PEP 440
+
+        Returns:
+            Latest compatible version string, or None if no compatible version found
+        """
+        spec_set = SpecifierSet(version_spec)
+
+        # Convert dataset versions to packaging.Version objects
+        # and filter compatible ones
+        compatible_versions = []
+        for v in self.versions:
+            pkg_version = Version(v.version)
+            if spec_set.contains(pkg_version):
+                compatible_versions.append(v)
+
+        if not compatible_versions:
             return None
 
-        return sorted(self.versions)[-2].version
+        # Return the latest compatible version
+        return max(compatible_versions).version
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "DatasetRecord":

{datachain-0.23.0 → datachain-0.24.0}/src/datachain/lib/dc/datasets.py

@@ -7,9 +7,6 @@ from datachain.error import (
     ProjectNotFoundError,
 )
 from datachain.lib.dataset_info import DatasetInfo
-from datachain.lib.file import (
-    File,
-)
 from datachain.lib.projects import get as get_project
 from datachain.lib.settings import Settings
 from datachain.lib.signal_schema import SignalSchema
@@ -34,7 +31,6 @@ def read_dataset(
     version: Optional[Union[str, int]] = None,
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
-    fallback_to_studio: bool = True,
     delta: Optional[bool] = False,
     delta_on: Optional[Union[str, Sequence[str]]] = (
         "file.path",
@@ -44,6 +40,7 @@ def read_dataset(
     delta_result_on: Optional[Union[str, Sequence[str]]] = None,
     delta_compare: Optional[Union[str, Sequence[str]]] = None,
     delta_retry: Optional[Union[bool, str]] = None,
+    update: bool = False,
 ) -> "DataChain":
     """Get data from a saved Dataset. It returns the chain itself.
     If dataset or version is not found locally, it will try to pull it from Studio.
@@ -55,11 +52,12 @@ def read_dataset(
             set; otherwise, default values will be applied.
         namespace : optional name of namespace in which dataset to read is created
         project : optional name of project in which dataset to read is created
-        version : dataset version
+        version : dataset version. Supports:
+            - Exact version strings: "1.2.3"
+            - Legacy integer versions: 1, 2, 3 (finds latest major version)
+            - Version specifiers (PEP 440): ">=1.0.0,<2.0.0", "~=1.4.2", "==1.2.*", etc.
         session : Session to use for the chain.
         settings : Settings to use for the chain.
-        fallback_to_studio : Try to pull dataset from Studio if not found locally.
-            Default is True.
         delta: If True, only process new or changed files instead of reprocessing
             everything. This saves time by skipping files that were already processed in
             previous versions. The optimization is working when a new version of the
@@ -79,6 +77,10 @@ def read_dataset(
               (error mode)
             - True: Reprocess records missing from the result dataset (missing mode)
             - None: No retry processing (default)
+        update: If True always checks for newer versions available on Studio, even if
+            some version of the dataset exists locally already. If False (default), it
+            will only fetch the dataset from Studio if it is not found locally.
+
 
     Example:
         ```py
@@ -92,11 +94,22 @@ def read_dataset(
         ```
 
         ```py
-        chain = dc.read_dataset("my_cats", fallback_to_studio=False)
+        chain = dc.read_dataset("my_cats", version="1.0.0")
         ```
 
         ```py
-        chain = dc.read_dataset("my_cats", version="1.0.0")
+        # Using version specifiers (PEP 440)
+        chain = dc.read_dataset("my_cats", version=">=1.0.0,<2.0.0")
+        ```
+
+        ```py
+        # Legacy integer version support (finds latest in major version)
+        chain = dc.read_dataset("my_cats", version=1)  # Latest 1.x.x version
+        ```
+
+        ```py
+        # Always check for newer versions matching a version specifier from Studio
+        chain = dc.read_dataset("my_cats", version=">=1.0.0", update=True)
         ```
 
         ```py
@@ -113,7 +126,6 @@ def read_dataset(
             version="1.0.0",
             session=session,
             settings=settings,
-            fallback_to_studio=True,
         )
         ```
     """
@@ -121,6 +133,8 @@ def read_dataset(
 
     from .datachain import DataChain
 
+    telemetry.send_event_once("class", "datachain_init", name=name, version=version)
+
     session = Session.get(session)
     catalog = session.catalog
 
@@ -131,31 +145,37 @@ def read_dataset(
     )
 
     if version is not None:
+        dataset = session.catalog.get_dataset_with_remote_fallback(
+            name, namespace_name, project_name, update=update
+        )
+
+        # Convert legacy integer versions to version specifiers
+        # For backward compatibility we still allow users to put version as integer
+        # in which case we convert it to a version specifier that finds the latest
+        # version where major part is equal to that input version.
+        # For example if user sets version=2, we convert it to ">=2.0.0,<3.0.0"
+        # which will find something like 2.4.3 (assuming 2.4.3 is the biggest among
+        # all 2.* dataset versions)
+        if isinstance(version, int):
+            version_spec = f">={version}.0.0,<{version + 1}.0.0"
+        else:
+            version_spec = str(version)
+
+        from packaging.specifiers import InvalidSpecifier, SpecifierSet
+
         try:
-            # for backward compatibility we still allow users to put version as integer
-            # in which case we are trying to find latest version where major part is
-            # equal to that input version. For example if user sets version=2, we could
-            # continue with something like 2.4.3 (assuming 2.4.3 is the biggest among
-            # all 2.* dataset versions). If dataset doesn't have any versions where
-            # major part is equal to that input, exception is thrown.
-            major = int(version)
-            try:
-                ds_project = get_project(project_name, namespace_name, session=session)
-            except ProjectNotFoundError:
-                raise DatasetNotFoundError(
-                    f"Dataset {name} not found in namespace {namespace_name} and",
-                    f" project {project_name}",
-                ) from None
-
-            dataset = session.catalog.get_dataset(name, ds_project)
-            latest_major = dataset.latest_major_version(major)
-            if not latest_major:
+            # Try to parse as version specifier
+            SpecifierSet(version_spec)
+            # If it's a valid specifier set, find the latest compatible version
+            latest_compatible = dataset.latest_compatible_version(version_spec)
+            if not latest_compatible:
                 raise DatasetVersionNotFoundError(
-                    f"Dataset {name} does not have version {version}"
+                    f"No dataset {name} version matching specifier {version_spec}"
                 )
-            version = latest_major
-        except ValueError:
-            # version is in new semver string format, continuing as normal
+            version = latest_compatible
+        except InvalidSpecifier:
+            # If not a valid specifier, treat as exact version string
+            # This handles cases like "1.2.3" which are exact versions, not specifiers
             pass
 
     if settings:
@@ -169,11 +189,8 @@ def read_dataset(
         namespace_name=namespace_name,
         version=version,  #  type: ignore[arg-type]
         session=session,
-        indexing_column_types=File._datachain_column_types,
-        fallback_to_studio=fallback_to_studio,
     )
 
-    telemetry.send_event_once("class", "datachain_init", name=name, version=version)
     signals_schema = SignalSchema({"sys": Sys})
     if query.feature_schema:
         signals_schema |= SignalSchema.deserialize(query.feature_schema)

{datachain-0.23.0 → datachain-0.24.0}/src/datachain/lib/dc/listings.py

@@ -127,12 +127,8 @@ def read_listing_dataset(
     if version is None:
         version = dataset.latest_version
 
-    query = DatasetQuery(
-        name=name,
-        session=session,
-        indexing_column_types=File._datachain_column_types,
-        fallback_to_studio=False,
-    )
+    query = DatasetQuery(name=name, session=session)
+
     if settings:
         cfg = {**settings}
         if "prefetch" not in cfg:

{datachain-0.23.0 → datachain-0.24.0}/src/datachain/lib/projects.py

@@ -54,7 +54,7 @@ def get(name: str, namespace: str, session: Optional[Session]) -> Project:
         ```py
         import datachain as dc
         from datachain.lib.projects import get as get_project
-        project  = get_project("my-project", "local")
+        project = get_project("my-project", "local")
         ```
     """
     return Session.get(session).catalog.metastore.get_project(name, namespace)

{datachain-0.23.0 → datachain-0.24.0}/src/datachain/query/dataset.py

@@ -1099,13 +1099,9 @@ class DatasetQuery:
         namespace_name: Optional[str] = None,
         catalog: Optional["Catalog"] = None,
         session: Optional[Session] = None,
-        indexing_column_types: Optional[dict[str, Any]] = None,
         in_memory: bool = False,
-        fallback_to_studio: bool = True,
         update: bool = False,
     ) -> None:
-        from datachain.remote.studio import is_token_set
-
         self.session = Session.get(session, catalog=catalog, in_memory=in_memory)
         self.catalog = catalog or self.session.catalog
         self.steps: list[Step] = []
@@ -1137,18 +1133,16 @@ class DatasetQuery:
             # not setting query step yet as listing dataset might not exist at
             # this point
             self.list_ds_name = name
-        elif fallback_to_studio and is_token_set():
+        else:
             self._set_starting_step(
                 self.catalog.get_dataset_with_remote_fallback(
                     name,
                     namespace_name=namespace_name,
                     project_name=project_name,
                     version=version,
+                    pull_dataset=True,
                 )
             )
-        else:
-            project = self.catalog.metastore.get_project(project_name, namespace_name)
-            self._set_starting_step(self.catalog.get_dataset(name, project=project))
 
     def _set_starting_step(self, ds: "DatasetRecord") -> None:
         if not self.version:

{datachain-0.23.0 → datachain-0.24.0}/src/datachain/remote/studio.py

@@ -78,10 +78,11 @@ def _parse_dates(obj: dict, date_fields: list[str]):
 
 
 class Response(Generic[T]):
-    def __init__(self, data: T, ok: bool, message: str) -> None:
+    def __init__(self, data: T, ok: bool, message: str, status: int) -> None:
         self.data = data
         self.ok = ok
         self.message = message
+        self.status = status
 
     def __repr__(self):
         return (
@@ -186,7 +187,7 @@ class StudioClient:
             message = "Indexing in progress"
         else:
             message = content.get("message", "")
-        return Response(response_data, ok, message)
+        return Response(response_data, ok, message, response.status_code)
 
     @retry_with_backoff(retries=3, errors=(HTTPError, Timeout))
     def _send_request(
@@ -236,7 +237,7 @@ class StudioClient:
         else:
             message = ""
 
-        return Response(data, ok, message)
+        return Response(data, ok, message, response.status_code)
 
     @staticmethod
     def _unpacker_hook(code, data):

{datachain-0.23.0 → datachain-0.24.0}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.23.0
+Version: 0.24.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.23.0 → datachain-0.24.0}/src/datachain.egg-info/SOURCES.txt

@@ -298,6 +298,8 @@ tests/func/test_pull.py
 tests/func/test_pytorch.py
 tests/func/test_query.py
 tests/func/test_read_database.py
+tests/func/test_read_dataset_remote.py
+tests/func/test_read_dataset_version_specifiers.py
 tests/func/test_retry.py
 tests/func/test_session.py
 tests/func/test_toolkit.py

{datachain-0.23.0 → datachain-0.24.0}/tests/conftest.py

@@ -904,3 +904,172 @@ def run_datachain_worker(datachain_job_id):
                 worker.wait(timeout=30)  # seconds
             except subprocess.TimeoutExpired:
                 os.kill(worker.pid, signal.SIGKILL)
+
+
+# Common constants for remote dataset testing
+REMOTE_DATASET_UUID = "20f5a2f1-fc9a-4e36-8b91-5a530f289451"
+REMOTE_DATASET_UUID_V2 = "30f5a2f1-fc9a-4e36-8b91-5a530f289452"
+REMOTE_NAMESPACE_UUID = "efbc3e84-3eb6-4be1-bec1-704b939e1fe4"
+REMOTE_PROJECT_UUID = "0ed3a6c6-f2f7-45aa-869b-39219c86a9d4"
+
+REMOTE_NAMESPACE_NAME = "dev"
+REMOTE_PROJECT_NAME = "animals"
+
+
+@pytest.fixture
+def remote_dataset_schema():
+    """Common schema for remote datasets."""
+    return {
+        "id": {"type": "UInt64"},
+        "sys__rand": {"type": "UInt64"},
+        "file__path": {"type": "String"},
+        "file__etag": {"type": "String"},
+        "file__version": {"type": "String"},
+        "file__is_latest": {"type": "Boolean"},
+        "file__last_modified": {"type": "DateTime"},
+        "file__size": {"type": "Int64"},
+        "file__location": {"type": "String"},
+        "file__source": {"type": "String"},
+        "version": {"type": "String"},
+    }
+
+
+@pytest.fixture
+def remote_file_feature_schema():
+    """Common File feature schema for remote datasets."""
+    return {
+        "file": "File@v1",
+        "version": "str",
+        "_custom_types": {
+            "File@v1": {
+                "schema_version": 2,
+                "name": "File@v1",
+                "fields": {
+                    "source": "str",
+                    "path": "str",
+                    "size": "int",
+                    "version": "str",
+                    "etag": "str",
+                    "is_latest": "bool",
+                    "last_modified": "datetime",
+                    "location": "Union[dict, list[dict], NoneType]",
+                },
+                "bases": [
+                    ["File", "datachain.lib.file", "File@v1"],
+                    ["DataModel", "datachain.lib.data_model", "DataModel@v1"],
+                    ["BaseModel", "pydantic.main", None],
+                    ["object", "builtins", None],
+                ],
+                "hidden_fields": [
+                    "source",
+                    "version",
+                    "etag",
+                    "is_latest",
+                    "last_modified",
+                    "location",
+                ],
+            }
+        },
+    }
+
+
+@pytest.fixture
+def remote_namespace():
+    """Remote namespace fixture for Studio API mocking."""
+    return {
+        "id": 1,
+        "uuid": REMOTE_NAMESPACE_UUID,
+        "name": REMOTE_NAMESPACE_NAME,
+        "descr": "Dev namespace",
+        "created_at": "2024-02-23T10:42:31.842944+00:00",
+    }
+
+
+@pytest.fixture
+def remote_project(remote_namespace):
+    """Remote project fixture for Studio API mocking."""
+    return {
+        "id": 1,
+        "uuid": REMOTE_PROJECT_UUID,
+        "name": REMOTE_PROJECT_NAME,
+        "descr": "Animals project",
+        "created_at": "2024-02-23T10:42:31.842944+00:00",
+        "namespace": remote_namespace,
+    }
+
+
+@pytest.fixture
+def compressed_parquet_data():
+    """
+    Factory fixture that creates lz4-compressed parquet for datasets.
+    Returns a function that can be called with different data.
+    """
+    import io
+    from datetime import datetime
+
+    import lz4.frame
+    import pandas as pd
+
+    def create_compressed_parquet(data, src_uri=None):
+        def _adapt_row(row):
+            """
+            Adjusting row values to match remote response
+            """
+            adapted = {}
+            for k, v in row.items():
+                if isinstance(v, datetime):
+                    adapted[k] = v.timestamp()
+                elif v is None:
+                    adapted[k] = ""
+                else:
+                    adapted[k] = v
+
+            adapted["sys__id"] = 1
+            adapted["sys__rand"] = 1
+            adapted["file__location"] = ""
+            adapted["file__source"] = src_uri or ""
+            adapted["file__version"] = ""
+            return adapted
+
+        adapted_data = [_adapt_row(row) for row in data]
+        df = pd.DataFrame.from_records(adapted_data)
+        buffer = io.BytesIO()
+        df.to_parquet(buffer, engine="auto")
+
+        return lz4.frame.compress(buffer.getvalue())
+
+    return create_compressed_parquet
+
+
+@pytest.fixture
+def dog_entries():
+    """Factory function to create version-specific dog entries."""
+    from tests.data import ENTRIES
+
+    def _create_dog_entries(version="1.0.0"):
+        return [
+            {
+                "file__path": e.path,
+                "file__etag": e.etag,
+                "file__version": e.version,
+                "file__is_latest": e.is_latest,
+                "file__last_modified": e.last_modified,
+                "file__size": e.size,
+                "version": version,
+            }
+            for e in ENTRIES
+            if e.name.startswith("dog")
+        ]
+
+    return _create_dog_entries
+
+
+@pytest.fixture
+def mock_parquet_data(compressed_parquet_data, dog_entries, version="1.0.0"):
+    return compressed_parquet_data(dog_entries(version))
+
+
+@pytest.fixture
+def mock_parquet_data_cloud(compressed_parquet_data, dog_entries, cloud_test_catalog):
+    src_uri = cloud_test_catalog.src_uri
+    return compressed_parquet_data(dog_entries("1.0.0"), src_uri)

{datachain-0.23.0 → datachain-0.24.0}/tests/func/test_dataset_query.py

@@ -7,9 +7,7 @@ import pytest
 import sqlalchemy
 
 from datachain.dataset import DatasetDependencyType, DatasetStatus
-from datachain.error import (
-    DatasetVersionNotFoundError,
-)
+from datachain.error import DatasetNotFoundError
 from datachain.lib.listing import parse_listing_uri
 from datachain.query import C, DatasetQuery, Object, Stream
 from datachain.sql.functions import path as pathfunc
@@ -70,7 +68,7 @@ def test_save_multiple_versions(cloud_test_catalog, animal_dataset):
     assert DatasetQuery(name=ds_name, version="1.0.1", catalog=catalog).count() == 3
     assert DatasetQuery(name=ds_name, version="1.0.2", catalog=catalog).count() == 3
 
-    with pytest.raises(DatasetVersionNotFoundError):
+    with pytest.raises(DatasetNotFoundError):
         DatasetQuery(name=ds_name, version="4.0.0", catalog=catalog).count()
 
 

{datachain-0.23.0 → datachain-0.24.0}/tests/func/test_delta.py

@@ -6,7 +6,7 @@ from PIL import Image
 
 import datachain as dc
 from datachain import func
-from datachain.error import DatasetVersionNotFoundError
+from datachain.error import DatasetNotFoundError
 from datachain.lib.dc import C
 from datachain.lib.file import File, ImageFile
 
@@ -278,10 +278,10 @@ def test_delta_update_no_diff(test_session, tmp_dir, tmp_path):
         "images/img9.jpg",
     ]
 
-    with pytest.raises(DatasetVersionNotFoundError) as exc_info:
+    with pytest.raises(DatasetNotFoundError) as exc_info:
         dc.read_dataset(ds_name, version="1.0.1")
 
-    assert str(exc_info.value) == f"Dataset {ds_name} does not have version 1.0.1"
+    assert str(exc_info.value) == f"Dataset {ds_name} version 1.0.1 not found"
 
 
 @pytest.fixture

{datachain-0.23.0 → datachain-0.24.0}/tests/func/test_ls.py

@@ -179,9 +179,10 @@ def test_ls_partial_indexing(cloud_test_catalog, cloud_type, capsys):
 
 
 class MockResponse:
-    def __init__(self, content, ok=True):
+    def __init__(self, content, status_code, ok=True):
         self.content = content
         self.ok = ok
+        self.status_code = status_code
 
 
 def mock_post(method, url, data=None, json=None, **kwargs):
@@ -196,7 +197,8 @@ def mock_post(method, url, data=None, json=None, **kwargs):
         for d in REMOTE_DATA[path]
     ]
     return MockResponse(
-        content=msgpack.packb({"data": data}, default=_pack_extended_types)
+        content=msgpack.packb({"data": data}, default=_pack_extended_types),
+        status_code=200,
     )