datachain 0.16.5__py3-none-any.whl → 0.17.0__py3-none-any.whl

datachain/dataset.py

@@ -12,6 +12,7 @@ from typing import (
 )
 from urllib.parse import urlparse
 
+from datachain import semver
 from datachain.error import DatasetVersionNotFoundError
 from datachain.sql.types import NAME_TYPES_MAPPING, SQLType
 
@@ -25,6 +26,8 @@ DATASET_PREFIX = "ds://"
 QUERY_DATASET_PREFIX = "ds_query_"
 LISTING_PREFIX = "lst__"
 
+DEFAULT_DATASET_VERSION = "1.0.0"
+
 
 # StorageURI represents a normalised URI to a valid storage location (full bucket or
 # absolute local path).
@@ -33,12 +36,12 @@ LISTING_PREFIX = "lst__"
 StorageURI = NewType("StorageURI", str)
 
 
-def parse_dataset_uri(uri: str) -> tuple[str, Optional[int]]:
+def parse_dataset_uri(uri: str) -> tuple[str, Optional[str]]:
     """
     Parse dataser uri to extract name and version out of it (if version is defined)
     Example:
-        Input: ds://zalando@v3
-        Output: (zalando, 3)
+        Input: ds://zalando@v3.0.1
+        Output: (zalando, 3.0.1)
     """
     p = urlparse(uri)
     if p.scheme != "ds":
@@ -51,16 +54,15 @@ def parse_dataset_uri(uri: str) -> tuple[str, Optional[int]]:
         raise Exception(
             "Wrong dataset uri format, it should be: ds://<name>@v<version>"
         )
-    version = int(s[1])
-    return name, version
+    return name, s[1]
 
 
-def create_dataset_uri(name: str, version: Optional[int] = None) -> str:
+def create_dataset_uri(name: str, version: Optional[str] = None) -> str:
     """
     Creates a dataset uri based on dataset name and optionally version
     Example:
-        Input: zalando, 3
-        Output: ds//zalando@v3
+        Input: zalando, 3.0.1
+        Output: ds//zalando@v3.0.1
     """
     uri = f"{DATASET_PREFIX}{name}"
     if version:
@@ -79,7 +81,7 @@ class DatasetDependency:
     id: int
     type: str
     name: str
-    version: str  # TODO change to int
+    version: str
     created_at: datetime
     dependencies: list[Optional["DatasetDependency"]]
 
@@ -102,7 +104,7 @@ class DatasetDependency:
         dataset_id: Optional[int],
         dataset_version_id: Optional[int],
         dataset_name: Optional[str],
-        dataset_version: Optional[int],
+        dataset_version: Optional[str],
         dataset_version_created_at: Optional[datetime],
     ) -> Optional["DatasetDependency"]:
         from datachain.client import Client
@@ -124,7 +126,7 @@ class DatasetDependency:
             dependency_type,
             dependency_name,
             (
-                str(dataset_version)  # type: ignore[arg-type]
+                dataset_version  # type: ignore[arg-type]
                 if dataset_version
                 else None
             ),
@@ -163,7 +165,7 @@ class DatasetVersion:
     id: int
     uuid: str
     dataset_id: int
-    version: int
+    version: str
     status: int
     feature_schema: dict
     created_at: datetime
@@ -185,7 +187,7 @@ class DatasetVersion:
         id: int,
         uuid: str,
         dataset_id: int,
-        version: int,
+        version: str,
         status: int,
         feature_schema: Optional[str],
         created_at: datetime,
@@ -222,6 +224,10 @@ class DatasetVersion:
             job_id,
         )
 
+    @property
+    def version_value(self) -> int:
+        return semver.value(self.version)
+
     def __eq__(self, other):
         if not isinstance(other, DatasetVersion):
             return False
@@ -230,7 +236,7 @@ class DatasetVersion:
     def __lt__(self, other):
         if not isinstance(other, DatasetVersion):
             return False
-        return self.version < other.version
+        return self.version_value < other.version_value
 
     def __hash__(self):
         return hash(f"{self.dataset_id}_{self.version}")
@@ -275,7 +281,7 @@ class DatasetListVersion:
     id: int
     uuid: str
     dataset_id: int
-    version: int
+    version: str
     status: int
     created_at: datetime
     finished_at: Optional[datetime]
@@ -292,7 +298,7 @@ class DatasetListVersion:
         id: int,
         uuid: str,
         dataset_id: int,
-        version: int,
+        version: str,
         status: int,
         created_at: datetime,
         finished_at: Optional[datetime],
@@ -323,6 +329,10 @@ class DatasetListVersion:
     def __hash__(self):
         return hash(f"{self.dataset_id}_{self.version}")
 
+    @property
+    def version_value(self) -> int:
+        return semver.value(self.version)
+
 
 @dataclass
 class DatasetRecord:
@@ -371,7 +381,7 @@ class DatasetRecord:
         version_id: int,
         version_uuid: str,
         version_dataset_id: int,
-        version: int,
+        version: str,
         version_status: int,
         version_feature_schema: Optional[str],
         version_created_at: datetime,
@@ -441,7 +451,7 @@ class DatasetRecord:
             for c_name, c_type in self.schema.items()
         }
 
-    def get_schema(self, version: int) -> dict[str, Union[SQLType, type[SQLType]]]:
+    def get_schema(self, version: str) -> dict[str, Union[SQLType, type[SQLType]]]:
         return self.get_version(version).schema if version else self.schema
 
     def update(self, **kwargs):
@@ -460,20 +470,23 @@ class DatasetRecord:
             self.versions = []
 
         self.versions = list(set(self.versions + other.versions))
-        self.versions.sort(key=lambda v: v.version)
+        self.versions.sort(key=lambda v: v.version_value)
         return self
 
-    def has_version(self, version: int) -> bool:
-        return version in self.versions_values
+    def has_version(self, version: str) -> bool:
+        return version in [v.version for v in self.versions]
 
-    def is_valid_next_version(self, version: int) -> bool:
+    def is_valid_next_version(self, version: str) -> bool:
         """
         Checks if a number can be a valid next latest version for dataset.
         The only rule is that it cannot be lower than current latest version
         """
-        return not (self.latest_version and self.latest_version >= version)
+        return not (
+            self.latest_version
+            and semver.value(self.latest_version) >= semver.value(version)
+        )
 
-    def get_version(self, version: int) -> DatasetVersion:
+    def get_version(self, version: str) -> DatasetVersion:
         if not self.has_version(version):
             raise DatasetVersionNotFoundError(
                 f"Dataset {self.name} does not have version {version}"
@@ -496,15 +509,15 @@ class DatasetRecord:
                 f"Dataset {self.name} does not have version with uuid {uuid}"
             ) from None
 
-    def remove_version(self, version: int) -> None:
+    def remove_version(self, version: str) -> None:
         if not self.versions or not self.has_version(version):
             return
 
         self.versions = [v for v in self.versions if v.version != version]
 
-    def identifier(self, version: int) -> str:
+    def identifier(self, version: str) -> str:
         """
-        Get identifier in the form my-dataset@v3
+        Get identifier in the form my-dataset@v3.0.1
         """
         if not self.has_version(version):
             raise DatasetVersionNotFoundError(
@@ -512,43 +525,73 @@ class DatasetRecord:
             )
         return f"{self.name}@v{version}"
 
-    def uri(self, version: int) -> str:
+    def uri(self, version: str) -> str:
         """
-        Dataset uri example: ds://dogs@v3
+        Dataset uri example: ds://dogs@v3.0.1
         """
         identifier = self.identifier(version)
         return f"{DATASET_PREFIX}{identifier}"
 
     @property
-    def versions_values(self) -> list[int]:
+    def next_version_major(self) -> str:
         """
-        Extracts actual versions from list of DatasetVersion objects
-        in self.versions attribute
+        Returns the next auto-incremented version if the major part is being bumped.
         """
         if not self.versions:
-            return []
+            return "1.0.0"
 
-        return sorted(v.version for v in self.versions)
+        major, minor, patch = semver.parse(self.latest_version)
+        return semver.create(major + 1, 0, 0)
 
     @property
-    def next_version(self) -> int:
-        """Returns what should be next autoincrement version of dataset"""
+    def next_version_minor(self) -> str:
+        """
+        Returns the next auto-incremented version if the minor part is being bumped.
+        """
         if not self.versions:
-            return 1
-        return max(self.versions_values) + 1
+            return "1.0.0"
+
+        major, minor, patch = semver.parse(self.latest_version)
+        return semver.create(major, minor + 1, 0)
 
     @property
-    def latest_version(self) -> int:
+    def next_version_patch(self) -> str:
+        """
+        Returns the next auto-incremented version if the patch part is being bumped.
+        """
+        if not self.versions:
+            return "1.0.0"
+
+        major, minor, patch = semver.parse(self.latest_version)
+        return semver.create(major, minor, patch + 1)
+
+    @property
+    def latest_version(self) -> str:
         """Returns latest version of a dataset"""
-        return max(self.versions_values)
+        return max(self.versions).version
+
+    def latest_major_version(self, major: int) -> Optional[str]:
+        """
+        Returns latest specific major version, e.g if dataset has versions:
+            - 1.4.1
+            - 2.0.1
+            - 2.1.1
+            - 2.4.0
+        and we call `.latest_major_version(2)` it will return: "2.4.0".
+        If no major version is find with input value, None will be returned
+        """
+        versions = [v for v in self.versions if semver.parse(v.version)[0] == major]
+        if not versions:
+            return None
+        return max(versions).version
 
     @property
-    def prev_version(self) -> Optional[int]:
+    def prev_version(self) -> Optional[str]:
         """Returns previous version of a dataset"""
         if len(self.versions) == 1:
             return None
 
-        return sorted(self.versions_values)[-2]
+        return sorted(self.versions)[-2].version
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "DatasetRecord":
@@ -577,7 +620,7 @@ class DatasetListRecord:
         version_id: int,
         version_uuid: str,
         version_dataset_id: int,
-        version: int,
+        version: str,
         version_status: int,
         version_created_at: datetime,
         version_finished_at: Optional[datetime],
@@ -626,11 +669,11 @@ class DatasetListRecord:
             self.versions = []
 
         self.versions = list(set(self.versions + other.versions))
-        self.versions.sort(key=lambda v: v.version)
+        self.versions.sort(key=lambda v: v.version_value)
         return self
 
     def latest_version(self) -> DatasetListVersion:
-        return max(self.versions, key=lambda v: v.version)
+        return max(self.versions, key=lambda v: v.version_value)
 
     @property
     def is_bucket_listing(self) -> bool:

datachain/lib/dataset_info.py

@@ -6,6 +6,7 @@ from uuid import uuid4
 from pydantic import Field, field_validator
 
 from datachain.dataset import (
+    DEFAULT_DATASET_VERSION,
     DatasetListRecord,
     DatasetListVersion,
     DatasetStatus,
@@ -22,7 +23,7 @@ if TYPE_CHECKING:
 class DatasetInfo(DataModel):
     name: str
     uuid: str = Field(default=str(uuid4()))
-    version: int = Field(default=1)
+    version: str = Field(default=DEFAULT_DATASET_VERSION)
     status: int = Field(default=DatasetStatus.CREATED)
     created_at: datetime = Field(default=TIME_ZERO)
     finished_at: Optional[datetime] = Field(default=None)

datachain/lib/dc/datachain.py

@@ -23,6 +23,7 @@ import sqlalchemy
 from pydantic import BaseModel
 from tqdm import tqdm
 
+from datachain import semver
 from datachain.dataset import DatasetRecord
 from datachain.func import literal
 from datachain.func.base import Function
@@ -214,7 +215,7 @@ class DataChain:
         return self._query.name
 
     @property
-    def version(self) -> Optional[int]:
+    def version(self) -> Optional[str]:
         """Version of the underlying dataset, if there is one."""
         return self._query.version
 
@@ -457,7 +458,7 @@ class DataChain:
     def save(  # type: ignore[override]
         self,
         name: str,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         description: Optional[str] = None,
         attrs: Optional[list[str]] = None,
         **kwargs,
@@ -466,11 +467,15 @@ class DataChain:
 
         Parameters:
             name : dataset name.
-            version : version of a dataset. Default - the last version that exist.
+            version : version of a dataset. If version is not specified and dataset
+                already exists, version patch increment will happen e.g 1.2.1 -> 1.2.2.
             description : description of a dataset.
             attrs : attributes of a dataset. They can be without value, e.g "NLP",
                 or with a value, e.g "location=US".
         """
+        if version is not None:
+            semver.validate(version)
+
         schema = self.signals_schema.clone_without_sys_signals().serialize()
         return self._evolve(
             query=self._query.save(

datachain/lib/dc/datasets.py

@@ -1,5 +1,6 @@
-from typing import TYPE_CHECKING, Optional, get_origin, get_type_hints
+from typing import TYPE_CHECKING, Optional, Union, get_origin, get_type_hints
 
+from datachain.error import DatasetVersionNotFoundError
 from datachain.lib.dataset_info import DatasetInfo
 from datachain.lib.file import (
     File,
@@ -22,7 +23,7 @@ if TYPE_CHECKING:
 
 def read_dataset(
     name: str,
-    version: Optional[int] = None,
+    version: Optional[Union[str, int]] = None,
     session: Optional[Session] = None,
     settings: Optional[dict] = None,
     fallback_to_studio: bool = True,
@@ -49,7 +50,7 @@ def read_dataset(
         ```
 
         ```py
-        chain = dc.read_dataset("my_cats", version=1)
+        chain = dc.read_dataset("my_cats", version="1.0.0")
         ```
 
         ```py
@@ -63,7 +64,7 @@ def read_dataset(
         }
         chain = dc.read_dataset(
             name="my_cats",
-            version=1,
+            version="1.0.0",
             session=session,
             settings=settings,
             fallback_to_studio=True,
@@ -74,9 +75,29 @@ def read_dataset(
 
     from .datachain import DataChain
 
+    if version is not None:
+        try:
+            # for backward compatibility we still allow users to put version as integer
+            # in which case we are trying to find latest version where major part is
+            # equal to that input version. For example if user sets version=2, we could
+            # continue with something like 2.4.3 (assuming 2.4.3 is the biggest among
+            # all 2.* dataset versions). If dataset doesn't have any versions where
+            # major part is equal to that input, exception is thrown.
+            major = int(version)
+            dataset = Session.get(session).catalog.get_dataset(name)
+            latest_major = dataset.latest_major_version(major)
+            if not latest_major:
+                raise DatasetVersionNotFoundError(
+                    f"Dataset {name} does not have version {version}"
+                )
+            version = latest_major
+        except ValueError:
+            # version is in new semver string format, continuing as normal
+            pass
+
     query = DatasetQuery(
         name=name,
-        version=version,
+        version=version,  #  type: ignore[arg-type]
         session=session,
         indexing_column_types=File._datachain_column_types,
         fallback_to_studio=fallback_to_studio,
@@ -179,7 +200,7 @@ def datasets(
 
 def delete_dataset(
     name: str,
-    version: Optional[int] = None,
+    version: Optional[str] = None,
     force: Optional[bool] = False,
     studio: Optional[bool] = False,
     session: Optional[Session] = None,
@@ -207,7 +228,7 @@ def delete_dataset(
 
         ```py
         import datachain as dc
-        dc.delete_dataset("cats", version=1)
+        dc.delete_dataset("cats", version="1.0.0")
         ```
     """
 

datachain/lib/dc/storage.py

@@ -5,6 +5,7 @@ from typing import (
     Union,
 )
 
+from datachain.error import DatasetNotFoundError
 from datachain.lib.file import (
     FileType,
     get_file_type,
@@ -97,7 +98,8 @@ def read_storage(
     if anon:
         client_config = (client_config or {}) | {"anon": True}
     session = Session.get(session, client_config=client_config, in_memory=in_memory)
-    cache = session.catalog.cache
+    catalog = session.catalog
+    cache = catalog.cache
     client_config = session.catalog.client_config
 
     uris = uri if isinstance(uri, (list, tuple)) else [uri]
@@ -130,6 +132,11 @@ def read_storage(
 
             def lst_fn(ds_name, lst_uri):
                 # disable prefetch for listing, as it pre-downloads all files
+                try:
+                    version = catalog.get_dataset(ds_name).next_version_major
+                except DatasetNotFoundError:
+                    version = None
+
                 (
                     read_records(
                         DataChain.DEFAULT_FILE_RECORD,
@@ -142,7 +149,8 @@ def read_storage(
                         list_bucket(lst_uri, cache, client_config=client_config),
                         output={f"{column}": file_type},
                     )
-                    .save(ds_name, listing=True)
+                    # for internal listing datasets, we always bump major version
+                    .save(ds_name, listing=True, version=version)
                 )
 
             dc._query.set_listing_fn(

datachain/lib/pytorch.py

@@ -43,7 +43,7 @@ class PytorchDataset(IterableDataset):
     def __init__(
         self,
         name: str,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         catalog: Optional["Catalog"] = None,
         transform: Optional["Transform"] = None,
         tokenizer: Optional[Callable] = None,
@@ -60,7 +60,7 @@ class PytorchDataset(IterableDataset):
 
         Args:
             name (str): Name of DataChain dataset to stream.
-            version (int): Version of DataChain dataset to stream.
+            version (str): Version of DataChain dataset to stream.
             catalog (Catalog): DataChain catalog to which dataset belongs.
             transform (Transform): Torchvision transforms to apply to the dataset.
             tokenizer (Callable): Tokenizer to use to tokenize text values.

datachain/listing.py

@@ -26,7 +26,7 @@ class Listing:
         warehouse: "AbstractWarehouse",
         client: "Client",
         dataset_name: Optional["str"] = None,
-        dataset_version: Optional[int] = None,
+        dataset_version: Optional[str] = None,
         column: str = "file",
     ):
         self.metastore = metastore

datachain/query/dataset.py

@@ -83,7 +83,7 @@ PartitionByType = Union[
     Function, ColumnElement, Sequence[Union[Function, ColumnElement]]
 ]
 JoinPredicateType = Union[str, ColumnClause, ColumnElement]
-DatasetDependencyType = tuple[str, int]
+DatasetDependencyType = tuple[str, str]
 
 logger = logging.getLogger("datachain")
 
@@ -168,7 +168,7 @@ class Step(ABC):
 class QueryStep:
     catalog: "Catalog"
     dataset_name: str
-    dataset_version: int
+    dataset_version: str
 
     def apply(self):
         def q(*columns):
@@ -1092,7 +1092,7 @@ class DatasetQuery:
     def __init__(
         self,
         name: str,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         catalog: Optional["Catalog"] = None,
         session: Optional[Session] = None,
         indexing_column_types: Optional[dict[str, Any]] = None,
@@ -1112,7 +1112,7 @@ class DatasetQuery:
         self.table = self.get_table()
         self.starting_step: Optional[QueryStep] = None
         self.name: Optional[str] = None
-        self.version: Optional[int] = None
+        self.version: Optional[str] = None
         self.feature_schema: Optional[dict] = None
         self.column_types: Optional[dict[str, Any]] = None
         self.before_steps: list[Callable] = []
@@ -1155,7 +1155,7 @@ class DatasetQuery:
     def __or__(self, other):
         return self.union(other)
 
-    def pull_dataset(self, name: str, version: Optional[int] = None) -> "DatasetRecord":
+    def pull_dataset(self, name: str, version: Optional[str] = None) -> "DatasetRecord":
         print("Dataset not found in local catalog, trying to get from studio")
 
         remote_ds_uri = f"{DATASET_PREFIX}{name}"
@@ -1185,8 +1185,8 @@ class DatasetQuery:
         it completely. If this is the case, name and version of underlying dataset
         will be defined.
         DatasetQuery instance can become attached in two scenarios:
-            1. ds = DatasetQuery(name="dogs", version=1) -> ds is attached to dogs
-            2. ds = ds.save("dogs", version=1) -> ds is attached to dogs dataset
+            1. ds = DatasetQuery(name="dogs", version="1.0.0") -> ds is attached to dogs
+            2. ds = ds.save("dogs", version="1.0.0") -> ds is attached to dogs dataset
         It can move to detached state if filter or similar methods are called on it,
         as then it no longer 100% represents underlying datasets.
         """
@@ -1663,7 +1663,7 @@ class DatasetQuery:
         )
         return query
 
-    def _add_dependencies(self, dataset: "DatasetRecord", version: int):
+    def _add_dependencies(self, dataset: "DatasetRecord", version: str):
         for dependency in self.dependencies:
             ds_dependency_name, ds_dependency_version = dependency
             self.catalog.metastore.add_dataset_dependency(
@@ -1685,7 +1685,7 @@ class DatasetQuery:
     def save(
         self,
         name: Optional[str] = None,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         feature_schema: Optional[dict] = None,
         description: Optional[str] = None,
         attrs: Optional[list[str]] = None,

datachain/query/session.py

@@ -69,7 +69,7 @@ class Session:
         self.catalog = catalog or get_catalog(
             client_config=client_config, in_memory=in_memory
         )
-        self.dataset_versions: list[tuple[DatasetRecord, int, bool]] = []
+        self.dataset_versions: list[tuple[DatasetRecord, str, bool]] = []
 
     def __enter__(self):
         # Push the current context onto the stack
@@ -90,7 +90,7 @@ class Session:
             Session.SESSION_CONTEXTS.pop()
 
     def add_dataset_version(
-        self, dataset: "DatasetRecord", version: int, listing: bool = False
+        self, dataset: "DatasetRecord", version: str, listing: bool = False
     ) -> None:
         self.dataset_versions.append((dataset, version, listing))
 

datachain/remote/studio.py

@@ -307,7 +307,7 @@ class StudioClient:
     def rm_dataset(
         self,
         name: str,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         force: Optional[bool] = False,
     ) -> Response[DatasetInfoData]:
         return self._send_request(
@@ -336,7 +336,7 @@ class StudioClient:
         return response
 
     def dataset_rows_chunk(
-        self, name: str, version: int, offset: int
+        self, name: str, version: str, offset: int
     ) -> Response[DatasetRowsData]:
         req_data = {"dataset_name": name, "dataset_version": version}
         return self._send_request_msgpack(
@@ -353,7 +353,7 @@ class StudioClient:
         )
 
     def export_dataset_table(
-        self, name: str, version: int
+        self, name: str, version: str
     ) -> Response[DatasetExportSignedUrls]:
         return self._send_request(
             "datachain/datasets/export",
@@ -362,7 +362,7 @@ class StudioClient:
         )
 
     def dataset_export_status(
-        self, name: str, version: int
+        self, name: str, version: str
     ) -> Response[DatasetExportStatus]:
         return self._send_request(
             "datachain/datasets/export-status",