datachain 0.16.5__py3-none-any.whl → 0.17.1__py3-none-any.whl

datachain/catalog/catalog.py

@@ -33,6 +33,7 @@ from datachain.cache import Cache
 from datachain.client import Client
 from datachain.dataset import (
     DATASET_PREFIX,
+    DEFAULT_DATASET_VERSION,
     QUERY_DATASET_PREFIX,
     DatasetDependency,
     DatasetListRecord,
@@ -154,9 +155,9 @@ class DatasetRowsFetcher(NodesThreadPool):
         metastore: "AbstractMetastore",
         warehouse: "AbstractWarehouse",
         remote_ds_name: str,
-        remote_ds_version: int,
+        remote_ds_version: str,
         local_ds_name: str,
-        local_ds_version: int,
+        local_ds_version: str,
         schema: dict[str, Union[SQLType, type[SQLType]]],
         max_threads: int = PULL_DATASET_MAX_THREADS,
         progress_bar=None,
@@ -286,7 +287,7 @@ class NodeGroup:
     # (not including the bucket name or s3:// prefix)
     source_path: str = ""
     dataset_name: Optional[str] = None
-    dataset_version: Optional[int] = None
+    dataset_version: Optional[str] = None
     instantiated_nodes: Optional[list[NodeWithPath]] = None
 
     @property
@@ -607,7 +608,7 @@ class Catalog:
         return lst, client, list_path
 
     def _remove_dataset_rows_and_warehouse_info(
-        self, dataset: DatasetRecord, version: int, **kwargs
+        self, dataset: DatasetRecord, version: str, **kwargs
     ):
         self.warehouse.drop_dataset_rows_table(dataset, version)
         self.update_dataset_version_with_warehouse_info(
@@ -767,7 +768,7 @@ class Catalog:
     def create_dataset(
         self,
         name: str,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         *,
         columns: Sequence[Column],
         feature_schema: Optional[dict] = None,
@@ -783,18 +784,17 @@ class Catalog:
         Creates new dataset of a specific version.
         If dataset is not yet created, it will create it with version 1
         If version is None, then next unused version is created.
-        If version is given, then it must be an unused version number.
+        If version is given, then it must be an unused version.
         """
         assert [c.name for c in columns if c.name != "sys__id"], f"got {columns=}"
         if not listing and Client.is_data_source_uri(name):
             raise RuntimeError(
                 "Cannot create dataset that starts with source prefix, e.g s3://"
             )
-        default_version = 1
+        default_version = DEFAULT_DATASET_VERSION
         try:
             dataset = self.get_dataset(name)
-            default_version = dataset.next_version
-
+            default_version = dataset.next_version_patch
             if (description or attrs) and (
                 dataset.description != description or dataset.attrs != attrs
             ):
@@ -846,7 +846,7 @@ class Catalog:
     def create_new_dataset_version(
         self,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         *,
         columns: Sequence[Column],
         sources="",
@@ -892,7 +892,7 @@ class Catalog:
         return dataset
 
     def update_dataset_version_with_warehouse_info(
-        self, dataset: DatasetRecord, version: int, rows_dropped=False, **kwargs
+        self, dataset: DatasetRecord, version: str, rows_dropped=False, **kwargs
     ) -> None:
         from datachain.query.dataset import DatasetQuery
 
@@ -959,7 +959,7 @@ class Catalog:
         return dataset
 
     def remove_dataset_version(
-        self, dataset: DatasetRecord, version: int, drop_rows: Optional[bool] = True
+        self, dataset: DatasetRecord, version: str, drop_rows: Optional[bool] = True
     ) -> None:
         """
         Deletes one single dataset version.
@@ -1037,82 +1037,11 @@ class Catalog:
 
         return self.get_dataset(name)
 
-    def register_dataset(
-        self,
-        dataset: DatasetRecord,
-        version: int,
-        target_dataset: DatasetRecord,
-        target_version: Optional[int] = None,
-    ) -> DatasetRecord:
-        """
-        Registers dataset version of one dataset as dataset version of another
-        one (it can be new version of existing one).
-        It also removes original dataset version
-        """
-        target_version = target_version or target_dataset.next_version
-
-        if not target_dataset.is_valid_next_version(target_version):
-            raise DatasetInvalidVersionError(
-                f"Version {target_version} must be higher than the current latest one"
-            )
-
-        dataset_version = dataset.get_version(version)
-        if not dataset_version:
-            raise DatasetVersionNotFoundError(
-                f"Dataset {dataset.name} does not have version {version}"
-            )
-
-        if not dataset_version.is_final_status():
-            raise ValueError("Cannot register dataset version in non final status")
-
-        # copy dataset version
-        target_dataset = self.metastore.create_dataset_version(
-            target_dataset,
-            target_version,
-            sources=dataset_version.sources,
-            status=dataset_version.status,
-            query_script=dataset_version.query_script,
-            error_message=dataset_version.error_message,
-            error_stack=dataset_version.error_stack,
-            script_output=dataset_version.script_output,
-            created_at=dataset_version.created_at,
-            finished_at=dataset_version.finished_at,
-            schema=dataset_version.serialized_schema,
-            num_objects=dataset_version.num_objects,
-            size=dataset_version.size,
-            preview=dataset_version.preview,
-            job_id=dataset_version.job_id,
-        )
-
-        # to avoid re-creating rows table, we are just renaming it for a new version
-        # of target dataset
-        self.warehouse.rename_dataset_table(
-            dataset.name,
-            target_dataset.name,
-            old_version=version,
-            new_version=target_version,
-        )
-        self.metastore.update_dataset_dependency_source(
-            dataset,
-            version,
-            new_source_dataset=target_dataset,
-            new_source_dataset_version=target_version,
-        )
-
-        if dataset.id == target_dataset.id:
-            # we are updating the same dataset so we need to refresh it to have newly
-            # added version in step before
-            dataset = self.get_dataset(dataset.name)
-
-        self.remove_dataset_version(dataset, version, drop_rows=False)
-
-        return self.get_dataset(target_dataset.name)
-
     def get_dataset(self, name: str) -> DatasetRecord:
         return self.metastore.get_dataset(name)
 
     def get_dataset_with_remote_fallback(
-        self, name: str, version: Optional[int] = None
+        self, name: str, version: Optional[str] = None
     ) -> DatasetRecord:
         try:
             ds = self.get_dataset(name)
@@ -1157,7 +1086,7 @@ class Catalog:
         return DatasetRecord.from_dict(dataset_info)
 
     def get_dataset_dependencies(
-        self, name: str, version: int, indirect=False
+        self, name: str, version: str, indirect=False
     ) -> list[Optional[DatasetDependency]]:
         dataset = self.get_dataset(name)
 
@@ -1175,7 +1104,7 @@ class Catalog:
             if d.is_dataset:
                 # only datasets can have dependencies
                 d.dependencies = self.get_dataset_dependencies(
-                    d.name, int(d.version), indirect=indirect
+                    d.name, d.version, indirect=indirect
                 )
 
         return direct_dependencies
@@ -1244,7 +1173,7 @@ class Catalog:
         ]
 
     def ls_dataset_rows(
-        self, name: str, version: int, offset=None, limit=None
+        self, name: str, version: str, offset=None, limit=None
     ) -> list[dict]:
         from datachain.query.dataset import DatasetQuery
 
@@ -1282,7 +1211,7 @@ class Catalog:
         self,
         bucket_uri: str,
         name: str,
-        version: int,
+        version: str,
         client_config=None,
     ) -> list[str]:
         dataset = self.get_dataset(name)
@@ -1291,14 +1220,14 @@ class Catalog:
             bucket_uri, dataset, version, client_config
         )
 
-    def dataset_table_export_file_names(self, name: str, version: int) -> list[str]:
+    def dataset_table_export_file_names(self, name: str, version: str) -> list[str]:
         dataset = self.get_dataset(name)
         return self.warehouse.dataset_table_export_file_names(dataset, version)
 
     def remove_dataset(
         self,
         name: str,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         force: Optional[bool] = False,
         studio: Optional[bool] = False,
     ):
@@ -1372,7 +1301,7 @@ class Catalog:
         remote_ds_uri: str,
         output: Optional[str] = None,
         local_ds_name: Optional[str] = None,
-        local_ds_version: Optional[int] = None,
+        local_ds_version: Optional[str] = None,
         cp: bool = False,
         force: bool = False,
         *,

datachain/cli/commands/datasets.py

@@ -127,7 +127,7 @@ def _datasets_tabulate_row(name, both, local_version, studio_version):
 def rm_dataset(
     catalog: "Catalog",
     name: str,
-    version: Optional[int] = None,
+    version: Optional[str] = None,
     force: Optional[bool] = False,
     studio: bool = False,
     local: bool = False,

datachain/cli/commands/show.py

@@ -10,7 +10,7 @@ if TYPE_CHECKING:
 def show(
     catalog: "Catalog",
     name: str,
-    version: Optional[int] = None,
+    version: Optional[str] = None,
     limit: int = 10,
     offset: int = 0,
     columns: Sequence[str] = (),

datachain/cli/parser/__init__.py

@@ -302,7 +302,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--version",
         action="store",
         default=None,
-        type=int,
+        type=str,
         help="Dataset version",
     )
     rm_dataset_parser.add_argument(
@@ -495,7 +495,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--version",
         action="store",
         default=None,
-        type=int,
+        type=str,
         help="Dataset version",
     )
     show_parser.add_argument("--schema", action="store_true", help="Show schema")

datachain/cli/parser/job.py

@@ -83,6 +83,36 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
         help="Python package requirements",
     )
 
+    studio_ls_help = "List jobs in Studio"
+    studio_ls_description = "List jobs in Studio."
+
+    studio_ls_parser = jobs_subparser.add_parser(
+        "ls",
+        parents=[parent_parser],
+        description=studio_ls_description,
+        help=studio_ls_help,
+        formatter_class=CustomHelpFormatter,
+    )
+
+    studio_ls_parser.add_argument(
+        "--status",
+        action="store",
+        help="Status to filter jobs by",
+    )
+
+    studio_ls_parser.add_argument(
+        "--team",
+        action="store",
+        default=None,
+        help="Team to list jobs for (default: from config)",
+    )
+    studio_ls_parser.add_argument(
+        "--limit",
+        type=int,
+        default=20,
+        help="Limit the number of jobs returned (default: 20)",
+    )
+
     studio_cancel_help = "Cancel a job in Studio"
     studio_cancel_description = "Cancel a running job in Studio."
 

datachain/data_storage/metastore.py

@@ -128,7 +128,7 @@ class AbstractMetastore(ABC, Serializable):
     def create_dataset_version(  # noqa: PLR0913
         self,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         status: int,
         sources: str = "",
         feature_schema: Optional[dict] = None,
@@ -158,13 +158,13 @@ class AbstractMetastore(ABC, Serializable):
 
     @abstractmethod
     def update_dataset_version(
-        self, dataset: DatasetRecord, version: int, **kwargs
+        self, dataset: DatasetRecord, version: str, **kwargs
     ) -> DatasetVersion:
         """Updates dataset version fields."""
 
     @abstractmethod
     def remove_dataset_version(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> DatasetRecord:
         """
         Deletes one single dataset version.
@@ -188,7 +188,7 @@ class AbstractMetastore(ABC, Serializable):
         self,
         dataset: DatasetRecord,
         status: int,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         error_message="",
         error_stack="",
         script_output="",
@@ -202,9 +202,9 @@ class AbstractMetastore(ABC, Serializable):
     def add_dataset_dependency(
         self,
         source_dataset_name: str,
-        source_dataset_version: int,
+        source_dataset_version: str,
         dataset_name: str,
-        dataset_version: int,
+        dataset_version: str,
     ) -> None:
         """Adds dataset dependency to dataset."""
 
@@ -212,21 +212,21 @@ class AbstractMetastore(ABC, Serializable):
     def update_dataset_dependency_source(
         self,
         source_dataset: DatasetRecord,
-        source_dataset_version: int,
+        source_dataset_version: str,
         new_source_dataset: Optional[DatasetRecord] = None,
-        new_source_dataset_version: Optional[int] = None,
+        new_source_dataset_version: Optional[str] = None,
     ) -> None:
         """Updates dataset dependency source."""
 
     @abstractmethod
     def get_direct_dataset_dependencies(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> list[Optional[DatasetDependency]]:
         """Gets direct dataset dependencies."""
 
     @abstractmethod
     def remove_dataset_dependencies(
-        self, dataset: DatasetRecord, version: Optional[int] = None
+        self, dataset: DatasetRecord, version: Optional[str] = None
     ) -> None:
         """
         When we remove dataset, we need to clean up it's dependencies as well.
@@ -234,7 +234,7 @@ class AbstractMetastore(ABC, Serializable):
 
     @abstractmethod
     def remove_dataset_dependants(
-        self, dataset: DatasetRecord, version: Optional[int] = None
+        self, dataset: DatasetRecord, version: Optional[str] = None
     ) -> None:
         """
         When we remove dataset, we need to clear its references in other dataset
@@ -370,7 +370,7 @@ class AbstractDBMetastore(AbstractMetastore):
                 ForeignKey(f"{cls.DATASET_TABLE}.id", ondelete="CASCADE"),
                 nullable=False,
             ),
-            Column("version", Integer, nullable=False),
+            Column("version", Text, nullable=False, default="1.0.0"),
             Column(
                 "status",
                 Integer,
@@ -554,7 +554,7 @@ class AbstractDBMetastore(AbstractMetastore):
     def create_dataset_version(  # noqa: PLR0913
         self,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         status: int,
         sources: str = "",
         feature_schema: Optional[dict] = None,
@@ -648,7 +648,7 @@ class AbstractDBMetastore(AbstractMetastore):
         return result_ds
 
     def update_dataset_version(
-        self, dataset: DatasetRecord, version: int, conn=None, **kwargs
+        self, dataset: DatasetRecord, version: str, conn=None, **kwargs
     ) -> DatasetVersion:
         """Updates dataset fields."""
         dataset_version = dataset.get_version(version)
@@ -758,7 +758,7 @@ class AbstractDBMetastore(AbstractMetastore):
         return ds
 
     def remove_dataset_version(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> DatasetRecord:
         """
         Deletes one single dataset version.
@@ -791,7 +791,7 @@ class AbstractDBMetastore(AbstractMetastore):
         self,
         dataset: DatasetRecord,
         status: int,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         error_message="",
         error_stack="",
         script_output="",
@@ -825,9 +825,9 @@ class AbstractDBMetastore(AbstractMetastore):
     def add_dataset_dependency(
         self,
         source_dataset_name: str,
-        source_dataset_version: int,
+        source_dataset_version: str,
         dataset_name: str,
-        dataset_version: int,
+        dataset_version: str,
     ) -> None:
         """Adds dataset dependency to dataset."""
         source_dataset = self.get_dataset(source_dataset_name)
@@ -847,9 +847,9 @@ class AbstractDBMetastore(AbstractMetastore):
     def update_dataset_dependency_source(
         self,
         source_dataset: DatasetRecord,
-        source_dataset_version: int,
+        source_dataset_version: str,
         new_source_dataset: Optional[DatasetRecord] = None,
-        new_source_dataset_version: Optional[int] = None,
+        new_source_dataset_version: Optional[str] = None,
     ) -> None:
         dd = self._datasets_dependencies
 
@@ -880,7 +880,7 @@ class AbstractDBMetastore(AbstractMetastore):
         """
 
     def get_direct_dataset_dependencies(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> list[Optional[DatasetDependency]]:
         d = self._datasets
         dd = self._datasets_dependencies
@@ -909,7 +909,7 @@ class AbstractDBMetastore(AbstractMetastore):
         return [self.dependency_class.parse(*r) for r in self.db.execute(query)]
 
     def remove_dataset_dependencies(
-        self, dataset: DatasetRecord, version: Optional[int] = None
+        self, dataset: DatasetRecord, version: Optional[str] = None
     ) -> None:
         """
         When we remove dataset, we need to clean up it's dependencies as well
@@ -928,7 +928,7 @@ class AbstractDBMetastore(AbstractMetastore):
         self.db.execute(q)
 
     def remove_dataset_dependants(
-        self, dataset: DatasetRecord, version: Optional[int] = None
+        self, dataset: DatasetRecord, version: Optional[str] = None
     ) -> None:
         """
         When we remove dataset, we need to clear its references in other dataset

datachain/data_storage/sqlite.py

@@ -25,6 +25,7 @@ from sqlalchemy.sql.selectable import Select
 from tqdm.auto import tqdm
 
 import datachain.sql.sqlite
+from datachain import semver
 from datachain.data_storage import AbstractDBMetastore, AbstractWarehouse
 from datachain.data_storage.db_engine import DatabaseEngine
 from datachain.data_storage.schema import DefaultSchema
@@ -486,7 +487,7 @@ class SQLiteWarehouse(AbstractWarehouse):
         return table
 
     def get_dataset_sources(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> list[StorageURI]:
         dr = self.dataset_rows(dataset, version)
         query = dr.select(dr.c("source", column="file")).distinct()
@@ -502,8 +503,8 @@ class SQLiteWarehouse(AbstractWarehouse):
         self,
         src: DatasetRecord,
         dst: DatasetRecord,
-        src_version: int,
-        dst_version: int,
+        src_version: str,
+        dst_version: str,
     ) -> None:
         dst_empty = False
 
@@ -534,7 +535,7 @@ class SQLiteWarehouse(AbstractWarehouse):
             dst_previous_versions = [
                 v.version
                 for v in dst.versions  # type: ignore [union-attr]
-                if v.version < dst_version
+                if semver.compare(v.version, dst_version) == -1
             ]
             if dst_previous_versions:
                 dst_version_latest = max(dst_previous_versions)
@@ -570,7 +571,7 @@ class SQLiteWarehouse(AbstractWarehouse):
                 conn=conn,
             )
 
-    def insert_dataset_rows(self, df, dataset: DatasetRecord, version: int) -> int:
+    def insert_dataset_rows(self, df, dataset: DatasetRecord, version: str) -> int:
         dr = self.dataset_rows(dataset, version)
         return self.db.insert_dataframe(dr.table.name, df)
 
@@ -595,7 +596,7 @@ class SQLiteWarehouse(AbstractWarehouse):
         return col_type.python_type
 
     def dataset_table_export_file_names(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> list[str]:
         raise NotImplementedError("Exporting dataset table not implemented for SQLite")
 
@@ -603,7 +604,7 @@ class SQLiteWarehouse(AbstractWarehouse):
         self,
         bucket_uri: str,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         client_config=None,
     ) -> list[str]:
         raise NotImplementedError("Exporting dataset table not implemented for SQLite")

datachain/data_storage/warehouse.py

@@ -176,7 +176,7 @@ class AbstractWarehouse(ABC, Serializable):
     def dataset_rows(
         self,
         dataset: DatasetRecord,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         column: str = "file",
     ):
         version = version or dataset.latest_version
@@ -253,7 +253,7 @@ class AbstractWarehouse(ABC, Serializable):
         name = parsed.path if parsed.scheme == "file" else parsed.netloc
         return parsed.scheme, name
 
-    def dataset_table_name(self, dataset_name: str, version: int) -> str:
+    def dataset_table_name(self, dataset_name: str, version: str) -> str:
         prefix = self.DATASET_TABLE_PREFIX
         if Client.is_data_source_uri(dataset_name):
             # for datasets that are created for bucket listing we use different prefix
@@ -282,7 +282,7 @@ class AbstractWarehouse(ABC, Serializable):
     def drop_dataset_rows_table(
         self,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         if_exists: bool = True,
     ) -> None:
         """Drops a dataset rows table for the given dataset name."""
@@ -295,8 +295,8 @@ class AbstractWarehouse(ABC, Serializable):
         self,
         src: "DatasetRecord",
         dst: "DatasetRecord",
-        src_version: int,
-        dst_version: int,
+        src_version: str,
+        dst_version: str,
     ) -> None:
         """
         Merges source dataset rows and current latest destination dataset rows
@@ -338,15 +338,15 @@ class AbstractWarehouse(ABC, Serializable):
 
     @abstractmethod
     def get_dataset_sources(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> list[StorageURI]: ...
 
     def rename_dataset_table(
         self,
         old_name: str,
         new_name: str,
-        old_version: int,
-        new_version: int,
+        old_version: str,
+        new_version: str,
     ) -> None:
         old_ds_table_name = self.dataset_table_name(old_name, old_version)
         new_ds_table_name = self.dataset_table_name(new_name, new_version)
@@ -362,7 +362,7 @@ class AbstractWarehouse(ABC, Serializable):
         return res[0]
 
     def dataset_stats(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> tuple[Optional[int], Optional[int]]:
         """
         Returns tuple with dataset stats: total number of rows and total dataset size.
@@ -399,7 +399,7 @@ class AbstractWarehouse(ABC, Serializable):
         """
 
     @abstractmethod
-    def insert_dataset_rows(self, df, dataset: DatasetRecord, version: int) -> int:
+    def insert_dataset_rows(self, df, dataset: DatasetRecord, version: str) -> int:
         """Inserts dataset rows directly into dataset table"""
 
     @abstractmethod
@@ -418,7 +418,7 @@ class AbstractWarehouse(ABC, Serializable):
 
     @abstractmethod
     def dataset_table_export_file_names(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> list[str]:
         """
         Returns list of file names that will be created when user runs dataset export
@@ -429,7 +429,7 @@ class AbstractWarehouse(ABC, Serializable):
         self,
         bucket_uri: str,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         client_config=None,
     ) -> list[str]:
         """