datachain 0.16.5__tar.gz → 0.17.1__tar.gz

{datachain-0.16.5/src/datachain.egg-info → datachain-0.17.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.16.5
+Version: 0.17.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -44,7 +44,7 @@ Requires-Dist: datamodel-code-generator>=0.25
 Requires-Dist: Pillow<12,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
-Requires-Dist: huggingface_hub
+Requires-Dist: huggingface_hub<0.31
 Requires-Dist: iterative-telemetry>=0.0.10
 Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21

datachain-0.17.1/docs/commands/job/ls.md

@@ -0,0 +1,84 @@
+# job ls
+
+List jobs in Studio.
+
+## Synopsis
+
+```usage
+usage: datachain job ls [-h] [-v] [-q] [--status STATUS] [--team TEAM] [--limit LIMIT]
+```
+
+## Description
+
+This command lists jobs in Studio. You can filter jobs by their status, specify a team, and limit the number of jobs returned. By default, it shows the 20 most recent jobs.
+
+
+## Options
+
+* `--status STATUS` - Status to filter jobs by
+* `--team TEAM` - Team to list jobs for (default: from config)
+* `--limit LIMIT` - Limit the number of jobs returned (default: 20)
+* `-h`, `--help` - Show the help message and exit
+* `-v`, `--verbose` - Be verbose
+* `-q`, `--quiet` - Be quiet
+
+## Status options
+
+You will be able to filter the job with following status:
+
+* `CREATED` - Job has been created but not yet scheduled
+* `SCHEDULED` - Job is scheduled to run at a future time
+* `QUEUED` - Job is in the queue waiting to be executed
+* `INIT` - Job is initializing and preparing to run
+* `RUNNING` - Job is currently executing
+* `COMPLETE` - Job has finished successfully
+* `FAILED` - Job has failed during execution
+* `CANCELING_SCHEDULED` - A scheduled job is being canceled
+* `CANCELING` - A running job is being canceled
+* `CANCELED` - Job has been canceled
+* `ACTIVE` - Job is in active state.
+* `INACTIVE` - Job is in inactive state.
+
+Note: The following statuses are considered active jobs:
+
+* `CREATED`
+* `SCHEDULED`
+* `QUEUED`
+* `INIT`
+* `RUNNING`
+* `CANCELING_SCHEDULED`
+* `CANCELING`
+
+
+## Examples
+
+1. List all jobs (default limit of 20):
+```bash
+datachain job ls
+```
+
+2. List jobs for a specific team:
+```bash
+datachain job ls --team my-team
+```
+
+3. List jobs with a specific status:
+```bash
+datachain job ls --status complete
+```
+
+4. List more jobs by increasing the limit:
+```bash
+datachain job ls --limit 50
+```
+
+5. List jobs with verbose output:
+```bash
+datachain job ls -v
+```
+
+## Notes
+
+* The default limit of 20 jobs helps manage the output size and performance
+* Jobs are typically listed in reverse chronological order (newest first)
+* Use the `--status` filter to find jobs in specific states (e.g., running, completed, failed)

{datachain-0.16.5 → datachain-0.17.1}/docs/examples.md

@@ -167,7 +167,7 @@ dialog-rating@v2
 By default, when a saved dataset is loaded, the latest version is fetched but another version can be requested:
 
 ```python
-ds = dc.read_dataset("dialog-rating", version=1)
+ds = dc.read_dataset("dialog-rating", version="1.0.0")
 ```
 
 ### Chain execution, optimization and parallelism

{datachain-0.16.5 → datachain-0.17.1}/mkdocs.yml

@@ -96,6 +96,7 @@ nav:
                   - run: commands/job/run.md
                   - logs: commands/job/logs.md
                   - cancel: commands/job/cancel.md
+                  - ls: commands/job/ls.md
       - 📡 Interacting with remote storage: references/remotes.md
       - 🤝 Contributing: contributing.md
 

{datachain-0.16.5 → datachain-0.17.1}/noxfile.py

@@ -90,6 +90,7 @@ def build(session: nox.Session) -> None:
 @nox.session(python=python_versions)
 def examples(session: nox.Session) -> None:
     session.install(".[examples]")
+    session.run("uv", "pip", "list")
     session.run(
         "pytest",
         "--durations=0",

{datachain-0.16.5 → datachain-0.17.1}/pyproject.toml

@@ -48,7 +48,7 @@ dependencies = [
   "Pillow>=10.0.0,<12",
   "msgpack>=1.0.4,<2",
   "psutil",
-  "huggingface_hub",
+  "huggingface_hub<0.31",  # fix for "Provider 'featherless-ai' not supported" error
   "iterative-telemetry>=0.0.10",
   "platformdirs",
   "dvc-studio-client>=0.21,<1",

{datachain-0.16.5 → datachain-0.17.1}/src/datachain/catalog/catalog.py

@@ -33,6 +33,7 @@ from datachain.cache import Cache
 from datachain.client import Client
 from datachain.dataset import (
     DATASET_PREFIX,
+    DEFAULT_DATASET_VERSION,
     QUERY_DATASET_PREFIX,
     DatasetDependency,
     DatasetListRecord,
@@ -154,9 +155,9 @@ class DatasetRowsFetcher(NodesThreadPool):
         metastore: "AbstractMetastore",
         warehouse: "AbstractWarehouse",
         remote_ds_name: str,
-        remote_ds_version: int,
+        remote_ds_version: str,
         local_ds_name: str,
-        local_ds_version: int,
+        local_ds_version: str,
         schema: dict[str, Union[SQLType, type[SQLType]]],
         max_threads: int = PULL_DATASET_MAX_THREADS,
         progress_bar=None,
@@ -286,7 +287,7 @@ class NodeGroup:
     # (not including the bucket name or s3:// prefix)
     source_path: str = ""
     dataset_name: Optional[str] = None
-    dataset_version: Optional[int] = None
+    dataset_version: Optional[str] = None
     instantiated_nodes: Optional[list[NodeWithPath]] = None
 
     @property
@@ -607,7 +608,7 @@ class Catalog:
         return lst, client, list_path
 
     def _remove_dataset_rows_and_warehouse_info(
-        self, dataset: DatasetRecord, version: int, **kwargs
+        self, dataset: DatasetRecord, version: str, **kwargs
     ):
         self.warehouse.drop_dataset_rows_table(dataset, version)
         self.update_dataset_version_with_warehouse_info(
@@ -767,7 +768,7 @@ class Catalog:
     def create_dataset(
         self,
         name: str,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         *,
         columns: Sequence[Column],
         feature_schema: Optional[dict] = None,
@@ -783,18 +784,17 @@ class Catalog:
         Creates new dataset of a specific version.
         If dataset is not yet created, it will create it with version 1
         If version is None, then next unused version is created.
-        If version is given, then it must be an unused version number.
+        If version is given, then it must be an unused version.
         """
         assert [c.name for c in columns if c.name != "sys__id"], f"got {columns=}"
         if not listing and Client.is_data_source_uri(name):
             raise RuntimeError(
                 "Cannot create dataset that starts with source prefix, e.g s3://"
             )
-        default_version = 1
+        default_version = DEFAULT_DATASET_VERSION
         try:
             dataset = self.get_dataset(name)
-            default_version = dataset.next_version
-
+            default_version = dataset.next_version_patch
             if (description or attrs) and (
                 dataset.description != description or dataset.attrs != attrs
             ):
@@ -846,7 +846,7 @@ class Catalog:
     def create_new_dataset_version(
         self,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         *,
         columns: Sequence[Column],
         sources="",
@@ -892,7 +892,7 @@ class Catalog:
         return dataset
 
     def update_dataset_version_with_warehouse_info(
-        self, dataset: DatasetRecord, version: int, rows_dropped=False, **kwargs
+        self, dataset: DatasetRecord, version: str, rows_dropped=False, **kwargs
     ) -> None:
         from datachain.query.dataset import DatasetQuery
 
@@ -959,7 +959,7 @@ class Catalog:
         return dataset
 
     def remove_dataset_version(
-        self, dataset: DatasetRecord, version: int, drop_rows: Optional[bool] = True
+        self, dataset: DatasetRecord, version: str, drop_rows: Optional[bool] = True
     ) -> None:
         """
         Deletes one single dataset version.
@@ -1037,82 +1037,11 @@ class Catalog:
 
         return self.get_dataset(name)
 
-    def register_dataset(
-        self,
-        dataset: DatasetRecord,
-        version: int,
-        target_dataset: DatasetRecord,
-        target_version: Optional[int] = None,
-    ) -> DatasetRecord:
-        """
-        Registers dataset version of one dataset as dataset version of another
-        one (it can be new version of existing one).
-        It also removes original dataset version
-        """
-        target_version = target_version or target_dataset.next_version
-
-        if not target_dataset.is_valid_next_version(target_version):
-            raise DatasetInvalidVersionError(
-                f"Version {target_version} must be higher than the current latest one"
-            )
-
-        dataset_version = dataset.get_version(version)
-        if not dataset_version:
-            raise DatasetVersionNotFoundError(
-                f"Dataset {dataset.name} does not have version {version}"
-            )
-
-        if not dataset_version.is_final_status():
-            raise ValueError("Cannot register dataset version in non final status")
-
-        # copy dataset version
-        target_dataset = self.metastore.create_dataset_version(
-            target_dataset,
-            target_version,
-            sources=dataset_version.sources,
-            status=dataset_version.status,
-            query_script=dataset_version.query_script,
-            error_message=dataset_version.error_message,
-            error_stack=dataset_version.error_stack,
-            script_output=dataset_version.script_output,
-            created_at=dataset_version.created_at,
-            finished_at=dataset_version.finished_at,
-            schema=dataset_version.serialized_schema,
-            num_objects=dataset_version.num_objects,
-            size=dataset_version.size,
-            preview=dataset_version.preview,
-            job_id=dataset_version.job_id,
-        )
-
-        # to avoid re-creating rows table, we are just renaming it for a new version
-        # of target dataset
-        self.warehouse.rename_dataset_table(
-            dataset.name,
-            target_dataset.name,
-            old_version=version,
-            new_version=target_version,
-        )
-        self.metastore.update_dataset_dependency_source(
-            dataset,
-            version,
-            new_source_dataset=target_dataset,
-            new_source_dataset_version=target_version,
-        )
-
-        if dataset.id == target_dataset.id:
-            # we are updating the same dataset so we need to refresh it to have newly
-            # added version in step before
-            dataset = self.get_dataset(dataset.name)
-
-        self.remove_dataset_version(dataset, version, drop_rows=False)
-
-        return self.get_dataset(target_dataset.name)
-
     def get_dataset(self, name: str) -> DatasetRecord:
         return self.metastore.get_dataset(name)
 
     def get_dataset_with_remote_fallback(
-        self, name: str, version: Optional[int] = None
+        self, name: str, version: Optional[str] = None
     ) -> DatasetRecord:
         try:
             ds = self.get_dataset(name)
@@ -1157,7 +1086,7 @@ class Catalog:
         return DatasetRecord.from_dict(dataset_info)
 
     def get_dataset_dependencies(
-        self, name: str, version: int, indirect=False
+        self, name: str, version: str, indirect=False
     ) -> list[Optional[DatasetDependency]]:
         dataset = self.get_dataset(name)
 
@@ -1175,7 +1104,7 @@ class Catalog:
             if d.is_dataset:
                 # only datasets can have dependencies
                 d.dependencies = self.get_dataset_dependencies(
-                    d.name, int(d.version), indirect=indirect
+                    d.name, d.version, indirect=indirect
                 )
 
         return direct_dependencies
@@ -1244,7 +1173,7 @@ class Catalog:
         ]
 
     def ls_dataset_rows(
-        self, name: str, version: int, offset=None, limit=None
+        self, name: str, version: str, offset=None, limit=None
     ) -> list[dict]:
         from datachain.query.dataset import DatasetQuery
 
@@ -1282,7 +1211,7 @@ class Catalog:
         self,
         bucket_uri: str,
         name: str,
-        version: int,
+        version: str,
         client_config=None,
     ) -> list[str]:
         dataset = self.get_dataset(name)
@@ -1291,14 +1220,14 @@ class Catalog:
             bucket_uri, dataset, version, client_config
         )
 
-    def dataset_table_export_file_names(self, name: str, version: int) -> list[str]:
+    def dataset_table_export_file_names(self, name: str, version: str) -> list[str]:
         dataset = self.get_dataset(name)
         return self.warehouse.dataset_table_export_file_names(dataset, version)
 
     def remove_dataset(
         self,
         name: str,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         force: Optional[bool] = False,
         studio: Optional[bool] = False,
     ):
@@ -1372,7 +1301,7 @@ class Catalog:
         remote_ds_uri: str,
         output: Optional[str] = None,
         local_ds_name: Optional[str] = None,
-        local_ds_version: Optional[int] = None,
+        local_ds_version: Optional[str] = None,
         cp: bool = False,
         force: bool = False,
         *,

{datachain-0.16.5 → datachain-0.17.1}/src/datachain/cli/commands/datasets.py

@@ -127,7 +127,7 @@ def _datasets_tabulate_row(name, both, local_version, studio_version):
 def rm_dataset(
     catalog: "Catalog",
     name: str,
-    version: Optional[int] = None,
+    version: Optional[str] = None,
     force: Optional[bool] = False,
     studio: bool = False,
     local: bool = False,

{datachain-0.16.5 → datachain-0.17.1}/src/datachain/cli/commands/show.py

@@ -10,7 +10,7 @@ if TYPE_CHECKING:
 def show(
     catalog: "Catalog",
     name: str,
-    version: Optional[int] = None,
+    version: Optional[str] = None,
     limit: int = 10,
     offset: int = 0,
     columns: Sequence[str] = (),

{datachain-0.16.5 → datachain-0.17.1}/src/datachain/cli/parser/__init__.py

@@ -302,7 +302,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--version",
         action="store",
         default=None,
-        type=int,
+        type=str,
         help="Dataset version",
     )
     rm_dataset_parser.add_argument(
@@ -495,7 +495,7 @@ def get_parser() -> ArgumentParser:  # noqa: PLR0915
         "--version",
         action="store",
         default=None,
-        type=int,
+        type=str,
         help="Dataset version",
     )
     show_parser.add_argument("--schema", action="store_true", help="Show schema")

{datachain-0.16.5 → datachain-0.17.1}/src/datachain/cli/parser/job.py

@@ -83,6 +83,36 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
         help="Python package requirements",
     )
 
+    studio_ls_help = "List jobs in Studio"
+    studio_ls_description = "List jobs in Studio."
+
+    studio_ls_parser = jobs_subparser.add_parser(
+        "ls",
+        parents=[parent_parser],
+        description=studio_ls_description,
+        help=studio_ls_help,
+        formatter_class=CustomHelpFormatter,
+    )
+
+    studio_ls_parser.add_argument(
+        "--status",
+        action="store",
+        help="Status to filter jobs by",
+    )
+
+    studio_ls_parser.add_argument(
+        "--team",
+        action="store",
+        default=None,
+        help="Team to list jobs for (default: from config)",
+    )
+    studio_ls_parser.add_argument(
+        "--limit",
+        type=int,
+        default=20,
+        help="Limit the number of jobs returned (default: 20)",
+    )
+
     studio_cancel_help = "Cancel a job in Studio"
     studio_cancel_description = "Cancel a running job in Studio."
 

{datachain-0.16.5 → datachain-0.17.1}/src/datachain/data_storage/metastore.py

@@ -128,7 +128,7 @@ class AbstractMetastore(ABC, Serializable):
     def create_dataset_version(  # noqa: PLR0913
         self,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         status: int,
         sources: str = "",
         feature_schema: Optional[dict] = None,
@@ -158,13 +158,13 @@ class AbstractMetastore(ABC, Serializable):
 
     @abstractmethod
     def update_dataset_version(
-        self, dataset: DatasetRecord, version: int, **kwargs
+        self, dataset: DatasetRecord, version: str, **kwargs
     ) -> DatasetVersion:
         """Updates dataset version fields."""
 
     @abstractmethod
     def remove_dataset_version(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> DatasetRecord:
         """
         Deletes one single dataset version.
@@ -188,7 +188,7 @@ class AbstractMetastore(ABC, Serializable):
         self,
         dataset: DatasetRecord,
         status: int,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         error_message="",
         error_stack="",
         script_output="",
@@ -202,9 +202,9 @@ class AbstractMetastore(ABC, Serializable):
     def add_dataset_dependency(
         self,
         source_dataset_name: str,
-        source_dataset_version: int,
+        source_dataset_version: str,
         dataset_name: str,
-        dataset_version: int,
+        dataset_version: str,
     ) -> None:
         """Adds dataset dependency to dataset."""
 
@@ -212,21 +212,21 @@ class AbstractMetastore(ABC, Serializable):
     def update_dataset_dependency_source(
         self,
         source_dataset: DatasetRecord,
-        source_dataset_version: int,
+        source_dataset_version: str,
         new_source_dataset: Optional[DatasetRecord] = None,
-        new_source_dataset_version: Optional[int] = None,
+        new_source_dataset_version: Optional[str] = None,
     ) -> None:
         """Updates dataset dependency source."""
 
     @abstractmethod
     def get_direct_dataset_dependencies(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> list[Optional[DatasetDependency]]:
         """Gets direct dataset dependencies."""
 
     @abstractmethod
     def remove_dataset_dependencies(
-        self, dataset: DatasetRecord, version: Optional[int] = None
+        self, dataset: DatasetRecord, version: Optional[str] = None
     ) -> None:
         """
         When we remove dataset, we need to clean up it's dependencies as well.
@@ -234,7 +234,7 @@ class AbstractMetastore(ABC, Serializable):
 
     @abstractmethod
     def remove_dataset_dependants(
-        self, dataset: DatasetRecord, version: Optional[int] = None
+        self, dataset: DatasetRecord, version: Optional[str] = None
     ) -> None:
         """
         When we remove dataset, we need to clear its references in other dataset
@@ -370,7 +370,7 @@ class AbstractDBMetastore(AbstractMetastore):
                 ForeignKey(f"{cls.DATASET_TABLE}.id", ondelete="CASCADE"),
                 nullable=False,
             ),
-            Column("version", Integer, nullable=False),
+            Column("version", Text, nullable=False, default="1.0.0"),
             Column(
                 "status",
                 Integer,
@@ -554,7 +554,7 @@ class AbstractDBMetastore(AbstractMetastore):
     def create_dataset_version(  # noqa: PLR0913
         self,
         dataset: DatasetRecord,
-        version: int,
+        version: str,
         status: int,
         sources: str = "",
         feature_schema: Optional[dict] = None,
@@ -648,7 +648,7 @@ class AbstractDBMetastore(AbstractMetastore):
         return result_ds
 
     def update_dataset_version(
-        self, dataset: DatasetRecord, version: int, conn=None, **kwargs
+        self, dataset: DatasetRecord, version: str, conn=None, **kwargs
     ) -> DatasetVersion:
         """Updates dataset fields."""
         dataset_version = dataset.get_version(version)
@@ -758,7 +758,7 @@ class AbstractDBMetastore(AbstractMetastore):
         return ds
 
     def remove_dataset_version(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> DatasetRecord:
         """
         Deletes one single dataset version.
@@ -791,7 +791,7 @@ class AbstractDBMetastore(AbstractMetastore):
         self,
         dataset: DatasetRecord,
         status: int,
-        version: Optional[int] = None,
+        version: Optional[str] = None,
         error_message="",
         error_stack="",
         script_output="",
@@ -825,9 +825,9 @@ class AbstractDBMetastore(AbstractMetastore):
     def add_dataset_dependency(
         self,
         source_dataset_name: str,
-        source_dataset_version: int,
+        source_dataset_version: str,
         dataset_name: str,
-        dataset_version: int,
+        dataset_version: str,
     ) -> None:
         """Adds dataset dependency to dataset."""
         source_dataset = self.get_dataset(source_dataset_name)
@@ -847,9 +847,9 @@ class AbstractDBMetastore(AbstractMetastore):
     def update_dataset_dependency_source(
         self,
         source_dataset: DatasetRecord,
-        source_dataset_version: int,
+        source_dataset_version: str,
         new_source_dataset: Optional[DatasetRecord] = None,
-        new_source_dataset_version: Optional[int] = None,
+        new_source_dataset_version: Optional[str] = None,
     ) -> None:
         dd = self._datasets_dependencies
 
@@ -880,7 +880,7 @@ class AbstractDBMetastore(AbstractMetastore):
         """
 
     def get_direct_dataset_dependencies(
-        self, dataset: DatasetRecord, version: int
+        self, dataset: DatasetRecord, version: str
     ) -> list[Optional[DatasetDependency]]:
         d = self._datasets
         dd = self._datasets_dependencies
@@ -909,7 +909,7 @@ class AbstractDBMetastore(AbstractMetastore):
         return [self.dependency_class.parse(*r) for r in self.db.execute(query)]
 
     def remove_dataset_dependencies(
-        self, dataset: DatasetRecord, version: Optional[int] = None
+        self, dataset: DatasetRecord, version: Optional[str] = None
     ) -> None:
         """
         When we remove dataset, we need to clean up it's dependencies as well
@@ -928,7 +928,7 @@ class AbstractDBMetastore(AbstractMetastore):
         self.db.execute(q)
 
     def remove_dataset_dependants(
-        self, dataset: DatasetRecord, version: Optional[int] = None
+        self, dataset: DatasetRecord, version: Optional[str] = None
     ) -> None:
         """
         When we remove dataset, we need to clear its references in other dataset