datachain 0.17.0__tar.gz → 0.17.2__tar.gz

{datachain-0.17.0/src/datachain.egg-info → datachain-0.17.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.17.0
+Version: 0.17.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -44,7 +44,7 @@ Requires-Dist: datamodel-code-generator>=0.25
 Requires-Dist: Pillow<12,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
-Requires-Dist: huggingface_hub
+Requires-Dist: huggingface_hub<0.31
 Requires-Dist: iterative-telemetry>=0.0.10
 Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21

datachain-0.17.2/docs/commands/job/ls.md

@@ -0,0 +1,84 @@
+# job ls
+
+List jobs in Studio.
+
+## Synopsis
+
+```usage
+usage: datachain job ls [-h] [-v] [-q] [--status STATUS] [--team TEAM] [--limit LIMIT]
+```
+
+## Description
+
+This command lists jobs in Studio. You can filter jobs by their status, specify a team, and limit the number of jobs returned. By default, it shows the 20 most recent jobs.
+
+
+## Options
+
+* `--status STATUS` - Status to filter jobs by
+* `--team TEAM` - Team to list jobs for (default: from config)
+* `--limit LIMIT` - Limit the number of jobs returned (default: 20)
+* `-h`, `--help` - Show the help message and exit
+* `-v`, `--verbose` - Be verbose
+* `-q`, `--quiet` - Be quiet
+
+## Status options
+
+You will be able to filter the job with following status:
+
+* `CREATED` - Job has been created but not yet scheduled
+* `SCHEDULED` - Job is scheduled to run at a future time
+* `QUEUED` - Job is in the queue waiting to be executed
+* `INIT` - Job is initializing and preparing to run
+* `RUNNING` - Job is currently executing
+* `COMPLETE` - Job has finished successfully
+* `FAILED` - Job has failed during execution
+* `CANCELING_SCHEDULED` - A scheduled job is being canceled
+* `CANCELING` - A running job is being canceled
+* `CANCELED` - Job has been canceled
+* `ACTIVE` - Job is in active state.
+* `INACTIVE` - Job is in inactive state.
+
+Note: The following statuses are considered active jobs:
+
+* `CREATED`
+* `SCHEDULED`
+* `QUEUED`
+* `INIT`
+* `RUNNING`
+* `CANCELING_SCHEDULED`
+* `CANCELING`
+
+
+## Examples
+
+1. List all jobs (default limit of 20):
+```bash
+datachain job ls
+```
+
+2. List jobs for a specific team:
+```bash
+datachain job ls --team my-team
+```
+
+3. List jobs with a specific status:
+```bash
+datachain job ls --status complete
+```
+
+4. List more jobs by increasing the limit:
+```bash
+datachain job ls --limit 50
+```
+
+5. List jobs with verbose output:
+```bash
+datachain job ls -v
+```
+
+## Notes
+
+* The default limit of 20 jobs helps manage the output size and performance
+* Jobs are typically listed in reverse chronological order (newest first)
+* Use the `--status` filter to find jobs in specific states (e.g., running, completed, failed)

{datachain-0.17.0 → datachain-0.17.2}/mkdocs.yml

@@ -96,6 +96,7 @@ nav:
                   - run: commands/job/run.md
                   - logs: commands/job/logs.md
                   - cancel: commands/job/cancel.md
+                  - ls: commands/job/ls.md
       - 📡 Interacting with remote storage: references/remotes.md
       - 🤝 Contributing: contributing.md
 

{datachain-0.17.0 → datachain-0.17.2}/noxfile.py

@@ -90,6 +90,7 @@ def build(session: nox.Session) -> None:
 @nox.session(python=python_versions)
 def examples(session: nox.Session) -> None:
     session.install(".[examples]")
+    session.run("uv", "pip", "list")
     session.run(
         "pytest",
         "--durations=0",

{datachain-0.17.0 → datachain-0.17.2}/pyproject.toml

@@ -48,7 +48,7 @@ dependencies = [
   "Pillow>=10.0.0,<12",
   "msgpack>=1.0.4,<2",
   "psutil",
-  "huggingface_hub",
+  "huggingface_hub<0.31",  # fix for "Provider 'featherless-ai' not supported" error
   "iterative-telemetry>=0.0.10",
   "platformdirs",
   "dvc-studio-client>=0.21,<1",

{datachain-0.17.0 → datachain-0.17.2}/src/datachain/cli/parser/job.py

@@ -83,6 +83,36 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
         help="Python package requirements",
     )
 
+    studio_ls_help = "List jobs in Studio"
+    studio_ls_description = "List jobs in Studio."
+
+    studio_ls_parser = jobs_subparser.add_parser(
+        "ls",
+        parents=[parent_parser],
+        description=studio_ls_description,
+        help=studio_ls_help,
+        formatter_class=CustomHelpFormatter,
+    )
+
+    studio_ls_parser.add_argument(
+        "--status",
+        action="store",
+        help="Status to filter jobs by",
+    )
+
+    studio_ls_parser.add_argument(
+        "--team",
+        action="store",
+        default=None,
+        help="Team to list jobs for (default: from config)",
+    )
+    studio_ls_parser.add_argument(
+        "--limit",
+        type=int,
+        default=20,
+        help="Limit the number of jobs returned (default: 20)",
+    )
+
     studio_cancel_help = "Cancel a job in Studio"
     studio_cancel_description = "Cancel a running job in Studio."
 

{datachain-0.17.0 → datachain-0.17.2}/src/datachain/data_storage/warehouse.py

@@ -258,7 +258,7 @@ class AbstractWarehouse(ABC, Serializable):
         if Client.is_data_source_uri(dataset_name):
             # for datasets that are created for bucket listing we use different prefix
             prefix = self.DATASET_SOURCE_TABLE_PREFIX
-        return f"{prefix}{dataset_name}_{version}"
+        return f"{prefix}{dataset_name}_{version.replace('.', '_')}"
 
     def temp_table_name(self) -> str:
         return self.TMP_TABLE_NAME_PREFIX + _random_string(6)

{datachain-0.17.0 → datachain-0.17.2}/src/datachain/lib/dc/values.py

@@ -42,6 +42,8 @@ def read_values(
     def _func_fr() -> Iterator[tuple_type]:  # type: ignore[valid-type]
         yield from tuples
 
+    _func_fr.__name__ = "read_values"
+
     chain = read_records(
         DataChain.DEFAULT_FILE_RECORD,
         session=session,

{datachain-0.17.0 → datachain-0.17.2}/src/datachain/query/dispatch.py

@@ -203,9 +203,10 @@ class UDFDispatcher:
         def get_inputs() -> Iterable["RowsOutput"]:
             warehouse = self.catalog.warehouse.clone()
             if ids_only:
-                yield from warehouse.dataset_rows_select_from_ids(
-                    self.query, input_rows, self.is_batching
-                )
+                for ids in batched(input_rows, DEFAULT_BATCH_SIZE):
+                    yield from warehouse.dataset_rows_select_from_ids(
+                        self.query, ids, self.is_batching
+                    )
             else:
                 yield from input_rows
 
@@ -425,8 +426,9 @@ class UDFWorker:
         warehouse = self.catalog.warehouse.clone()
         while (batch := get_from_queue(self.task_queue)) != STOP_SIGNAL:
             if ids_only:
-                yield from warehouse.dataset_rows_select_from_ids(
-                    self.query, batch, self.is_batching
-                )
+                for ids in batched(batch, DEFAULT_BATCH_SIZE):
+                    yield from warehouse.dataset_rows_select_from_ids(
+                        self.query, ids, self.is_batching
+                    )
             else:
                 yield from batch

{datachain-0.17.0 → datachain-0.17.2}/src/datachain/remote/studio.py

@@ -29,7 +29,7 @@ DatasetExportStatus = Optional[dict[str, Any]]
 DatasetExportSignedUrls = Optional[list[str]]
 FileUploadData = Optional[dict[str, Any]]
 JobData = Optional[dict[str, Any]]
-
+JobListData = dict[str, Any]
 logger = logging.getLogger("datachain")
 
 DATASET_ROWS_CHUNK_SIZE = 8192
@@ -402,6 +402,17 @@ class StudioClient:
         }
         return self._send_request("datachain/job", data)
 
+    def get_jobs(
+        self,
+        status: Optional[str] = None,
+        limit: int = 20,
+    ) -> Response[JobListData]:
+        return self._send_request(
+            "datachain/jobs",
+            {"status": status, "limit": limit} if status else {"limit": limit},
+            method="GET",
+        )
+
     def cancel_job(
         self,
         job_id: str,

{datachain-0.17.0 → datachain-0.17.2}/src/datachain/studio.py

@@ -3,6 +3,8 @@ import os
 import sys
 from typing import TYPE_CHECKING, Optional
 
+import tabulate
+
 from datachain.config import Config, ConfigLevel
 from datachain.dataset import QUERY_DATASET_PREFIX
 from datachain.error import DataChainError
@@ -44,6 +46,10 @@ def process_jobs_args(args: "Namespace"):
         return cancel_job(args.id, args.team)
     if args.cmd == "logs":
         return show_job_logs(args.id, args.team)
+
+    if args.cmd == "ls":
+        return list_jobs(args.status, args.team, args.limit)
+
     raise DataChainError(f"Unknown command '{args.cmd}'.")
 
 
@@ -240,13 +246,13 @@ def show_logs_from_client(client, job_id):
         raise DataChainError(response.message)
 
     response_data = response.data
-    if response_data:
+    if response_data and response_data.get("dataset_versions"):
         dataset_versions = response_data.get("dataset_versions", [])
         print("\n\n>>>> Dataset versions created during the job:")
         for version in dataset_versions:
             print(f"    - {version.get('dataset_name')}@v{version.get('version')}")
     else:
-        print("No dataset versions created during the job.")
+        print("\n\nNo dataset versions created during the job.")
 
 
 def create_job(
@@ -337,6 +343,31 @@ def cancel_job(job_id: str, team_name: Optional[str]):
     print(f"Job {job_id} canceled")
 
 
+def list_jobs(status: Optional[str], team_name: Optional[str], limit: int):
+    client = StudioClient(team=team_name)
+    response = client.get_jobs(status, limit)
+    if not response.ok:
+        raise DataChainError(response.message)
+
+    jobs = response.data.get("jobs", [])
+    if not jobs:
+        print("No jobs found")
+        return
+
+    rows = [
+        {
+            "ID": job.get("id"),
+            "Name": job.get("name"),
+            "Status": job.get("status"),
+            "Created at": job.get("created_at"),
+            "Created by": job.get("created_by"),
+        }
+        for job in jobs
+    ]
+
+    print(tabulate.tabulate(rows, headers="keys", tablefmt="grid"))
+
+
 def show_job_logs(job_id: str, team_name: Optional[str]):
     token = Config().read().get("studio", {}).get("token")
     if not token:

{datachain-0.17.0 → datachain-0.17.2/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.17.0
+Version: 0.17.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -44,7 +44,7 @@ Requires-Dist: datamodel-code-generator>=0.25
 Requires-Dist: Pillow<12,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
-Requires-Dist: huggingface_hub
+Requires-Dist: huggingface_hub<0.31
 Requires-Dist: iterative-telemetry>=0.0.10
 Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21

{datachain-0.17.0 → datachain-0.17.2}/src/datachain.egg-info/SOURCES.txt

@@ -33,6 +33,7 @@ docs/commands/auth/team.md
 docs/commands/auth/token.md
 docs/commands/job/cancel.md
 docs/commands/job/logs.md
+docs/commands/job/ls.md
 docs/commands/job/run.md
 docs/css/github-permalink-style.css
 docs/overrides/main.html

{datachain-0.17.0 → datachain-0.17.2}/src/datachain.egg-info/requires.txt

@@ -25,7 +25,7 @@ datamodel-code-generator>=0.25
 Pillow<12,>=10.0.0
 msgpack<2,>=1.0.4
 psutil
-huggingface_hub
+huggingface_hub<0.31
 iterative-telemetry>=0.0.10
 platformdirs
 dvc-studio-client<1,>=0.21

{datachain-0.17.0 → datachain-0.17.2}/tests/func/test_warehouse.py

@@ -1,6 +1,6 @@
 def test_dataset_stats_no_table(cloud_test_catalog, dogs_dataset):
     catalog = cloud_test_catalog.catalog
-    catalog.warehouse.drop_dataset_rows_table(dogs_dataset, 1)
-    num_objects, size = catalog.warehouse.dataset_stats(dogs_dataset, 1)
+    catalog.warehouse.drop_dataset_rows_table(dogs_dataset, version="1.0.0")
+    num_objects, size = catalog.warehouse.dataset_stats(dogs_dataset, version="1.0.0")
     assert num_objects is None
     assert size is None