datachain 0.17.0__py3-none-any.whl → 0.17.2__py3-none-any.whl

datachain/cli/parser/job.py

@@ -83,6 +83,36 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
         help="Python package requirements",
     )
 
+    studio_ls_help = "List jobs in Studio"
+    studio_ls_description = "List jobs in Studio."
+
+    studio_ls_parser = jobs_subparser.add_parser(
+        "ls",
+        parents=[parent_parser],
+        description=studio_ls_description,
+        help=studio_ls_help,
+        formatter_class=CustomHelpFormatter,
+    )
+
+    studio_ls_parser.add_argument(
+        "--status",
+        action="store",
+        help="Status to filter jobs by",
+    )
+
+    studio_ls_parser.add_argument(
+        "--team",
+        action="store",
+        default=None,
+        help="Team to list jobs for (default: from config)",
+    )
+    studio_ls_parser.add_argument(
+        "--limit",
+        type=int,
+        default=20,
+        help="Limit the number of jobs returned (default: 20)",
+    )
+
     studio_cancel_help = "Cancel a job in Studio"
     studio_cancel_description = "Cancel a running job in Studio."
 

datachain/data_storage/warehouse.py

@@ -258,7 +258,7 @@ class AbstractWarehouse(ABC, Serializable):
         if Client.is_data_source_uri(dataset_name):
             # for datasets that are created for bucket listing we use different prefix
             prefix = self.DATASET_SOURCE_TABLE_PREFIX
-        return f"{prefix}{dataset_name}_{version}"
+        return f"{prefix}{dataset_name}_{version.replace('.', '_')}"
 
     def temp_table_name(self) -> str:
         return self.TMP_TABLE_NAME_PREFIX + _random_string(6)

datachain/lib/dc/values.py

@@ -42,6 +42,8 @@ def read_values(
     def _func_fr() -> Iterator[tuple_type]:  # type: ignore[valid-type]
         yield from tuples
 
+    _func_fr.__name__ = "read_values"
+
     chain = read_records(
         DataChain.DEFAULT_FILE_RECORD,
         session=session,

datachain/query/dispatch.py

@@ -203,9 +203,10 @@ class UDFDispatcher:
         def get_inputs() -> Iterable["RowsOutput"]:
             warehouse = self.catalog.warehouse.clone()
             if ids_only:
-                yield from warehouse.dataset_rows_select_from_ids(
-                    self.query, input_rows, self.is_batching
-                )
+                for ids in batched(input_rows, DEFAULT_BATCH_SIZE):
+                    yield from warehouse.dataset_rows_select_from_ids(
+                        self.query, ids, self.is_batching
+                    )
             else:
                 yield from input_rows
 
@@ -425,8 +426,9 @@ class UDFWorker:
         warehouse = self.catalog.warehouse.clone()
         while (batch := get_from_queue(self.task_queue)) != STOP_SIGNAL:
             if ids_only:
-                yield from warehouse.dataset_rows_select_from_ids(
-                    self.query, batch, self.is_batching
-                )
+                for ids in batched(batch, DEFAULT_BATCH_SIZE):
+                    yield from warehouse.dataset_rows_select_from_ids(
+                        self.query, ids, self.is_batching
+                    )
             else:
                 yield from batch

datachain/remote/studio.py

@@ -29,7 +29,7 @@ DatasetExportStatus = Optional[dict[str, Any]]
 DatasetExportSignedUrls = Optional[list[str]]
 FileUploadData = Optional[dict[str, Any]]
 JobData = Optional[dict[str, Any]]
-
+JobListData = dict[str, Any]
 logger = logging.getLogger("datachain")
 
 DATASET_ROWS_CHUNK_SIZE = 8192
@@ -402,6 +402,17 @@ class StudioClient:
         }
         return self._send_request("datachain/job", data)
 
+    def get_jobs(
+        self,
+        status: Optional[str] = None,
+        limit: int = 20,
+    ) -> Response[JobListData]:
+        return self._send_request(
+            "datachain/jobs",
+            {"status": status, "limit": limit} if status else {"limit": limit},
+            method="GET",
+        )
+
     def cancel_job(
         self,
         job_id: str,

datachain/studio.py

@@ -3,6 +3,8 @@ import os
 import sys
 from typing import TYPE_CHECKING, Optional
 
+import tabulate
+
 from datachain.config import Config, ConfigLevel
 from datachain.dataset import QUERY_DATASET_PREFIX
 from datachain.error import DataChainError
@@ -44,6 +46,10 @@ def process_jobs_args(args: "Namespace"):
         return cancel_job(args.id, args.team)
     if args.cmd == "logs":
         return show_job_logs(args.id, args.team)
+
+    if args.cmd == "ls":
+        return list_jobs(args.status, args.team, args.limit)
+
     raise DataChainError(f"Unknown command '{args.cmd}'.")
 
 
@@ -240,13 +246,13 @@ def show_logs_from_client(client, job_id):
         raise DataChainError(response.message)
 
     response_data = response.data
-    if response_data:
+    if response_data and response_data.get("dataset_versions"):
         dataset_versions = response_data.get("dataset_versions", [])
         print("\n\n>>>> Dataset versions created during the job:")
         for version in dataset_versions:
             print(f"    - {version.get('dataset_name')}@v{version.get('version')}")
     else:
-        print("No dataset versions created during the job.")
+        print("\n\nNo dataset versions created during the job.")
 
 
 def create_job(
@@ -337,6 +343,31 @@ def cancel_job(job_id: str, team_name: Optional[str]):
     print(f"Job {job_id} canceled")
 
 
+def list_jobs(status: Optional[str], team_name: Optional[str], limit: int):
+    client = StudioClient(team=team_name)
+    response = client.get_jobs(status, limit)
+    if not response.ok:
+        raise DataChainError(response.message)
+
+    jobs = response.data.get("jobs", [])
+    if not jobs:
+        print("No jobs found")
+        return
+
+    rows = [
+        {
+            "ID": job.get("id"),
+            "Name": job.get("name"),
+            "Status": job.get("status"),
+            "Created at": job.get("created_at"),
+            "Created by": job.get("created_by"),
+        }
+        for job in jobs
+    ]
+
+    print(tabulate.tabulate(rows, headers="keys", tablefmt="grid"))
+
+
 def show_job_logs(job_id: str, team_name: Optional[str]):
     token = Config().read().get("studio", {}).get("token")
     if not token:

{datachain-0.17.0.dist-info → datachain-0.17.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.17.0
+Version: 0.17.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -44,7 +44,7 @@ Requires-Dist: datamodel-code-generator>=0.25
 Requires-Dist: Pillow<12,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
-Requires-Dist: huggingface_hub
+Requires-Dist: huggingface_hub<0.31
 Requires-Dist: iterative-telemetry>=0.0.10
 Requires-Dist: platformdirs
 Requires-Dist: dvc-studio-client<1,>=0.21

{datachain-0.17.0.dist-info → datachain-0.17.2.dist-info}/RECORD

@@ -14,7 +14,7 @@ datachain/progress.py,sha256=lRzxoYP4Qv2XBwD78sOkmYRzHFpZ2ExVNJF8wAeICtY,770
 datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/script_meta.py,sha256=V-LaFOZG84pD0Zc0NvejYdzwDgzITv6yHvAHggDCnuY,4978
 datachain/semver.py,sha256=t_3Y5OGLEthrstBwuwrf5pXVquEuRFu3ZoGe3ajfJB8,1715
-datachain/studio.py,sha256=kUNm7iUSMKWIkZs8w_b9VrcK45YrsY91dEe6kPz8AWE,10113
+datachain/studio.py,sha256=5MSDk-pM2Na0TlK0eCStmZ47hNBvlY2WPKzCQFmBPpM,10948
 datachain/telemetry.py,sha256=0A4IOPPp9VlP5pyW9eBfaTK3YhHGzHl7dQudQjUAx9A,994
 datachain/utils.py,sha256=DNqOi-Ydb7InyWvD9m7_yailxz6-YGpZzh00biQaHNo,15305
 datachain/catalog/__init__.py,sha256=cMZzSz3VoUi-6qXSVaHYN-agxQuAcz2XSqnEPZ55crE,353
@@ -32,7 +32,7 @@ datachain/cli/commands/misc.py,sha256=c0DmkOLwcDI2YhA8ArOuLJk6aGzSMZCiKL_E2JGibV
 datachain/cli/commands/query.py,sha256=Xzfgh14nPVH-sclqX1tpZqgfdTugw5s_44v0D33z6FA,1505
 datachain/cli/commands/show.py,sha256=Cf8wBs12h-xtdOzjU5GTDy2C8rF5HJSF0hDJYER1zH8,1606
 datachain/cli/parser/__init__.py,sha256=sjCIcosAtZqa0m50GMQHqmCkZSYxKyZNwQ29XwRQlP0,15913
-datachain/cli/parser/job.py,sha256=1TfM8qkGn0e-zixnn40Z5-Pv9h_mZncPaDAOyzjIf5k,3601
+datachain/cli/parser/job.py,sha256=dpRY6KQgooCXhQGQiebPuNuGouKk5jclkpafpQ2TDvQ,4373
 datachain/cli/parser/studio.py,sha256=Y-1OlQGecLVi9QofvWUfSlPd2ISyaESf7QFGZqGsrdw,3609
 datachain/cli/parser/utils.py,sha256=rETdD-9Hq9A4OolgfT7jQw4aoawtbfmkdtH6E7nkhpI,2888
 datachain/client/__init__.py,sha256=1kDpCPoibMXi1gExR4lTLc5pi-k6M5TANiwtXkPoLhU,49
@@ -50,7 +50,7 @@ datachain/data_storage/metastore.py,sha256=vo2ab-U_-BKfeFYTmvpbCoMyMZEVxrVqM9Djj
 datachain/data_storage/schema.py,sha256=asZYz1cg_WKfe2Q-k5W51E2z2CzHU5B4QEDZDMFr8yo,9346
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
 datachain/data_storage/sqlite.py,sha256=bwZAB_NUMT2WMv5tPQnnLFA0P-PiQtxzSaQ1q6xDxOU,24590
-datachain/data_storage/warehouse.py,sha256=3MFurg3Gm24xalWI03e_uHkznPxfhzTXjX7AhrKMiXQ,31516
+datachain/data_storage/warehouse.py,sha256=RkdX1cunfmpDkRYRdOGNy0kLw7RekIokVl3Dd0i-hrA,31534
 datachain/diff/__init__.py,sha256=YkGdiDbZIMhAZ2SJ4eSe00HU67VP1P6SL2L_t0ODYMs,9425
 datachain/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/fs/reference.py,sha256=A8McpXF0CqbXPqanXuvpKu50YLB3a2ZXA3YAPxtBXSM,914
@@ -108,7 +108,7 @@ datachain/lib/dc/parquet.py,sha256=zYcSgrWwyEDW9UxGUSVdIVsCu15IGEf0xL8KfWQqK94,1
 datachain/lib/dc/records.py,sha256=J1I69J2gFIBjRTGr2LG-5qn_rTVzRLcr2y3tVDrmHdg,3068
 datachain/lib/dc/storage.py,sha256=Z0rqBNoss8RVHo8I3Bpc5hJBLTJfmJHRbV0SA_tR5LI,5635
 datachain/lib/dc/utils.py,sha256=VawOAlJSvAtZbsMg33s5tJe21TRx1Km3QggI1nN6tnw,3984
-datachain/lib/dc/values.py,sha256=cBQubhmPNEDMJldUXzGh-UKbdim4P6O2B91Gp39roKw,1389
+datachain/lib/dc/values.py,sha256=7l1n352xWrEdql2NhBcZ3hj8xyPglWiY4qHjFPjn6iw,1428
 datachain/model/__init__.py,sha256=R9faX5OHV1xh2EW-g2MPedwbtEqt3LodJRyluB-QylI,189
 datachain/model/bbox.py,sha256=cQNHuQuVsh6bW3n3Hj40F2Cc20cExQ9Lg_q7R2jxUMI,9324
 datachain/model/pose.py,sha256=rjquA6M-I-Y30Xm6YSkGv1OY52hJZmR2AuxbIpE5uD0,3865
@@ -121,7 +121,7 @@ datachain/model/ultralytics/segment.py,sha256=koq1HASo29isf0in6oSlzmU4IzsmOXe87F
 datachain/query/__init__.py,sha256=7DhEIjAA8uZJfejruAVMZVcGFmvUpffuZJwgRqNwe-c,263
 datachain/query/batch.py,sha256=-goxLpE0EUvaDHu66rstj53UnfHpYfBUGux8GSpJ93k,4306
 datachain/query/dataset.py,sha256=v9xVYyft06KNvsPrA-j59krZXwRAunKvyYyJsINZpxM,59114
-datachain/query/dispatch.py,sha256=xNAZ5HNx48vY8Azhh3j-H_WSeAlRTQKNWKJbOAEhyPY,15398
+datachain/query/dispatch.py,sha256=15M3zlTUFKM6D2ijITX4o5QxCkRe2klkODsIDi3aQOg,15544
 datachain/query/metrics.py,sha256=DOK5HdNVaRugYPjl8qnBONvTkwjMloLqAr7Mi3TjCO0,858
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
 datachain/query/queue.py,sha256=v0UeK4ilmdiRoJ5OdjB5qpnHTYDxRP4vhVp5Iw_toaI,3512
@@ -130,7 +130,7 @@ datachain/query/session.py,sha256=3nyOvPmLiA86IdHc3BL6Dt_himtHVvaDz_I1h3hZ_gI,65
 datachain/query/udf.py,sha256=e753bDJzTNjGFQn1WGTvOAWSwjDbrFI1-_DDWkWN2ls,1343
 datachain/query/utils.py,sha256=HaSDNH_XGvp_NIcXjcB7j4vJRPi4_tbztDWclYelHY4,1208
 datachain/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/remote/studio.py,sha256=opSos4cG7GdPIJtIuNifYHdn-IpWP8lvN49a_rEuB1Y,13187
+datachain/remote/studio.py,sha256=4oypHea2NYB7br_BJTYPAmh3FH8KV835mq01Pwexrrg,13524
 datachain/sql/__init__.py,sha256=6SQRdbljO3d2hx3EAVXEZrHQKv5jth0Jh98PogT59No,262
 datachain/sql/selectable.py,sha256=cTc60qVoAwqqss0Vop8Lt5Z-ROnM1XrQmL_GLjRxhXs,1765
 datachain/sql/types.py,sha256=ASSPkmM5EzdRindqj2O7WHLXq8VHAgFYedG8lYfGvVI,14045
@@ -152,9 +152,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.17.0.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.17.0.dist-info/METADATA,sha256=P5M0UjmMc5FrB37aNkR3qS3hoNwfT8ItegWSAuiOhOU,11331
-datachain-0.17.0.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-datachain-0.17.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.17.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.17.0.dist-info/RECORD,,
+datachain-0.17.2.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.17.2.dist-info/METADATA,sha256=JEQUpUwLjYtMDRL9nRObwbo3bnYo2MNgQVomGf-JBpI,11336
+datachain-0.17.2.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+datachain-0.17.2.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.17.2.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.17.2.dist-info/RECORD,,

{datachain-0.17.0.dist-info → datachain-0.17.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.4.0)
 Root-Is-Purelib: true
 Tag: py3-none-any