datachain 0.26.3__py3-none-any.whl → 0.26.4__py3-none-any.whl

datachain/cli/parser/job.py

@@ -17,7 +17,12 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
     )
 
     studio_run_help = "Run a job in Studio"
-    studio_run_description = "Run a job in Studio."
+    studio_run_description = "Run a job in Studio. \n"
+    studio_run_description += (
+        "When using --start-time or --cron,"
+        " the job is scheduled as a task and will not show logs immediately."
+        " The job will be executed according to the schedule."
+    )
 
     studio_run_parser = jobs_subparser.add_parser(
         "run",
@@ -96,6 +101,14 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
         help="Priority for the job in range 0-5. "
         "Lower value is higher priority (default: 5)",
     )
+    studio_run_parser.add_argument(
+        "--start-time",
+        action="store",
+        help="Start time in ISO format or natural language for the cron task.",
+    )
+    studio_run_parser.add_argument(
+        "--cron", action="store", help="Cron expression for the cron task."
+    )
 
     studio_ls_help = "List jobs in Studio"
     studio_ls_description = "List jobs in Studio."

datachain/lib/dc/datachain.py

@@ -2388,7 +2388,7 @@ class DataChain:
         placement: FileExportPlacement = "fullpath",
         link_type: Literal["copy", "symlink"] = "copy",
         num_threads: Optional[int] = EXPORT_FILES_MAX_THREADS,
-        anon: bool = False,
+        anon: Optional[bool] = None,
         client_config: Optional[dict] = None,
     ) -> None:
         """Export files from a specified signal to a directory. Files can be
@@ -2403,7 +2403,11 @@ class DataChain:
                 Falls back to `'copy'` if symlinking fails.
             num_threads : number of threads to use for exporting files.
                 By default it uses 5 threads.
-            anon: If true, we will treat cloud bucket as public one
+            anon: If True, we will treat cloud bucket as public one. Default behavior
+                depends on the previous session configuration (e.g. happens in the
+                initial `read_storage`) and particular cloud storage client
+                implementation (e.g. S3 fallbacks to anonymous access if no credentials
+                were found).
             client_config: Optional configuration for the destination storage client
 
         Example:
@@ -2421,8 +2425,8 @@ class DataChain:
         ):
             raise ValueError("Files with the same name found")
 
-        if anon:
-            client_config = (client_config or {}) | {"anon": True}
+        if anon is not None:
+            client_config = (client_config or {}) | {"anon": anon}
 
         progress_bar = tqdm(
             desc=f"Exporting files to {output}: ",

datachain/lib/dc/storage.py

@@ -33,7 +33,7 @@ def read_storage(
     recursive: Optional[bool] = True,
     column: str = "file",
     update: bool = False,
-    anon: bool = False,
+    anon: Optional[bool] = None,
     delta: Optional[bool] = False,
     delta_on: Optional[Union[str, Sequence[str]]] = (
         "file.path",
@@ -124,8 +124,8 @@ def read_storage(
 
     file_type = get_file_type(type)
 
-    if anon:
-        client_config = (client_config or {}) | {"anon": True}
+    if anon is not None:
+        client_config = (client_config or {}) | {"anon": anon}
     session = Session.get(session, client_config=client_config, in_memory=in_memory)
     catalog = session.catalog
     cache = catalog.cache

datachain/lib/file.py

@@ -717,6 +717,23 @@ class ImageFile(File):
         destination = stringify_path(destination)
 
         client: Client = self._catalog.get_client(destination, **(client_config or {}))
+
+        # If format is not provided, determine it from the file extension
+        if format is None:
+            from pathlib import PurePosixPath
+
+            from PIL import Image as PilImage
+
+            ext = PurePosixPath(destination).suffix.lower()
+            format = PilImage.registered_extensions().get(ext)
+
+        if not format:
+            raise FileError(
+                f"Can't determine format for destination '{destination}'",
+                self.source,
+                self.path,
+            )
+
         with client.fs.open(destination, mode="wb") as f:
             self.read().save(f, format=format)
 

datachain/remote/studio.py

@@ -429,6 +429,8 @@ class StudioClient:
         repository: Optional[str] = None,
         priority: Optional[int] = None,
         cluster: Optional[str] = None,
+        start_time: Optional[str] = None,
+        cron: Optional[str] = None,
     ) -> Response[JobData]:
         data = {
             "query": query,
@@ -442,6 +444,8 @@ class StudioClient:
             "repository": repository,
             "priority": priority,
             "compute_cluster_name": cluster,
+            "start_after": start_time,
+            "cron_expression": cron,
         }
         return self._send_request("datachain/job", data)
 

datachain/studio.py

@@ -1,8 +1,10 @@
 import asyncio
 import os
 import sys
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Optional
 
+import dateparser
 import tabulate
 
 from datachain.config import Config, ConfigLevel
@@ -42,6 +44,8 @@ def process_jobs_args(args: "Namespace"):
             args.req_file,
             args.priority,
             args.cluster,
+            args.start_time,
+            args.cron,
         )
 
     if args.cmd == "cancel":
@@ -262,6 +266,31 @@ def save_config(hostname, token, level=ConfigLevel.GLOBAL):
     return config.config_file()
 
 
+def parse_start_time(start_time_str: Optional[str]) -> Optional[str]:
+    if not start_time_str:
+        return None
+
+    try:
+        # Parse the datetime string using dateparser
+        parsed_datetime = dateparser.parse(start_time_str)
+
+        if parsed_datetime is None:
+            raise DataChainError(
+                f"Could not parse datetime string: '{start_time_str}'. "
+                f"Supported formats include: '2024-01-15 14:30:00', 'tomorrow 3pm', "
+                f"'monday 9am', '2024-01-15T14:30:00Z', 'in 2 hours', etc."
+            )
+
+        # Convert to ISO format string
+        return parsed_datetime.isoformat()
+    except Exception as e:
+        raise DataChainError(
+            f"Invalid datetime format for start_time: '{start_time_str}'. "
+            f"Supported formats include: '2024-01-15 14:30:00', 'tomorrow 3pm', "
+            f"'monday 9am', '2024-01-15T14:30:00Z', 'in 2 hours', etc. Error: {e}"
+        ) from e
+
+
 def show_logs_from_client(client, job_id):
     # Sync usage
     async def _run():
@@ -310,6 +339,8 @@ def create_job(
     req_file: Optional[str] = None,
     priority: Optional[int] = None,
     cluster: Optional[str] = None,
+    start_time: Optional[str] = None,
+    cron: Optional[str] = None,
 ):
     query_type = "PYTHON" if query_file.endswith(".py") else "SHELL"
     with open(query_file) as f:
@@ -328,6 +359,11 @@ def create_job(
     client = StudioClient(team=team_name)
     file_ids = upload_files(client, files) if files else []
 
+    # Parse start_time if provided
+    parsed_start_time = parse_start_time(start_time)
+    if cron and parsed_start_time is None:
+        parsed_start_time = datetime.now(timezone.utc).isoformat()
+
     response = client.create_job(
         query=query,
         query_type=query_type,
@@ -340,6 +376,8 @@ def create_job(
         requirements=requirements,
         priority=priority,
         cluster=cluster,
+        start_time=parsed_start_time,
+        cron=cron,
     )
     if not response.ok:
         raise DataChainError(response.message)
@@ -348,6 +386,11 @@ def create_job(
         raise DataChainError("Failed to create job")
 
     job_id = response.data.get("job", {}).get("id")
+
+    if parsed_start_time or cron:
+        print(f"Job {job_id} is scheduled as a task in Studio.")
+        return 0
+
     print(f"Job {job_id} created")
     print("Open the job in Studio at", response.data.get("job", {}).get("url"))
     print("=" * 40)

{datachain-0.26.3.dist-info → datachain-0.26.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.26.3
+Version: 0.26.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -26,6 +26,7 @@ Requires-Dist: packaging
 Requires-Dist: pyarrow
 Requires-Dist: typing-extensions
 Requires-Dist: python-dateutil>=2
+Requires-Dist: dateparser>=1.0.0
 Requires-Dist: attrs>=21.3.0
 Requires-Dist: fsspec>=2024.2.0
 Requires-Dist: s3fs>=2024.2.0
@@ -100,6 +101,7 @@ Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
 Requires-Dist: mypy==1.17.0; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: types-dateparser; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"

{datachain-0.26.3.dist-info → datachain-0.26.4.dist-info}/RECORD

@@ -17,7 +17,7 @@ datachain/project.py,sha256=90D4GpJSA3t0fayYZbzrL3sk4U7EJhQo8psnWvdI7_o,2280
 datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/script_meta.py,sha256=V-LaFOZG84pD0Zc0NvejYdzwDgzITv6yHvAHggDCnuY,4978
 datachain/semver.py,sha256=UB8GHPBtAP3UJGeiuJoInD7SK-DnB93_Xd1qy_CQ9cU,2074
-datachain/studio.py,sha256=bLok-eJNFRHQScEyAyA_Fas52dmijd5r-73KudWxV4k,13337
+datachain/studio.py,sha256=w5RyntqSl6qOs2mbw4Dc7SpZNNEN97xpvjxfJL0rO7M,14850
 datachain/telemetry.py,sha256=0A4IOPPp9VlP5pyW9eBfaTK3YhHGzHl7dQudQjUAx9A,994
 datachain/utils.py,sha256=DNqOi-Ydb7InyWvD9m7_yailxz6-YGpZzh00biQaHNo,15305
 datachain/catalog/__init__.py,sha256=cMZzSz3VoUi-6qXSVaHYN-agxQuAcz2XSqnEPZ55crE,353
@@ -35,7 +35,7 @@ datachain/cli/commands/misc.py,sha256=c0DmkOLwcDI2YhA8ArOuLJk6aGzSMZCiKL_E2JGibV
 datachain/cli/commands/query.py,sha256=Xzfgh14nPVH-sclqX1tpZqgfdTugw5s_44v0D33z6FA,1505
 datachain/cli/commands/show.py,sha256=Cf8wBs12h-xtdOzjU5GTDy2C8rF5HJSF0hDJYER1zH8,1606
 datachain/cli/parser/__init__.py,sha256=NPB6ssP4CCt7G1SWZ_8oNQEH2C1lktWgkyHYXDQJZNc,15073
-datachain/cli/parser/job.py,sha256=_wqOOxGRXG_-xuQ35FaLUOwjw6w8HviWvoEpZZ7VBzI,5289
+datachain/cli/parser/job.py,sha256=2_g46bx_p7DnqZoYsXY2rHlB07BjBCuRPzpGP-Duk-s,5804
 datachain/cli/parser/studio.py,sha256=Bo__LKM7qhJGgkyX8M_bCvgZ2Gvqq6r_X4t1NdtaBIY,3881
 datachain/cli/parser/utils.py,sha256=rETdD-9Hq9A4OolgfT7jQw4aoawtbfmkdtH6E7nkhpI,2888
 datachain/client/__init__.py,sha256=1kDpCPoibMXi1gExR4lTLc5pi-k6M5TANiwtXkPoLhU,49
@@ -75,7 +75,7 @@ datachain/lib/audio.py,sha256=J7XJ14ItPF9y6pN-tmMV9In9X9rgwlBwzyzdGOUkPGk,4376
 datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
 datachain/lib/data_model.py,sha256=JPHPO6z-pehyiY-qNBAnp8u015xUHrijPKbGkMHS6lo,3493
 datachain/lib/dataset_info.py,sha256=7w-DoKOyIVoOtWGCgciMLcP5CiAWJB3rVI-vUDF80k0,3311
-datachain/lib/file.py,sha256=tHBBacsh1580UPFC6fAINBNwNiyymNgzj89rpsz1LKc,40817
+datachain/lib/file.py,sha256=vlSFsmj0ltvQWG6_isfWwNZt5u002bwrl70J2KbdvDE,41335
 datachain/lib/hf.py,sha256=dadHs2dsi4ALwXz92Y3T7AUgq3wQF4mBydWqHCMjvks,6880
 datachain/lib/image.py,sha256=erWvZW5M3emnbl6_fGAOPyKm-1EKbt3vOdWPfe3Oo7U,3265
 datachain/lib/listing.py,sha256=U-2stsTEwEsq4Y80dqGfktGzkmB5-ZntnL1_rzXlH0k,7089
@@ -104,7 +104,7 @@ datachain/lib/convert/values_to_tuples.py,sha256=j5yZMrVUH6W7b-7yUvdCTGI7JCUAYUO
 datachain/lib/dc/__init__.py,sha256=TFci5HTvYGjBesNUxDAnXaX36PnzPEUSn5a6JxB9o0U,872
 datachain/lib/dc/csv.py,sha256=q6a9BpapGwP6nwy6c5cklxQumep2fUp9l2LAjtTJr6s,4411
 datachain/lib/dc/database.py,sha256=g5M6NjYR1T0vKte-abV-3Ejnm-HqxTIMir5cRi_SziE,6051
-datachain/lib/dc/datachain.py,sha256=ap54lcuj71tvp0zX1jiFFiEWvA5UPeyYJRJkd2APmlI,92897
+datachain/lib/dc/datachain.py,sha256=mLE5v4KhzEQm7HVWBTxY6EwJ2J-YeFVcLUY4I21216c,93212
 datachain/lib/dc/datasets.py,sha256=P6CIJizD2IYFwOQG5D3VbQRjDmUiRH0ysdtb551Xdm8,15098
 datachain/lib/dc/hf.py,sha256=MJWO-NL4jAD6CEAmXsyeqXEyvefRLMhyxhT9jKT5vMU,2324
 datachain/lib/dc/json.py,sha256=dNijfJ-H92vU3soyR7X1IiDrWhm6yZIGG3bSnZkPdAE,2733
@@ -112,7 +112,7 @@ datachain/lib/dc/listings.py,sha256=V379Cb-7ZyquM0w7sWArQZkzInZy4GB7QQ1ZfowKzQY,
 datachain/lib/dc/pandas.py,sha256=ObueUXDUFKJGu380GmazdG02ARpKAHPhSaymfmOH13E,1489
 datachain/lib/dc/parquet.py,sha256=zYcSgrWwyEDW9UxGUSVdIVsCu15IGEf0xL8KfWQqK94,1782
 datachain/lib/dc/records.py,sha256=FpPbApWopUri1gIaSMsfXN4fevja4mjmfb6Q5eiaGxI,3116
-datachain/lib/dc/storage.py,sha256=8xiV3c6k-sG14RGwNJCp0AbV6L0mNDsTVZ-Est-ccnw,7672
+datachain/lib/dc/storage.py,sha256=FXroEdxOZfbuEBIWfWTkbGwrI0D4_mrLZSRsIQm0WFE,7693
 datachain/lib/dc/utils.py,sha256=VawOAlJSvAtZbsMg33s5tJe21TRx1Km3QggI1nN6tnw,3984
 datachain/lib/dc/values.py,sha256=7l1n352xWrEdql2NhBcZ3hj8xyPglWiY4qHjFPjn6iw,1428
 datachain/model/__init__.py,sha256=R9faX5OHV1xh2EW-g2MPedwbtEqt3LodJRyluB-QylI,189
@@ -136,7 +136,7 @@ datachain/query/session.py,sha256=gKblltJAVQAVSTswAgWGDgGbpmFlFzFVkIQojDCjgXM,68
 datachain/query/udf.py,sha256=e753bDJzTNjGFQn1WGTvOAWSwjDbrFI1-_DDWkWN2ls,1343
 datachain/query/utils.py,sha256=HaSDNH_XGvp_NIcXjcB7j4vJRPi4_tbztDWclYelHY4,1208
 datachain/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/remote/studio.py,sha256=oJp2KD9eO8zQDnPfNpAALZYsOlBfqVKKRTeCkEpcsYk,15196
+datachain/remote/studio.py,sha256=vsuqCAO65PBJKGLMxOvc3Bmieo2TJwcfc9YclxkzmFk,15350
 datachain/sql/__init__.py,sha256=6SQRdbljO3d2hx3EAVXEZrHQKv5jth0Jh98PogT59No,262
 datachain/sql/selectable.py,sha256=cTc60qVoAwqqss0Vop8Lt5Z-ROnM1XrQmL_GLjRxhXs,1765
 datachain/sql/types.py,sha256=ASSPkmM5EzdRindqj2O7WHLXq8VHAgFYedG8lYfGvVI,14045
@@ -158,9 +158,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.26.3.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.26.3.dist-info/METADATA,sha256=HdG_quEq0rfrdKJJ_teSViVCXbXI3SxLlnh6tu2Mgfs,13543
-datachain-0.26.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.26.3.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.26.3.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.26.3.dist-info/RECORD,,
+datachain-0.26.4.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.26.4.dist-info/METADATA,sha256=oWaaj_Avr95dDdM_txeheiOefsoHuXTu0QR71hTN634,13624
+datachain-0.26.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.26.4.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.26.4.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.26.4.dist-info/RECORD,,