datachain 0.26.3__tar.gz → 0.26.4__tar.gz

{datachain-0.26.3 → datachain-0.26.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.26.3
+Version: 0.26.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -26,6 +26,7 @@ Requires-Dist: packaging
 Requires-Dist: pyarrow
 Requires-Dist: typing-extensions
 Requires-Dist: python-dateutil>=2
+Requires-Dist: dateparser>=1.0.0
 Requires-Dist: attrs>=21.3.0
 Requires-Dist: fsspec>=2024.2.0
 Requires-Dist: s3fs>=2024.2.0
@@ -100,6 +101,7 @@ Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
 Requires-Dist: mypy==1.17.0; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: types-dateparser; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"

{datachain-0.26.3 → datachain-0.26.4}/docs/commands/job/run.md

@@ -5,15 +5,15 @@ Run a job in Studio.
 ## Synopsis
 
 ```usage
-usage: datachain job run [-h] [-v] [-q] [--team TEAM] [--env-file ENV_FILE] [--env ENV [ENV ...]]
-                         [--workers WORKERS] [--files FILES [FILES ...]] [--python-version PYTHON_VERSION]
-                         [--req-file REQ_FILE] [--req REQ [REQ ...]]
+usage: datachain job run [-h] [-v] [-q] [--team TEAM] [--env-file ENV_FILE] [--env ENV [ENV ...]] [--cluster CLUSTER] [--workers WORKERS]
+                         [--files FILES [FILES ...]] [--python-version PYTHON_VERSION] [--repository REPOSITORY] [--req-file REQ_FILE] [--req REQ [REQ ...]]
+                         [--priority PRIORITY] [--start-time START_TIME] [--cron CRON]
                          file
 ```
 
 ## Description
 
-This command runs a job in Studio using the specified query file. You can configure various aspects of the job including environment variables, Python version, dependencies, and more.
+This command runs a job in Studio using the specified query file. You can configure various aspects of the job including environment variables, Python version, dependencies, and more. When using --start-time or --cron, the job is scheduled as a task and will not show logs immediately. The job will be executed according to the schedule.
 
 ## Arguments
 
@@ -28,10 +28,12 @@ This command runs a job in Studio using the specified query file. You can config
 * `--workers WORKERS` - Number of workers for the job
 * `--files FILES` - Additional files to include in the job
 * `--python-version PYTHON_VERSION` - Python version for the job (e.g., 3.9, 3.10, 3.11)
+* `--repository REPOSITORY` - Repository URL to clone before running the job
 * `--req-file REQ_FILE` - Python requirements file
 * `--req REQ` - Python package requirements
 * `--priority PRIORITY` - Priority for the job in range 0-5. Lower value is higher priority (default: 5)
-* `--repository URL` - Repository URL to clone before running the job.
+* `--start-time START_TIME` - Start time in ISO format or natural language for the cron task.
+* `--cron CRON` - Cron expression for the cron task.
 * `-h`, `--help` - Show the help message and exit.
 * `-v`, `--verbose` - Be verbose.
 * `-q`, `--quiet` - Be quiet.
@@ -66,17 +68,11 @@ datachain job run --env API_KEY=123 --req pandas numpy query.py
 6. Run a job with a repository (will be cloned in the job working directory):
 ```bash
 datachain job run --repository https://github.com/iterative/datachain query.py
-```
-
-To specify a branch / revision:
 
-```bash
+# To specify a branch / revision:
 datachain job run --repository https://github.com/iterative/datachain@main query.py
-```
-
-Git URLs are also supported:
 
-```bash
+# Git URLs are also supported:
 datachain job run --repository git@github.com:iterative/datachain.git@main query.py
 ```
 
@@ -90,7 +86,43 @@ datachain job run --priority 2 query.py
 # Get the cluster id using following command
 datachain job clusters
 # Use the id  of an active clusters from above
-datachain job run --cluster-id 1 query.py
+datachain job run --cluster 1 query.py
+```
+
+9. Schedule a job to run once at a specific time
+```bash
+# Run job tomorrow at 3pm
+datachain job run --start-time "tomorrow 3pm" query.py
+
+# Run job in 2 hours
+datachain job run --start-time "in 2 hours" query.py
+
+# Run job on Monday at 9am
+datachain job run --start-time "monday 9am" query.py
+
+# Run job at a specific date and time
+datachain job run --start-time "2024-01-15 14:30:00" query.py
+```
+
+10. Schedule a recurring job using cron expression
+```bash
+# Run job daily at midnight
+datachain job run --cron "0 0 * * *" query.py
+
+# Run job every Monday at 9am
+datachain job run --cron "0 9 * * 1" query.py
+
+# Run job every hour
+datachain job run --cron "0 * * * *" query.py
+
+# Run job every month
+datachain job run --cron "@monthly" query.py
+```
+
+11. Schedule a recurring job with a start time
+```bash
+# Start the cron job after tomorrow 3pm
+datachain job run --start-time "tomorrow 3pm" --cron "0 0 * * *" query.py
 ```
 
 ## Notes
@@ -99,3 +131,14 @@ datachain job run --cluster-id 1 query.py
 * To cancel a running job, use the `datachain job cancel` command
 * The job will continue running in Studio even after you stop viewing the logs
 * You can get the list of compute clusters using `datachain job clusters` command.
+* When using `--start-time` or `--cron` options, the job is scheduled as a task and will not show logs immediately. The job will be executed according to the schedule.
+* The `--start-time` option supports natural language parsing using the dateparser library, allowing flexible time expressions like "tomorrow 3pm", "in 2 hours", "monday 9am", etc.
+* Cron expressions follow the standard format: minute hour day-of-month month day-of-week (e.g., "0 0 * * *" for daily at midnight) or Vixie cron-style “@” keyword expressions.
+* Following options for Vixie cron-style expressions are supported:
+    * @midnight
+    * @hourly
+    * @daily
+    * @weekly
+    * @monthly
+    * @yearly
+    * @annually

{datachain-0.26.3 → datachain-0.26.4}/pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
   "pyarrow",
   "typing-extensions",
   "python-dateutil>=2",
+  "dateparser>=1.0.0",
   "attrs>=21.3.0",
   "fsspec>=2024.2.0",
   "s3fs>=2024.2.0",
@@ -116,6 +117,7 @@ dev = [
   "datachain[docs,tests]",
   "mypy==1.17.0",
   "types-python-dateutil",
+  "types-dateparser",
   "types-pytz",
   "types-PyYAML",
   "types-requests",

{datachain-0.26.3 → datachain-0.26.4}/src/datachain/cli/parser/job.py

@@ -17,7 +17,12 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
     )
 
     studio_run_help = "Run a job in Studio"
-    studio_run_description = "Run a job in Studio."
+    studio_run_description = "Run a job in Studio. \n"
+    studio_run_description += (
+        "When using --start-time or --cron,"
+        " the job is scheduled as a task and will not show logs immediately."
+        " The job will be executed according to the schedule."
+    )
 
     studio_run_parser = jobs_subparser.add_parser(
         "run",
@@ -96,6 +101,14 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
         help="Priority for the job in range 0-5. "
         "Lower value is higher priority (default: 5)",
     )
+    studio_run_parser.add_argument(
+        "--start-time",
+        action="store",
+        help="Start time in ISO format or natural language for the cron task.",
+    )
+    studio_run_parser.add_argument(
+        "--cron", action="store", help="Cron expression for the cron task."
+    )
 
     studio_ls_help = "List jobs in Studio"
     studio_ls_description = "List jobs in Studio."

{datachain-0.26.3 → datachain-0.26.4}/src/datachain/lib/dc/datachain.py

@@ -2388,7 +2388,7 @@ class DataChain:
         placement: FileExportPlacement = "fullpath",
         link_type: Literal["copy", "symlink"] = "copy",
         num_threads: Optional[int] = EXPORT_FILES_MAX_THREADS,
-        anon: bool = False,
+        anon: Optional[bool] = None,
         client_config: Optional[dict] = None,
     ) -> None:
         """Export files from a specified signal to a directory. Files can be
@@ -2403,7 +2403,11 @@ class DataChain:
                 Falls back to `'copy'` if symlinking fails.
             num_threads : number of threads to use for exporting files.
                 By default it uses 5 threads.
-            anon: If true, we will treat cloud bucket as public one
+            anon: If True, we will treat cloud bucket as public one. Default behavior
+                depends on the previous session configuration (e.g. happens in the
+                initial `read_storage`) and particular cloud storage client
+                implementation (e.g. S3 fallbacks to anonymous access if no credentials
+                were found).
             client_config: Optional configuration for the destination storage client
 
         Example:
@@ -2421,8 +2425,8 @@ class DataChain:
         ):
             raise ValueError("Files with the same name found")
 
-        if anon:
-            client_config = (client_config or {}) | {"anon": True}
+        if anon is not None:
+            client_config = (client_config or {}) | {"anon": anon}
 
         progress_bar = tqdm(
             desc=f"Exporting files to {output}: ",

{datachain-0.26.3 → datachain-0.26.4}/src/datachain/lib/dc/storage.py

@@ -33,7 +33,7 @@ def read_storage(
     recursive: Optional[bool] = True,
     column: str = "file",
     update: bool = False,
-    anon: bool = False,
+    anon: Optional[bool] = None,
     delta: Optional[bool] = False,
     delta_on: Optional[Union[str, Sequence[str]]] = (
         "file.path",
@@ -124,8 +124,8 @@ def read_storage(
 
     file_type = get_file_type(type)
 
-    if anon:
-        client_config = (client_config or {}) | {"anon": True}
+    if anon is not None:
+        client_config = (client_config or {}) | {"anon": anon}
     session = Session.get(session, client_config=client_config, in_memory=in_memory)
     catalog = session.catalog
     cache = catalog.cache

{datachain-0.26.3 → datachain-0.26.4}/src/datachain/lib/file.py

@@ -717,6 +717,23 @@ class ImageFile(File):
         destination = stringify_path(destination)
 
         client: Client = self._catalog.get_client(destination, **(client_config or {}))
+
+        # If format is not provided, determine it from the file extension
+        if format is None:
+            from pathlib import PurePosixPath
+
+            from PIL import Image as PilImage
+
+            ext = PurePosixPath(destination).suffix.lower()
+            format = PilImage.registered_extensions().get(ext)
+
+        if not format:
+            raise FileError(
+                f"Can't determine format for destination '{destination}'",
+                self.source,
+                self.path,
+            )
+
         with client.fs.open(destination, mode="wb") as f:
             self.read().save(f, format=format)
 

{datachain-0.26.3 → datachain-0.26.4}/src/datachain/remote/studio.py

@@ -429,6 +429,8 @@ class StudioClient:
         repository: Optional[str] = None,
         priority: Optional[int] = None,
         cluster: Optional[str] = None,
+        start_time: Optional[str] = None,
+        cron: Optional[str] = None,
     ) -> Response[JobData]:
         data = {
             "query": query,
@@ -442,6 +444,8 @@ class StudioClient:
             "repository": repository,
             "priority": priority,
             "compute_cluster_name": cluster,
+            "start_after": start_time,
+            "cron_expression": cron,
         }
         return self._send_request("datachain/job", data)
 

{datachain-0.26.3 → datachain-0.26.4}/src/datachain/studio.py

@@ -1,8 +1,10 @@
 import asyncio
 import os
 import sys
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Optional
 
+import dateparser
 import tabulate
 
 from datachain.config import Config, ConfigLevel
@@ -42,6 +44,8 @@ def process_jobs_args(args: "Namespace"):
             args.req_file,
             args.priority,
             args.cluster,
+            args.start_time,
+            args.cron,
         )
 
     if args.cmd == "cancel":
@@ -262,6 +266,31 @@ def save_config(hostname, token, level=ConfigLevel.GLOBAL):
     return config.config_file()
 
 
+def parse_start_time(start_time_str: Optional[str]) -> Optional[str]:
+    if not start_time_str:
+        return None
+
+    try:
+        # Parse the datetime string using dateparser
+        parsed_datetime = dateparser.parse(start_time_str)
+
+        if parsed_datetime is None:
+            raise DataChainError(
+                f"Could not parse datetime string: '{start_time_str}'. "
+                f"Supported formats include: '2024-01-15 14:30:00', 'tomorrow 3pm', "
+                f"'monday 9am', '2024-01-15T14:30:00Z', 'in 2 hours', etc."
+            )
+
+        # Convert to ISO format string
+        return parsed_datetime.isoformat()
+    except Exception as e:
+        raise DataChainError(
+            f"Invalid datetime format for start_time: '{start_time_str}'. "
+            f"Supported formats include: '2024-01-15 14:30:00', 'tomorrow 3pm', "
+            f"'monday 9am', '2024-01-15T14:30:00Z', 'in 2 hours', etc. Error: {e}"
+        ) from e
+
+
 def show_logs_from_client(client, job_id):
     # Sync usage
     async def _run():
@@ -310,6 +339,8 @@ def create_job(
     req_file: Optional[str] = None,
     priority: Optional[int] = None,
     cluster: Optional[str] = None,
+    start_time: Optional[str] = None,
+    cron: Optional[str] = None,
 ):
     query_type = "PYTHON" if query_file.endswith(".py") else "SHELL"
     with open(query_file) as f:
@@ -328,6 +359,11 @@ def create_job(
     client = StudioClient(team=team_name)
     file_ids = upload_files(client, files) if files else []
 
+    # Parse start_time if provided
+    parsed_start_time = parse_start_time(start_time)
+    if cron and parsed_start_time is None:
+        parsed_start_time = datetime.now(timezone.utc).isoformat()
+
     response = client.create_job(
         query=query,
         query_type=query_type,
@@ -340,6 +376,8 @@ def create_job(
         requirements=requirements,
         priority=priority,
         cluster=cluster,
+        start_time=parsed_start_time,
+        cron=cron,
     )
     if not response.ok:
         raise DataChainError(response.message)
@@ -348,6 +386,11 @@ def create_job(
         raise DataChainError("Failed to create job")
 
     job_id = response.data.get("job", {}).get("id")
+
+    if parsed_start_time or cron:
+        print(f"Job {job_id} is scheduled as a task in Studio.")
+        return 0
+
     print(f"Job {job_id} created")
     print("Open the job in Studio at", response.data.get("job", {}).get("url"))
     print("=" * 40)

{datachain-0.26.3 → datachain-0.26.4}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.26.3
+Version: 0.26.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
@@ -26,6 +26,7 @@ Requires-Dist: packaging
 Requires-Dist: pyarrow
 Requires-Dist: typing-extensions
 Requires-Dist: python-dateutil>=2
+Requires-Dist: dateparser>=1.0.0
 Requires-Dist: attrs>=21.3.0
 Requires-Dist: fsspec>=2024.2.0
 Requires-Dist: s3fs>=2024.2.0
@@ -100,6 +101,7 @@ Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
 Requires-Dist: mypy==1.17.0; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: types-dateparser; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"

{datachain-0.26.3 → datachain-0.26.4}/src/datachain.egg-info/SOURCES.txt

@@ -305,6 +305,7 @@ tests/func/test_read_dataset_remote.py
 tests/func/test_read_dataset_version_specifiers.py
 tests/func/test_retry.py
 tests/func/test_session.py
+tests/func/test_studio_datetime_parsing.py
 tests/func/test_toolkit.py
 tests/func/test_video.py
 tests/func/test_warehouse.py

{datachain-0.26.3 → datachain-0.26.4}/src/datachain.egg-info/requires.txt

@@ -7,6 +7,7 @@ packaging
 pyarrow
 typing-extensions
 python-dateutil>=2
+dateparser>=1.0.0
 attrs>=21.3.0
 fsspec>=2024.2.0
 s3fs>=2024.2.0
@@ -43,6 +44,7 @@ soundfile
 datachain[docs,tests]
 mypy==1.17.0
 types-python-dateutil
+types-dateparser
 types-pytz
 types-PyYAML
 types-requests

{datachain-0.26.3 → datachain-0.26.4}/tests/func/test_image.py

@@ -42,6 +42,34 @@ def test_image_save(tmp_path, image_file, format):
     assert img.size == (256, 256)
 
 
+def test_image_save_no_extension(tmp_path, image_file):
+    image_file = image_file.as_image_file()
+    filename = f"{tmp_path}/test"
+    with pytest.raises(FileError):
+        image_file.save(filename)
+
+
+@pytest.mark.parametrize("format", [None, "JPEG", "PNG"])
+def test_image_save_cloud(cloud_test_catalog_upload, image_file, format):
+    """Test saving ImageFile to different cloud filesystems (S3, GCS, Azure)."""
+    ctc = cloud_test_catalog_upload
+    image_file = image_file.as_image_file()
+    image_file._set_stream(ctc.catalog)
+
+    # Save to cloud storage with the specified format
+    cloud_filename = f"{ctc.src_uri}/test_image.jpg"
+    image_file.save(cloud_filename, format=format, client_config=ctc.client_config)
+
+    # Verify the saved file by reading it back
+    saved_image_file = ImageFile(path="test_image.jpg", source=ctc.src_uri)
+    saved_image_file._set_stream(ctc.catalog)
+
+    # Read the saved image and verify it
+    saved_img = saved_image_file.read()
+    assert saved_img.format == (format or "JPEG")
+    assert saved_img.size == (256, 256)
+
+
 def test_get_info(image_file):
     info = image_file.as_image_file().get_info()
     assert info.model_dump() == {"width": 256, "height": 256, "format": "JPEG"}

datachain-0.26.4/tests/func/test_studio_datetime_parsing.py

@@ -0,0 +1,107 @@
+from datetime import datetime
+
+import pytest
+
+from datachain.error import DataChainError
+from datachain.studio import parse_start_time
+
+
+def test_parse_start_time_none():
+    """Test that None input returns None."""
+    assert parse_start_time(None) is None
+
+
+def test_parse_start_time_empty_string():
+    """Test that empty string input returns None."""
+    assert parse_start_time("") is None
+
+
+def test_parse_start_time_iso_format():
+    """Test parsing ISO format datetime strings."""
+    # Test ISO format with timezone
+    result = parse_start_time("2024-01-15T14:30:00Z")
+    assert result == "2024-01-15T14:30:00+00:00"
+
+    # Test ISO format without timezone
+    result = parse_start_time("2024-01-15T14:30:00")
+    assert result.startswith("2024-01-15T14:30:00")
+
+
+def test_parse_start_time_standard_format():
+    """Test parsing standard datetime format."""
+    result = parse_start_time("2024-01-15 14:30:00")
+    assert result.startswith("2024-01-15T14:30:00")
+
+
+def test_parse_start_time_natural_language():
+    """Test parsing natural language datetime strings."""
+    # Test natural language formats that dateparser supports
+    result = parse_start_time("tomorrow 3pm")
+    assert result is not None
+    assert isinstance(result, str)
+
+    result = parse_start_time("monday 9am")
+    assert result is not None
+    assert isinstance(result, str)
+
+    result = parse_start_time("in 2 hours")
+    assert result is not None
+    assert isinstance(result, str)
+
+    result = parse_start_time("next week")
+    assert result is not None
+    assert isinstance(result, str)
+
+
+def test_parse_start_time_various_formats():
+    """Test parsing various datetime formats."""
+    test_cases = [
+        "2024-01-15 14:30:00",
+        "2024-01-15T14:30:00Z",
+        "2024-01-15T14:30:00+00:00",
+        "Jan 15, 2024 2:30 PM",
+        "15/01/2024 14:30",
+        "2024-01-15",
+        "tomorrow",
+        "next week",
+        "in 2 hours",
+        "monday 9am",
+        "tomorrow 3pm",
+    ]
+
+    for test_case in test_cases:
+        result = parse_start_time(test_case)
+        assert result is not None
+        assert isinstance(result, str)
+        # Verify it's a valid ISO format
+        try:
+            datetime.fromisoformat(result.replace("Z", "+00:00"))
+        except ValueError:
+            pytest.fail(f"Failed to parse result '{result}' as ISO format")
+
+
+def test_parse_start_time_invalid_format():
+    """Test that invalid datetime formats raise DataChainError."""
+    invalid_formats = [
+        "not a date",
+        "invalid datetime string",
+        "2024-13-45 25:70:99",  # Invalid date/time values
+    ]
+
+    for invalid_format in invalid_formats:
+        with pytest.raises(DataChainError) as exc_info:
+            parse_start_time(invalid_format)
+
+        assert "Invalid datetime format" in str(exc_info.value)
+        assert invalid_format in str(exc_info.value)
+
+
+def test_parse_start_time_timezone_handling():
+    """Test timezone handling in datetime parsing."""
+    # Test with explicit timezone
+    result = parse_start_time("2024-01-15 14:30:00 UTC")
+    assert result is not None
+
+    # Test with local timezone (should be preserved)
+    result = parse_start_time("2024-01-15 14:30:00")
+    assert result is not None

{datachain-0.26.3 → datachain-0.26.4}/tests/test_cli_studio.py

@@ -440,9 +440,56 @@ def test_studio_run(capsys, mocker, tmp_dir):
         "repository": "https://github.com/iterative/datachain",
         "priority": 5,
         "compute_cluster_name": "default",
+        "start_after": None,
+        "cron_expression": None,
     }
 
 
+def test_studio_run_task(capsys, mocker, tmp_dir, studio_token):
+    mocker.patch(
+        "datachain.remote.studio.websockets.connect", side_effect=mocked_connect
+    )
+
+    with requests_mock.mock() as m:
+        m.post(
+            f"{STUDIO_URL}/api/datachain/job",
+            json={"job": {"id": 1, "url": "https://example.com"}},
+        )
+        m.get(
+            f"{STUDIO_URL}/api/datachain/datasets/dataset_job_versions?job_id=1&team_name=team_name",
+            json={
+                "dataset_versions": [
+                    {"dataset_name": "dataset_name", "version": "1.0.0"}
+                ]
+            },
+        )
+        (tmp_dir / "example_query.py").write_text("print(1)")
+
+        assert (
+            main(
+                [
+                    "job",
+                    "run",
+                    "example_query.py",
+                    "--start-time",
+                    "tomorrow 3pm",
+                    "--cron",
+                    "0 0 * * *",
+                ]
+            )
+            == 0
+        )
+    first_request = m.request_history[0]
+    assert first_request.method == "POST"
+    assert first_request.url == f"{STUDIO_URL}/api/datachain/job"
+    request_json = first_request.json()
+    assert request_json["start_after"] is not None
+    assert request_json["cron_expression"] is not None
+
+    assert request_json["start_after"] is not None
+    assert request_json["cron_expression"] == "0 0 * * *"
+
+
 @pytest.mark.parametrize(
     "status,expected_exit_code", [("FAILED", 1), ("CANCELLED", 2), ("COMPLETED", 0)]
 )