datachain 0.20.0__tar.gz → 0.20.2__tar.gz

{datachain-0.20.0 → datachain-0.20.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.20.0
+Version: 0.20.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.20.0 → datachain-0.20.2}/docs/commands/job/run.md

@@ -24,7 +24,7 @@ This command runs a job in Studio using the specified query file. You can config
 * `--team TEAM` - Team to run job for (default: from config)
 * `--env-file ENV_FILE` - File with environment variables for the job
 * `--env ENV` - Environment variables in KEY=VALUE format
-* `--cluster-id CLUSTER_ID` - Compute cluster ID to run the job on
+* `--cluster CLUSTER` - Compute cluster to run the job on
 * `--workers WORKERS` - Number of workers for the job
 * `--files FILES` - Additional files to include in the job
 * `--python-version PYTHON_VERSION` - Python version for the job (e.g., 3.9, 3.10, 3.11)

{datachain-0.20.0 → datachain-0.20.2}/examples/llm_and_nlp/hf-dataset-llm-eval.py

@@ -1,3 +1,5 @@
+import os
+
 from huggingface_hub import InferenceClient
 from requests import HTTPError
 
@@ -23,6 +25,7 @@ def eval_dialog(
 ) -> DialogEval:
     try:
         completion = client.chat_completion(
+            model="meta-llama/Llama-3.3-70B-Instruct",
             messages=[
                 {
                     "role": "user",
@@ -31,9 +34,10 @@ def eval_dialog(
             ],
             response_format={"type": "json", "value": DialogEval.model_json_schema()},
         )
-    except HTTPError:
+    except HTTPError as e:
         return DialogEval(
-            result="Error", reason="Error while interacting with the Hugging Face API."
+            result="Error",
+            reason=f"Error while interacting with the Hugging Face API. {e}",
         )
 
     message = completion.choices[0].message
@@ -48,9 +52,15 @@ def eval_dialog(
 # Save to HF as Parquet. Dataset can be previewed here:
 # https://huggingface.co/datasets/dvcorg/test-datachain-llm-eval/viewer
 (
-    dc.read_csv("hf://datasets/infinite-dataset-hub/MobilePlanAssistant/data.csv")
-    .settings(parallel=10)
-    .setup(client=lambda: InferenceClient("meta-llama/Llama-3.1-70B-Instruct"))
+    dc.read_csv(
+        "hf://datasets/infinite-dataset-hub/MobilePlanAssistant/data.csv", source=False
+    )
+    .settings(parallel=True)
+    .setup(
+        client=lambda: InferenceClient(
+            provider="hf-inference", api_key=os.environ["HF_TOKEN"]
+        )
+    )
     .map(response=eval_dialog)
     .to_parquet("hf://datasets/dvcorg/test-datachain-llm-eval/data.parquet")
 )

{datachain-0.20.0 → datachain-0.20.2}/src/datachain/cli/parser/job.py

@@ -52,10 +52,10 @@ def add_jobs_parser(subparsers, parent_parser) -> None:
     )
 
     studio_run_parser.add_argument(
-        "--cluster-id",
-        type=int,
+        "--cluster",
+        type=str,
         action="store",
-        help="Compute cluster ID to run the job on",
+        help="Compute cluster to run the job on",
     )
 
     studio_run_parser.add_argument(

{datachain-0.20.0 → datachain-0.20.2}/src/datachain/client/hf.py

@@ -21,6 +21,9 @@ def _wrap_class(sync_fs_class):
     asynchronous to False by default. This is similar to other Async FS
     we initialize. E.g. it means we don't break things in Jupyter where code
     run in async.
+
+    This also fixes write operations by ensuring they are properly forwarded
+    to the underlying filesystem without async buffering issues.
     """
     from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
 
@@ -29,6 +32,13 @@ def _wrap_class(sync_fs_class):
             sync_fs = sync_fs_class(*args, **kwargs)
             super().__init__(sync_fs, asynchronous=False)
 
+        def open(self, path, mode="rb", **kwargs):
+            # Override open to ensure write operations work correctly.
+            # It seems to be a bug in the fsspec wrapper. It avoids
+            # wrapping open() explicitly but also doesn't redirect it to
+            # sync filesystem.
+            return self.sync_fs.open(path, mode, **kwargs)
+
     GeneratedAsyncFileSystemWrapper.__name__ = f"Async{sync_fs_class.__name__}Wrapper"
     return GeneratedAsyncFileSystemWrapper
 

{datachain-0.20.0 → datachain-0.20.2}/src/datachain/remote/studio.py

@@ -427,7 +427,7 @@ class StudioClient:
         requirements: Optional[str] = None,
         repository: Optional[str] = None,
         priority: Optional[int] = None,
-        cluster_id: Optional[int] = None,
+        cluster: Optional[str] = None,
     ) -> Response[JobData]:
         data = {
             "query": query,
@@ -440,7 +440,7 @@ class StudioClient:
             "requirements": requirements,
             "repository": repository,
             "priority": priority,
-            "compute_cluster_id": cluster_id,
+            "compute_cluster_name": cluster,
         }
         return self._send_request("datachain/job", data)
 

{datachain-0.20.0 → datachain-0.20.2}/src/datachain/studio.py

@@ -41,7 +41,7 @@ def process_jobs_args(args: "Namespace"):
             args.req,
             args.req_file,
             args.priority,
-            args.cluster_id,
+            args.cluster,
         )
 
     if args.cmd == "cancel":
@@ -300,7 +300,7 @@ def create_job(
     req: Optional[list[str]] = None,
     req_file: Optional[str] = None,
     priority: Optional[int] = None,
-    cluster_id: Optional[int] = None,
+    cluster: Optional[str] = None,
 ):
     query_type = "PYTHON" if query_file.endswith(".py") else "SHELL"
     with open(query_file) as f:
@@ -330,7 +330,7 @@ def create_job(
         repository=repository,
         requirements=requirements,
         priority=priority,
-        cluster_id=cluster_id,
+        cluster=cluster,
     )
     if not response.ok:
         raise DataChainError(response.message)

{datachain-0.20.0 → datachain-0.20.2}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.20.0
+Version: 0.20.2
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.20.0 → datachain-0.20.2}/tests/test_cli_studio.py

@@ -398,8 +398,8 @@ def test_studio_run(capsys, mocker, tmp_dir):
                     "stupidity",
                     "--repository",
                     "https://github.com/iterative/datachain",
-                    "--cluster-id",
-                    "1",
+                    "--cluster",
+                    "default",
                 ]
             )
             == 0
@@ -438,5 +438,5 @@ def test_studio_run(capsys, mocker, tmp_dir):
         "team_name": "team_name",
         "repository": "https://github.com/iterative/datachain",
         "priority": 5,
-        "compute_cluster_id": 1,
+        "compute_cluster_name": "default",
     }