datachain 0.7.8__tar.gz → 0.7.9__tar.gz

{datachain-0.7.8/src/datachain.egg-info → datachain-0.7.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.7.8
+Version: 0.7.9
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -98,7 +98,7 @@ Requires-Dist: unstructured[embed-huggingface,pdf]<0.16.0; extra == "examples"
 Requires-Dist: pdfplumber==0.11.4; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
 Requires-Dist: onnx==1.16.1; extra == "examples"
-Requires-Dist: ultralytics==8.3.29; extra == "examples"
+Requires-Dist: ultralytics==8.3.37; extra == "examples"
 
 ================
 |logo| DataChain

{datachain-0.7.8 → datachain-0.7.9}/examples/llm_and_nlp/hf-dataset-llm-eval.py

@@ -15,9 +15,11 @@ class DialogEval(DataModel):
 
 # DataChain function to evaluate dialog.
 # DataChain is using types for inputs, results to automatically infer schema.
-def eval_dialog(user_input: str, bot_response: str) -> DialogEval:
-    client = InferenceClient("meta-llama/Llama-3.1-70B-Instruct")
-
+def eval_dialog(
+    client: InferenceClient,
+    user_input: str,
+    bot_response: str,
+) -> DialogEval:
     completion = client.chat_completion(
         messages=[
             {
@@ -44,6 +46,7 @@ def eval_dialog(user_input: str, bot_response: str) -> DialogEval:
         "hf://datasets/infinite-dataset-hub/MobilePlanAssistant/data.csv"
     )
     .settings(parallel=10)
+    .setup(client=lambda: InferenceClient("meta-llama/Llama-3.1-70B-Instruct"))
     .map(response=eval_dialog)
     .to_parquet("hf://datasets/dvcorg/test-datachain-llm-eval/data.parquet")
 )

{datachain-0.7.8 → datachain-0.7.9}/pyproject.toml

@@ -111,7 +111,7 @@ examples = [
   "pdfplumber==0.11.4",
   "huggingface_hub[hf_transfer]",
   "onnx==1.16.1",
-  "ultralytics==8.3.29"
+  "ultralytics==8.3.37"
 ]
 
 [project.urls]

{datachain-0.7.8 → datachain-0.7.9}/src/datachain/cli.py

@@ -16,7 +16,7 @@ from tabulate import tabulate
 from datachain import Session, utils
 from datachain.cli_utils import BooleanOptionalAction, CommaSeparatedArgs, KeyValueArgs
 from datachain.config import Config
-from datachain.error import DataChainError
+from datachain.error import DataChainError, DatasetNotFoundError
 from datachain.lib.dc import DataChain
 from datachain.studio import (
     edit_studio_dataset,
@@ -1056,7 +1056,10 @@ def rm_dataset(
     all, local, studio = _determine_flavors(studio, local, all, token)
 
     if all or local:
-        catalog.remove_dataset(name, version=version, force=force)
+        try:
+            catalog.remove_dataset(name, version=version, force=force)
+        except DatasetNotFoundError:
+            print("Dataset not found in local", file=sys.stderr)
 
     if (all or studio) and token:
         remove_studio_dataset(team, name, version, force)
@@ -1077,7 +1080,10 @@ def edit_dataset(
     all, local, studio = _determine_flavors(studio, local, all, token)
 
     if all or local:
-        catalog.edit_dataset(name, new_name, description, labels)
+        try:
+            catalog.edit_dataset(name, new_name, description, labels)
+        except DatasetNotFoundError:
+            print("Dataset not found in local", file=sys.stderr)
 
     if (all or studio) and token:
         edit_studio_dataset(team, name, new_name, description, labels)

{datachain-0.7.8 → datachain-0.7.9}/src/datachain/data_storage/metastore.py

@@ -725,9 +725,10 @@ class AbstractDBMetastore(AbstractMetastore):
 
     def list_datasets(self) -> Iterator["DatasetListRecord"]:
         """Lists all datasets."""
-        yield from self._parse_dataset_list(
-            self.db.execute(self._base_list_datasets_query())
+        query = self._base_list_datasets_query().order_by(
+            self._datasets.c.name, self._datasets_versions.c.version
         )
+        yield from self._parse_dataset_list(self.db.execute(query))
 
     def list_datasets_by_prefix(
         self, prefix: str, conn=None

{datachain-0.7.8 → datachain-0.7.9}/src/datachain/lib/dc.py

@@ -1446,6 +1446,7 @@ class DataChain:
             tokenizer=tokenizer,
             tokenizer_kwargs=tokenizer_kwargs,
             num_samples=num_samples,
+            dc_settings=chain._settings,
         )
 
     def remove_file_signals(self) -> "Self":  # noqa: D102

{datachain-0.7.8 → datachain-0.7.9}/src/datachain/lib/pytorch.py

@@ -10,8 +10,10 @@ from torchvision.transforms import v2
 from tqdm import tqdm
 
 from datachain import Session
+from datachain.asyn import AsyncMapper
 from datachain.catalog import Catalog, get_catalog
 from datachain.lib.dc import DataChain
+from datachain.lib.settings import Settings
 from datachain.lib.text import convert_text
 
 if TYPE_CHECKING:
@@ -30,6 +32,8 @@ def label_to_int(value: str, classes: list) -> int:
 
 
 class PytorchDataset(IterableDataset):
+    prefetch: int = 2
+
     def __init__(
         self,
         name: str,
@@ -39,6 +43,7 @@ class PytorchDataset(IterableDataset):
         tokenizer: Optional[Callable] = None,
         tokenizer_kwargs: Optional[dict[str, Any]] = None,
         num_samples: int = 0,
+        dc_settings: Optional[Settings] = None,
     ):
         """
         Pytorch IterableDataset that streams DataChain datasets.
@@ -66,6 +71,11 @@ class PytorchDataset(IterableDataset):
             catalog = get_catalog()
         self._init_catalog(catalog)
 
+        dc_settings = dc_settings or Settings()
+        self.cache = dc_settings.cache
+        if (prefetch := dc_settings.prefetch) is not None:
+            self.prefetch = prefetch
+
     def _init_catalog(self, catalog: "Catalog"):
         # For compatibility with multiprocessing,
         # we can only store params in __init__(), as Catalog isn't picklable
@@ -82,51 +92,58 @@ class PytorchDataset(IterableDataset):
         wh = wh_cls(*wh_args, **wh_kwargs)
         return Catalog(ms, wh, **self._catalog_params)
 
-    def __iter__(self) -> Iterator[Any]:
-        if self.catalog is None:
-            self.catalog = self._get_catalog()
-        session = Session.get(catalog=self.catalog)
-        total_rank, total_workers = self.get_rank_and_workers()
+    def _rows_iter(self, total_rank: int, total_workers: int):
+        catalog = self._get_catalog()
+        session = Session("PyTorch", catalog=catalog)
         ds = DataChain.from_dataset(
             name=self.name, version=self.version, session=session
-        )
+        ).settings(cache=self.cache, prefetch=self.prefetch)
         ds = ds.remove_file_signals()
 
         if self.num_samples > 0:
             ds = ds.sample(self.num_samples)
         ds = ds.chunk(total_rank, total_workers)
+        yield from ds.collect()
+
+    def __iter__(self) -> Iterator[Any]:
+        total_rank, total_workers = self.get_rank_and_workers()
+        rows = self._rows_iter(total_rank, total_workers)
+        if self.prefetch > 0:
+            from datachain.lib.udf import _prefetch_input
+
+            rows = AsyncMapper(_prefetch_input, rows, workers=self.prefetch).iterate()
+
         desc = f"Parsed PyTorch dataset for rank={total_rank} worker"
-        with tqdm(desc=desc, unit=" rows") as pbar:
-            for row_features in ds.collect():
-                row = []
-                for fr in row_features:
-                    if hasattr(fr, "read"):
-                        row.append(fr.read())  # type: ignore[unreachable]
-                    else:
-                        row.append(fr)
-                # Apply transforms
-                if self.transform:
-                    try:
-                        if isinstance(self.transform, v2.Transform):
-                            row = self.transform(row)
-                        for i, val in enumerate(row):
-                            if isinstance(val, Image.Image):
-                                row[i] = self.transform(val)
-                    except ValueError:
-                        logger.warning(
-                            "Skipping transform due to unsupported data types."
-                        )
-                        self.transform = None
-                if self.tokenizer:
-                    for i, val in enumerate(row):
-                        if isinstance(val, str) or (
-                            isinstance(val, list) and isinstance(val[0], str)
-                        ):
-                            row[i] = convert_text(
-                                val, self.tokenizer, self.tokenizer_kwargs
-                            ).squeeze(0)  # type: ignore[union-attr]
-                yield row
-                pbar.update(1)
+        with tqdm(rows, desc=desc, unit=" rows", position=total_rank) as rows_it:
+            yield from map(self._process_row, rows_it)
+
+    def _process_row(self, row_features):
+        row = []
+        for fr in row_features:
+            if hasattr(fr, "read"):
+                row.append(fr.read())  # type: ignore[unreachable]
+            else:
+                row.append(fr)
+        # Apply transforms
+        if self.transform:
+            try:
+                if isinstance(self.transform, v2.Transform):
+                    row = self.transform(row)
+                for i, val in enumerate(row):
+                    if isinstance(val, Image.Image):
+                        row[i] = self.transform(val)
+            except ValueError:
+                logger.warning("Skipping transform due to unsupported data types.")
+                self.transform = None
+        if self.tokenizer:
+            for i, val in enumerate(row):
+                if isinstance(val, str) or (
+                    isinstance(val, list) and isinstance(val[0], str)
+                ):
+                    row[i] = convert_text(
+                        val, self.tokenizer, self.tokenizer_kwargs
+                    ).squeeze(0)  # type: ignore[union-attr]
+        return row
 
     @staticmethod
     def get_rank_and_workers() -> tuple[int, int]:

{datachain-0.7.8 → datachain-0.7.9}/src/datachain/remote/studio.py

@@ -119,18 +119,27 @@ class StudioClient:
                 "\tpip install 'datachain[remote]'"
             ) from None
 
-    def _send_request_msgpack(self, route: str, data: dict[str, Any]) -> Response[Any]:
+    def _send_request_msgpack(
+        self, route: str, data: dict[str, Any], method: Optional[str] = "POST"
+    ) -> Response[Any]:
         import msgpack
         import requests
 
-        response = requests.post(
-            f"{self.url}/{route}",
-            json={**data, "team_name": self.team},
+        kwargs = (
+            {"params": {**data, "team_name": self.team}}
+            if method == "GET"
+            else {"json": {**data, "team_name": self.team}}
+        )
+
+        response = requests.request(
+            method=method,  # type: ignore[arg-type]
+            url=f"{self.url}/{route}",
             headers={
                 "Content-Type": "application/json",
                 "Authorization": f"token {self.token}",
             },
             timeout=self.timeout,
+            **kwargs,  # type: ignore[arg-type]
         )
         ok = response.ok
         if not ok:
@@ -148,7 +157,9 @@ class StudioClient:
         return Response(response_data, ok, message)
 
     @retry_with_backoff(retries=5)
-    def _send_request(self, route: str, data: dict[str, Any]) -> Response[Any]:
+    def _send_request(
+        self, route: str, data: dict[str, Any], method: Optional[str] = "POST"
+    ) -> Response[Any]:
         """
         Function that communicate Studio API.
         It will raise an exception, and try to retry, if 5xx status code is
@@ -157,14 +168,21 @@ class StudioClient:
         """
         import requests
 
-        response = requests.post(
-            f"{self.url}/{route}",
-            json={**data, "team_name": self.team},
+        kwargs = (
+            {"params": {**data, "team_name": self.team}}
+            if method == "GET"
+            else {"json": {**data, "team_name": self.team}}
+        )
+
+        response = requests.request(
+            method=method,  # type: ignore[arg-type]
+            url=f"{self.url}/{route}",
             headers={
                 "Content-Type": "application/json",
                 "Authorization": f"token {self.token}",
             },
             timeout=self.timeout,
+            **kwargs,  # type: ignore[arg-type]
         )
         try:
             response.raise_for_status()
@@ -222,7 +240,7 @@ class StudioClient:
             yield path, response
 
     def ls_datasets(self) -> Response[LsData]:
-        return self._send_request("datachain/ls-datasets", {})
+        return self._send_request("datachain/datasets", {}, method="GET")
 
     def edit_dataset(
         self,
@@ -232,20 +250,14 @@ class StudioClient:
         labels: Optional[list[str]] = None,
     ) -> Response[DatasetInfoData]:
         body = {
+            "new_name": new_name,
             "dataset_name": name,
+            "description": description,
+            "labels": labels,
         }
 
-        if new_name is not None:
-            body["new_name"] = new_name
-
-        if description is not None:
-            body["description"] = description
-
-        if labels is not None:
-            body["labels"] = labels  # type: ignore[assignment]
-
         return self._send_request(
-            "datachain/edit-dataset",
+            "datachain/datasets",
             body,
         )
 
@@ -256,12 +268,13 @@ class StudioClient:
         force: Optional[bool] = False,
     ) -> Response[DatasetInfoData]:
         return self._send_request(
-            "datachain/rm-dataset",
+            "datachain/datasets",
             {
                 "dataset_name": name,
                 "version": version,
                 "force": force,
             },
+            method="DELETE",
         )
 
     def dataset_info(self, name: str) -> Response[DatasetInfoData]:
@@ -272,7 +285,9 @@ class StudioClient:
 
             return dataset_info
 
-        response = self._send_request("datachain/dataset-info", {"dataset_name": name})
+        response = self._send_request(
+            "datachain/datasets/info", {"dataset_name": name}, method="GET"
+        )
         if response.ok:
             response.data = _parse_dataset_info(response.data)
         return response
@@ -282,14 +297,16 @@ class StudioClient:
     ) -> Response[DatasetRowsData]:
         req_data = {"dataset_name": name, "dataset_version": version}
         return self._send_request_msgpack(
-            "datachain/dataset-rows",
+            "datachain/datasets/rows",
             {**req_data, "offset": offset, "limit": DATASET_ROWS_CHUNK_SIZE},
+            method="GET",
         )
 
     def dataset_stats(self, name: str, version: int) -> Response[DatasetStatsData]:
         response = self._send_request(
-            "datachain/dataset-stats",
+            "datachain/datasets/stats",
             {"dataset_name": name, "dataset_version": version},
+            method="GET",
         )
         if response.ok:
             response.data = DatasetStats(**response.data)
@@ -299,16 +316,18 @@ class StudioClient:
         self, name: str, version: int
     ) -> Response[DatasetExportSignedUrls]:
         return self._send_request(
-            "datachain/dataset-export",
+            "datachain/datasets/export",
             {"dataset_name": name, "dataset_version": version},
+            method="GET",
         )
 
     def dataset_export_status(
         self, name: str, version: int
     ) -> Response[DatasetExportStatus]:
         return self._send_request(
-            "datachain/dataset-export-status",
+            "datachain/datasets/export-status",
             {"dataset_name": name, "dataset_version": version},
+            method="GET",
         )
 
     def upload_file(self, file_name: str, content: bytes) -> Response[FileUploadData]:

{datachain-0.7.8 → datachain-0.7.9}/src/datachain/studio.py

@@ -155,7 +155,7 @@ def edit_studio_dataset(
     if not response.ok:
         raise_remote_error(response.message)
 
-    print(f"Dataset {name} updated")
+    print(f"Dataset '{name}' updated in Studio")
 
 
 def remove_studio_dataset(
@@ -169,7 +169,7 @@ def remove_studio_dataset(
     if not response.ok:
         raise_remote_error(response.message)
 
-    print(f"Dataset {name} removed")
+    print(f"Dataset '{name}' removed from Studio")
 
 
 def save_config(hostname, token):

{datachain-0.7.8 → datachain-0.7.9/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.7.8
+Version: 0.7.9
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -98,7 +98,7 @@ Requires-Dist: unstructured[embed-huggingface,pdf]<0.16.0; extra == "examples"
 Requires-Dist: pdfplumber==0.11.4; extra == "examples"
 Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
 Requires-Dist: onnx==1.16.1; extra == "examples"
-Requires-Dist: ultralytics==8.3.29; extra == "examples"
+Requires-Dist: ultralytics==8.3.37; extra == "examples"
 
 ================
 |logo| DataChain

{datachain-0.7.8 → datachain-0.7.9}/src/datachain.egg-info/requires.txt

@@ -55,7 +55,7 @@ unstructured[embed-huggingface,pdf]<0.16.0
 pdfplumber==0.11.4
 huggingface_hub[hf_transfer]
 onnx==1.16.1
-ultralytics==8.3.29
+ultralytics==8.3.37
 
 [hf]
 numba>=0.60.0

{datachain-0.7.8 → datachain-0.7.9}/tests/conftest.py

@@ -649,7 +649,7 @@ def studio_datasets(requests_mock):
         },
     ]
 
-    requests_mock.post(f"{STUDIO_URL}/api/datachain/ls-datasets", json=datasets)
+    requests_mock.get(f"{STUDIO_URL}/api/datachain/datasets", json=datasets)
 
 
 @pytest.fixture

{datachain-0.7.8 → datachain-0.7.9}/tests/func/test_catalog.py

@@ -772,6 +772,38 @@ def test_dataset_stats(test_session):
     assert dataset_version2.size == 18
 
 
+def test_ls_datasets_ordered(test_session):
+    ids = [1, 2, 3]
+    values = tuple(zip(["a", "b", "c"], ids))
+
+    assert not list(test_session.catalog.ls_datasets())
+
+    dc = DataChain.from_values(
+        ids=ids,
+        file=[File(path=name, size=size) for name, size in values],
+        session=test_session,
+    )
+    dc.save("cats")
+    dc.save("dogs")
+    dc.save("cats")
+    dc.save("cats")
+    dc.save("cats")
+    datasets = list(test_session.catalog.ls_datasets())
+
+    assert [
+        (d.name, v.version)
+        for d in datasets
+        for v in d.versions
+        if not d.name.startswith("session_")
+    ] == [
+        ("cats", 1),
+        ("cats", 2),
+        ("cats", 3),
+        ("cats", 4),
+        ("dogs", 1),
+    ]
+
+
 def test_ls_datasets_no_json(test_session):
     ids = [1, 2, 3]
     values = tuple(zip(["a", "b", "c"], [1, 2, 3]))

{datachain-0.7.8 → datachain-0.7.9}/tests/func/test_ls.py

@@ -193,7 +193,7 @@ class MockResponse:
         self.ok = ok
 
 
-def mock_post(url, data=None, json=None, **kwargs):
+def mock_post(method, url, data=None, json=None, **kwargs):
     source = json["source"]
     path = re.sub(r"\w+://[^/]+/?", "", source).rstrip("/")
     data = [
@@ -238,7 +238,7 @@ dog3
 def test_ls_remote_sources(cloud_type, capsys, monkeypatch, studio_config):
     src = f"{cloud_type}://bucket"
     with monkeypatch.context() as m:
-        m.setattr("requests.post", mock_post)
+        m.setattr("requests.request", mock_post)
         ls([src, f"{src}/dogs/others", f"{src}/dogs"], studio=True)
     captured = capsys.readouterr()
     assert captured.out == ls_remote_sources_output.format(src=src)

{datachain-0.7.8 → datachain-0.7.9}/tests/func/test_pull.py

@@ -150,28 +150,28 @@ def remote_dataset_chunk_url():
 
 @pytest.fixture
 def remote_dataset_info(requests_mock, remote_dataset):
-    requests_mock.post(f"{STUDIO_URL}/api/datachain/dataset-info", json=remote_dataset)
+    requests_mock.get(f"{STUDIO_URL}/api/datachain/datasets/info", json=remote_dataset)
 
 
 @pytest.fixture
 def remote_dataset_stats(requests_mock):
-    requests_mock.post(
-        f"{STUDIO_URL}/api/datachain/dataset-stats",
+    requests_mock.get(
+        f"{STUDIO_URL}/api/datachain/datasets/stats",
         json={"num_objects": 5, "size": 1000},
     )
 
 
 @pytest.fixture
 def dataset_export(requests_mock, remote_dataset_chunk_url):
-    requests_mock.post(
-        f"{STUDIO_URL}/api/datachain/dataset-export", json=[remote_dataset_chunk_url]
+    requests_mock.get(
+        f"{STUDIO_URL}/api/datachain/datasets/export", json=[remote_dataset_chunk_url]
     )
 
 
 @pytest.fixture
 def dataset_export_status(requests_mock):
-    requests_mock.post(
-        f"{STUDIO_URL}/api/datachain/dataset-export-status",
+    requests_mock.get(
+        f"{STUDIO_URL}/api/datachain/datasets/export-status",
         json={"status": "completed"},
     )
 
@@ -303,8 +303,8 @@ def test_pull_dataset_not_found_in_remote(
     requests_mock,
     cloud_test_catalog,
 ):
-    requests_mock.post(
-        f"{STUDIO_URL}/api/datachain/dataset-info",
+    requests_mock.get(
+        f"{STUDIO_URL}/api/datachain/datasets/info",
         status_code=404,
         json={"message": "Dataset not found"},
     )
@@ -322,8 +322,8 @@ def test_pull_dataset_error_on_fetching_stats(
     cloud_test_catalog,
     remote_dataset_info,
 ):
-    requests_mock.post(
-        f"{STUDIO_URL}/api/datachain/dataset-stats",
+    requests_mock.get(
+        f"{STUDIO_URL}/api/datachain/datasets/stats",
         status_code=400,
         json={"message": "Internal error"},
     )
@@ -345,8 +345,8 @@ def test_pull_dataset_exporting_dataset_failed_in_remote(
     dataset_export,
     export_status,
 ):
-    requests_mock.post(
-        f"{STUDIO_URL}/api/datachain/dataset-export-status",
+    requests_mock.get(
+        f"{STUDIO_URL}/api/datachain/datasets/export-status",
         json={"status": export_status},
     )
 

{datachain-0.7.8 → datachain-0.7.9}/tests/test_cli_studio.py

@@ -169,7 +169,7 @@ def test_studio_datasets(capsys, studio_datasets, mocker):
 
 def test_studio_edit_dataset(capsys, mocker):
     with requests_mock.mock() as m:
-        m.post(f"{STUDIO_URL}/api/datachain/edit-dataset", json={})
+        m.post(f"{STUDIO_URL}/api/datachain/datasets", json={})
 
         # Studio token is required
         assert (
@@ -217,6 +217,8 @@ def test_studio_edit_dataset(capsys, mocker):
             "dataset_name": "name",
             "new_name": "new-name",
             "team_name": "team_name",
+            "description": None,
+            "labels": None,
         }
 
         # With all arguments
@@ -251,7 +253,7 @@ def test_studio_edit_dataset(capsys, mocker):
 
 def test_studio_rm_dataset(capsys, mocker):
     with requests_mock.mock() as m:
-        m.post(f"{STUDIO_URL}/api/datachain/rm-dataset", json={})
+        m.delete(f"{STUDIO_URL}/api/datachain/datasets", json={})
 
         # Studio token is required
         assert main(["datasets", "rm", "name", "--team", "team_name", "--studio"]) == 1