dataverse-sdk 2.2.1__tar.gz → 2.2.2__tar.gz

{dataverse_sdk-2.2.1 → dataverse_sdk-2.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataverse-sdk
-Version: 2.2.1
+Version: 2.2.2
 Summary: Dataverse SDK For Python
 Home-page: 
 Author: LinkerVision
@@ -100,7 +100,7 @@ The following sections provide examples for the most common DataVerse tasks incl
 * [Edit Project](#edit-project)
 * [Update Alias](#update-ontology-alias)
 * [Create Dataset](#create-dataset)
-* [Get Dataset](#get-dataset)
+* [List Dataset](#list-and-get-dataset)
 * [List Dataslices](#list-and-get-dataslices)
 * [Export Dataslice](#export-dataslice-and-download)
 * [List Models](#list-models)
@@ -410,7 +410,18 @@ python tools/import_dataset_from_local.py -host https://staging.visionai.linkerv
 ```
 <br>
 
-### Get Dataset
+### List and Get Dataset
+
+The `list_datasets` method would return the list of dataset under the given project
+```Python
+project = client.get_project(project_id=1)
+datasets:list = project.list_datasets()
+```
+OR
+```Python
+datasets:list = client.list_datasets(project_id=1, client_alias=client.alias )
+```
+
 
 The `get_dataset` method retrieves the dataset info from the connected site. The `dataset_id` parameter is the unique integer ID of the dataset, not its "name" property.
 
@@ -561,8 +572,9 @@ python tools/export_dataslice.py -host https://staging.visionai.linkervision.ai/
 ```
 
 ### Export Large Dataslice and download files
-
-python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export model name / groundtruth} --target_folder {folder path} --export-format {coco, visionai ...etc}
+```
+python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
+``````
 
 ## Links to language repos
 

{dataverse_sdk-2.2.1 → dataverse_sdk-2.2.2}/README.md

@@ -73,7 +73,7 @@ The following sections provide examples for the most common DataVerse tasks incl
 * [Edit Project](#edit-project)
 * [Update Alias](#update-ontology-alias)
 * [Create Dataset](#create-dataset)
-* [Get Dataset](#get-dataset)
+* [List Dataset](#list-and-get-dataset)
 * [List Dataslices](#list-and-get-dataslices)
 * [Export Dataslice](#export-dataslice-and-download)
 * [List Models](#list-models)
@@ -383,7 +383,18 @@ python tools/import_dataset_from_local.py -host https://staging.visionai.linkerv
 ```
 <br>
 
-### Get Dataset
+### List and Get Dataset
+
+The `list_datasets` method would return the list of dataset under the given project
+```Python
+project = client.get_project(project_id=1)
+datasets:list = project.list_datasets()
+```
+OR
+```Python
+datasets:list = client.list_datasets(project_id=1, client_alias=client.alias )
+```
+
 
 The `get_dataset` method retrieves the dataset info from the connected site. The `dataset_id` parameter is the unique integer ID of the dataset, not its "name" property.
 
@@ -534,8 +545,9 @@ python tools/export_dataslice.py -host https://staging.visionai.linkervision.ai/
 ```
 
 ### Export Large Dataslice and download files
-
-python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export model name / groundtruth} --target_folder {folder path} --export-format {coco, visionai ...etc}
+```
+python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
+``````
 
 ## Links to language repos
 

{dataverse_sdk-2.2.1 → dataverse_sdk-2.2.2}/dataverse_sdk/apis/backend.py

@@ -12,6 +12,7 @@ from requests import sessions
 from requests.adapters import HTTPAdapter, Retry
 
 from ..exceptions.client import DataverseExceptionBase
+from ..utils.utils import chunks
 
 logger = logging.getLogger(__name__)
 
@@ -232,6 +233,15 @@ class BackendAPI:
         )
         return resp.json()["results"]
 
+    def list_datasets(self, project_id: int, **kwargs) -> list:
+        kwargs["project"] = project_id
+        resp = self.send_request(
+            url=f"{self.host}/api/datasets/?{urlencode(kwargs)}",
+            method="get",
+            headers=self.headers,
+        )
+        return resp.json()["results"]
+
     def list_dataslices(self, project_id: int, **kwargs) -> list:
         kwargs["project"] = project_id
         resp = self.send_request(
@@ -613,39 +623,63 @@ class AsyncBackendAPI:
             return None
 
     async def get_datarows(
-        self, batch_size: int = 20, order_by: str = "id", **kwargs
-    ) -> AsyncGenerator[list[dict], None, None]:
+        self,
+        batch_size: int = 20,
+        order_by: str = "id",
+        id_set_list: Optional[list] = None,
+        **kwargs,
+    ) -> AsyncGenerator[list[dict]]:
         if "offset" in kwargs or "limit" in kwargs:
             raise ValueError("Specifying offset or limit directly is not allowed.")
-
         kwargs["order_by"] = order_by
-        dataslice_set = kwargs.pop("dataslice_set", [])
-        query_params = {
-            **kwargs,
-            "order_by": order_by,
-            "dataslice_set": dataslice_set,
-            "limit": batch_size,
-        }
-        query_string = urlencode(query_params, doseq=True)
-
         id_gt = 0
-        while True:
-            url = f"{self.host}/api/datarows/?{query_string}&id__gt={id_gt}"
-            resp: dict = await self.async_send_request(
-                url=url,
-                method="get",
-                headers=self.headers,
-            )
-            json_data = resp
-            if json_data["count"] == 0:
-                break
-            if not json_data["results"]:
-                break
-
-            # Get last datarow id
-            datarows = json_data["results"]
-            id_gt = datarows[-1]["id"]
-            yield datarows
+        if id_set_list:
+            for id_chunks in chunks(id_set_list, batch_size):
+                while True:
+                    kwargs.update(
+                        {
+                            "id_set": ",".join([str(id_) for id_ in id_chunks]),
+                            "limit": batch_size,
+                            "id__gt": id_gt,
+                        }
+                    )
+                    url = f"{self.host}/api/datarows/?{urlencode(kwargs)}"
+                    resp: dict = await self.async_send_request(
+                        url=url,
+                        method="get",
+                        headers=self.headers,
+                    )
+                    json_data = resp
+                    datarows = json_data["results"]
+                    if not datarows:
+                        break
+                    # Get last datarow id
+                    id_gt = datarows[-1]["id"]
+                    yield datarows
+        else:
+            dataslice_set = kwargs.pop("dataslice_set", [])
+            query_params = {
+                **kwargs,
+                "dataslice_set": dataslice_set,
+                "limit": batch_size,
+            }
+            query_string = urlencode(query_params, doseq=True)
+
+            id_gt = 0
+            while True:
+                url = f"{self.host}/api/datarows/?{query_string}&id__gt={id_gt}"
+                resp: dict = await self.async_send_request(
+                    url=url,
+                    method="get",
+                    headers=self.headers,
+                )
+                json_data = resp
+                if not json_data["results"]:
+                    break
+                # Get last datarow id
+                datarows = json_data["results"]
+                id_gt = datarows[-1]["id"]
+                yield datarows
 
     async def get_datarows_flat_parent(
         self, batch_size: int = 20, order_by: str = "id", **kwargs

{dataverse_sdk-2.2.1 → dataverse_sdk-2.2.2}/dataverse_sdk/client.py

@@ -1149,6 +1149,24 @@ of this project OR has been added before"
             raise ClientConnectionError(f"Failed to get the models: {e}")
         return dataslice_list
 
+    @staticmethod
+    def list_datasets(
+        project_id: int,
+        client: Optional["DataverseClient"] = None,
+        client_alias: Optional[str] = None,
+    ) -> list:
+        api, client_alias = DataverseClient._get_api_client(
+            client=client, client_alias=client_alias
+        )
+        try:
+            dataset_list: list = api.list_datasets(project_id=project_id)
+        except DataverseExceptionBase:
+            logging.exception("Got api error from Dataverse")
+            raise
+        except Exception as e:
+            raise ClientConnectionError(f"Failed to get the models: {e}")
+        return dataset_list
+
     @staticmethod
     def list_models(
         project_id: int,

{dataverse_sdk-2.2.1 → dataverse_sdk-2.2.2}/dataverse_sdk/export/exporter.py

@@ -139,9 +139,7 @@ class Exporter:
             datarow_id_set: set[int] = set()
             for id_chunks in chunks(datarow_id_list, 1000):
                 gen: AsyncGenerator = curation_api.get_datarows(
-                    id_set=",".join(
-                        str(_id) for _id in id_chunks
-                    ),  # id_set="1,2,3,4,5"
+                    id_set_list=id_chunks,
                     batch_size=BATCH_SIZE,
                     fields="id,items,vlm_items,url,frame_id,image_width,image_height,sensor_name,original_url",
                 )

{dataverse_sdk-2.2.1 → dataverse_sdk-2.2.2}/dataverse_sdk/schemas/client.py

@@ -2,6 +2,7 @@ import re
 from typing import Optional, Union
 
 from pydantic import BaseModel, ConfigDict, field_validator
+from pydantic_core.core_schema import ValidationInfo
 
 from .common import (
     AnnotationFormat,
@@ -111,10 +112,10 @@ class QuestionClass(BaseModel):
         return value
 
     @field_validator("answer_type")
-    def answer_type_validator(cls, value, values, **kwargs):
-        if value == AttributeType.OPTION and not values.get("answer_options"):
+    def answer_type_validator(cls, value, values: ValidationInfo, **kwargs):
+        if value == AttributeType.OPTION and not values.data.get("answer_options"):
             raise ValueError(
-                f"* {values} Need to assign value for `answer_options` "
+                f"* {values.data} Need to assign value for `answer_options` "
                 + "if the Answer type is option"
             )
         return value
@@ -255,6 +256,14 @@ class Project(BaseModel):
         )
         return project
 
+    def list_datasets(self) -> list:
+        from ..client import DataverseClient
+
+        dataset_list: list = DataverseClient.list_datasets(
+            project_id=self.id, client_alias=self.client_alias
+        )
+        return dataset_list
+
     def list_dataslices(self) -> list:
         from ..client import DataverseClient
 

{dataverse_sdk-2.2.1 → dataverse_sdk-2.2.2}/dataverse_sdk/utils/utils.py

@@ -72,3 +72,9 @@ def download_file_from_url(url: str, save_path: str):
 
     except requests.exceptions.RequestException as e:
         print(f"An error occurred while downloading the file: {e}")
+
+
+def chunks(lst: list, n: int):
+    """Yield successive n-sized chunks from lst."""
+    for i in range(0, len(lst), n):
+        yield lst[i : i + n]

{dataverse_sdk-2.2.1 → dataverse_sdk-2.2.2}/dataverse_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataverse-sdk
-Version: 2.2.1
+Version: 2.2.2
 Summary: Dataverse SDK For Python
 Home-page: 
 Author: LinkerVision
@@ -100,7 +100,7 @@ The following sections provide examples for the most common DataVerse tasks incl
 * [Edit Project](#edit-project)
 * [Update Alias](#update-ontology-alias)
 * [Create Dataset](#create-dataset)
-* [Get Dataset](#get-dataset)
+* [List Dataset](#list-and-get-dataset)
 * [List Dataslices](#list-and-get-dataslices)
 * [Export Dataslice](#export-dataslice-and-download)
 * [List Models](#list-models)
@@ -410,7 +410,18 @@ python tools/import_dataset_from_local.py -host https://staging.visionai.linkerv
 ```
 <br>
 
-### Get Dataset
+### List and Get Dataset
+
+The `list_datasets` method would return the list of dataset under the given project
+```Python
+project = client.get_project(project_id=1)
+datasets:list = project.list_datasets()
+```
+OR
+```Python
+datasets:list = client.list_datasets(project_id=1, client_alias=client.alias )
+```
+
 
 The `get_dataset` method retrieves the dataset info from the connected site. The `dataset_id` parameter is the unique integer ID of the dataset, not its "name" property.
 
@@ -561,8 +572,9 @@ python tools/export_dataslice.py -host https://staging.visionai.linkervision.ai/
 ```
 
 ### Export Large Dataslice and download files
-
-python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export model name / groundtruth} --target_folder {folder path} --export-format {coco, visionai ...etc}
+```
+python tools/export_dataslice_large.py -host https://visionai.linkervision.ai/dataverse/curation -e {your-account-email} -p {PASSWORD} -s {service-id} -dataslice {dataslice_id} --anno {export-model-name / groundtruth} --target_folder {folder path} --export-format {coco, visionai, yolo, vlm ...etc}
+``````
 
 ## Links to language repos
 

{dataverse_sdk-2.2.1 → dataverse_sdk-2.2.2}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 AUTHOR = "LinkerVision"
 PACKAGE_NAME = "dataverse-sdk"
-PACKAGE_VERSION = "2.2.1"
+PACKAGE_VERSION = "2.2.2"
 DESC = "Dataverse SDK For Python"
 with open("README.md", encoding="utf-8") as fh:
     long_description = fh.read()