scale-nucleus 0.17.7__tar.gz → 0.17.9__tar.gz

{scale_nucleus-0.17.7 → scale_nucleus-0.17.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scale-nucleus
-Version: 0.17.7
+Version: 0.17.9
 Summary: The official Python client library for Nucleus, the Data Platform for AI
 Home-page: https://scale.com/nucleus
 License: MIT

{scale_nucleus-0.17.7 → scale_nucleus-0.17.9}/nucleus/annotation.py

@@ -33,6 +33,7 @@ from .constants import (
     POLYGON_TYPE,
     POSITION_KEY,
     REFERENCE_ID_KEY,
+    TASK_ID_KEY,
     TAXONOMY_NAME_KEY,
     TRACK_REFERENCE_ID_KEY,
     TYPE_KEY,
@@ -158,6 +159,7 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
     metadata: Optional[Dict] = None
     embedding_vector: Optional[list] = None
     track_reference_id: Optional[str] = None
+    _task_id: Optional[str] = field(default=None, repr=False)
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -178,6 +180,7 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
             metadata=payload.get(METADATA_KEY, {}),
             embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            _task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -195,6 +198,7 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
             METADATA_KEY: self.metadata,
             EMBEDDING_VECTOR_KEY: self.embedding_vector,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self._task_id,
         }
 
     def __eq__(self, other):
@@ -209,6 +213,7 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
             and sorted(self.metadata.items()) == sorted(other.metadata.items())
             and self.embedding_vector == other.embedding_vector
             and self.track_reference_id == other.track_reference_id
+            and self._task_id == other._task_id
         )
 
 
@@ -275,6 +280,7 @@ class LineAnnotation(Annotation):
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
     track_reference_id: Optional[str] = None
+    _task_id: Optional[str] = field(default=None, repr=False)
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -304,6 +310,7 @@ class LineAnnotation(Annotation):
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            _task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -317,6 +324,7 @@ class LineAnnotation(Annotation):
             ANNOTATION_ID_KEY: self.annotation_id,
             METADATA_KEY: self.metadata,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self._task_id,
         }
         return payload
 
@@ -367,6 +375,7 @@ class PolygonAnnotation(Annotation):
     metadata: Optional[Dict] = None
     embedding_vector: Optional[list] = None
     track_reference_id: Optional[str] = None
+    _task_id: Optional[str] = field(default=None, repr=False)
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -397,6 +406,7 @@ class PolygonAnnotation(Annotation):
             metadata=payload.get(METADATA_KEY, {}),
             embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            _task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -411,6 +421,7 @@ class PolygonAnnotation(Annotation):
             METADATA_KEY: self.metadata,
             EMBEDDING_VECTOR_KEY: self.embedding_vector,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self._task_id,
         }
         return payload
 
@@ -507,6 +518,7 @@ class KeypointsAnnotation(Annotation):
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
     track_reference_id: Optional[str] = None
+    _task_id: Optional[str] = field(default=None, repr=False)
 
     def __post_init__(self):
         self.metadata = self.metadata or {}
@@ -559,6 +571,7 @@ class KeypointsAnnotation(Annotation):
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            _task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -574,6 +587,7 @@ class KeypointsAnnotation(Annotation):
             ANNOTATION_ID_KEY: self.annotation_id,
             METADATA_KEY: self.metadata,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self._task_id,
         }
         return payload
 
@@ -678,6 +692,7 @@ class CuboidAnnotation(Annotation):  # pylint: disable=R0902
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
     track_reference_id: Optional[str] = None
+    _task_id: Optional[str] = field(default=None, repr=False)
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -694,6 +709,7 @@ class CuboidAnnotation(Annotation):  # pylint: disable=R0902
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            _task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -713,7 +729,8 @@ class CuboidAnnotation(Annotation):  # pylint: disable=R0902
             payload[METADATA_KEY] = self.metadata
         if self.track_reference_id:
             payload[TRACK_REFERENCE_ID_KEY] = self.track_reference_id
-
+        if self._task_id:
+            payload[TASK_ID_KEY] = self._task_id
         return payload
 
 
@@ -926,6 +943,7 @@ class CategoryAnnotation(Annotation):
     taxonomy_name: Optional[str] = None
     metadata: Optional[Dict] = None
     track_reference_id: Optional[str] = None
+    _task_id: Optional[str] = field(default=None, repr=False)
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -938,6 +956,7 @@ class CategoryAnnotation(Annotation):
             taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            _task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -948,6 +967,7 @@ class CategoryAnnotation(Annotation):
             REFERENCE_ID_KEY: self.reference_id,
             METADATA_KEY: self.metadata,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self._task_id,
         }
         if self.taxonomy_name is not None:
             payload[TAXONOMY_NAME_KEY] = self.taxonomy_name
@@ -963,6 +983,7 @@ class MultiCategoryAnnotation(Annotation):
     taxonomy_name: Optional[str] = None
     metadata: Optional[Dict] = None
     track_reference_id: Optional[str] = None
+    _task_id: Optional[str] = field(default=None, repr=False)
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -975,6 +996,7 @@ class MultiCategoryAnnotation(Annotation):
             taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            _task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -985,6 +1007,7 @@ class MultiCategoryAnnotation(Annotation):
             REFERENCE_ID_KEY: self.reference_id,
             METADATA_KEY: self.metadata,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self._task_id,
         }
         if self.taxonomy_name is not None:
             payload[TAXONOMY_NAME_KEY] = self.taxonomy_name
@@ -1023,6 +1046,7 @@ class SceneCategoryAnnotation(Annotation):
     reference_id: str
     taxonomy_name: Optional[str] = None
     metadata: Optional[Dict] = field(default_factory=dict)
+    _task_id: Optional[str] = field(default=None, repr=False)
 
     @classmethod
     def from_json(cls, payload: dict):
@@ -1031,6 +1055,7 @@ class SceneCategoryAnnotation(Annotation):
             reference_id=payload[REFERENCE_ID_KEY],
             taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
+            _task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -1040,6 +1065,7 @@ class SceneCategoryAnnotation(Annotation):
             GEOMETRY_KEY: {},
             REFERENCE_ID_KEY: self.reference_id,
             METADATA_KEY: self.metadata,
+            TASK_ID_KEY: self._task_id,
         }
         if self.taxonomy_name is not None:
             payload[TAXONOMY_NAME_KEY] = self.taxonomy_name
@@ -1057,9 +1083,7 @@ class AnnotationList:
         default_factory=list
     )
     cuboid_annotations: List[CuboidAnnotation] = field(default_factory=list)
-    category_annotations: List[CategoryAnnotation] = field(
-        default_factory=list
-    )
+    category_annotations: List[CategoryAnnotation] = field(default_factory=list)
     multi_category_annotations: List[MultiCategoryAnnotation] = field(
         default_factory=list
     )

{scale_nucleus-0.17.7 → scale_nucleus-0.17.9}/nucleus/annotation_uploader.py

@@ -176,9 +176,7 @@ class AnnotationUploader:
         """
 
         def fn():
-            request_json = construct_segmentation_payload(
-                segmentations, update
-            )
+            request_json = construct_segmentation_payload(segmentations, update)
             form_data = [
                 FileFormField(
                     name=SERIALIZED_REQUEST_KEY,
@@ -212,13 +210,15 @@ class AnnotationUploader:
 
         return fn
 
-    @staticmethod
-    def check_for_duplicate_ids(annotations: Iterable[Annotation]):
-        """Do not allow annotations to have the same (annotation_id, reference_id) tuple"""
+    def check_for_duplicate_ids(self, annotations: Iterable[Annotation]):
+        """Do not allow annotations to have the same (annotation_id, reference_id, task_id) tuple"""
 
-        # some annotations like CategoryAnnotation do not have annotation_id attribute, and as such, we allow duplicates
         tuple_ids = [
-            (ann.reference_id, ann.annotation_id)  # type: ignore
+            (
+                ann.reference_id,
+                ann.annotation_id,
+                getattr(ann, "_task_id", None),
+            )
             for ann in annotations
             if hasattr(ann, "annotation_id")
         ]
@@ -226,7 +226,7 @@ class AnnotationUploader:
         duplicates = {key for key, value in tuple_count.items() if value > 1}
         if len(duplicates) > 0:
             raise DuplicateIDError(
-                f"Duplicate annotations with the same (reference_id, annotation_id) properties found.\n"
+                f"Duplicate annotations with the same (reference_id, annotation_id, task_id) properties found.\n"
                 f"Duplicates: {duplicates}\n"
                 f"To fix this, avoid duplicate annotations, or specify a different annotation_id attribute "
                 f"for the failing items."
@@ -255,3 +255,20 @@ class PredictionUploader(AnnotationUploader):
             self._route = (
                 f"dataset/{dataset_id}/model/{model_id}/uploadPredictions"
             )
+
+    def check_for_duplicate_ids(self, annotations: Iterable[Annotation]):
+        """Do not allow predictions to have the same (annotation_id, reference_id) tuple"""
+        tuple_ids = [
+            (pred.annotation_id, pred.reference_id)  # type: ignore
+            for pred in annotations
+            if hasattr(pred, "annotation_id") and hasattr(pred, "reference_id")
+        ]
+        tuple_count = Counter(tuple_ids)
+        duplicates = {key for key, value in tuple_count.items() if value > 1}
+        if len(duplicates) > 0:
+            raise DuplicateIDError(
+                f"Duplicate predictions with the same (annotation_id, reference_id) properties found.\n"
+                f"Duplicates: {duplicates}\n"
+                f"To fix this, avoid duplicate predictions, or specify a different annotation_id attribute "
+                f"for the failing items."
+            )

{scale_nucleus-0.17.7 → scale_nucleus-0.17.9}/nucleus/constants.py

@@ -148,6 +148,7 @@ STATUS_KEY = "status"
 SUCCESS_STATUS_CODES = [200, 201, 202]
 SLICE_TAGS_KEY = "slice_tags"
 TAXONOMY_NAME_KEY = "taxonomy_name"
+TASK_ID_KEY = "task_id"
 TRACK_REFERENCE_ID_KEY = "track_reference_id"
 TRACK_REFERENCE_IDS_KEY = "track_reference_ids"
 TRACKS_KEY = "tracks"

{scale_nucleus-0.17.7 → scale_nucleus-0.17.9}/nucleus/dataset.py

@@ -332,8 +332,7 @@ class Dataset:
         dataset_item_jsons = response.get(DATASET_ITEMS_KEY, None)
 
         return [
-            DatasetItem.from_json(item_json)
-            for item_json in dataset_item_jsons
+            DatasetItem.from_json(item_json) for item_json in dataset_item_jsons
         ]
 
     @property
@@ -699,9 +698,7 @@ class Dataset:
                 asynchronous
             ), "In order to avoid timeouts, you must set asynchronous=True when uploading videos."
 
-            return self._append_video_scenes(
-                video_scenes, update, asynchronous
-            )
+            return self._append_video_scenes(video_scenes, update, asynchronous)
 
         if len(dataset_items) > WARN_FOR_LARGE_UPLOAD and not asynchronous:
             print(
@@ -1450,13 +1447,14 @@ class Dataset:
         return convert_export_payload(api_payload[EXPORTED_ROWS])
 
     def scene_and_annotation_generator(
-        self, slice_id=None, page_size: int = 10
+        self, slice_id=None, page_size: int = 10, only_most_recent_tasks=True
     ):
         """Provides a generator of all Scenes and Annotations in the dataset grouped by scene.
 
         Args:
             slice_id: Optional slice ID to filter the scenes and annotations.
             page_size: Number of scenes to fetch per page. Default is 10.
+            only_most_recent_tasks: If True, only the annotations corresponding to the most recent task for each item is returned.
 
         Returns:
             Generator where each element is a nested dict containing scene and annotation information of the dataset structured as a JSON.
@@ -1509,6 +1507,7 @@ class Dataset:
             result_key=EXPORT_FOR_TRAINING_KEY,
             page_size=page_size,
             sliceId=slice_id,
+            onlyMostRecentTask=only_most_recent_tasks,
         )
 
         for data in json_generator:
@@ -1518,12 +1517,14 @@ class Dataset:
         self,
         query: Optional[str] = None,
         use_mirrored_images: bool = False,
+        only_most_recent_tasks: bool = True,
     ) -> Iterable[Dict[str, Union[DatasetItem, Dict[str, List[Annotation]]]]]:
         """Provides a generator of all DatasetItems and Annotations in the dataset.
 
         Args:
             query: Structured query compatible with the `Nucleus query language <https://nucleus.scale.com/docs/query-language-reference>`_.
             use_mirrored_images: If True, returns the location of the mirrored image hosted in Scale S3. Useful when the original image is no longer available.
+            only_most_recent_tasks: If True, only the annotations corresponding to the most recent task for each item is returned.
 
         Returns:
             Generator where each element is a dict containing the DatasetItem
@@ -1550,6 +1551,7 @@ class Dataset:
             page_size=10000,  # max ES page size
             query=query,
             chip=use_mirrored_images,
+            onlyMostRecentTask=only_most_recent_tasks,
         )
         for data in json_generator:
             for ia in convert_export_payload([data], has_predictions=False):
@@ -2356,10 +2358,7 @@ class Dataset:
         )
 
         if len(items) > 0:
-            if (
-                len(items) > GLOB_SIZE_THRESHOLD_CHECK
-                and not skip_size_warning
-            ):
+            if len(items) > GLOB_SIZE_THRESHOLD_CHECK and not skip_size_warning:
                 raise Exception(
                     f"Found over {GLOB_SIZE_THRESHOLD_CHECK} items in {dirname}. If this is intended,"
                     f" set skip_size_warning=True when calling this function."
@@ -2406,3 +2405,26 @@ class Dataset:
             route=f"dataset/{self.id}/model/{model.id}/pointcloud/{pointcloud_ref_id}/uploadLSSPrediction",
             requests_command=requests.post,
         )
+
+    def export_class_labels(self, slice_id: Optional[str] = None):
+        """Fetches a list of class labels for the dataset.
+
+        Args:
+            slice_id (str | None): The ID of the slice to export class labels for. If None, export class labels for the entire dataset.
+
+        Returns:
+            A list of class labels for the dataset.
+        """
+        if slice_id:
+            api_payload = self._client.make_request(
+                payload=None,
+                route=f"slice/{slice_id}/class_labels",
+                requests_command=requests.get,
+            )
+        else:
+            api_payload = self._client.make_request(
+                payload=None,
+                route=f"dataset/{self.id}/class_labels",
+                requests_command=requests.get,
+            )
+        return api_payload.get("data", [])

{scale_nucleus-0.17.7 → scale_nucleus-0.17.9}/nucleus/slice.py

@@ -168,9 +168,7 @@ class Slice:
     @property
     def pending_job_count(self) -> Optional[int]:
         if self._pending_job_count is None:
-            self._pending_job_count = self.info().get(
-                "pending_job_count", None
-            )
+            self._pending_job_count = self.info().get("pending_job_count", None)
         return self._pending_job_count
 
     @classmethod
@@ -410,9 +408,13 @@ class Slice:
 
     def items_and_annotation_generator(
         self,
+        use_mirrored_images: bool = False,
     ) -> Iterable[Dict[str, Union[DatasetItem, Dict[str, List[Annotation]]]]]:
         """Provides a generator of all DatasetItems and Annotations in the slice.
 
+        Args:
+            use_mirrored_images: If True, returns the location of the mirrored image hosted in Scale S3. Useful when the original image is no longer available.
+
         Returns:
             Generator where each element is a dict containing the DatasetItem
             and all of its associated Annotations, grouped by type (e.g. box).
@@ -436,6 +438,7 @@ class Slice:
             endpoint=f"slice/{self.id}/exportForTrainingPage",
             result_key=EXPORT_FOR_TRAINING_KEY,
             page_size=10000,  # max ES page size
+            chip=use_mirrored_images,
         )
         for data in json_generator:
             for ia in convert_export_payload([data], has_predictions=False):
@@ -700,6 +703,15 @@ class Slice:
         )
         return api_payload
 
+    def export_class_labels(self):
+        """Fetches a list of class labels for the slice."""
+        api_payload = self._client.make_request(
+            payload=None,
+            route=f"slice/{self.id}/class_labels",
+            requests_command=requests.get,
+        )
+        return api_payload.get("data", [])
+
 
 def check_annotations_are_in_slice(
     annotations: List[Annotation], slice_to_check: Slice

{scale_nucleus-0.17.7 → scale_nucleus-0.17.9}/pyproject.toml

@@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"]  # Easy ignore for getting it running
 
 [tool.poetry]
 name = "scale-nucleus"
-version = "0.17.7"
+version = "0.17.9"
 description = "The official Python client library for Nucleus, the Data Platform for AI"
 license =  "MIT"
 authors = ["Scale AI Nucleus Team <nucleusapi@scaleapi.com>"]