scale-nucleus 0.17.6b4__py3-none-any.whl → 0.17.8__py3-none-any.whl

nucleus/annotation.py

@@ -33,6 +33,7 @@ from .constants import (
     POLYGON_TYPE,
     POSITION_KEY,
     REFERENCE_ID_KEY,
+    TASK_ID_KEY,
     TAXONOMY_NAME_KEY,
     TRACK_REFERENCE_ID_KEY,
     TYPE_KEY,
@@ -158,6 +159,7 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
     metadata: Optional[Dict] = None
     embedding_vector: Optional[list] = None
     track_reference_id: Optional[str] = None
+    task_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -178,6 +180,7 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
             metadata=payload.get(METADATA_KEY, {}),
             embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -195,6 +198,7 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
             METADATA_KEY: self.metadata,
             EMBEDDING_VECTOR_KEY: self.embedding_vector,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self.task_id,
         }
 
     def __eq__(self, other):
@@ -209,6 +213,7 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
             and sorted(self.metadata.items()) == sorted(other.metadata.items())
             and self.embedding_vector == other.embedding_vector
             and self.track_reference_id == other.track_reference_id
+            and self.task_id == other.task_id
         )
 
 
@@ -275,6 +280,7 @@ class LineAnnotation(Annotation):
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
     track_reference_id: Optional[str] = None
+    task_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -304,6 +310,7 @@ class LineAnnotation(Annotation):
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -317,6 +324,7 @@ class LineAnnotation(Annotation):
             ANNOTATION_ID_KEY: self.annotation_id,
             METADATA_KEY: self.metadata,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self.task_id,
         }
         return payload
 
@@ -367,6 +375,7 @@ class PolygonAnnotation(Annotation):
     metadata: Optional[Dict] = None
     embedding_vector: Optional[list] = None
     track_reference_id: Optional[str] = None
+    task_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -397,6 +406,7 @@ class PolygonAnnotation(Annotation):
             metadata=payload.get(METADATA_KEY, {}),
             embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -411,6 +421,7 @@ class PolygonAnnotation(Annotation):
             METADATA_KEY: self.metadata,
             EMBEDDING_VECTOR_KEY: self.embedding_vector,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self.task_id,
         }
         return payload
 
@@ -507,6 +518,7 @@ class KeypointsAnnotation(Annotation):
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
     track_reference_id: Optional[str] = None
+    task_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata or {}
@@ -559,6 +571,7 @@ class KeypointsAnnotation(Annotation):
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -574,6 +587,7 @@ class KeypointsAnnotation(Annotation):
             ANNOTATION_ID_KEY: self.annotation_id,
             METADATA_KEY: self.metadata,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self.task_id,
         }
         return payload
 
@@ -678,6 +692,7 @@ class CuboidAnnotation(Annotation):  # pylint: disable=R0902
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
     track_reference_id: Optional[str] = None
+    task_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -694,6 +709,7 @@ class CuboidAnnotation(Annotation):  # pylint: disable=R0902
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -926,6 +942,7 @@ class CategoryAnnotation(Annotation):
     taxonomy_name: Optional[str] = None
     metadata: Optional[Dict] = None
     track_reference_id: Optional[str] = None
+    task_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -938,6 +955,7 @@ class CategoryAnnotation(Annotation):
             taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -948,6 +966,7 @@ class CategoryAnnotation(Annotation):
             REFERENCE_ID_KEY: self.reference_id,
             METADATA_KEY: self.metadata,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self.task_id,
         }
         if self.taxonomy_name is not None:
             payload[TAXONOMY_NAME_KEY] = self.taxonomy_name
@@ -963,6 +982,7 @@ class MultiCategoryAnnotation(Annotation):
     taxonomy_name: Optional[str] = None
     metadata: Optional[Dict] = None
     track_reference_id: Optional[str] = None
+    task_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -975,6 +995,7 @@ class MultiCategoryAnnotation(Annotation):
             taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
+            task_id=payload.get(TASK_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -985,6 +1006,7 @@ class MultiCategoryAnnotation(Annotation):
             REFERENCE_ID_KEY: self.reference_id,
             METADATA_KEY: self.metadata,
             TRACK_REFERENCE_ID_KEY: self.track_reference_id,
+            TASK_ID_KEY: self.task_id,
         }
         if self.taxonomy_name is not None:
             payload[TAXONOMY_NAME_KEY] = self.taxonomy_name

nucleus/annotation_uploader.py

@@ -214,19 +214,19 @@ class AnnotationUploader:
 
     @staticmethod
     def check_for_duplicate_ids(annotations: Iterable[Annotation]):
-        """Do not allow annotations to have the same (annotation_id, reference_id) tuple"""
+        """Do not allow annotations to have the same (annotation_id, reference_id, task_id) tuple"""
 
         # some annotations like CategoryAnnotation do not have annotation_id attribute, and as such, we allow duplicates
         tuple_ids = [
-            (ann.reference_id, ann.annotation_id)  # type: ignore
+            (ann.reference_id, ann.annotation_id, ann.task_id)  # type: ignore
             for ann in annotations
-            if hasattr(ann, "annotation_id")
+            if hasattr(ann, "annotation_id") and hasattr(ann, "task_id")
         ]
         tuple_count = Counter(tuple_ids)
         duplicates = {key for key, value in tuple_count.items() if value > 1}
         if len(duplicates) > 0:
             raise DuplicateIDError(
-                f"Duplicate annotations with the same (reference_id, annotation_id) properties found.\n"
+                f"Duplicate annotations with the same (reference_id, annotation_id, task_id) properties found.\n"
                 f"Duplicates: {duplicates}\n"
                 f"To fix this, avoid duplicate annotations, or specify a different annotation_id attribute "
                 f"for the failing items."

nucleus/constants.py

@@ -114,8 +114,7 @@ MODEL_PREDICTION_ID_KEY = "model_prediction_id"
 MODEL_PREDICTION_LABEL_KEY = "model_prediction_label"
 NAME_KEY = "name"
 NEW_ITEMS = "new_items"
-# NUCLEUS_ENDPOINT = "https://api.scale.com/v1/nucleus"
-NUCLEUS_ENDPOINT = "http://localhost:3003/nucleus"
+NUCLEUS_ENDPOINT = "https://api.scale.com/v1/nucleus"
 NUM_SENSORS_KEY = "num_sensors"
 ORIGINAL_IMAGE_URL_KEY = "original_image_url"
 PAGE_SIZE_KEY = "pageSize"
@@ -149,6 +148,7 @@ STATUS_KEY = "status"
 SUCCESS_STATUS_CODES = [200, 201, 202]
 SLICE_TAGS_KEY = "slice_tags"
 TAXONOMY_NAME_KEY = "taxonomy_name"
+TASK_ID_KEY = "task_id"
 TRACK_REFERENCE_ID_KEY = "track_reference_id"
 TRACK_REFERENCE_IDS_KEY = "track_reference_ids"
 TRACKS_KEY = "tracks"

nucleus/dataset.py

@@ -1449,11 +1449,15 @@ class Dataset:
         )
         return convert_export_payload(api_payload[EXPORTED_ROWS])
 
-    def scene_and_annotation_generator(self, page_size: int = 10):
+    def scene_and_annotation_generator(
+        self, slice_id=None, page_size: int = 10, only_most_recent_tasks=True
+    ):
         """Provides a generator of all Scenes and Annotations in the dataset grouped by scene.
 
         Args:
+            slice_id: Optional slice ID to filter the scenes and annotations.
             page_size: Number of scenes to fetch per page. Default is 10.
+            only_most_recent_tasks: If True, only the annotations corresponding to the most recent task for each item is returned.
 
         Returns:
             Generator where each element is a nested dict containing scene and annotation information of the dataset structured as a JSON.
@@ -1505,6 +1509,8 @@ class Dataset:
             endpoint=f"dataset/{self.id}/{endpoint_name}",
             result_key=EXPORT_FOR_TRAINING_KEY,
             page_size=page_size,
+            sliceId=slice_id,
+            onlyMostRecentTask=only_most_recent_tasks,
         )
 
         for data in json_generator:
@@ -1514,12 +1520,14 @@ class Dataset:
         self,
         query: Optional[str] = None,
         use_mirrored_images: bool = False,
+        only_most_recent_tasks: bool = True,
     ) -> Iterable[Dict[str, Union[DatasetItem, Dict[str, List[Annotation]]]]]:
         """Provides a generator of all DatasetItems and Annotations in the dataset.
 
         Args:
             query: Structured query compatible with the `Nucleus query language <https://nucleus.scale.com/docs/query-language-reference>`_.
             use_mirrored_images: If True, returns the location of the mirrored image hosted in Scale S3. Useful when the original image is no longer available.
+            only_most_recent_tasks: If True, only the annotations corresponding to the most recent task for each item is returned.
 
         Returns:
             Generator where each element is a dict containing the DatasetItem
@@ -1546,6 +1554,7 @@ class Dataset:
             page_size=10000,  # max ES page size
             query=query,
             chip=use_mirrored_images,
+            onlyMostRecentTask=only_most_recent_tasks,
         )
         for data in json_generator:
             for ia in convert_export_payload([data], has_predictions=False):

nucleus/slice.py

@@ -410,9 +410,13 @@ class Slice:
 
     def items_and_annotation_generator(
         self,
+        use_mirrored_images: bool = False,
     ) -> Iterable[Dict[str, Union[DatasetItem, Dict[str, List[Annotation]]]]]:
         """Provides a generator of all DatasetItems and Annotations in the slice.
 
+        Args:
+            use_mirrored_images: If True, returns the location of the mirrored image hosted in Scale S3. Useful when the original image is no longer available.
+
         Returns:
             Generator where each element is a dict containing the DatasetItem
             and all of its associated Annotations, grouped by type (e.g. box).
@@ -436,6 +440,7 @@ class Slice:
             endpoint=f"slice/{self.id}/exportForTrainingPage",
             result_key=EXPORT_FOR_TRAINING_KEY,
             page_size=10000,  # max ES page size
+            chip=use_mirrored_images,
         )
         for data in json_generator:
             for ia in convert_export_payload([data], has_predictions=False):

{scale_nucleus-0.17.6b4.dist-info → scale_nucleus-0.17.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scale-nucleus
-Version: 0.17.6b4
+Version: 0.17.8
 Summary: The official Python client library for Nucleus, the Data Platform for AI
 Home-page: https://scale.com/nucleus
 License: MIT
@@ -14,7 +14,6 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: launch
 Provides-Extra: metrics
 Requires-Dist: Pillow (>=7.1.2)

{scale_nucleus-0.17.6b4.dist-info → scale_nucleus-0.17.8.dist-info}/RECORD

@@ -11,22 +11,22 @@ cli/reference.py,sha256=RuHVhmGTZNe0MfwpL96YjJdaH0OJzg98rz4xeIu4hJU,256
 cli/slices.py,sha256=nxq_Zg1m5oXuhz0ibyHkElvyVWt1AcE9tG-fN4CQxF8,1397
 cli/tests.py,sha256=NiwEVGuF08_jlCiKEIjKhwq55NvyU4xvPEJW5MJmdZg,4590
 nucleus/__init__.py,sha256=RSXlW0oL5ThX9LdRb5Eng6W9Fn_H9bqNZhAQrFQWMx8,49712
-nucleus/annotation.py,sha256=qogOLOmNwv2o13kNEZkIRbI2lkp1owNZ2OnRsFJUZwU,42904
-nucleus/annotation_uploader.py,sha256=ipXw7QhJQXqaftHrbJn8oCGcm6aXyoL0GhQA9EcZWD4,9580
+nucleus/annotation.py,sha256=s79v-BmI3MjJiRoZCEoAeFUKn43Pghl63ZJI3P7mHc4,43803
+nucleus/annotation_uploader.py,sha256=CXvIjDNuQczGk8poNLimP7s2owRwEG7xxycMjFm0pYI,9639
 nucleus/async_job.py,sha256=yjPDwyyLIrF0K67anGB40xux1AMhWrq1X_hPvQ_ewzc,6890
 nucleus/async_utils.py,sha256=ayqajeSonX68fre3u8AoNRYT8GFGPd4_iu6YPQTvpvU,8226
 nucleus/autocurate.py,sha256=kI0vRqad_An8SN5JX6sSdGP_vNHJI2Pq4NINHuhNf2U,1080
 nucleus/camera_params.py,sha256=fl17aaSAZDAJIWo6F2HFvM6HKGcQh9fXvo4t3RzGMc4,3726
 nucleus/chip_utils.py,sha256=1J1NHCh0ZptW8cdeuLWFM_cXuwQVSQFtSF8kXU8s2tI,6743
 nucleus/connection.py,sha256=q212plDtWoonfXWMVaCqTZBPZTy8dnNSGj0YeAR1Qmk,2990
-nucleus/constants.py,sha256=p7urHHYcp0Kxd-NUjkXs6dkrwnm2lyEj5vRapdQGT6Y,5519
+nucleus/constants.py,sha256=6Ce2RUb2S9vd4ww6sDG27ueE_HChIPVHC0CwQ60Czig,5490
 nucleus/data_transfer_object/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucleus/data_transfer_object/dataset_details.py,sha256=1YGvfKkPSqDrXK_y5mBXyRThY07tU-nwOCYTkYCSl6k,214
 nucleus/data_transfer_object/dataset_info.py,sha256=5P_gpvAyaqXxj2ZQuzLkGN2XROaN9Me56OLybCmO3R4,940
 nucleus/data_transfer_object/dataset_size.py,sha256=oe-dXaMLpsQRDcJQRZ9Ja8JTagYz4dviZuTognEylp0,111
 nucleus/data_transfer_object/job_status.py,sha256=hxvyNdrdVdj3UpEfwvryKC_QCJQEC9ru6IPjhPFcK44,2038
 nucleus/data_transfer_object/scenes_list.py,sha256=iTHE6vA47bRB6ciyEU4LArUXEXco4ArnGvZTGTeK8xs,432
-nucleus/dataset.py,sha256=pmtUpYKWgSReT-lxA1dODCL3VzY_8Tq7yuyNWL7CkUk,94080
+nucleus/dataset.py,sha256=azWq0ftX2QOx2pxYDqFuwRPo48DK9I_F_AO_g3xPPIs,94663
 nucleus/dataset_item.py,sha256=y9ia47i31lX2wvw6EkVAxeHburMrrZpuyjEGlstWa2A,10166
 nucleus/dataset_item_uploader.py,sha256=BD0FTgimEFYmDbnOLIaQZS3OLDfLe5wumADDmgMX598,6684
 nucleus/deprecation_warning.py,sha256=5C9dVusR5UkUQnW2MrRkIXCfbc8ULc7xOaB134agNKk,976
@@ -61,7 +61,7 @@ nucleus/pydantic_base.py,sha256=ZBUVrf948qzaxSuTaiDWxPC_Y8AOBdLKfi52ozGpGWk,1388
 nucleus/quaternion.py,sha256=TAnwj4arQXoTeofFgZMdZsCyxAMnu23N6to0F1WFNwk,1111
 nucleus/retry_strategy.py,sha256=daKZqjZYCh87WtXoVUuR9BZu2TTE-CtOFEYZ-d6xVMY,312
 nucleus/scene.py,sha256=qZQD7QdF6Ics8kuszsl278NCowKVnAkVNGHvPr5luRo,26937
-nucleus/slice.py,sha256=VlFRE4NSLDcus86EfSHniWGLKBdjEokrnrmBtUz8pT0,28056
+nucleus/slice.py,sha256=DvZQZS9HvQku9Tj7SHLaE7bv5x7Z72_4T_cio-d2hdA,28312
 nucleus/test_launch_integration.py,sha256=oFKLZWjFGeUvwVV0XAAjP1Y_oKFkaouh_SXVPXtCvcE,10688
 nucleus/track.py,sha256=ROmOyzYZKrHVTnLBhnk-qEBtklD_EDsSnRcGYE8xG4E,3247
 nucleus/upload_response.py,sha256=wR_pfZCBju1vGiGqbVgk8zhM6GhD3ebYxyGBm8y0GvY,3287
@@ -85,8 +85,8 @@ nucleus/validate/scenario_test.py,sha256=pCmM157dblSciZCDTw-f47Fpy3OUZFgXmokdhIL
 nucleus/validate/scenario_test_evaluation.py,sha256=Q0WzaEE9uUbPVc4EHlCoKjhJcqMNt4QbyiiJx12VOR0,4075
 nucleus/validate/scenario_test_metric.py,sha256=AhVFOB1ULwBqlZ2X_Au1TXy4iQELljtzR4ZpeLB35So,1209
 nucleus/validate/utils.py,sha256=VjdIJj9Pii4z4L6xbvClAc7ra_J7cX0vWB_J2X6yrGE,185
-scale_nucleus-0.17.6b4.dist-info/LICENSE,sha256=jaTGyQSQIZeWMo5iyYqgbAYHR9Bdy7nOzgE-Up3m_-g,1075
-scale_nucleus-0.17.6b4.dist-info/METADATA,sha256=vCvgoP7nRtfkxA9t2Wgz22S4Q76ZUYBVSFm5V-iE6Ws,7973
-scale_nucleus-0.17.6b4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-scale_nucleus-0.17.6b4.dist-info/entry_points.txt,sha256=fmqEzh6NZQyg9eFMILnWabKT8OWQTMSCdDzMiVq2zYs,32
-scale_nucleus-0.17.6b4.dist-info/RECORD,,
+scale_nucleus-0.17.8.dist-info/LICENSE,sha256=jaTGyQSQIZeWMo5iyYqgbAYHR9Bdy7nOzgE-Up3m_-g,1075
+scale_nucleus-0.17.8.dist-info/METADATA,sha256=XZWcJybnh0CG0lxxioOUf7KC-s2HNpkduNqvRmxU7k0,7920
+scale_nucleus-0.17.8.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+scale_nucleus-0.17.8.dist-info/entry_points.txt,sha256=fmqEzh6NZQyg9eFMILnWabKT8OWQTMSCdDzMiVq2zYs,32
+scale_nucleus-0.17.8.dist-info/RECORD,,

{scale_nucleus-0.17.6b4.dist-info → scale_nucleus-0.17.8.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 1.6.1
 Root-Is-Purelib: true
 Tag: py3-none-any