scale-nucleus 0.1.24__py3-none-any.whl → 0.6.4__py3-none-any.whl

nucleus/job.py

@@ -1,7 +1,9 @@
-from dataclasses import dataclass
 import time
+from dataclasses import dataclass
 from typing import Dict, List
+
 import requests
+
 from nucleus.constants import (
     JOB_CREATION_TIME_KEY,
     JOB_ID_KEY,
@@ -9,12 +11,33 @@ from nucleus.constants import (
     JOB_TYPE_KEY,
     STATUS_KEY,
 )
+from nucleus.utils import replace_double_slashes
 
 JOB_POLLING_INTERVAL = 5
 
 
 @dataclass
 class AsyncJob:
+    """Object used to check the status or errors of a long running asynchronous operation.
+
+    ::
+
+        import nucleus
+
+        client = nucleus.NucleusClient(YOUR_SCALE_API_KEY)
+        dataset = client.get_dataset("ds_bwkezj6g5c4g05gqp1eg")
+
+        # When kicking off an asynchronous job, store the return value as a variable
+        job = dataset.append(items=YOUR_DATASET_ITEMS, asynchronous=True)
+
+        # Poll for status or errors
+        print(job.status())
+        print(job.errors())
+
+        # Block until job finishes
+        job.sleep_until_complete()
+    """
+
     job_id: str
     job_last_known_status: str
     job_type: str
@@ -22,6 +45,23 @@ class AsyncJob:
     client: "NucleusClient"  # type: ignore # noqa: F821
 
     def status(self) -> Dict[str, str]:
+        """Fetches status of the job and an informative message on job progress.
+
+        Returns:
+            A dict of the job ID, status (one of Running, Completed, or Errored),
+            an informative message on the job progress, and number of both completed
+            and total steps.
+            ::
+
+                {
+                    "job_id": "job_c19xcf9mkws46gah0000",
+                    "status": "Completed",
+                    "message": "Job completed successfully.",
+                    "job_progress": "0.33",
+                    "completed_steps": "1",
+                    "total_steps:": "3",
+                }
+        """
         response = self.client.make_request(
             payload={},
             route=f"job/{self.job_id}",
@@ -31,30 +71,57 @@ class AsyncJob:
         return response
 
     def errors(self) -> List[str]:
-        return self.client.make_request(
+        """Fetches a list of the latest errors generated by the asynchronous job.
+
+        Useful for debugging failed or partially successful jobs.
+
+        Returns:
+            A list of strings containing the 10,000 most recently generated errors.
+            ::
+
+                [
+                    '{"annotation":{"label":"car","type":"box","geometry":{"x":50,"y":60,"width":70,"height":80},"referenceId":"bad_ref_id","annotationId":"attempted_annot_upload","metadata":{}},"error":"Item with id bad_ref_id doesn\'t exist."}'
+                ]
+        """
+        errors = self.client.make_request(
             payload={},
             route=f"job/{self.job_id}/errors",
             requests_command=requests.get,
         )
+        return [replace_double_slashes(error) for error in errors]
 
     def sleep_until_complete(self, verbose_std_out=True):
+        """Blocks until the job completes or errors.
+
+        Parameters:
+            verbose_std_out (Optional[bool]): Whether or not to verbosely log while
+              sleeping. Defaults to True.
+        """
+        start_time = time.perf_counter()
         while 1:
             status = self.status()
             time.sleep(JOB_POLLING_INTERVAL)
 
             if verbose_std_out:
-                print(f"Status at {time.ctime()}: {status}")
+                print(
+                    f"Status at {time.perf_counter() - start_time} s: {status}"
+                )
             if status["status"] == "Running":
                 continue
 
             break
 
+        if verbose_std_out:
+            print(
+                f"Finished at {time.perf_counter() - start_time} s: {status}"
+            )
         final_status = status
         if final_status["status"] == "Errored":
             raise JobError(final_status, self)
 
     @classmethod
     def from_json(cls, payload: dict, client):
+        # TODO: make private
         return cls(
             job_id=payload[JOB_ID_KEY],
             job_last_known_status=payload[JOB_LAST_KNOWN_STATUS_KEY],
@@ -74,4 +141,5 @@ class JobError(Exception):
             f"The final status message was: {final_status_message} \n"
             f"For more detailed error messages you can call {str(job)}.errors()"
         )
+        message = replace_double_slashes(message)
         super().__init__(message)

nucleus/logger.py

@@ -0,0 +1,9 @@
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+logging.basicConfig()
+logging.getLogger(
+    requests.packages.urllib3.__package__  # pylint: disable=no-member
+).setLevel(logging.ERROR)

nucleus/metadata_manager.py

@@ -0,0 +1,45 @@
+from enum import Enum
+from typing import TYPE_CHECKING, Dict
+
+if TYPE_CHECKING:
+    from . import NucleusClient
+
+
+# Wording set to match with backend enum
+class ExportMetadataType(Enum):
+    SCENES = "scene"
+    DATASET_ITEMS = "item"
+
+
+class MetadataManager:
+    """
+    Helper class for managing metadata updates on a scene or dataset item.
+    Do not call directly, use the dataset class methods: `update_scene_metadata` or `update_item_metadata`
+    """
+
+    def __init__(
+        self,
+        dataset_id: str,
+        client: "NucleusClient",
+        raw_mappings: Dict[str, dict],
+        level: ExportMetadataType,
+    ):
+        self.dataset_id = dataset_id
+        self._client = client
+        self.raw_mappings = raw_mappings
+        self.level = level
+
+        self._payload = self._format_mappings()
+
+    def _format_mappings(self):
+        payload = []
+        for ref_id, meta in self.raw_mappings.items():
+            payload.append({"reference_id": ref_id, "metadata": meta})
+        return payload
+
+    def update(self):
+        payload = {"metadata": self._payload, "level": self.level.value}
+        resp = self._client.make_request(
+            payload=payload, route=f"dataset/{self.dataset_id}/metadata"
+        )
+        return resp

nucleus/metrics/__init__.py

@@ -0,0 +1,10 @@
+from .base import Metric, ScalarResult
+from .categorization_metrics import CategorizationF1
+from .polygon_metrics import (
+    PolygonAveragePrecision,
+    PolygonIOU,
+    PolygonMAP,
+    PolygonMetric,
+    PolygonPrecision,
+    PolygonRecall,
+)

nucleus/metrics/base.py

@@ -0,0 +1,117 @@
+import sys
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Iterable, List
+
+from nucleus.annotation import AnnotationList
+from nucleus.prediction import PredictionList
+
+
+class MetricResult(ABC):
+    """Base MetricResult class"""
+
+
+@dataclass
+class ScalarResult(MetricResult):
+    """A scalar result contains the value of an evaluation, as well as its weight.
+    The weight is useful when aggregating metrics where each dataset item may hold a
+    different relative weight. For example, when calculating precision over a dataset,
+    the denominator of the precision is the number of annotations, and therefore the weight
+    can be set as the number of annotations.
+
+    Attributes:
+        value (float): The value of the evaluation result
+        weight (float): The weight of the evaluation result.
+    """
+
+    value: float
+    weight: float = 1.0
+
+    @staticmethod
+    def aggregate(results: Iterable["ScalarResult"]) -> "ScalarResult":
+        """Aggregates results using a weighted average."""
+        results = list(filter(lambda x: x.weight != 0, results))
+        total_weight = sum([result.weight for result in results])
+        total_value = sum([result.value * result.weight for result in results])
+        value = total_value / max(total_weight, sys.float_info.epsilon)
+        return ScalarResult(value, total_weight)
+
+
+class Metric(ABC):
+    """Abstract class for defining a metric, which takes a list of annotations
+    and predictions and returns a scalar.
+
+    To create a new concrete Metric, override the `__call__` function
+    with logic to define a metric between annotations and predictions. ::
+
+        from nucleus import BoxAnnotation, CuboidPrediction, Point3D
+        from nucleus.annotation import AnnotationList
+        from nucleus.prediction import PredictionList
+        from nucleus.metrics import Metric, MetricResult
+        from nucleus.metrics.polygon_utils import BoxOrPolygonAnnotation, BoxOrPolygonPrediction
+
+        class MyMetric(Metric):
+            def __call__(
+                self, annotations: AnnotationList, predictions: PredictionList
+            ) -> MetricResult:
+                value = (len(annotations) - len(predictions)) ** 2
+                weight = len(annotations)
+                return MetricResult(value, weight)
+
+        box = BoxAnnotation(
+            label="car",
+            x=0,
+            y=0,
+            width=10,
+            height=10,
+            reference_id="image_1",
+            annotation_id="image_1_car_box_1",
+            metadata={"vehicle_color": "red"}
+        )
+
+        cuboid = CuboidPrediction(
+            label="car",
+            position=Point3D(100, 100, 10),
+            dimensions=Point3D(5, 10, 5),
+            yaw=0,
+            reference_id="pointcloud_1",
+            confidence=0.8,
+            annotation_id="pointcloud_1_car_cuboid_1",
+            metadata={"vehicle_color": "green"}
+        )
+
+        metric = MyMetric()
+        annotations = AnnotationList(box_annotations=[box])
+        predictions = PredictionList(cuboid_predictions=[cuboid])
+        metric(annotations, predictions)
+    """
+
+    @abstractmethod
+    def __call__(
+        self, annotations: AnnotationList, predictions: PredictionList
+    ) -> MetricResult:
+        """A metric must override this method and return a metric result, given annotations and predictions."""
+
+    @abstractmethod
+    def aggregate_score(self, results: List[MetricResult]) -> ScalarResult:
+        """A metric must define how to aggregate results from single items to a single ScalarResult.
+
+        E.g. to calculate a R2 score with sklearn you could define a custom metric class ::
+
+            class R2Result(MetricResult):
+                y_true: float
+                y_pred: float
+
+
+        And then define an aggregate_score ::
+
+            def aggregate_score(self, results: List[MetricResult]) -> ScalarResult:
+                y_trues = []
+                y_preds = []
+                for result in results:
+                    y_true.append(result.y_true)
+                    y_preds.append(result.y_pred)
+                r2_score = sklearn.metrics.r2_score(y_trues, y_preds)
+                return ScalarResult(r2_score)
+
+        """

nucleus/metrics/categorization_metrics.py

@@ -0,0 +1,197 @@
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import List, Set, Tuple, Union
+
+from sklearn.metrics import f1_score
+
+from nucleus.annotation import AnnotationList, CategoryAnnotation
+from nucleus.metrics.base import Metric, MetricResult, ScalarResult
+from nucleus.metrics.filters import confidence_filter
+from nucleus.prediction import CategoryPrediction, PredictionList
+
+F1_METHODS = {"micro", "macro", "samples", "weighted", "binary"}
+
+
+def to_taxonomy_labels(
+    anns_or_preds: Union[List[CategoryAnnotation], List[CategoryPrediction]]
+) -> Set[str]:
+    """Transforms annotation or prediction lists to taxonomy labels by joining them with a seperator (->)"""
+    labels = set()
+    for item in anns_or_preds:
+        taxonomy_label = (
+            f"{item.taxonomy_name}->{item.label}"
+            if item.taxonomy_name
+            else item.label
+        )
+        labels.add(taxonomy_label)
+    return labels
+
+
+@dataclass
+class CategorizationResult(MetricResult):
+    annotations: List[CategoryAnnotation]
+    predictions: List[CategoryPrediction]
+
+    @property
+    def value(self):
+        annotation_labels = to_taxonomy_labels(self.annotations)
+        prediction_labels = to_taxonomy_labels(self.predictions)
+
+        # TODO: Change task.py interface such that we can return label matching
+        # NOTE: Returning 1 if all taxonomy labels match else 0
+        value = f1_score(list(annotation_labels), list(prediction_labels), average="macro")
+        return value
+
+
+class CategorizationMetric(Metric):
+    """Abstract class for metrics related to Categorization
+
+    The Categorization class automatically filters incoming annotations and
+    predictions for only categorization annotations. It also filters
+    predictions whose confidence is less than the provided confidence_threshold.
+    """
+
+    def __init__(
+        self,
+        confidence_threshold: float = 0.0,
+    ):
+        """Initializes CategorizationMetric abstract object.
+
+        Args:
+            confidence_threshold: minimum confidence threshold for predictions to be taken into account for evaluation. Must be in [0, 1]. Default 0.0
+        """
+        assert 0 <= confidence_threshold <= 1
+        self.confidence_threshold = confidence_threshold
+
+    @abstractmethod
+    def eval(
+        self,
+        annotations: List[
+            CategoryAnnotation
+        ],  # TODO(gunnar): List to conform with other APIs or single instance?
+        predictions: List[CategoryPrediction],
+    ) -> CategorizationResult:
+        # Main evaluation function that subclasses must override.
+        # TODO(gunnar): Allow passing multiple predictions and selecting highest confidence? Allows us to show next
+        #  contender. Are top-5 scores something that we care about?
+        # TODO(gunnar): How do we handle multi-head classification?
+        pass
+
+    @abstractmethod
+    def aggregate_score(self, results: List[CategorizationResult]) -> ScalarResult:  # type: ignore[override]
+        pass
+
+    def __call__(
+        self, annotations: AnnotationList, predictions: PredictionList
+    ) -> CategorizationResult:
+        if self.confidence_threshold > 0:
+            predictions = confidence_filter(
+                predictions, self.confidence_threshold
+            )
+
+        cat_annotations, cat_predictions = self._filter_common_taxonomies(
+            annotations.category_annotations, predictions.category_predictions
+        )
+
+        result = self.eval(
+            cat_annotations,
+            cat_predictions,
+        )
+        return result
+
+    def _filter_common_taxonomies(
+        self,
+        annotations: List[CategoryAnnotation],
+        predictions: List[CategoryPrediction],
+    ) -> Tuple[List[CategoryAnnotation], List[CategoryPrediction]]:
+        annotated_taxonomies = {ann.taxonomy_name for ann in annotations}
+        matching_predictions, matching_taxonomies = self._filter_in_taxonomies(
+            predictions, annotated_taxonomies
+        )
+        matching_annotations, _ = self._filter_in_taxonomies(
+            annotations, matching_taxonomies
+        )
+
+        return matching_annotations, matching_predictions  # type: ignore
+
+    def _filter_in_taxonomies(
+        self,
+        anns_or_preds: Union[
+            List[CategoryAnnotation], List[CategoryPrediction]
+        ],
+        filter_on_taxonomies: Set[Union[None, str]],
+    ) -> Tuple[
+        Union[List[CategoryAnnotation], List[CategoryPrediction]],
+        Set[Union[None, str]],
+    ]:
+        matching_predictions = []
+        matching_taxonomies = set()
+        for pred in anns_or_preds:
+            if pred.taxonomy_name in filter_on_taxonomies:
+                matching_predictions.append(pred)
+                matching_taxonomies.add(pred.taxonomy_name)
+        return matching_predictions, matching_taxonomies
+
+
+class CategorizationF1(CategorizationMetric):
+    """Evaluation method that matches categories and returns a CategorizationF1Result that aggregates to the F1 score"""
+
+    def __init__(
+        self, confidence_threshold: float = 0.0, f1_method: str = "macro"
+    ):
+        """
+        Args:
+            confidence_threshold: minimum confidence threshold for predictions to be taken into account for evaluation. Must be in [0, 1]. Default 0.0
+            f1_method: {'micro', 'macro', 'samples','weighted', 'binary'}, \
+                default='macro'
+            This parameter is required for multiclass/multilabel targets.
+            If ``None``, the scores for each class are returned. Otherwise, this
+            determines the type of averaging performed on the data:
+
+            ``'binary'``:
+                Only report results for the class specified by ``pos_label``.
+                This is applicable only if targets (``y_{true,pred}``) are binary.
+            ``'micro'``:
+                Calculate metrics globally by counting the total true positives,
+                false negatives and false positives.
+            ``'macro'``:
+                Calculate metrics for each label, and find their unweighted
+                mean.  This does not take label imbalance into account.
+            ``'weighted'``:
+                Calculate metrics for each label, and find their average weighted
+                by support (the number of true instances for each label). This
+                alters 'macro' to account for label imbalance; it can result in an
+                F-score that is not between precision and recall.
+            ``'samples'``:
+                Calculate metrics for each instance, and find their average (only
+                meaningful for multilabel classification where this differs from
+                :func:`accuracy_score`).
+        """
+        super().__init__(confidence_threshold)
+        assert (
+            f1_method in F1_METHODS
+        ), f"Invalid f1_method {f1_method}, expected one of {F1_METHODS}"
+        self.f1_method = f1_method
+
+    def eval(
+        self,
+        annotations: List[CategoryAnnotation],
+        predictions: List[CategoryPrediction],
+    ) -> CategorizationResult:
+        """
+        Notes: This is a little weird eval function. It essentially only does matching of annotation to label and
+        the actual metric computation happens in the aggregate step since F1 score only makes sense on a collection.
+        """
+
+        return CategorizationResult(
+            annotations=annotations, predictions=predictions
+        )
+
+    def aggregate_score(self, results: List[CategorizationResult]) -> ScalarResult:  # type: ignore[override]
+        gt = []
+        predicted = []
+        for result in results:
+            gt.extend(list(to_taxonomy_labels(result.annotations)))
+            predicted.extend(list(to_taxonomy_labels(result.predictions)))
+        value = f1_score(gt, predicted, average=self.f1_method)
+        return ScalarResult(value)

nucleus/metrics/errors.py

@@ -0,0 +1,7 @@
+class PolygonAnnotationTypeError(Exception):
+    def __init__(
+        self,
+        message="Annotation was expected to be of type 'BoxAnnotation' or 'PolygonAnnotation'.",
+    ):
+        self.message = message
+        super().__init__(self.message)

nucleus/metrics/filters.py

@@ -0,0 +1,40 @@
+from typing import List
+
+from nucleus.prediction import PredictionList
+
+from .polygon_utils import (
+    BoxOrPolygonAnnoOrPred,
+    polygon_annotation_to_geometry,
+)
+
+
+def polygon_area_filter(
+    polygons: List[BoxOrPolygonAnnoOrPred], min_area: float, max_area: float
+) -> List[BoxOrPolygonAnnoOrPred]:
+    filter_fn = (
+        lambda polygon: min_area
+        <= polygon_annotation_to_geometry(polygon).signed_area
+        <= max_area
+    )
+    return list(filter(filter_fn, polygons))
+
+
+def confidence_filter(
+    predictions: PredictionList, min_confidence: float
+) -> PredictionList:
+    predictions_copy = PredictionList()
+    filter_fn = (
+        lambda prediction: not hasattr(prediction, "confidence")
+        or prediction.confidence >= min_confidence
+    )
+    for attr in predictions.__dict__:
+        predictions_copy.__dict__[attr] = list(
+            filter(filter_fn, predictions.__dict__[attr])
+        )
+    return predictions_copy
+
+
+def polygon_label_filter(
+    polygons: List[BoxOrPolygonAnnoOrPred], label: str
+) -> List[BoxOrPolygonAnnoOrPred]:
+    return list(filter(lambda polygon: polygon.label == label, polygons))