scale-nucleus 0.12b1__py3-none-any.whl → 0.14.14b0__py3-none-any.whl

nucleus/metrics/segmentation_utils.py

@@ -1,16 +1,29 @@
+import logging
+from collections import defaultdict
+from typing import List, Sequence, Tuple, Union
+
 import numpy as np
-from rasterio import features
+from scipy.optimize import linear_sum_assignment
+
+from nucleus import Point, PolygonPrediction, Segment
+from nucleus.metrics.custom_types import BoxOrPolygonAnnotation
+from nucleus.metrics.polygon_utils import polygon_annotation_to_shape
+from nucleus.package_not_installed import (  # pylint: disable=ungrouped-imports
+    PackageNotInstalled,
+)
 
-from nucleus import Point, PolygonPrediction
+FALSE_POSITIVES = "__non_max_false_positive"
 
 try:
     from shapely import geometry
-except ModuleNotFoundError:
-    from nucleus.shapely_not_installed import (  # pylint: disable=ungrouped-imports
-        ShapelyNotInstalled,
-    )
+except (ModuleNotFoundError, OSError):
+    geometry = PackageNotInstalled
 
-    geometry = ShapelyNotInstalled
+
+try:
+    from rasterio import features
+except (ModuleNotFoundError, OSError):
+    rasterio = PackageNotInstalled
 
 
 def instance_mask_to_polys(instance_mask: np.ndarray, background_code=None):
@@ -20,7 +33,7 @@ def instance_mask_to_polys(instance_mask: np.ndarray, background_code=None):
         (instance_mask != background_code) if background_code else None
     )
     for shape, value in features.shapes(
-        instance_mask.astype(np.int16),
+        instance_mask.astype(np.int32),
         mask=not_background_mask,
     ):
         poly = geometry.shape(shape)
@@ -46,3 +59,221 @@ def transform_poly_codes_to_poly_preds(
         )
         polygon_predictions.append(pred)
     return polygon_predictions
+
+
+def max_iou_match_from_confusion(confusion):
+    """Calculate iou from confusion matrix and do linear sum assignment to get strongest candiaate for each GT
+
+    Returns:
+        iou_matrix with same dims as confusion and 1-d best match rows, 1-d best match cols
+    """
+    iou = np.zeros(confusion.shape, dtype=np.float)
+    with np.errstate(divide="ignore", invalid="ignore"):
+        for i in range(confusion.shape[0]):
+            for j in range(confusion.shape[1]):
+                intersection = confusion[i, j]
+                union = (
+                    confusion[i, :].sum()
+                    + confusion[:, j].sum()
+                    - intersection
+                )
+                if union > 0:
+                    iou[i, j] = intersection / union
+    iou = np.nan_to_num(iou)
+    iou_match_row, iou_match_col = linear_sum_assignment(-iou)
+    return iou, iou_match_row, iou_match_col
+
+
+def fast_confusion_matrix(
+    label_true: np.ndarray, label_pred: np.ndarray, n_class: int
+) -> np.ndarray:
+    """Calculates confusion matrix - fast!
+
+    Outputs a confusion matrix where each row is GT confusion and column is prediction confusion
+    Example:
+        _fast_hist(np.array([0, 1, 2, 3], dtype=np.int32), np.array([0, 1, 1, 1], dtype=np.int32), n_class=4)
+        array([[1, 0, 0, 0],
+               [0, 1, 0, 0],
+               [0, 1, 0, 0],
+               [0, 1, 0, 0]])
+    """
+    mask = (label_true >= 0) & (label_true < n_class)
+    hist = np.bincount(
+        n_class * label_true[mask].astype(int) + label_pred[mask],
+        minlength=n_class ** 2,
+    ).reshape(n_class, n_class)
+    return hist
+
+
+def non_max_suppress_confusion(confusion: np.ndarray, iou_threshold):
+    """Uses linear sum assignment to find biggest pixel-wise IOU match. Secondary matches are moved to last column
+    as false positives (since they are outside of instance boundaries).
+
+    TODO(gunnar): Change logic to only move suppressed TP to FP so we can maintain the original confusion
+
+    Arguments:
+        confusion: Confusion matrix
+        iou_threshold: Detections under iou threshold are considered false positives
+
+    Returns:
+        Non max suppressed confusion matrix (NxN) with dimension ((N+1)x(N+1)) where the last column are suppressed
+        positives
+
+    """
+    original_count = confusion.sum()
+    iou, max_iou_row, max_iou_col = max_iou_match_from_confusion(confusion)
+    # Prepare the new confusion with +1 added to the shape
+    non_max_suppressed = np.zeros(np.add(confusion.shape, 1), dtype=np.int64)
+
+    # ----  IOU filtering from diagonal
+    keep_diagonal = iou.diagonal() >= iou_threshold
+    # Move false positives from diag to new false_positive class
+    move_indexes = np.where(~keep_diagonal)
+
+    # log iou suppressed as FPs
+    non_max_suppressed[:, -1].put(
+        move_indexes, confusion.diagonal().take(move_indexes)
+    )
+    # Zero false positives on diagonal
+    keep_indexes = np.where(keep_diagonal)
+    new_diagonal = np.zeros(len(confusion.diagonal()))
+    new_diagonal.put(keep_indexes, confusion.diagonal()[keep_indexes])
+    np.fill_diagonal(confusion, new_diagonal)
+    # ----
+
+    # -- move max over
+    non_max_suppressed[max_iou_row, max_iou_col] = confusion[
+        max_iou_row, max_iou_col
+    ]
+    confusion[max_iou_row, max_iou_col] = np.zeros(len(max_iou_col))
+    # --
+
+    # -- move left on diagonal to FPs
+    non_max_suppressed[:, -1] = (
+        np.r_[confusion.diagonal(), np.zeros(1)] + non_max_suppressed[:, -1]
+    )
+    np.fill_diagonal(confusion, np.zeros(len(confusion.diagonal())))
+    # --
+
+    # -- move valid confusions over
+    valid_confusion = confusion > 0
+    valid_row, valid_col = np.where(confusion > 0)
+    flat_idxs = valid_col + valid_row * non_max_suppressed.shape[1]
+    non_max_suppressed.put(flat_idxs, confusion[valid_confusion])
+    # --
+    assert original_count == non_max_suppressed.sum()
+    return non_max_suppressed
+
+
+def rasterize_polygons_to_segmentation_mask(
+    annotations: Sequence[BoxOrPolygonAnnotation], shape: Tuple
+) -> Tuple[np.ndarray, List[Segment]]:
+    polys = [polygon_annotation_to_shape(a) for a in annotations]
+    segments = [
+        Segment(ann.label, index=idx + 1, metadata=ann.metadata)
+        for idx, ann in enumerate(annotations)
+    ]
+    poly_vals = [
+        (poly, segment.index) for poly, segment in zip(polys, segments)
+    ]
+    rasterized = features.rasterize(
+        poly_vals,
+        out_shape=shape,
+        fill=0,
+        out=None,
+        all_touched=False,
+        dtype=None,
+    )
+    return rasterized, segments
+
+
+def convert_to_instance_seg_confusion(confusion, annotation, prediction):
+    pred_index_to_label = {s.index: s.label for s in prediction.annotations}
+
+    gt_label_to_old_indexes = defaultdict(set)
+    for segment in annotation.annotations:
+        gt_label_to_old_indexes[segment.label].add(segment.index)
+
+    pr_label_to_old_indexes = defaultdict(set)
+    for segment in prediction.annotations:
+        pr_label_to_old_indexes[segment.label].add(segment.index)
+
+    new_labels = list(
+        dict.fromkeys(
+            list(pr_label_to_old_indexes)[:-1]
+            + list(gt_label_to_old_indexes)[:-1]
+        )
+    )
+    # NOTE: We make sure that FALSE_POSITIVES are at the back
+    false_positive_label = list(pr_label_to_old_indexes.keys())[-1]
+    new_labels.append(false_positive_label)
+    non_taxonomy_classes = {len(new_labels) - 1}
+
+    num_classes = len(new_labels)
+    new_confusion = np.zeros(
+        (num_classes, num_classes),
+        dtype=np.int32,
+    )
+
+    for gt_class_idx, from_label in enumerate(new_labels):
+        from_indexes = gt_label_to_old_indexes[from_label]
+        tp, fp = 0, 0
+        if len(from_indexes) == 0:
+            logging.warning(
+                "No annotations with label '%s', interpreted as false positives.",
+                from_label,
+            )
+            non_taxonomy_classes.add(gt_class_idx)
+            # NOTE: If the index is not in the gt segments it comes from the predictions, we get the "old_indexes"
+            # from there even though they are all FPs
+            from_indexes = pr_label_to_old_indexes[from_label]
+        for gt_instance_idx in from_indexes:
+            max_col = np.argmax(
+                confusion[gt_instance_idx, :]
+            )  # TODO: Get from IOU
+            if confusion[gt_instance_idx, max_col] == 0:
+                continue
+
+            for pred_class_idx, to_label in enumerate(new_labels):
+                to_indexes = pr_label_to_old_indexes[to_label]
+                if from_label == to_label:
+                    if pred_index_to_label.get(max_col, None) == from_label:
+                        tp += confusion[gt_instance_idx, max_col]
+                        fp_indexes = to_indexes - {max_col}
+                    else:
+                        fp_indexes = to_indexes
+                    fp += (
+                        confusion[gt_instance_idx, :]
+                        .take(list(fp_indexes))
+                        .sum()
+                        + confusion[gt_instance_idx, -1]
+                    )
+                else:
+                    new_confusion[gt_class_idx, pred_class_idx] += (
+                        confusion[gt_instance_idx, :]
+                        .take(list(to_indexes))
+                        .sum()
+                    )
+
+        new_confusion[gt_class_idx, gt_class_idx] = tp
+        new_confusion[gt_class_idx, -1] = fp
+
+    assert confusion.sum() == new_confusion.sum()
+    return new_confusion, new_labels, non_taxonomy_classes
+
+
+def setup_iou_thresholds(iou_thresholds: Union[Sequence[float], str] = "coco"):
+    supported_iou_setups = {"coco"}
+    if isinstance(iou_thresholds, (list, np.ndarray)):
+        return np.array(iou_thresholds, np.float_)
+    elif isinstance(iou_thresholds, str):
+        if iou_thresholds in supported_iou_setups:
+            return np.arange(0.5, 1.0, 0.05)
+        else:
+            raise RuntimeError(
+                f"Got invalid configuration value: {iou_thresholds}, expected one of: {supported_iou_setups}"
+            )
+    else:
+        raise RuntimeError(
+            f"Got invalid configuration: {iou_thresholds}. Expected list of floats or one of: {supported_iou_setups}"
+        )

nucleus/model.py

@@ -2,7 +2,7 @@ from typing import Dict, List, Optional, Union
 
 import requests
 
-from .constants import METADATA_KEY, NAME_KEY, REFERENCE_ID_KEY
+from .constants import METADATA_KEY, MODEL_TAGS_KEY, NAME_KEY, REFERENCE_ID_KEY
 from .dataset import Dataset
 from .job import AsyncJob
 from .model_run import ModelRun
@@ -27,7 +27,7 @@ class Model:
 
     Within Nucleus, Models work in the following way:
 
-    1. You first :meth:`create a Model<NucleusClient.add_model>`. You can do this
+    1. You first :meth:`create a Model<NucleusClient.create_model>`. You can do this
        just once and reuse the model on multiple datasets.
     2. You then :meth:`upload predictions <Dataset.upload_predictions>` to a dataset.
     3. Trigger :meth:`calculation of metrics <Dataset.calculate_evaluation_metrics>`
@@ -68,7 +68,7 @@ class Model:
             class_pdf={"label": 0.2, "other_label": 0.8},
         )
 
-        model = client.add_model(
+        model = client.create_model(
             name="My Model", reference_id="My-CNN", metadata={"timestamp": "121012401"}
         )
 
@@ -77,7 +77,7 @@ class Model:
 
         # For large ingestions, we recommend asynchronous ingestion
         job = dataset.upload_predictions(
-            [prediction_1, prediction_2], asynchronous=True
+            model, [prediction_1, prediction_2], asynchronous=True
         )
         # Check current status
         job.status()
@@ -89,21 +89,29 @@ class Model:
         dataset.calculate_evaluation_metrics(model)
 
     Models cannot be instantiated directly and instead must be created via API
-    endpoint, using :meth:`NucleusClient.add_model`.
+    endpoint, using :meth:`NucleusClient.create_model`.
     """
 
     def __init__(
-        self, model_id, name, reference_id, metadata, client, bundle_name=None
+        self,
+        model_id,
+        name,
+        reference_id,
+        metadata,
+        client,
+        bundle_name=None,
+        tags=None,
     ):
         self.id = model_id
         self.name = name
         self.reference_id = reference_id
         self.metadata = metadata
         self.bundle_name = bundle_name
+        self.tags = tags if tags else []
         self._client = client
 
     def __repr__(self):
-        return f"Model(model_id='{self.id}', name='{self.name}', reference_id='{self.reference_id}', metadata={self.metadata}, bundle_name={self.bundle_name}, client={self._client})"
+        return f"Model(model_id='{self.id}', name='{self.name}', reference_id='{self.reference_id}', metadata={self.metadata}, bundle_name={self.bundle_name}, tags={self.tags}, client={self._client})"
 
     def __eq__(self, other):
         return (
@@ -202,9 +210,9 @@ class Model:
             model.run("ds_123456")
 
         Args:
-            dataset_id: id of dataset to run inference on
-            job_id: nucleus job used to track async job progress
-            slice_id: (optional) id of slice of the dataset to run inference on
+            dataset_id: The ID of the dataset to run inference on.
+            job_id: The ID of the :class:`AsyncJob` used to track job progress.
+            slice_id: The ID of the slice of the dataset to run inference on.
         """
         response = self._client.make_request(
             {"dataset_id": dataset_id, "slice_id": slice_id},
@@ -213,3 +221,51 @@ class Model:
         )
 
         return response
+
+    def add_tags(self, tags: List[str]):
+        """Tag the model with custom tag names. ::
+
+            import nucleus
+            client = nucleus.NucleusClient("YOUR_SCALE_API_KEY")
+            model = client.list_models()[0]
+
+            model.add_tags(["tag_A", "tag_B"])
+
+        Args:
+            tags: list of tag names
+        """
+        response: requests.Response = self._client.make_request(
+            {MODEL_TAGS_KEY: tags},
+            f"model/{self.id}/tag",
+            requests_command=requests.post,
+            return_raw_response=True,
+        )
+
+        if response.ok:
+            self.tags.extend(tags)
+
+        return response.json()
+
+    def remove_tags(self, tags: List[str]):
+        """Remove tag(s) from the model. ::
+
+            import nucleus
+            client = nucleus.NucleusClient("YOUR_SCALE_API_KEY")
+            model = client.list_models()[0]
+
+            model.remove_tags(["tag_x"])
+
+        Args:
+            tags: list of tag names to remove
+        """
+        response: requests.Response = self._client.make_request(
+            {MODEL_TAGS_KEY: tags},
+            f"model/{self.id}/tag",
+            requests_command=requests.delete,
+            return_raw_response=True,
+        )
+
+        if response.ok:
+            self.tags = list(filter(lambda t: t not in tags, self.tags))
+
+        return response.json()

nucleus/model_run.py

@@ -8,7 +8,7 @@ For example::
     client = nucleus.NucleusClient(YOUR_SCALE_API_KEY)
     prediction_1 = nucleus.BoxPrediction(label="label", x=0, y=0, width=10, height=10, reference_id="1", confidence=0.9, class_pdf={'label': 0.9, 'other_label': 0.1})
     prediction_2 = nucleus.BoxPrediction(label="label", x=0, y=0, width=10, height=10, reference_id="2", confidence=0.2, class_pdf={'label': 0.2, 'other_label': 0.8})
-    model = client.add_model(name="My Model", reference_id="My-CNN", metadata={"timestamp": "121012401"})
+    model = client.create_model(name="My Model", reference_id="My-CNN", metadata={"timestamp": "121012401"})
     response = dataset.upload_predictions(model, [prediction_1, prediction_2])
 """
 

nucleus/{shapely_not_installed.py → package_not_installed.py}

@@ -1,7 +1,7 @@
 import sys
 
 
-class ShapelyNotInstalled:
+class PackageNotInstalled:
     def __init__(self, *args, **kwargs):
         self.raise_error_msg()
 
@@ -23,6 +23,6 @@ class ShapelyNotInstalled:
         else:
             platform_specific_msg = "GEOS package will need to be installed see (https://trac.osgeo.org/geos/)"
         raise ModuleNotFoundError(
-            f"Module 'shapely' not found. Install optionally with `scale-nucleus[shapely]` or when developing "
-            f"`poetry install -E shapely`. {platform_specific_msg}"
+            f"Module 'shapely' not found. Install optionally with `scale-nucleus[metrics]` or when developing "
+            f"`poetry install -E metrics`. {platform_specific_msg}"
         )

nucleus/payload_constructor.py

@@ -17,6 +17,7 @@ from .constants import (
     METADATA_KEY,
     MODEL_BUNDLE_NAME_KEY,
     MODEL_ID_KEY,
+    MODEL_TAGS_KEY,
     NAME_KEY,
     REFERENCE_ID_KEY,
     SCENES_KEY,
@@ -127,6 +128,7 @@ def construct_model_creation_payload(
     reference_id: str,
     metadata: Optional[Dict],
     bundle_name: Optional[str],
+    tags: Optional[List[str]],
 ) -> dict:
     payload = {
         NAME_KEY: name,
@@ -136,6 +138,8 @@ def construct_model_creation_payload(
 
     if bundle_name:
         payload[MODEL_BUNDLE_NAME_KEY] = bundle_name
+    if tags:
+        payload[MODEL_TAGS_KEY] = tags
 
     return payload
 

nucleus/prediction.py

@@ -75,7 +75,7 @@ class SegmentationPrediction(SegmentationAnnotation):
         from nucleus import SegmentationPrediction
 
         segmentation = SegmentationPrediction(
-            mask_url="s3://your-bucket-name/pred-seg-masks/image_2_pred_mask_id1.png",
+            mask_url="s3://your-bucket-name/pred-seg-masks/image_2_pred_mask_id_1.png",
             annotations=[
                 Segment(label="grass", index="1"),
                 Segment(label="road", index="2"),
@@ -88,7 +88,7 @@ class SegmentationPrediction(SegmentationAnnotation):
 
     Parameters:
         mask_url (str): A URL pointing to the segmentation prediction mask which is
-          accessible to Scale. This URL can be a path to a local file.
+          accessible to Scale. This "URL" can also be a path to a local file.
           The mask is an HxW int8 array saved in PNG format,
           with each pixel value ranging from [0, N), where N is the number of
           possible classes (for semantic segmentation) or instances (for instance
@@ -226,7 +226,7 @@ class LinePrediction(LineAnnotation):
 
     Parameters:
         label (str): The label for this prediction (e.g. car, pedestrian, bicycle).
-        vertices List[:class:`Point`]: The list of points making up the line.
+        vertices (List[:class:`Point`]): The list of points making up the line.
         reference_id (str): User-defined ID of the image to which to apply this
             annotation.
         confidence: 0-1 indicating the confidence of the prediction.
@@ -600,6 +600,9 @@ class PredictionList:
         default_factory=list
     )
 
+    def items(self):
+        return self.__dict__.items()
+
     def add_predictions(self, predictions: List[Prediction]):
         for prediction in predictions:
             if isinstance(prediction, BoxPrediction):

nucleus/scene.py

@@ -1,4 +1,5 @@
 import json
+import warnings
 from abc import ABC
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional, Union
@@ -470,8 +471,14 @@ class VideoScene(ABC):
     items: List[DatasetItem] = field(default_factory=list)
     metadata: Optional[dict] = field(default_factory=dict)
     upload_to_scale: Optional[bool] = True
+    attachment_type: Optional[str] = None
 
     def __post_init__(self):
+        if self.attachment_type:
+            warnings.warn(
+                "The attachment_type parameter is no longer required and will be deprecated soon.",
+                DeprecationWarning,
+            )
         if self.metadata is None:
             self.metadata = {}