scale-nucleus 0.12b9__tar.gz → 0.14.5b1__tar.gz

{scale-nucleus-0.12b9 → scale-nucleus-0.14.5b1}/PKG-INFO

@@ -1,31 +1,34 @@
 Metadata-Version: 2.1
 Name: scale-nucleus
-Version: 0.12b9
+Version: 0.14.5b1
 Summary: The official Python client library for Nucleus, the Data Platform for AI
 Home-page: https://scale.com/nucleus
 License: MIT
 Author: Scale AI Nucleus Team
 Author-email: nucleusapi@scaleapi.com
-Requires-Python: >=3.6.2,<4.0.0
+Requires-Python: >=3.6.2,<4.0
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
-Provides-Extra: shapely
+Provides-Extra: launch
+Provides-Extra: metrics
 Requires-Dist: Pillow (>=7.1.2)
-Requires-Dist: Shapely (>=1.8.0); extra == "shapely"
+Requires-Dist: Shapely (>=1.8.0); extra == "metrics"
 Requires-Dist: aiohttp (>=3.7.4,<4.0.0)
 Requires-Dist: click (>=7.1.2,<9.0)
 Requires-Dist: dataclasses (>=0.7,<0.8); python_full_version >= "3.6.1" and python_version < "3.7"
 Requires-Dist: nest-asyncio (>=1.5.1,<2.0.0)
-Requires-Dist: numpy (>=1.19.5,<2.0.0)
+Requires-Dist: numpy (>=1.19.5); python_version >= "3.6" and python_version < "3.10"
+Requires-Dist: numpy (>=1.22.0); python_version >= "3.10"
 Requires-Dist: pydantic (>=1.8.2,<2.0.0)
-Requires-Dist: rasterio (>=1.2.10,<2.0.0)
+Requires-Dist: rasterio (>=1.2.0); extra == "metrics"
 Requires-Dist: requests (>=2.23.0,<3.0.0)
-Requires-Dist: rich (>=10.15.2,<11.0.0)
-Requires-Dist: s3fs (>=2022.1.0)
+Requires-Dist: rich (>=10.15.2)
+Requires-Dist: s3fs (>=2021.9.0); extra == "metrics"
+Requires-Dist: scale-launch (>=0.1.0); (python_version >= "3.7" and python_version < "4.0") and (extra == "launch")
 Requires-Dist: scikit-learn (>=0.24.0)
 Requires-Dist: scipy (>=1.4.1)
 Requires-Dist: shellingham (>=1.4.0,<2.0.0)
@@ -224,7 +227,7 @@ sphinx-autobuild . ./_build/html --watch ../nucleus
 
 ## Custom Metrics using Shapely in scale-validate
 
-Certain metrics use `shapely` which is added as an optional dependency.
+Certain metrics use `Shapely` and `rasterio` which is added as optional dependencies.
 
 ```bash
 pip install scale-nucleus[metrics]
@@ -241,5 +244,5 @@ apt-get install libgeos-dev
 
 To develop it locally use
 
-`poetry install --extras shapely`
+`poetry install --extras metrics`
 

{scale-nucleus-0.12b9 → scale-nucleus-0.14.5b1}/README.md

@@ -188,7 +188,7 @@ sphinx-autobuild . ./_build/html --watch ../nucleus
 
 ## Custom Metrics using Shapely in scale-validate
 
-Certain metrics use `shapely` which is added as an optional dependency.
+Certain metrics use `Shapely` and `rasterio` which is added as optional dependencies.
 
 ```bash
 pip install scale-nucleus[metrics]
@@ -205,4 +205,4 @@ apt-get install libgeos-dev
 
 To develop it locally use
 
-`poetry install --extras shapely`
+`poetry install --extras metrics`

{scale-nucleus-0.12b9 → scale-nucleus-0.14.5b1}/nucleus/__init__.py

@@ -40,7 +40,7 @@ __all__ = [
 
 import os
 import warnings
-from typing import Dict, List, Optional, Sequence, Union
+from typing import Any, Dict, List, Optional, Sequence, Union
 
 import pkg_resources
 import pydantic
@@ -91,6 +91,7 @@ from .constants import (
     KEEP_HISTORY_KEY,
     MESSAGE_KEY,
     MODEL_RUN_ID_KEY,
+    MODEL_TAGS_KEY,
     NAME_KEY,
     NUCLEUS_ENDPOINT,
     PREDICTIONS_IGNORED_KEY,
@@ -218,6 +219,7 @@ class NucleusClient:
                 reference_id=model["ref_id"],
                 metadata=model["metadata"] or None,
                 client=self,
+                tags=model.get(MODEL_TAGS_KEY, []),
             )
             for model in model_objects["models"]
         ]
@@ -484,6 +486,7 @@ class NucleusClient:
         reference_id: str,
         metadata: Optional[Dict] = None,
         bundle_name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
     ) -> Model:
         """Adds a :class:`Model` to Nucleus.
 
@@ -495,13 +498,15 @@ class NucleusClient:
             metadata: An arbitrary dictionary of additional data about this model
               that can be stored and retrieved. For example, you can store information
               about the hyperparameters used in training this model.
+            bundle_name: Optional name of bundle attached to this model
+            tags: Optional list of tags to attach to this model
 
         Returns:
             :class:`Model`: The newly created model as an object.
         """
         response = self.make_request(
             construct_model_creation_payload(
-                name, reference_id, metadata, bundle_name
+                name, reference_id, metadata, bundle_name, tags
             ),
             "models/add",
         )
@@ -516,6 +521,197 @@ class NucleusClient:
             metadata=metadata,
             bundle_name=bundle_name,
             client=self,
+            tags=tags,
+        )
+
+    def create_launch_model(
+        self,
+        name: str,
+        reference_id: str,
+        bundle_args: Dict[str, Any],
+        metadata: Optional[Dict] = None,
+    ) -> Model:
+        """
+        Adds a :class:`Model` to Nucleus, as well as a Launch bundle from a given function.
+
+        Parameters:
+            name: A human-readable name for the model.
+            reference_id: Unique, user-controlled ID for the model. This can be
+              used, for example, to link to an external storage of models which
+              may have its own id scheme.
+            bundle_args: Dict for kwargs for the creation of a Launch bundle,
+              more details on the keys below.
+            metadata: An arbitrary dictionary of additional data about this model
+              that can be stored and retrieved. For example, you can store information
+              about the hyperparameters used in training this model.
+
+        Returns:
+            :class:`Model`: The newly created model as an object.
+
+        Details on `bundle_args`:
+                Grabs a s3 signed url and uploads a model bundle to Scale Launch.
+
+        A model bundle consists of exactly {predict_fn_or_cls}, {load_predict_fn + model}, or {load_predict_fn + load_model_fn}.
+        Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.
+
+        Parameters:
+            model_bundle_name: Name of model bundle you want to create. This acts as a unique identifier.
+            predict_fn_or_cls: Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call.
+                I.e. `predict_fn_or_cls(REQUEST) -> RESPONSE`.
+            model: Typically a trained Neural Network, e.g. a Pytorch module
+            load_predict_fn: Function that when called with model, returns a function that carries out inference
+                I.e. `load_predict_fn(model) -> func; func(REQUEST) -> RESPONSE`
+            load_model_fn: Function that when run, loads a model, e.g. a Pytorch module
+                I.e. `load_predict_fn(load_model_fn()) -> func; func(REQUEST) -> RESPONSE`
+            bundle_url: Only for self-hosted mode. Desired location of bundle.
+            Overrides any value given by self.bundle_location_fn
+            requirements: A list of python package requirements, e.g.
+                ["tensorflow==2.3.0", "tensorflow-hub==0.11.0"]. If no list has been passed, will default to the currently
+                imported list of packages.
+            app_config: Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
+            env_params: A dictionary that dictates environment information e.g.
+                the use of pytorch or tensorflow, which cuda/cudnn versions to use.
+                Specifically, the dictionary should contain the following keys:
+                "framework_type": either "tensorflow" or "pytorch".
+                "pytorch_version": Version of pytorch, e.g. "1.5.1", "1.7.0", etc. Only applicable if framework_type is pytorch
+                "cuda_version": Version of cuda used, e.g. "11.0".
+                "cudnn_version" Version of cudnn used, e.g. "cudnn8-devel".
+                "tensorflow_version": Version of tensorflow, e.g. "2.3.0". Only applicable if framework_type is tensorflow
+            globals_copy: Dictionary of the global symbol table. Normally provided by `globals()` built-in function.
+        """
+        from launch import LaunchClient
+
+        launch_client = LaunchClient(api_key=self.api_key)
+
+        model_exists = any(model.name == name for model in self.list_models())
+        bundle_exists = any(
+            bundle.name == name + "-nucleus-autogen"
+            for bundle in launch_client.list_model_bundles()
+        )
+
+        if bundle_exists or model_exists:
+            raise ModelCreationError(
+                "Bundle with the given name already exists, please try a different name"
+            )
+
+        kwargs = {
+            "model_bundle_name": name + "-nucleus-autogen",
+            **bundle_args,
+        }
+
+        bundle = launch_client.create_model_bundle(**kwargs)
+        return self.create_model(
+            name,
+            reference_id,
+            metadata,
+            bundle.name,
+        )
+
+    def create_launch_model_from_dir(
+        self,
+        name: str,
+        reference_id: str,
+        bundle_from_dir_args: Dict[str, Any],
+        metadata: Optional[Dict] = None,
+    ) -> Model:
+        """
+        Adds a :class:`Model` to Nucleus, as well as a Launch bundle from a directory.
+
+        Parameters:
+            name: A human-readable name for the model.
+            reference_id: Unique, user-controlled ID for the model. This can be
+              used, for example, to link to an external storage of models which
+              may have its own id scheme.
+            bundle_from_dir_args: Dict for kwargs for the creation of a bundle from directory,
+              more details on the keys below.
+            metadata: An arbitrary dictionary of additional data about this model
+              that can be stored and retrieved. For example, you can store information
+              about the hyperparameters used in training this model.
+
+        Returns:
+            :class:`Model`: The newly created model as an object.
+
+        Details on `bundle_from_dir_args`
+        Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch.
+        In this mode, a bundle is just local code instead of a serialized object.
+
+        For example, if you have a directory structure like so, and your current working directory is also `my_root`:
+
+        ```
+        my_root/
+            my_module1/
+                __init__.py
+                ...files and directories
+                my_inference_file.py
+            my_module2/
+                __init__.py
+                ...files and directories
+        ```
+
+        then calling `create_model_bundle_from_dirs` with `base_paths=["my_module1", "my_module2"]` essentially
+        creates a zip file without the root directory, e.g.:
+
+        ```
+        my_module1/
+            __init__.py
+            ...files and directories
+            my_inference_file.py
+        my_module2/
+            __init__.py
+            ...files and directories
+        ```
+
+        and these contents will be unzipped relative to the server side `PYTHONPATH`. Bear these points in mind when
+        referencing Python module paths for this bundle. For instance, if `my_inference_file.py` has `def f(...)`
+        as the desired inference loading function, then the `load_predict_fn_module_path` argument should be
+        `my_module1.my_inference_file.f`.
+
+
+        Keys for `bundle_from_dir_args`:
+            model_bundle_name: Name of model bundle you want to create. This acts as a unique identifier.
+            base_paths: The paths on the local filesystem where the bundle code lives.
+            requirements_path: A path on the local filesystem where a requirements.txt file lives.
+            env_params: A dictionary that dictates environment information e.g.
+                the use of pytorch or tensorflow, which cuda/cudnn versions to use.
+                Specifically, the dictionary should contain the following keys:
+                "framework_type": either "tensorflow" or "pytorch".
+                "pytorch_version": Version of pytorch, e.g. "1.5.1", "1.7.0", etc. Only applicable if framework_type is pytorch
+                "cuda_version": Version of cuda used, e.g. "11.0".
+                "cudnn_version" Version of cudnn used, e.g. "cudnn8-devel".
+                "tensorflow_version": Version of tensorflow, e.g. "2.3.0". Only applicable if framework_type is tensorflow
+            load_predict_fn_module_path: A python module path for a function that, when called with the output of
+                load_model_fn_module_path, returns a function that carries out inference.
+            load_model_fn_module_path: A python module path for a function that returns a model. The output feeds into
+                the function located at load_predict_fn_module_path.
+            app_config: Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
+        """
+        from launch import LaunchClient
+
+        launch_client = LaunchClient(api_key=self.api_key)
+
+        model_exists = any(model.name == name for model in self.list_models())
+        bundle_exists = any(
+            bundle.name == name + "-nucleus-autogen"
+            for bundle in launch_client.list_model_bundles()
+        )
+
+        if bundle_exists or model_exists:
+            raise ModelCreationError(
+                "Bundle with the given name already exists, please try a different name"
+            )
+
+        kwargs = {
+            "model_bundle_name": name + "-nucleus-autogen",
+            **bundle_from_dir_args,
+        }
+
+        bundle = launch_client.create_model_bundle_from_dir(**kwargs)
+
+        return self.create_model(
+            name,
+            reference_id,
+            metadata,
+            bundle.name,
         )
 
     @deprecated(
@@ -837,7 +1033,6 @@ class NucleusClient:
         Returns:
             Response payload as JSON dict.
         """
-        print(payload, route)
         if payload is None:
             payload = {}
         if requests_command is requests.get:

{scale-nucleus-0.12b9 → scale-nucleus-0.14.5b1}/nucleus/annotation.py

@@ -180,6 +180,19 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
             EMBEDDING_VECTOR_KEY: self.embedding_vector,
         }
 
+    def __eq__(self, other):
+        return (
+            self.label == other.label
+            and self.x == other.x
+            and self.y == other.y
+            and self.width == other.width
+            and self.height == other.height
+            and self.reference_id == other.reference_id
+            and self.annotation_id == other.annotation_id
+            and sorted(self.metadata.items()) == sorted(other.metadata.items())
+            and self.embedding_vector == other.embedding_vector
+        )
+
 
 @dataclass
 class Point:
@@ -426,7 +439,7 @@ class KeypointsAnnotation(Annotation):
             label="face",
             keypoints=[Keypoint(100, 100), Keypoint(120, 120), Keypoint(visible=False), Keypoint(0, 0)],
             names=["point1", "point2", "point3", "point4"],
-            skeleton=[[0, 1], [1, 2], [1, 3], [2, 4]],
+            skeleton=[[0, 1], [1, 2], [1, 3], [2, 3]],
             reference_id="image_2",
             annotation_id="image_2_face_keypoints_1",
             metadata={"face_direction": "forward"},
@@ -473,11 +486,17 @@ class KeypointsAnnotation(Annotation):
                     )
                 seen.add(name)
 
+        max_segment_index = len(self.keypoints) - 1
         for segment in self.skeleton:
             if len(segment) != 2:
                 raise ValueError(
                     "The keypoints skeleton must contain a list of line segments with exactly 2 indices"
                 )
+            for index in segment:
+                if index > max_segment_index:
+                    raise ValueError(
+                        f"The skeleton index {index} is not a valid keypoint index"
+                    )
 
     @classmethod
     def from_json(cls, payload: dict):
@@ -920,6 +939,9 @@ class AnnotationList:
                 ), f"Unexpected annotation type: {type(annotation)}"
                 self.segmentation_annotations.append(annotation)
 
+    def items(self):
+        return self.__dict__.items()
+
     def __len__(self):
         return (
             len(self.box_annotations)

{scale-nucleus-0.12b9 → scale-nucleus-0.14.5b1}/nucleus/constants.py

@@ -27,6 +27,7 @@ ANNOTATION_UPDATE_KEY = "update"
 AUTOTAGS_KEY = "autotags"
 AUTOTAG_SCORE_THRESHOLD = "score_threshold"
 EXPORTED_ROWS = "exportedRows"
+EXPORTED_SCALE_TASK_INFO_ROWS = "exportedScaleTaskInfoRows"
 CAMERA_MODEL_KEY = "camera_model"
 CAMERA_PARAMS_KEY = "camera_params"
 CLASS_PDF_KEY = "class_pdf"
@@ -89,6 +90,7 @@ MAX_PAYLOAD_SIZE = 0x1FFFFFE8  # Set to max string size since we currently conve
 MESSAGE_KEY = "message"
 METADATA_KEY = "metadata"
 MODEL_BUNDLE_NAME_KEY = "bundle_name"
+MODEL_TAGS_KEY = "tags"
 MODEL_ID_KEY = "model_id"
 MODEL_RUN_ID_KEY = "model_run_id"
 NAME_KEY = "name"
@@ -105,11 +107,14 @@ POINTCLOUD_LOCATION_KEY = "pointcloud_location"
 POINTCLOUD_URL_KEY = "pointcloud_url"
 POSITION_KEY = "position"
 PREDICTIONS_IGNORED_KEY = "predictions_ignored"
+PREDICTIONS_KEY = "predictions"
 PREDICTIONS_PROCESSED_KEY = "predictions_processed"
 REFERENCE_IDS_KEY = "reference_ids"
 REFERENCE_ID_KEY = "reference_id"
 BACKEND_REFERENCE_ID_KEY = "ref_id"  # TODO(355762): Our backend returns this instead of the "proper" key sometimes.
 REQUEST_ID_KEY = "requestId"
+SCALE_TASK_INFO_KEY = "scale_task_info"
+SCENE_KEY = "scene"
 SCENES_KEY = "scenes"
 SERIALIZED_REQUEST_KEY = "serialized_request"
 SEGMENTATIONS_KEY = "segmentations"

{scale-nucleus-0.12b9 → scale-nucleus-0.14.5b1}/nucleus/dataset.py

@@ -18,6 +18,7 @@ from nucleus.utils import (
     convert_export_payload,
     format_dataset_item_response,
     format_prediction_response,
+    format_scale_task_info_response,
     paginate_generator,
     serialize_and_write_to_presigned_url,
 )
@@ -1278,6 +1279,36 @@ class Dataset:
         )
         return format_prediction_response({ANNOTATIONS_KEY: json_response})
 
+    def export_scale_task_info(self):
+        """Fetches info for all linked Scale tasks of items/scenes in the dataset.
+
+        Returns:
+            A list of dicts, each with two keys, respectively mapping to items/scenes
+            and info on their corresponding Scale tasks within the dataset::
+
+                List[{
+                    "item" | "scene": Union[:class:`DatasetItem`, :class:`Scene`],
+                    "scale_task_info": {
+                        "task_id": str,
+                        "subtask_id": str,
+                        "task_status": str,
+                        "task_audit_status": str,
+                        "task_audit_review_comment": Optional[str],
+                        "project_name": str,
+                        "batch": str,
+                        "created_at": str,
+                        "completed_at": Optional[str]
+                    }[]
+                }]
+
+        """
+        response = self._client.make_request(
+            payload=None,
+            route=f"dataset/{self.id}/exportScaleTaskInfo",
+            requests_command=requests.get,
+        )
+        return format_scale_task_info_response(response)
+
     def calculate_evaluation_metrics(self, model, options: dict = None):
         """Starts computation of evaluation metrics for a model on the dataset.
 

{scale-nucleus-0.12b9 → scale-nucleus-0.14.5b1}/nucleus/metrics/__init__.py

@@ -18,7 +18,6 @@ from .polygon_metrics import (
     PolygonRecall,
 )
 from .segmentation_metrics import (
-    SegmentationAveragePrecision,
     SegmentationFWAVACC,
     SegmentationIOU,
     SegmentationMAP,

{scale-nucleus-0.12b9 → scale-nucleus-0.14.5b1}/nucleus/metrics/base.py

@@ -4,10 +4,13 @@ from dataclasses import dataclass
 from typing import Iterable, List, Optional, Union
 
 from nucleus.annotation import AnnotationList
+from nucleus.metrics.errors import EverythingFilteredError
 from nucleus.metrics.filtering import (
     ListOfAndFilters,
     ListOfOrAndFilters,
-    apply_filters,
+    compose_helpful_filtering_error,
+    filter_annotation_list,
+    filter_prediction_list,
 )
 from nucleus.prediction import PredictionList
 
@@ -133,64 +136,16 @@ class Metric(ABC):
     def __call__(
         self, annotations: AnnotationList, predictions: PredictionList
     ) -> MetricResult:
-        annotations = self._filter_annotations(annotations)
-        predictions = self._filter_predictions(predictions)
-        return self.call_metric(annotations, predictions)
-
-    def _filter_annotations(self, annotations: AnnotationList):
-        if (
-            self.annotation_filters is None
-            or len(self.annotation_filters) == 0
-        ):
-            return annotations
-        annotations.box_annotations = apply_filters(
-            annotations.box_annotations, self.annotation_filters
+        filtered_anns = filter_annotation_list(
+            annotations, self.annotation_filters
         )
-        annotations.line_annotations = apply_filters(
-            annotations.line_annotations, self.annotation_filters
+        filtered_preds = filter_prediction_list(
+            predictions, self.prediction_filters
         )
-        annotations.polygon_annotations = apply_filters(
-            annotations.polygon_annotations, self.annotation_filters
+        self._raise_if_everything_filtered(
+            annotations, filtered_anns, predictions, filtered_preds
         )
-        annotations.cuboid_annotations = apply_filters(
-            annotations.cuboid_annotations, self.annotation_filters
-        )
-        annotations.category_annotations = apply_filters(
-            annotations.category_annotations, self.annotation_filters
-        )
-        annotations.multi_category_annotations = apply_filters(
-            annotations.multi_category_annotations, self.annotation_filters
-        )
-        annotations.segmentation_annotations = apply_filters(
-            annotations.segmentation_annotations, self.annotation_filters
-        )
-        return annotations
-
-    def _filter_predictions(self, predictions: PredictionList):
-        if (
-            self.prediction_filters is None
-            or len(self.prediction_filters) == 0
-        ):
-            return predictions
-        predictions.box_predictions = apply_filters(
-            predictions.box_predictions, self.prediction_filters
-        )
-        predictions.line_predictions = apply_filters(
-            predictions.line_predictions, self.prediction_filters
-        )
-        predictions.polygon_predictions = apply_filters(
-            predictions.polygon_predictions, self.prediction_filters
-        )
-        predictions.cuboid_predictions = apply_filters(
-            predictions.cuboid_predictions, self.prediction_filters
-        )
-        predictions.category_predictions = apply_filters(
-            predictions.category_predictions, self.prediction_filters
-        )
-        predictions.segmentation_predictions = apply_filters(
-            predictions.segmentation_predictions, self.prediction_filters
-        )
-        return predictions
+        return self.call_metric(filtered_anns, filtered_preds)
 
     @abstractmethod
     def aggregate_score(self, results: List[MetricResult]) -> ScalarResult:
@@ -215,3 +170,26 @@ class Metric(ABC):
                 return ScalarResult(r2_score)
 
         """
+
+    def _raise_if_everything_filtered(
+        self,
+        annotations: AnnotationList,
+        filtered_annotations: AnnotationList,
+        predictions: PredictionList,
+        filtered_predictions: PredictionList,
+    ):
+        msg = []
+        if len(filtered_annotations) == 0:
+            msg.extend(
+                compose_helpful_filtering_error(
+                    annotations, self.annotation_filters
+                )
+            )
+        if len(filtered_predictions) == 0:
+            msg.extend(
+                compose_helpful_filtering_error(
+                    predictions, self.prediction_filters
+                )
+            )
+        if msg:
+            raise EverythingFilteredError("\n".join(msg))