code-loader 1.0.214.dev1__tar.gz → 1.0.214.dev2__tar.gz

{code_loader-1.0.214.dev1 → code_loader-1.0.214.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.214.dev1
+Version: 1.0.214.dev2
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.214.dev1 → code_loader-1.0.214.dev2}/code_loader/contract/datasetclasses.py

@@ -155,6 +155,9 @@ class ElementInstance:
     # masks funnel (utils.to_numpy_return_masks_wrapper) so the engine copies flags rather than
     # deriving them, exactly as it does for sample-level metadata.
     instance_metadata_is_none: Optional[Dict[str, bool]] = None
+    # Per-instance metric values keyed by the registered name of a tensorleap_custom_instances_metric.
+    # Only meaningful on false-positive instances (they have no ground-truth row for the metric to run on).
+    instance_metrics: Optional[Dict[str, float]] = None
 
 SectionCallableInterface = Callable[[Union[int, str], PreprocessResponse], npt.NDArray[np.float32]]
 InstanceCallableInterface = Callable[[Union[int, str], PreprocessResponse, int], Optional[ElementInstance]]
@@ -412,6 +415,7 @@ class DatasetIntegrationSetup:
     instance_masks: List[ElementInstanceMasksHandler] = field(default_factory=list)
     false_positive_instances: Optional[FalsePositiveInstancesHandler] = None
     false_positive_empty_gt: Dict[str, Callable] = field(default_factory=dict)
+    false_positive_custom_latent_spaces: Dict[str, Callable] = field(default_factory=dict)
     ground_truths: List[GroundTruthHandler] = field(default_factory=list)
     metadata: List[MetadataHandler] = field(default_factory=list)
     prediction_types: List[PredictionTypeHandler] = field(default_factory=list)

{code_loader-1.0.214.dev1 → code_loader-1.0.214.dev2}/code_loader/inner_leap_binder/leapbinder.py

@@ -392,6 +392,61 @@ class LeapBinder:
                     f"tensorleap_false_positive_empty_gt for '{name}' returned dtype {empty.dtype}, "
                     f"which cannot be cast to float32.")
 
+    def set_false_positive_custom_latent_space(self, function: Callable, name: str) -> None:
+        """
+        Set the custom latent space hook for false-positive instance rows. It receives
+        (sample_id, preprocess, fp_instance) and returns a 1-D vector for the instance-aware
+        custom latent space registered under `name`.
+        """
+        if name in self.setup_container.false_positive_custom_latent_spaces:
+            raise Exception(f"tensorleap_false_positive_custom_latent_space for '{name}' is already defined")
+        self.setup_container.false_positive_custom_latent_spaces[name] = function
+
+    def check_false_positive_custom_latent_spaces(
+            self, preprocess_result: Optional[Dict[DataStateEnum, PreprocessResponse]] = None) -> None:
+        """Every FP latent-space hook must name an instance-aware custom latent space and, when a
+        preprocess result is given, return the same shape as it (no-op without an FP encoder)."""
+        if self.setup_container.false_positive_instances is None:
+            return
+        for name in self.setup_container.false_positive_custom_latent_spaces:
+            handler = self.setup_container.custom_latent_spaces.get(name)
+            if handler is None or not handler.instance_aware:
+                raise Exception(
+                    f"tensorleap_false_positive_custom_latent_space validation failed: '{name}' is not an "
+                    f"instance custom latent space (tensorleap_instance_custom_latent_space)")
+        if preprocess_result is not None:
+            self._check_false_positive_custom_latent_space_shapes(preprocess_result)
+
+    def _check_false_positive_custom_latent_space_shapes(
+            self, preprocess_result: Dict[DataStateEnum, PreprocessResponse]) -> None:
+        """Call each FP latent-space hook with a real GT ElementInstance and compare its shape with
+        the instance custom latent space's. FP and GT rows of one latent space are usually stored in
+        different bulks, so a width mismatch would otherwise be dropped silently at load time.
+        Skipped when there is no GT instance to probe with (no mask encoder, or no training sample
+        with an instance)."""
+        fp_hooks = self.setup_container.false_positive_custom_latent_spaces
+        if not fp_hooks or not self.setup_container.instance_masks:
+            return
+        training = preprocess_result.get(DataStateEnum.training)
+        if training is None or not training.sample_ids_to_instance_mappings:
+            return
+        sample_id = next((sid for sid, instance_ids in training.sample_ids_to_instance_mappings.items()
+                          if instance_ids), None)
+        if sample_id is None:
+            return
+        gt_instance = self.setup_container.instance_masks[0].function(sample_id, training, 0)
+        if gt_instance is None:
+            return
+        for name, fp_hook in fp_hooks.items():
+            instance_ls = cast(InstanceSectionCallableInterface,
+                               self.setup_container.custom_latent_spaces[name].function)
+            gt_shape = np.asarray(instance_ls(sample_id, training, 0)).shape
+            fp_shape = np.asarray(fp_hook(sample_id, training, gt_instance)).shape
+            if fp_shape != gt_shape:
+                raise Exception(
+                    f"tensorleap_false_positive_custom_latent_space validation failed: '{name}' returns shape "
+                    f"{fp_shape} but the instance custom latent space returns {gt_shape}")
+
     def add_custom_loss(self, function: CustomCallableInterface, name: str) -> None:
         """
         Add a custom loss function to the setup.
@@ -1075,6 +1130,7 @@ class LeapBinder:
         self.check_preprocess(preprocess_result)
         self.check_handlers(preprocess_result)
         self.check_false_positive_empty_gt(preprocess_result)
+        self.check_false_positive_custom_latent_spaces(preprocess_result)
         self.check_simulations()
         self.validate_ignore_latent_spaces()
         print("Successful!")

{code_loader-1.0.214.dev1 → code_loader-1.0.214.dev2}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -3603,6 +3603,54 @@ def tensorleap_false_positive_empty_gt(gt_name: str):
     return decorating_function
 
 
+def tensorleap_false_positive_custom_latent_space(name: str):
+    """Compute the instance custom latent space `name` (a tensorleap_instance_custom_latent_space)
+    for a false-positive instance. Signature: fn(sample_id, preprocess, fp_instance) -> 1-D np.ndarray."""
+    def decorating_function(user_function: Callable):
+        def _validate_input_args(sample_id, preprocess_response, fp_instance):
+            assert isinstance(sample_id, (str, int)), \
+                (f'tensorleap_false_positive_custom_latent_space validation failed: '
+                 f'Argument sample_id should be str or int. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_false_positive_custom_latent_space validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
+            assert isinstance(fp_instance, ElementInstance), \
+                (f'tensorleap_false_positive_custom_latent_space validation failed: '
+                 f'Argument fp_instance should be an ElementInstance. Got {type(fp_instance)}.')
+
+        def _validate_result(result):
+            assert isinstance(result, np.ndarray), \
+                (f'tensorleap_false_positive_custom_latent_space validation failed: '
+                 f'The return type should be a numpy array. Got {type(result)}.')
+            assert result.ndim == 1, \
+                (f'tensorleap_false_positive_custom_latent_space validation failed: '
+                 f'The returned array should be 1-D. Got shape {result.shape}.')
+
+        def inner_without_validate(sample_id, preprocess_response, fp_instance):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+            try:
+                result = user_function(sample_id, preprocess_response, fp_instance)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+            return result
+
+        leap_binder.set_false_positive_custom_latent_space(inner_without_validate, name)
+
+        def inner(sample_id, preprocess_response, fp_instance):
+            if os.environ.get(mapping_runtime_mode_env_var_mame):
+                return np.zeros(0, dtype=np.float32)
+
+            _validate_input_args(sample_id, preprocess_response, fp_instance)
+            result = inner_without_validate(sample_id, preprocess_response, fp_instance)
+            _validate_result(result)
+            return result
+
+        return inner
+
+    return decorating_function
+
+
 def tensorleap_instances_length_encoder(name: str):
     def decorating_function(user_function: InstanceLengthCallableInterface):
         def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse):

{code_loader-1.0.214.dev1 → code_loader-1.0.214.dev2}/code_loader/leaploader.py

@@ -653,7 +653,9 @@ class LeapLoader(LeapLoaderBase):
             return None
         test_result = DatasetTestResultPayload(handler.name)
         try:
-            global_leap_binder.check_false_positive_empty_gt(self._preprocess_result())
+            preprocess_result = self._preprocess_result()
+            global_leap_binder.check_false_positive_empty_gt(preprocess_result)
+            global_leap_binder.check_false_positive_custom_latent_spaces(preprocess_result)
         except Exception as e:
             line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
             error_string = f"{repr(e)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
@@ -1825,7 +1827,7 @@ class LeapLoader(LeapLoaderBase):
             row = {name: array[i] for name, array in predictions.items()}
             try:
                 instances = handler.function(self._native_sample_id(sample_id), preprocess_response, row)
-                self._validate_false_positive_instances(instances)
+                self._validate_false_positive_instances(instances, set(self._instance_metric_handler_by_name()))
             except Exception as e:
                 print(f"tensorleap_false_positive_instances_encoder failed for sample {sample_id!r} "
                       f"(state {state.name}): {e!r}")
@@ -1835,7 +1837,7 @@ class LeapLoader(LeapLoaderBase):
         return results
 
     @staticmethod
-    def _validate_false_positive_instances(instances: Any) -> None:
+    def _validate_false_positive_instances(instances: Any, registered_instance_metrics: Set[str]) -> None:
         prefix = "tensorleap_false_positive_instances_encoder validation failed: "
         if not isinstance(instances, list):
             raise Exception(f"{prefix}The return type should be a list of ElementInstance. "
@@ -1844,6 +1846,21 @@ class LeapLoader(LeapLoaderBase):
             if not isinstance(instance, ElementInstance):
                 raise Exception(f"{prefix}Element {j} of the returned list should be an ElementInstance. "
                                 f"Got {type(instance)}.")
+            if instance.instance_metrics is None:
+                continue
+            if not isinstance(instance.instance_metrics, dict):
+                raise Exception(f"{prefix}instance_metrics of element {j} should be a dict. "
+                                f"Got {type(instance.instance_metrics)}.")
+            for key, value in instance.instance_metrics.items():
+                if key not in registered_instance_metrics:
+                    raise Exception(f"{prefix}instance_metrics key {key!r} of element {j} is not the name of a "
+                                    f"registered tensorleap_custom_instances_metric. "
+                                    f"Registered: {sorted(registered_instance_metrics)}.")
+                try:
+                    instance.instance_metrics[key] = float(value)
+                except (TypeError, ValueError):
+                    raise Exception(f"{prefix}instance_metrics[{key!r}] of element {j} should be castable to "
+                                    f"float. Got {type(value)}.")
 
     def get_false_positive_empty_gt(self, state: DataStateEnum,
                                     sample_id: Union[str, int]) -> Dict[str, npt.NDArray[np.float32]]:
@@ -1859,6 +1876,26 @@ class LeapLoader(LeapLoaderBase):
             empty_gt[gt_name] = value.astype(np.float32, copy=False)
         return empty_gt
 
+    def get_false_positive_custom_latent_spaces(self, state: DataStateEnum, sample_id: Union[str, int],
+                                                fp_instance: ElementInstance) -> Dict[str, npt.NDArray[np.float32]]:
+        self.exec_script()
+        hooks = global_leap_binder.setup_container.false_positive_custom_latent_spaces
+        if not hooks:
+            return {}
+        preprocess_response = self._preprocess_result()[state]
+        sample_id = self._native_sample_id(sample_id)
+        vectors: Dict[str, npt.NDArray[np.float32]] = {}
+        for name, function in hooks.items():
+            value = function(sample_id, preprocess_response, fp_instance)
+            if not isinstance(value, np.ndarray):
+                raise Exception(f"tensorleap_false_positive_custom_latent_space validation failed: "
+                                f"The return type for '{name}' should be a numpy array. Got {type(value)}.")
+            if value.ndim != 1:
+                raise Exception(f"tensorleap_false_positive_custom_latent_space validation failed: "
+                                f"The return value for '{name}' should be a 1-D array. Got shape {value.shape}.")
+            vectors[name] = value.astype(np.float32, copy=False)
+        return vectors
+
     def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str]]:
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result.get(state)

{code_loader-1.0.214.dev1 → code_loader-1.0.214.dev2}/code_loader/leaploaderbase.py

@@ -87,6 +87,10 @@ class LeapLoaderBase:
                                     sample_id: Union[str, int]) -> Dict[str, npt.NDArray[np.float32]]:
         raise NotImplementedError("get_false_positive_empty_gt is not implemented by this loader")
 
+    def get_false_positive_custom_latent_spaces(self, state: DataStateEnum, sample_id: Union[str, int],
+                                                fp_instance: ElementInstance) -> Dict[str, npt.NDArray[np.float32]]:
+        return {}
+
     def get_metadata_multiple_samples(self, state: DataStateEnum, sample_ids: Union[List[int], List[str]],
                                       requested_metadata_names: Optional[List[str]] = None
                                       ) -> Tuple[Dict[str, Union[List[str], List[int], List[bool],

{code_loader-1.0.214.dev1 → code_loader-1.0.214.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.214.dev1"
+version = "1.0.214.dev2"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"