code-loader 1.0.212__tar.gz → 1.0.214__tar.gz

{code_loader-1.0.212 → code_loader-1.0.214}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.212
+Version: 1.0.214
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.212 → code_loader-1.0.214}/code_loader/contract/datasetclasses.py

@@ -155,6 +155,9 @@ class ElementInstance:
     # masks funnel (utils.to_numpy_return_masks_wrapper) so the engine copies flags rather than
     # deriving them, exactly as it does for sample-level metadata.
     instance_metadata_is_none: Optional[Dict[str, bool]] = None
+    # Per-instance metric values keyed by the registered name of a tensorleap_custom_instances_metric.
+    # Only meaningful on false-positive instances (they have no ground-truth row for the metric to run on).
+    instance_metrics: Optional[Dict[str, float]] = None
 
 SectionCallableInterface = Callable[[Union[int, str], PreprocessResponse], npt.NDArray[np.float32]]
 InstanceCallableInterface = Callable[[Union[int, str], PreprocessResponse, int], Optional[ElementInstance]]
@@ -307,6 +310,11 @@ class ElementInstanceMasksHandler:
     name: str
     function: InstanceCallableInterface
 
+@dataclass
+class FalsePositiveInstancesHandler:
+    name: str
+    function: Callable
+
 @dataclass
 class GroundTruthHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None
@@ -405,6 +413,9 @@ class DatasetIntegrationSetup:
     visualizers: List[VisualizerHandler] = field(default_factory=list)
     inputs: List[InputHandler] = field(default_factory=list)
     instance_masks: List[ElementInstanceMasksHandler] = field(default_factory=list)
+    false_positive_instances: Optional[FalsePositiveInstancesHandler] = None
+    false_positive_empty_gt: Dict[str, Callable] = field(default_factory=dict)
+    false_positive_custom_latent_spaces: Dict[str, Callable] = field(default_factory=dict)
     ground_truths: List[GroundTruthHandler] = field(default_factory=list)
     metadata: List[MetadataHandler] = field(default_factory=list)
     prediction_types: List[PredictionTypeHandler] = field(default_factory=list)

{code_loader-1.0.212 → code_loader-1.0.214}/code_loader/inner_leap_binder/leapbinder.py

@@ -16,7 +16,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
-    ElementInstanceMasksHandler, InstanceCallableInterface, InstanceSectionCallableInterface, \
+    ElementInstanceMasksHandler, FalsePositiveInstancesHandler, InstanceCallableInterface, InstanceSectionCallableInterface, \
     CustomLatentSpaceHandler, InstanceMetricHandler, MAX_CUSTOM_LATENT_SPACES, \
     SimulationHandler, _simulation_context, AutoregressiveStepHandler, AutoregressiveStepCallableInterface, \
     AUTOREGRESSIVE_LATENT_SPACE_AGGREGATIONS, AUTOREGRESSIVE_IMPLICIT_ARG_NAMES, \
@@ -28,7 +28,8 @@ from code_loader.contract.responsedataclasses import DatasetTestResultPayload, L
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
 from code_loader.default_losses import loss_name_to_function
 from code_loader.default_metrics import metrics_names_to_functions_and_direction
-from code_loader.utils import to_numpy_return_wrapper, get_shape, to_numpy_return_masks_wrapper
+from code_loader.utils import to_numpy_return_wrapper, get_shape, to_numpy_return_masks_wrapper, \
+    to_numpy_return_fp_masks_wrapper
 from code_loader.contract.sim_config import parse_sim_config, sample_sim_params
 from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
     default_graph_visualizer, \
@@ -340,6 +341,112 @@ class LeapBinder:
         self.setup_container.instance_masks.append(ElementInstanceMasksHandler(name, function))
 
 
+    def set_false_positive_instances(self, function: Callable, name: str) -> None:
+        """
+        Set the false-positive instances handler. It receives (sample_id, preprocess, predictions)
+        and returns a list of ElementInstance, one per false-positive prediction.
+        """
+        if self.setup_container.false_positive_instances is not None:
+            raise Exception("only one tensorleap_false_positive_instances_encoder can be defined")
+        function = to_numpy_return_fp_masks_wrapper(function)
+        self.setup_container.false_positive_instances = FalsePositiveInstancesHandler(name, function)
+
+    def set_false_positive_empty_gt(self, function: Callable, gt_name: str) -> None:
+        """
+        Set the empty ground truth for false-positive instance rows of the given GT encoder.
+        It receives (sample_id, preprocess) and returns an array shaped like the GT encoder's output.
+        """
+        if gt_name in self.setup_container.false_positive_empty_gt:
+            raise Exception(f"tensorleap_false_positive_empty_gt for '{gt_name}' is already defined")
+        self.setup_container.false_positive_empty_gt[gt_name] = function
+
+    def check_false_positive_empty_gt(self, preprocess_result: Dict[DataStateEnum, PreprocessResponse]) -> None:
+        """Validate the empty-GT functions against the GT encoders (no-op without an FP encoder)."""
+        if self.setup_container.false_positive_instances is None:
+            return
+        gt_handlers = {handler.name: handler for handler in self.setup_container.ground_truths}
+        empty_gt = self.setup_container.false_positive_empty_gt
+        missing = [name for name in gt_handlers if name not in empty_gt]
+        if missing:
+            raise Exception(
+                f"tensorleap_false_positive_instances_encoder requires a tensorleap_false_positive_empty_gt "
+                f"for every ground truth encoder. Missing for: {missing}.")
+        unknown = [name for name in empty_gt if name not in gt_handlers]
+        if unknown:
+            raise Exception(
+                f"tensorleap_false_positive_empty_gt names {unknown} do not match any registered ground truth "
+                f"encoder. Registered ground truth encoders: {list(gt_handlers)}.")
+        training = preprocess_result.get(DataStateEnum.training)
+        if training is None or not training.sample_ids or training.is_grouped:
+            return
+        sample_id = cast(Union[int, str], training.sample_ids[0])
+        for name, handler in gt_handlers.items():
+            gt = np.asarray(handler.function(sample_id, training))
+            empty = np.asarray(empty_gt[name](sample_id, training))
+            if empty.shape != gt.shape:
+                raise Exception(
+                    f"tensorleap_false_positive_empty_gt for '{name}' returned shape {empty.shape}, but the "
+                    f"ground truth encoder returns shape {gt.shape}.")
+            if not np.can_cast(empty.dtype, np.float32, casting='same_kind'):
+                raise Exception(
+                    f"tensorleap_false_positive_empty_gt for '{name}' returned dtype {empty.dtype}, "
+                    f"which cannot be cast to float32.")
+
+    def set_false_positive_custom_latent_space(self, function: Callable, name: str) -> None:
+        """
+        Set the custom latent space hook for false-positive instance rows. It receives
+        (sample_id, preprocess, fp_instance) and returns a 1-D vector for the instance-aware
+        custom latent space registered under `name`.
+        """
+        if name in self.setup_container.false_positive_custom_latent_spaces:
+            raise Exception(f"tensorleap_false_positive_custom_latent_space for '{name}' is already defined")
+        self.setup_container.false_positive_custom_latent_spaces[name] = function
+
+    def check_false_positive_custom_latent_spaces(
+            self, preprocess_result: Optional[Dict[DataStateEnum, PreprocessResponse]] = None) -> None:
+        """Every FP latent-space hook must name an instance-aware custom latent space and, when a
+        preprocess result is given, return the same shape as it (no-op without an FP encoder)."""
+        if self.setup_container.false_positive_instances is None:
+            return
+        for name in self.setup_container.false_positive_custom_latent_spaces:
+            handler = self.setup_container.custom_latent_spaces.get(name)
+            if handler is None or not handler.instance_aware:
+                raise Exception(
+                    f"tensorleap_false_positive_custom_latent_space validation failed: '{name}' is not an "
+                    f"instance custom latent space (tensorleap_instance_custom_latent_space)")
+        if preprocess_result is not None:
+            self._check_false_positive_custom_latent_space_shapes(preprocess_result)
+
+    def _check_false_positive_custom_latent_space_shapes(
+            self, preprocess_result: Dict[DataStateEnum, PreprocessResponse]) -> None:
+        """Call each FP latent-space hook with a real GT ElementInstance and compare its shape with
+        the instance custom latent space's. FP and GT rows of one latent space are usually stored in
+        different bulks, so a width mismatch would otherwise be dropped silently at load time.
+        Skipped when there is no GT instance to probe with (no mask encoder, or no training sample
+        with an instance)."""
+        fp_hooks = self.setup_container.false_positive_custom_latent_spaces
+        if not fp_hooks or not self.setup_container.instance_masks:
+            return
+        training = preprocess_result.get(DataStateEnum.training)
+        if training is None or not training.sample_ids_to_instance_mappings:
+            return
+        sample_id = next((sid for sid, instance_ids in training.sample_ids_to_instance_mappings.items()
+                          if instance_ids), None)
+        if sample_id is None:
+            return
+        gt_instance = self.setup_container.instance_masks[0].function(sample_id, training, 0)
+        if gt_instance is None:
+            return
+        for name, fp_hook in fp_hooks.items():
+            instance_ls = cast(InstanceSectionCallableInterface,
+                               self.setup_container.custom_latent_spaces[name].function)
+            gt_shape = np.asarray(instance_ls(sample_id, training, 0)).shape
+            fp_shape = np.asarray(fp_hook(sample_id, training, gt_instance)).shape
+            if fp_shape != gt_shape:
+                raise Exception(
+                    f"tensorleap_false_positive_custom_latent_space validation failed: '{name}' returns shape "
+                    f"{fp_shape} but the instance custom latent space returns {gt_shape}")
+
     def add_custom_loss(self, function: CustomCallableInterface, name: str) -> None:
         """
         Add a custom loss function to the setup.
@@ -798,6 +905,10 @@ class LeapBinder:
         for state, preprocess_response in preprocess_result.items():
             if preprocess_response.length is None or preprocess_response.length <= 0:
                 raise Exception('Invalid dataset length')
+            if preprocess_response.is_grouped and self.setup_container.false_positive_instances is not None:
+                raise Exception(
+                    'tensorleap_false_positive_instances_encoder is not supported with a grouped '
+                    f'preprocess response (grouped state: {state.name}). Use flat sample_ids.')
             if unlabeled_preprocess_handler is not None and state == DataStateEnum.unlabeled:
                 unlabeled_preprocess_handler.data_length = preprocess_response.length
             else:
@@ -1018,6 +1129,8 @@ class LeapBinder:
         preprocess_result = self.get_preprocess_result()
         self.check_preprocess(preprocess_result)
         self.check_handlers(preprocess_result)
+        self.check_false_positive_empty_gt(preprocess_result)
+        self.check_false_positive_custom_latent_spaces(preprocess_result)
         self.check_simulations()
         self.validate_ignore_latent_spaces()
         print("Successful!")
@@ -1058,7 +1171,7 @@ class LeapBinder:
                 "Simulations are not supported together with tensorleap_autoregressive_step yet: "
                 "the autoregressive entry points do not resolve synthetic sample ids. Remove the "
                 "simulation decorators or the autoregressive hook.")
-        if self.setup_container.instance_masks:
+        if self.setup_container.instance_masks or self.setup_container.false_positive_instances is not None:
             raise Exception(
                 "Element instances are not supported together with tensorleap_autoregressive_step: "
                 "instance generation masks the sample's encoded inputs, which do not exist in an "

{code_loader-1.0.212 → code_loader-1.0.214}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -3254,29 +3254,59 @@ def tensorleap_simulation(name: str, sim_params: dict):
     return decorating_function
 
 
+def _register_builtin_instance_metadata() -> None:
+    """Idempotently register the builtin instance metadata handler.
+
+    Shared by the element-instance preprocess and the false-positive encoder, since either alone
+    makes the engine produce instance samples.
+    """
+    if any(h.name == "builtin_instance_metadata" for h in leap_binder.setup_container.metadata):
+        return
+
+    def builtin_instance_metadata(idx: str, preprocess: PreprocessResponse) -> Dict[str, str]:
+        result = {'is_instance': '0', 'original_sample_id': idx, 'instance_name': 'none'}
+        if leap_binder.setup_container.false_positive_instances is not None:
+            result['is_false_positive'] = '0'
+        return result
+
+    leap_binder.set_metadata(builtin_instance_metadata, "builtin_instance_metadata")
+
+
+def _register_builtin_instance_extra_metadata(metadata_types: Dict[str, DatasetMetadataType]) -> None:
+    """Register (or extend) the `builtin_instance_extra_metadata` column declaration.
+
+    Shared by the element-instance preprocess (keys probed from the GT mask encoder) and the
+    false-positive encoder (keys declared with `instance_metadata_types`). Idempotent and merging:
+    set_metadata appends without deduping, and the probe path runs inside preprocess, so a second
+    call must never register a duplicate handler; a key already declared keeps its first type.
+    """
+    if not metadata_types:
+        return
+    existing = next((handler for handler in leap_binder.setup_container.metadata
+                     if handler.name == "builtin_instance_extra_metadata"), None)
+    if existing is not None:
+        if isinstance(existing.metadata_type, dict):
+            for name, metadata_type in metadata_types.items():
+                existing.metadata_type.setdefault(name, metadata_type)
+        return
+    declared: Dict[str, DatasetMetadataType] = dict(metadata_types)
+
+    def builtin_instance_extra_metadata(idx: str, preprocess: PreprocessResponse) -> Dict[str, None]:
+        # Sample rows declare the columns without claiming values; the engine injects the
+        # real per-instance ones. A fresh dict, because nulling the probed
+        # ElementInstance's own dict in place destroyed any encoder that returned a
+        # reference into preprocess.data. Read at call time, so keys merged in later count.
+        return {name: None for name in declared}
+
+    leap_binder.set_metadata(builtin_instance_extra_metadata,
+                             "builtin_instance_extra_metadata", declared)
+
+
 def tensorleap_element_instance_preprocess(
         instance_length_encoder: InstanceLengthCallableInterface,
         instance_mask_encoder: InstanceCallableInterface,
         instance_metadata_types: Optional[Dict[str, DatasetMetadataType]] = None):
     def decorating_function(user_function: Callable[[], List[PreprocessResponse]]):
-        def register_instance_extra_metadata(metadata_types: Dict[str, DatasetMetadataType]) -> None:
-            # Idempotent: set_metadata appends without deduping, and the probe path runs inside
-            # preprocess, so a second preprocess call would register a duplicate handler.
-            if any(handler.name == "builtin_instance_extra_metadata"
-                   for handler in leap_binder.setup_container.metadata):
-                return
-            names = list(metadata_types)
-
-            def builtin_instance_extra_metadata(idx: str, preprocess: PreprocessResponse) -> Dict[str, None]:
-                # Sample rows declare the columns without claiming values; the engine injects the
-                # real per-instance ones. A fresh dict, because nulling the probed
-                # ElementInstance's own dict in place destroyed any encoder that returned a
-                # reference into preprocess.data.
-                return {name: None for name in names}
-
-            leap_binder.set_metadata(builtin_instance_extra_metadata,
-                                     "builtin_instance_extra_metadata", metadata_types)
-
         def resolve_instance_metadata_types(sample_id: Union[int, str], preprocess_response: PreprocessResponse,
                                             idx: int) -> Optional[Dict[str, DatasetMetadataType]]:
             # The probe enumerates the KEYS; the declaration, where given, supplies the TYPE.
@@ -3346,7 +3376,7 @@ def tensorleap_element_instance_preprocess(
                             resolved_types = resolve_instance_metadata_types(
                                 sample_id, preprocess_response, idx)
                             if resolved_types:
-                                register_instance_extra_metadata(resolved_types)
+                                _register_builtin_instance_extra_metadata(resolved_types)
                             found_instance_metadata = True
 
                         instance_to_sample_ids_mappings[instance_id] = sample_id
@@ -3357,11 +3387,8 @@ def tensorleap_element_instance_preprocess(
                 preprocess_response.sample_ids = all_sample_ids
             return result
 
-        def builtin_instance_metadata(idx: str, preprocess: PreprocessResponse) -> Dict[str, str]:
-            return {'is_instance': '0', 'original_sample_id': idx, 'instance_name': 'none'}
-
         leap_binder.set_preprocess(user_function_instance)
-        leap_binder.set_metadata(builtin_instance_metadata, "builtin_instance_metadata")
+        _register_builtin_instance_metadata()
 
         def _validate_input_args(*args, **kwargs):
             assert len(args) == 0 and len(kwargs) == 0, \
@@ -3473,6 +3500,157 @@ def tensorleap_instances_masks_encoder(name: str):
     return decorating_function
 
 
+def tensorleap_false_positive_instances_encoder(
+        name: str, instance_metadata_types: Optional[Dict[str, DatasetMetadataType]] = None):
+    """Register the false-positive instances encoder.
+
+    `instance_metadata_types` declares the keys (and types) of the `instance_metadata` the returned
+    ElementInstances may carry. They become `builtin_instance_extra_metadata_<key>` columns, merged
+    with any keys the element-instance preprocess declares; a key not declared here (or by the
+    GT instances) is dropped from false-positive rows by the engine.
+    """
+    def decorating_function(user_function: Callable):
+        def _validate_input_args(sample_id, preprocess_response, predictions):
+            assert isinstance(sample_id, (str, int)), \
+                (f'tensorleap_false_positive_instances_encoder validation failed: '
+                 f'Argument sample_id should be str or int. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_false_positive_instances_encoder validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
+            assert isinstance(predictions, dict), \
+                (f'tensorleap_false_positive_instances_encoder validation failed: '
+                 f'Argument predictions should be a dict. Got {type(predictions)}.')
+
+        def _validate_result(result):
+            assert isinstance(result, list), \
+                (f'tensorleap_false_positive_instances_encoder validation failed: '
+                 f'The return type should be a list of ElementInstance. Got {type(result)}.')
+            for i, instance in enumerate(result):
+                assert isinstance(instance, ElementInstance), \
+                    (f'tensorleap_false_positive_instances_encoder validation failed: '
+                     f'Element {i} of the returned list should be an ElementInstance. Got {type(instance)}.')
+
+        def inner_without_validate(sample_id, preprocess_response, predictions):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+
+            try:
+                result = user_function(sample_id, preprocess_response, predictions)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+
+            return result
+
+        leap_binder.set_false_positive_instances(inner_without_validate, name)
+        _register_builtin_instance_metadata()
+        if instance_metadata_types:
+            _register_builtin_instance_extra_metadata(instance_metadata_types)
+
+        def inner(sample_id, preprocess_response, predictions):
+            if os.environ.get(mapping_runtime_mode_env_var_mame):
+                return []
+
+            _validate_input_args(sample_id, preprocess_response, predictions)
+
+            result = inner_without_validate(sample_id, preprocess_response, predictions)
+
+            _validate_result(result)
+            return result
+
+        return inner
+
+    return decorating_function
+
+
+def tensorleap_false_positive_empty_gt(gt_name: str):
+    """Declare the empty ground truth used for false-positive instance rows of GT encoder `gt_name`."""
+    def decorating_function(user_function: Callable):
+        def _validate_input_args(sample_id, preprocess_response):
+            assert isinstance(sample_id, (str, int)), \
+                (f'tensorleap_false_positive_empty_gt validation failed: '
+                 f'Argument sample_id should be str or int. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_false_positive_empty_gt validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
+
+        def _validate_result(result):
+            assert isinstance(result, np.ndarray), \
+                (f'tensorleap_false_positive_empty_gt validation failed: '
+                 f'The return type should be a numpy array. Got {type(result)}.')
+
+        def inner_without_validate(sample_id, preprocess_response):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+            try:
+                result = user_function(sample_id, preprocess_response)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+            return result
+
+        leap_binder.set_false_positive_empty_gt(inner_without_validate, gt_name)
+
+        def inner(sample_id, preprocess_response):
+            if os.environ.get(mapping_runtime_mode_env_var_mame):
+                return np.zeros(0, dtype=np.float32)
+
+            _validate_input_args(sample_id, preprocess_response)
+            result = inner_without_validate(sample_id, preprocess_response)
+            _validate_result(result)
+            return result
+
+        return inner
+
+    return decorating_function
+
+
+def tensorleap_false_positive_custom_latent_space(name: str):
+    """Compute the instance custom latent space `name` (a tensorleap_instance_custom_latent_space)
+    for a false-positive instance. Signature: fn(sample_id, preprocess, fp_instance) -> 1-D np.ndarray."""
+    def decorating_function(user_function: Callable):
+        def _validate_input_args(sample_id, preprocess_response, fp_instance):
+            assert isinstance(sample_id, (str, int)), \
+                (f'tensorleap_false_positive_custom_latent_space validation failed: '
+                 f'Argument sample_id should be str or int. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_false_positive_custom_latent_space validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
+            assert isinstance(fp_instance, ElementInstance), \
+                (f'tensorleap_false_positive_custom_latent_space validation failed: '
+                 f'Argument fp_instance should be an ElementInstance. Got {type(fp_instance)}.')
+
+        def _validate_result(result):
+            assert isinstance(result, np.ndarray), \
+                (f'tensorleap_false_positive_custom_latent_space validation failed: '
+                 f'The return type should be a numpy array. Got {type(result)}.')
+            assert result.ndim == 1, \
+                (f'tensorleap_false_positive_custom_latent_space validation failed: '
+                 f'The returned array should be 1-D. Got shape {result.shape}.')
+
+        def inner_without_validate(sample_id, preprocess_response, fp_instance):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+            try:
+                result = user_function(sample_id, preprocess_response, fp_instance)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+            return result
+
+        leap_binder.set_false_positive_custom_latent_space(inner_without_validate, name)
+
+        def inner(sample_id, preprocess_response, fp_instance):
+            if os.environ.get(mapping_runtime_mode_env_var_mame):
+                return np.zeros(0, dtype=np.float32)
+
+            _validate_input_args(sample_id, preprocess_response, fp_instance)
+            result = inner_without_validate(sample_id, preprocess_response, fp_instance)
+            _validate_result(result)
+            return result
+
+        return inner
+
+    return decorating_function
+
+
 def tensorleap_instances_length_encoder(name: str):
     def decorating_function(user_function: InstanceLengthCallableInterface):
         def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse):

{code_loader-1.0.212 → code_loader-1.0.214}/code_loader/leaploader.py

@@ -434,6 +434,9 @@ class LeapLoader(LeapLoaderBase):
                 autoregressive_test_payload = self._check_autoregressive_step()
                 if autoregressive_test_payload is not None:
                     test_payloads.append(autoregressive_test_payload)
+                false_positive_test_payload = self._check_false_positive_empty_gt()
+                if false_positive_test_payload is not None:
+                    test_payloads.append(false_positive_test_payload)
                 is_valid = all([payload.is_passed for payload in test_payloads])
                 setup_response = self.get_dataset_setup_response(handlers_test_payloads)
 
@@ -642,6 +645,24 @@ class LeapLoader(LeapLoaderBase):
             result_payloads.append(test_result)
         return result_payloads
 
+    def _check_false_positive_empty_gt(self) -> Optional[DatasetTestResultPayload]:
+        """Push-time check that every GT encoder has a matching tensorleap_false_positive_empty_gt
+        of the right shape. None (no payload) without a false-positive encoder."""
+        handler = global_leap_binder.setup_container.false_positive_instances
+        if handler is None:
+            return None
+        test_result = DatasetTestResultPayload(handler.name)
+        try:
+            preprocess_result = self._preprocess_result()
+            global_leap_binder.check_false_positive_empty_gt(preprocess_result)
+            global_leap_binder.check_false_positive_custom_latent_spaces(preprocess_result)
+        except Exception as e:
+            line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
+            error_string = f"{repr(e)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
+            test_result.display[TestingSectionEnum.Errors.name] = error_string
+            test_result.is_passed = False
+        return test_result
+
     def _check_autoregressive_step(self) -> Optional[DatasetTestResultPayload]:
         handler = global_leap_binder.setup_container.autoregressive_step
         if handler is None:
@@ -1056,7 +1077,10 @@ class LeapLoader(LeapLoaderBase):
         encoder cannot be masked. Preprocessing has already run by the time this is called (the
         lengths above come from it), so reading the mappings costs nothing extra.
         """
+        self.exec_script()
         setup = global_leap_binder.setup_container
+        if setup.false_positive_instances is not None:
+            return True
         if not setup.instance_masks:
             return False
         try:
@@ -1768,6 +1792,110 @@ class LeapLoader(LeapLoaderBase):
         return [prediction_type.name for prediction_type in
                 global_leap_binder.setup_container.prediction_types]
 
+    def has_false_positive_instances(self) -> bool:
+        self.exec_script()
+        return global_leap_binder.setup_container.false_positive_instances is not None
+
+    def _native_sample_id(self, sample_id: Union[str, int]) -> Union[str, int]:
+        # numpy ints / stringified ids from the engine -> the type the preprocess declared.
+        return self.get_sample_id_type()(sample_id)
+
+    def get_false_positive_instances(self, state: DataStateEnum, sample_ids: List[Union[str, int]],
+                                     predictions: Dict[str, npt.NDArray[np.float32]]
+                                     ) -> List[Optional[List[ElementInstance]]]:
+        """Run the false-positive encoder once per sample. `predictions` arrays carry a leading
+        batch axis of len(sample_ids); row i is handed to the user function for sample_ids[i].
+
+        Returns one entry per sample id, in order: the sample's list of ElementInstance (`[]` when
+        it has no false positives), or None when the user function raised for that sample or
+        returned something that is not a list of ElementInstance. A failed sample does not fail
+        the others; its error is printed with the sample id. A batch-size mismatch between
+        `predictions` and `sample_ids`, or a missing encoder, still raises: those are caller errors.
+        """
+        self.exec_script()
+        handler = global_leap_binder.setup_container.false_positive_instances
+        if handler is None:
+            raise Exception("No tensorleap_false_positive_instances_encoder is registered.")
+        preprocess_response = self._preprocess_result()[state]
+        for name, array in predictions.items():
+            if len(array) != len(sample_ids):
+                raise Exception(
+                    f"predictions['{name}'] has batch size {len(array)} but {len(sample_ids)} sample ids were given.")
+
+        results: List[Optional[List[ElementInstance]]] = []
+        for i, sample_id in enumerate(sample_ids):
+            row = {name: array[i] for name, array in predictions.items()}
+            try:
+                instances = handler.function(self._native_sample_id(sample_id), preprocess_response, row)
+                self._validate_false_positive_instances(instances, set(self._instance_metric_handler_by_name()))
+            except Exception as e:
+                print(f"tensorleap_false_positive_instances_encoder failed for sample {sample_id!r} "
+                      f"(state {state.name}): {e!r}")
+                results.append(None)
+                continue
+            results.append(instances)
+        return results
+
+    @staticmethod
+    def _validate_false_positive_instances(instances: Any, registered_instance_metrics: Set[str]) -> None:
+        prefix = "tensorleap_false_positive_instances_encoder validation failed: "
+        if not isinstance(instances, list):
+            raise Exception(f"{prefix}The return type should be a list of ElementInstance. "
+                            f"Got {type(instances)}.")
+        for j, instance in enumerate(instances):
+            if not isinstance(instance, ElementInstance):
+                raise Exception(f"{prefix}Element {j} of the returned list should be an ElementInstance. "
+                                f"Got {type(instance)}.")
+            if instance.instance_metrics is None:
+                continue
+            if not isinstance(instance.instance_metrics, dict):
+                raise Exception(f"{prefix}instance_metrics of element {j} should be a dict. "
+                                f"Got {type(instance.instance_metrics)}.")
+            for key, value in instance.instance_metrics.items():
+                if key not in registered_instance_metrics:
+                    raise Exception(f"{prefix}instance_metrics key {key!r} of element {j} is not the name of a "
+                                    f"registered tensorleap_custom_instances_metric. "
+                                    f"Registered: {sorted(registered_instance_metrics)}.")
+                try:
+                    instance.instance_metrics[key] = float(value)
+                except (TypeError, ValueError):
+                    raise Exception(f"{prefix}instance_metrics[{key!r}] of element {j} should be castable to "
+                                    f"float. Got {type(value)}.")
+
+    def get_false_positive_empty_gt(self, state: DataStateEnum,
+                                    sample_id: Union[str, int]) -> Dict[str, npt.NDArray[np.float32]]:
+        self.exec_script()
+        preprocess_response = self._preprocess_result()[state]
+        sample_id = self._native_sample_id(sample_id)
+        empty_gt: Dict[str, npt.NDArray[np.float32]] = {}
+        for gt_name, function in global_leap_binder.setup_container.false_positive_empty_gt.items():
+            value = function(sample_id, preprocess_response)
+            if not isinstance(value, np.ndarray):
+                raise Exception(f"tensorleap_false_positive_empty_gt validation failed: "
+                                f"The return type for '{gt_name}' should be a numpy array. Got {type(value)}.")
+            empty_gt[gt_name] = value.astype(np.float32, copy=False)
+        return empty_gt
+
+    def get_false_positive_custom_latent_spaces(self, state: DataStateEnum, sample_id: Union[str, int],
+                                                fp_instance: ElementInstance) -> Dict[str, npt.NDArray[np.float32]]:
+        self.exec_script()
+        hooks = global_leap_binder.setup_container.false_positive_custom_latent_spaces
+        if not hooks:
+            return {}
+        preprocess_response = self._preprocess_result()[state]
+        sample_id = self._native_sample_id(sample_id)
+        vectors: Dict[str, npt.NDArray[np.float32]] = {}
+        for name, function in hooks.items():
+            value = function(sample_id, preprocess_response, fp_instance)
+            if not isinstance(value, np.ndarray):
+                raise Exception(f"tensorleap_false_positive_custom_latent_space validation failed: "
+                                f"The return type for '{name}' should be a numpy array. Got {type(value)}.")
+            if value.ndim != 1:
+                raise Exception(f"tensorleap_false_positive_custom_latent_space validation failed: "
+                                f"The return value for '{name}' should be a 1-D array. Got shape {value.shape}.")
+            vectors[name] = value.astype(np.float32, copy=False)
+        return vectors
+
     def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str]]:
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result.get(state)

{code_loader-1.0.212 → code_loader-1.0.214}/code_loader/leaploaderbase.py

@@ -9,7 +9,7 @@ import numpy.typing as npt
 
 from code_loader.contract.datasetclasses import DatasetSample, LeapData, \
     PredictionTypeHandler, CustomLayerHandler, VisualizerHandlerData, MetricHandlerData, MetricCallableReturnType, \
-    CustomLossHandlerData
+    CustomLossHandlerData, ElementInstance
 from code_loader.contract.enums import DataStateEnum, DataStateType
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetSetup, ModelSetup
@@ -72,6 +72,25 @@ class LeapLoaderBase:
     def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str]]:
         pass
 
+    # Not abstract: a loader that predates false-positive instances (or a stand-in) has none.
+    def has_false_positive_instances(self) -> bool:
+        return False
+
+    def get_false_positive_instances(self, state: DataStateEnum, sample_ids: List[Union[str, int]],
+                                     predictions: Dict[str, npt.NDArray[np.float32]]
+                                     ) -> List[Optional[List[ElementInstance]]]:
+        """One entry per sample id: its false-positive instances, or None if the encoder failed
+        for that sample (see LeapLoader.get_false_positive_instances)."""
+        raise NotImplementedError("get_false_positive_instances is not implemented by this loader")
+
+    def get_false_positive_empty_gt(self, state: DataStateEnum,
+                                    sample_id: Union[str, int]) -> Dict[str, npt.NDArray[np.float32]]:
+        raise NotImplementedError("get_false_positive_empty_gt is not implemented by this loader")
+
+    def get_false_positive_custom_latent_spaces(self, state: DataStateEnum, sample_id: Union[str, int],
+                                                fp_instance: ElementInstance) -> Dict[str, npt.NDArray[np.float32]]:
+        return {}
+
     def get_metadata_multiple_samples(self, state: DataStateEnum, sample_ids: Union[List[int], List[str]],
                                       requested_metadata_names: Optional[List[str]] = None
                                       ) -> Tuple[Dict[str, Union[List[str], List[int], List[bool],

{code_loader-1.0.212 → code_loader-1.0.214}/code_loader/utils.py

@@ -53,6 +53,22 @@ def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -
     return numpy_encoder_function
 
 
+def to_numpy_return_fp_masks_wrapper(encoder_function: Callable) -> Callable:
+    def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse, predictions: Dict[str, Any]) -> List[ElementInstance]:
+        result = encoder_function(idx, samples, predictions)
+        if not isinstance(result, list):
+            return result  # malformed: the loader reports it with a clear message
+        for instance in result:
+            if not isinstance(instance, ElementInstance):
+                continue
+            instance.mask = np.array(instance.mask)
+            if instance.instance_metadata is not None and instance.instance_metadata_is_none is None:
+                instance.instance_metadata_is_none = {
+                    k: is_absent_metadata_value(v) for k, v in instance.instance_metadata.items()}
+        return result
+    return numpy_encoder_function
+
+
 # The engine downcasts every tensor to float16 before pushing it to Redis, and metrics and losses
 # read their tensors back from there — so on the platform they are handed float16 arrays. A local
 # run feeds them the float32 arrays straight from the encoders and the model, which is why a

{code_loader-1.0.212 → code_loader-1.0.214}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.212"
+version = "1.0.214"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"