code-loader 1.0.174.dev3__tar.gz → 1.0.174.dev5__tar.gz

{code_loader-1.0.174.dev3 → code_loader-1.0.174.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.174.dev3
+Version: 1.0.174.dev5
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.174.dev3 → code_loader-1.0.174.dev5}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -879,7 +879,8 @@ def tensorleap_custom_metric(name: str,
 
 def tensorleap_custom_instances_metric(name: str,
                              direction: Union[MetricDirection, Dict[str, MetricDirection]] = _UNSET,
-                             compute_insights: Optional[Union[bool, Dict[str, bool]]] = None):
+                             compute_insights: Optional[Union[bool, Dict[str, bool]]] = None,
+                             connects_to=None):
     name_to_unique_name = defaultdict(set)
 
     def decorating_function(
@@ -942,21 +943,21 @@ def tensorleap_custom_instances_metric(name: str,
                                 err_message +
                                 f"Invalid type for compute_insights['{k}']: expected bool, got type {type(v).__name__}."
                             )
-            # if connects_to is not None:
-            #     valid_types = (str, list, tuple, set)
-            #     if not isinstance(connects_to, valid_types):
-            #         raise TypeError(
-            #             err_message +
-            #             f"`connects_to` must be one of {valid_types}, got type {type(connects_to).__name__}."
-            #         )
-            #     if isinstance(connects_to, (list, tuple, set)):
-            #         invalid_elems = [f"{type(e).__name__}" for e in connects_to if not isinstance(e, str)]
-            #         if invalid_elems:
-            #             raise TypeError(
-            #                 err_message +
-            #                 f"All elements in `connects_to` must be strings, "
-            #                 f"but found element types: {invalid_elems}."
-            #             )
+            if connects_to is not None:
+                valid_types = (str, list, tuple, set)
+                if not isinstance(connects_to, valid_types):
+                    raise TypeError(
+                        err_message +
+                        f"`connects_to` must be one of {valid_types}, got type {type(connects_to).__name__}."
+                    )
+                if isinstance(connects_to, (list, tuple, set)):
+                    invalid_elems = [f"{type(e).__name__}" for e in connects_to if not isinstance(e, str)]
+                    if invalid_elems:
+                        raise TypeError(
+                            err_message +
+                            f"All elements in `connects_to` must be strings, "
+                            f"but found element types: {invalid_elems}."
+                        )
 
         _validate_decorators_signature()
 
@@ -1038,9 +1039,9 @@ def tensorleap_custom_instances_metric(name: str,
 
         leap_binder.add_custom_instance_metric(inner_without_validate, name, direction, compute_insights)
 
-        # if connects_to is not None:
-        #     arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
-        #     _add_mapping_connections(connects_to, arg_names, NodeMappingType.Metric, name)
+        if connects_to is not None:
+            arg_names = leap_binder.setup_container.instance_metrics[-1].metric_handler_data.arg_names
+            _add_mapping_connections(connects_to, arg_names, NodeMappingType.Metric, name)
 
         def inner(*args, **kwargs):
             if not _call_from_tl_platform:
@@ -1071,7 +1072,7 @@ def tensorleap_custom_instances_metric(name: str,
 
             return None
 
-        mapping_inner.arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
+        mapping_inner.arg_names = leap_binder.setup_container.instance_metrics[-1].metric_handler_data.arg_names
         mapping_inner.name = name
 
         def final_inner(*args, **kwargs):

{code_loader-1.0.174.dev3 → code_loader-1.0.174.dev5}/code_loader/leaploader.py

@@ -16,7 +16,7 @@ from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandle
     PreprocessResponse, VisualizerHandler, LeapData, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
     MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse, \
-    ElementInstance, custom_latent_space_attribute, DatasetIntegrationSetup
+    ElementInstance, custom_latent_space_attribute, DatasetIntegrationSetup, InstanceMetricHandler
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
@@ -109,6 +109,24 @@ class LeapLoader(LeapLoaderBase):
             for metric_handler in setup.metrics
         }
 
+    @lru_cache()
+    def instance_metric_by_name(self) -> Dict[str, MetricHandlerData]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            handler.metric_handler_data.name: handler.metric_handler_data
+            for handler in setup.instance_metrics
+        }
+
+    @lru_cache()
+    def _instance_metric_handler_by_name(self) -> Dict[str, InstanceMetricHandler]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            handler.metric_handler_data.name: handler
+            for handler in setup.instance_metrics
+        }
+
     @lru_cache()
     def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
         self.exec_script()
@@ -292,6 +310,20 @@ class LeapLoader(LeapLoaderBase):
 
         return metric_handler.function(**input_tensors_by_arg_name)
 
+    def run_instance_metric(self, metric_name: str, sample_ids: np.array, state: DataStateEnum,
+                            input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+                            ) -> Dict[int, npt.NDArray[np.float32]]:
+        self._preprocess_result()
+
+        handler = self._instance_metric_handler_by_name()[metric_name]
+        preprocess_response_arg_name = self._get_preprocess_response_arg_name(handler.function)
+
+        if preprocess_response_arg_name is not None:
+            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(
+                sample_ids, self._preprocess_result()[state])
+
+        return handler.function(**input_tensors_by_arg_name)
+
     @staticmethod
     def _get_preprocess_response_arg_name(
             func: Callable) -> Optional[str]:

{code_loader-1.0.174.dev3 → code_loader-1.0.174.dev5}/code_loader/leaploaderbase.py

@@ -44,6 +44,10 @@ class LeapLoaderBase:
     def metric_by_name(self) -> Dict[str, MetricHandlerData]:
         pass
 
+    @abstractmethod
+    def instance_metric_by_name(self) -> Dict[str, MetricHandlerData]:
+        pass
+
     @abstractmethod
     def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
         pass
@@ -100,6 +104,12 @@ class LeapLoaderBase:
                    input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
         pass
 
+    @abstractmethod
+    def run_instance_metric(self, metric_name: str, sample_ids: np.array, state: DataStateEnum,
+                            input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+                            ) -> Dict[int, npt.NDArray[np.float32]]:
+        pass
+
     @abstractmethod
     def run_custom_loss(self, custom_loss_name: str, sample_ids: np.array, state: DataStateEnum,
                         input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):

{code_loader-1.0.174.dev3 → code_loader-1.0.174.dev5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.174.dev3"
+version = "1.0.174.dev5"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"