code-loader 1.0.174.dev4__tar.gz → 1.0.174.dev5__tar.gz

{code_loader-1.0.174.dev4 → code_loader-1.0.174.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.174.dev4
+Version: 1.0.174.dev5
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.174.dev4 → code_loader-1.0.174.dev5}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1040,7 +1040,7 @@ def tensorleap_custom_instances_metric(name: str,
         leap_binder.add_custom_instance_metric(inner_without_validate, name, direction, compute_insights)
 
         if connects_to is not None:
-            arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
+            arg_names = leap_binder.setup_container.instance_metrics[-1].metric_handler_data.arg_names
             _add_mapping_connections(connects_to, arg_names, NodeMappingType.Metric, name)
 
         def inner(*args, **kwargs):
@@ -1072,7 +1072,7 @@ def tensorleap_custom_instances_metric(name: str,
 
             return None
 
-        mapping_inner.arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
+        mapping_inner.arg_names = leap_binder.setup_container.instance_metrics[-1].metric_handler_data.arg_names
         mapping_inner.name = name
 
         def final_inner(*args, **kwargs):

{code_loader-1.0.174.dev4 → code_loader-1.0.174.dev5}/code_loader/leaploader.py

@@ -16,7 +16,7 @@ from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandle
     PreprocessResponse, VisualizerHandler, LeapData, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
     MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse, \
-    ElementInstance, custom_latent_space_attribute, DatasetIntegrationSetup
+    ElementInstance, custom_latent_space_attribute, DatasetIntegrationSetup, InstanceMetricHandler
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
@@ -109,6 +109,24 @@ class LeapLoader(LeapLoaderBase):
             for metric_handler in setup.metrics
         }
 
+    @lru_cache()
+    def instance_metric_by_name(self) -> Dict[str, MetricHandlerData]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            handler.metric_handler_data.name: handler.metric_handler_data
+            for handler in setup.instance_metrics
+        }
+
+    @lru_cache()
+    def _instance_metric_handler_by_name(self) -> Dict[str, InstanceMetricHandler]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            handler.metric_handler_data.name: handler
+            for handler in setup.instance_metrics
+        }
+
     @lru_cache()
     def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
         self.exec_script()
@@ -292,6 +310,20 @@ class LeapLoader(LeapLoaderBase):
 
         return metric_handler.function(**input_tensors_by_arg_name)
 
+    def run_instance_metric(self, metric_name: str, sample_ids: np.array, state: DataStateEnum,
+                            input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+                            ) -> Dict[int, npt.NDArray[np.float32]]:
+        self._preprocess_result()
+
+        handler = self._instance_metric_handler_by_name()[metric_name]
+        preprocess_response_arg_name = self._get_preprocess_response_arg_name(handler.function)
+
+        if preprocess_response_arg_name is not None:
+            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(
+                sample_ids, self._preprocess_result()[state])
+
+        return handler.function(**input_tensors_by_arg_name)
+
     @staticmethod
     def _get_preprocess_response_arg_name(
             func: Callable) -> Optional[str]:

{code_loader-1.0.174.dev4 → code_loader-1.0.174.dev5}/code_loader/leaploaderbase.py

@@ -44,6 +44,10 @@ class LeapLoaderBase:
     def metric_by_name(self) -> Dict[str, MetricHandlerData]:
         pass
 
+    @abstractmethod
+    def instance_metric_by_name(self) -> Dict[str, MetricHandlerData]:
+        pass
+
     @abstractmethod
     def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
         pass
@@ -100,6 +104,12 @@ class LeapLoaderBase:
                    input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
         pass
 
+    @abstractmethod
+    def run_instance_metric(self, metric_name: str, sample_ids: np.array, state: DataStateEnum,
+                            input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+                            ) -> Dict[int, npt.NDArray[np.float32]]:
+        pass
+
     @abstractmethod
     def run_custom_loss(self, custom_loss_name: str, sample_ids: np.array, state: DataStateEnum,
                         input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):

{code_loader-1.0.174.dev4 → code_loader-1.0.174.dev5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.174.dev4"
+version = "1.0.174.dev5"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"