code-loader 1.0.175__tar.gz → 1.0.176__tar.gz

{code_loader-1.0.175 → code_loader-1.0.176}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.175
+Version: 1.0.176
 Summary: 
+Home-page: https://github.com/tensorleap/code-loader
 License: MIT
 Author: dorhar
 Author-email: doron.harnoy@tensorleap.ai
@@ -19,7 +20,6 @@ Requires-Dist: numpy (>=2.3.2,<3.0.0) ; python_version >= "3.11" and python_vers
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
-Project-URL: Homepage, https://github.com/tensorleap/code-loader
 Project-URL: Repository, https://github.com/tensorleap/code-loader
 Description-Content-Type: text/markdown
 

{code_loader-1.0.175 → code_loader-1.0.176}/code_loader/contract/datasetclasses.py

@@ -171,6 +171,12 @@ class MetricHandler:
     function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]
 
 
+@dataclass
+class InstanceMetricHandler:
+    metric_handler_data: MetricHandlerData
+    function: CustomMultipleReturnCallableInterfaceMultiArgs
+
+
 @dataclass
 class RawInputsForHeatmap:
     raw_input_by_vizualizer_arg_name: Dict[str, npt.NDArray[np.float32]]
@@ -258,6 +264,7 @@ class DatasetIntegrationSetup:
     prediction_types: List[PredictionTypeHandler] = field(default_factory=list)
     custom_loss_handlers: List[CustomLossHandler] = field(default_factory=list)
     metrics: List[MetricHandler] = field(default_factory=list)
+    instance_metrics: List[InstanceMetricHandler] = field(default_factory=list)
     custom_layers: Dict[str, CustomLayerHandler] = field(default_factory=dict)
     custom_latent_space: Optional[CustomLatentSpaceHandler] = None
 

{code_loader-1.0.175 → code_loader-1.0.176}/code_loader/inner_leap_binder/leapbinder.py

@@ -12,7 +12,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
-    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler
+    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler, InstanceMetricHandler
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload, LeapAnalysisConfiguration
@@ -335,6 +335,47 @@ class LeapBinder:
         metric_handler_data = MetricHandlerData(name, regular_arg_names, direction, compute_insights)
         self.setup_container.metrics.append(MetricHandler(metric_handler_data, function))
 
+    def add_custom_instance_metric(self,
+                          function: CustomMultipleReturnCallableInterfaceMultiArgs,
+                          name: str,
+                          direction: Optional[
+                              Union[MetricDirection, Dict[str, MetricDirection]]] = MetricDirection.Downward,
+                          compute_insights: Optional[Union[bool, Dict[str, bool]]] = None) -> None:
+        """
+        Add a custom metric to the setup.
+
+        Args:
+        function (CustomMultipleReturnCallableInterfaceMultiArgs): The custom metric function returning Dict[int, np.ndarray].
+        name (str): The name of the custom metric.
+        direction (Optional[Union[MetricDirection, Dict[str, MetricDirection]]]): The direction of the metric, either
+        MetricDirection.Upward or MetricDirection.Downward, in case custom metric return a dictionary of metrics we can
+        supply a dictionary of directions correspondingly.
+            - MetricDirection.Upward: Indicates that higher values of the metric are better and should be maximized.
+            - MetricDirection.Downward: Indicates that lower values of the metric are better and should be minimized.
+        compute_insights (Union[bool, Dict[str, bool]]): Whether to compute insights or not. in case custom metric
+        return a dictionary of metrics we can supply a dictionary of values correspondingly
+
+
+
+        Example:
+            def custom_metric_function(y_true, y_pred):
+                return np.mean(np.abs(y_true - y_pred))
+
+            leap_binder.add_custom_metric(custom_metric_function, name='custom_metric', direction=MetricDirection.Downward)
+        """
+
+        regular_arg_names = inspect.getfullargspec(function)[0]
+        preprocess_response_arg_name = None
+        for arg_name, arg_type in inspect.getfullargspec(function).annotations.items():
+            if arg_type == SamplePreprocessResponse:
+                if preprocess_response_arg_name is not None:
+                    raise Exception("only one argument can be of type SamplePreprocessResponse")
+                preprocess_response_arg_name = arg_name
+                regular_arg_names.remove(arg_name)
+
+        metric_handler_data = MetricHandlerData(name, regular_arg_names, direction, compute_insights)
+        self.setup_container.instance_metrics.append(InstanceMetricHandler(metric_handler_data, function))
+
     def add_prediction(self, name: str, labels: List[str], channel_dim: int = -1, prediction_index: Optional[int]=None) -> None:
         """
         Add prediction labels to the setup.

{code_loader-1.0.175 → code_loader-1.0.176}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -877,6 +877,215 @@ def tensorleap_custom_metric(name: str,
     return decorating_function
 
 
+def tensorleap_custom_instances_metric(name: str,
+                             direction: Union[MetricDirection, Dict[str, MetricDirection]] = _UNSET,
+                             compute_insights: Optional[Union[bool, Dict[str, bool]]] = None,
+                             connects_to=None):
+    name_to_unique_name = defaultdict(set)
+
+    def decorating_function(
+            user_function: CustomMultipleReturnCallableInterfaceMultiArgs):
+        nonlocal direction
+
+        direction_was_provided = direction is not _UNSET
+
+        def _validate_decorators_signature():
+            err_message = f"{user_function.__name__} validation failed.\n"
+            if not isinstance(name, str):
+                raise TypeError(err_message + f"`name` must be a string, got type {type(name).__name__}.")
+            valid_directions = {MetricDirection.Upward, MetricDirection.Downward}
+            if direction is _UNSET:
+                pass
+            elif isinstance(direction, MetricDirection):
+                if direction not in valid_directions:
+                    raise ValueError(
+                        err_message +
+                        f"Invalid MetricDirection: {direction}. Must be one of {valid_directions}, "
+                        f"got type {type(direction).__name__}."
+                    )
+            elif isinstance(direction, dict):
+                if not all(isinstance(k, str) for k in direction.keys()):
+                    invalid_keys = {k: type(k).__name__ for k in direction.keys() if not isinstance(k, str)}
+                    raise TypeError(
+                        err_message +
+                        f"All keys in `direction` must be strings, got invalid key types: {invalid_keys}."
+                    )
+                for k, v in direction.items():
+                    if v not in valid_directions:
+                        raise ValueError(
+                            err_message +
+                            f"Invalid direction for key '{k}': {v}. Must be one of {valid_directions}, "
+                            f"got type {type(v).__name__}."
+                        )
+            else:
+                raise TypeError(
+                    err_message +
+                    f"`direction` must be a MetricDirection or a Dict[str, MetricDirection], "
+                    f"got type {type(direction).__name__}."
+                )
+            if compute_insights is not None:
+                if not isinstance(compute_insights, (bool, dict)):
+                    raise TypeError(
+                        err_message +
+                        f"`compute_insights` must be a bool or a Dict[str, bool], "
+                        f"got type {type(compute_insights).__name__}."
+                    )
+                if isinstance(compute_insights, dict):
+                    if not all(isinstance(k, str) for k in compute_insights.keys()):
+                        invalid_keys = {k: type(k).__name__ for k in compute_insights.keys() if not isinstance(k, str)}
+                        raise TypeError(
+                            err_message +
+                            f"All keys in `compute_insights` must be strings, got invalid key types: {invalid_keys}."
+                        )
+                    for k, v in compute_insights.items():
+                        if not isinstance(v, bool):
+                            raise TypeError(
+                                err_message +
+                                f"Invalid type for compute_insights['{k}']: expected bool, got type {type(v).__name__}."
+                            )
+            if connects_to is not None:
+                valid_types = (str, list, tuple, set)
+                if not isinstance(connects_to, valid_types):
+                    raise TypeError(
+                        err_message +
+                        f"`connects_to` must be one of {valid_types}, got type {type(connects_to).__name__}."
+                    )
+                if isinstance(connects_to, (list, tuple, set)):
+                    invalid_elems = [f"{type(e).__name__}" for e in connects_to if not isinstance(e, str)]
+                    if invalid_elems:
+                        raise TypeError(
+                            err_message +
+                            f"All elements in `connects_to` must be strings, "
+                            f"but found element types: {invalid_elems}."
+                        )
+
+        _validate_decorators_signature()
+
+        for metric_handler in leap_binder.setup_container.instance_metrics:
+            if metric_handler.metric_handler_data.name == name:
+                raise Exception(f'Metric with name {name} already exists. '
+                                f'Please choose another')
+
+        def _validate_input_args(*args, **kwargs) -> None:
+            assert len(args) + len(kwargs) > 0, (
+                f"{user_function.__name__}() validation failed: "
+                f"Expected at least one positional|key-word argument of type np.ndarray, "
+                f"but received none. "
+                f"Correct usage example: tensorleap_custom_metric(input_array: np.ndarray, ...)"
+            )
+            for i, arg in enumerate(args):
+                assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
+                    f'{user_function.__name__}() validation failed: '
+                    f'Argument #{i} should be a numpy array. Got {type(arg)}.')
+                if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
+                    assert arg.shape[0] == leap_binder.batch_size_to_validate, \
+                        (f'{user_function.__name__}() validation failed: Argument #{i} '
+                         f'first dim should be as the batch size. Got {arg.shape[0]} '
+                         f'instead of {leap_binder.batch_size_to_validate}')
+
+            for _arg_name, arg in kwargs.items():
+                assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
+                    f'{user_function.__name__}() validation failed: '
+                    f'Argument {_arg_name} should be a numpy array. Got {type(arg)}.')
+                if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
+                    assert arg.shape[0] == leap_binder.batch_size_to_validate, \
+                        (f'{user_function.__name__}() validation failed: Argument {_arg_name} '
+                         f'first dim should be as the batch size. Got {arg.shape[0]} '
+                         f'instead of {leap_binder.batch_size_to_validate}')
+
+        def _validate_result(result) -> None:
+            nonlocal direction
+            supported_types_message = (f'{user_function.__name__}() validation failed: '
+                                       f'{user_function.__name__}() has returned unsupported type.\nSupported type is Dict[List[NDArray[np.float32]]]')
+
+            def _validate_single_metric(single_metric_result, key=None):
+                assert isinstance(single_metric_result,
+                                  np.ndarray), f'{supported_types_message}\nGot {type(single_metric_result)}.'
+                assert len(single_metric_result.shape) == 1, (f'{user_function.__name__}() validation failed: '
+                                                              f'The return shape should be 1D. Got {len(single_metric_result.shape)}D.')
+
+                if leap_binder.batch_size_to_validate:
+                    assert len(single_metric_result) == leap_binder.batch_size_to_validate, \
+                        f'{user_function.__name__}() validation failed: The return len {f"of srt{key} value" if key is not None else ""} should be as the batch size.'
+
+            assert isinstance(result, dict), f'{supported_types_message}\nGot {type(result)}.'
+
+            result_keys = set(result.keys())
+            for key, value in result.items():
+                _validate_single_metric(value, key)
+
+                assert isinstance(key, int), \
+                    (f'{user_function.__name__}() validation failed: '
+                     f'Keys in the return dict should be of type int (instance number). Got {type(key)}.')
+
+            leap_binder.setup_container.instance_metrics[-1].metric_handler_data.direction = direction
+
+        @functools.wraps(user_function)
+        def inner_without_validate(*args, **kwargs):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+
+            try:
+                result = user_function(*args, **kwargs)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+
+            return result
+
+        try:
+            inner_without_validate.__signature__ = inspect.signature(user_function)
+        except (TypeError, ValueError):
+            pass
+
+        leap_binder.add_custom_instance_metric(inner_without_validate, name, direction, compute_insights)
+
+        if connects_to is not None:
+            arg_names = leap_binder.setup_container.instance_metrics[-1].metric_handler_data.arg_names
+            _add_mapping_connections(connects_to, arg_names, NodeMappingType.Metric, name)
+
+        def inner(*args, **kwargs):
+            if not _call_from_tl_platform:
+                set_current('tensorleap_custom_instances_metric')
+            _validate_input_args(*args, **kwargs)
+
+            result = inner_without_validate(*args, **kwargs)
+
+            _validate_result(result)
+            if not _call_from_tl_platform:
+                update_env_params_func("tensorleap_custom_instances_metric", "v")
+            return result
+
+        def mapping_inner(*args, **kwargs):
+            user_unique_name = mapping_inner.name
+            if 'user_unique_name' in kwargs:
+                user_unique_name = kwargs['user_unique_name']
+
+            ordered_connections = [kwargs[n] for n in mapping_inner.arg_names if n in kwargs]
+            ordered_connections = list(args) + ordered_connections
+
+            if user_unique_name in name_to_unique_name[mapping_inner.name]:
+                user_unique_name = f'{user_unique_name}_{len(name_to_unique_name[mapping_inner.name])}'
+            name_to_unique_name[mapping_inner.name].add(user_unique_name)
+
+            _add_mapping_connection(user_unique_name, ordered_connections, mapping_inner.arg_names,
+                                    mapping_inner.name, NodeMappingType.Metric)
+
+            return None
+
+        mapping_inner.arg_names = leap_binder.setup_container.instance_metrics[-1].metric_handler_data.arg_names
+        mapping_inner.name = name
+
+        def final_inner(*args, **kwargs):
+            if os.environ.get(mapping_runtime_mode_env_var_mame):
+                return mapping_inner(*args, **kwargs)
+            else:
+                return inner(*args, **kwargs)
+
+        return final_inner
+
+    return decorating_function
+
+
 def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
                                  heatmap_function: Optional[Callable[..., npt.NDArray[np.float32]]] = None,
                                  connects_to=None):

{code_loader-1.0.175 → code_loader-1.0.176}/code_loader/leaploader.py

@@ -16,7 +16,7 @@ from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandle
     PreprocessResponse, VisualizerHandler, LeapData, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
     MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse, \
-    ElementInstance, custom_latent_space_attribute, DatasetIntegrationSetup
+    ElementInstance, custom_latent_space_attribute, DatasetIntegrationSetup, InstanceMetricHandler
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
@@ -109,6 +109,24 @@ class LeapLoader(LeapLoaderBase):
             for metric_handler in setup.metrics
         }
 
+    @lru_cache()
+    def instance_metric_by_name(self) -> Dict[str, MetricHandlerData]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            handler.metric_handler_data.name: handler.metric_handler_data
+            for handler in setup.instance_metrics
+        }
+
+    @lru_cache()
+    def _instance_metric_handler_by_name(self) -> Dict[str, InstanceMetricHandler]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            handler.metric_handler_data.name: handler
+            for handler in setup.instance_metrics
+        }
+
     @lru_cache()
     def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
         self.exec_script()
@@ -293,6 +311,20 @@ class LeapLoader(LeapLoaderBase):
 
         return metric_handler.function(**input_tensors_by_arg_name)
 
+    def run_instance_metric(self, metric_name: str, sample_ids: np.array, state: DataStateEnum,
+                            input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+                            ) -> Dict[int, npt.NDArray[np.float32]]:
+        self._preprocess_result()
+
+        handler = self._instance_metric_handler_by_name()[metric_name]
+        preprocess_response_arg_name = self._get_preprocess_response_arg_name(handler.function)
+
+        if preprocess_response_arg_name is not None:
+            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(
+                sample_ids, self._preprocess_result()[state])
+
+        return handler.function(**input_tensors_by_arg_name)
+
     @staticmethod
     def _get_preprocess_response_arg_name(
             func: Callable) -> Optional[str]:

{code_loader-1.0.175 → code_loader-1.0.176}/code_loader/leaploaderbase.py

@@ -44,6 +44,10 @@ class LeapLoaderBase:
     def metric_by_name(self) -> Dict[str, MetricHandlerData]:
         pass
 
+    @abstractmethod
+    def instance_metric_by_name(self) -> Dict[str, MetricHandlerData]:
+        pass
+
     @abstractmethod
     def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
         pass
@@ -100,6 +104,12 @@ class LeapLoaderBase:
                    input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
         pass
 
+    @abstractmethod
+    def run_instance_metric(self, metric_name: str, sample_ids: np.array, state: DataStateEnum,
+                            input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+                            ) -> Dict[int, npt.NDArray[np.float32]]:
+        pass
+
     @abstractmethod
     def run_custom_loss(self, custom_loss_name: str, sample_ids: np.array, state: DataStateEnum,
                         input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):

{code_loader-1.0.175 → code_loader-1.0.176}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.175"
+version = "1.0.176"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"