code-loader 1.0.173.dev2__tar.gz → 1.0.174.dev1__tar.gz

{code_loader-1.0.173.dev2 → code_loader-1.0.174.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.173.dev2
+Version: 1.0.174.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.173.dev2 → code_loader-1.0.174.dev1}/code_loader/contract/datasetclasses.py

@@ -171,6 +171,12 @@ class MetricHandler:
     function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]
 
 
+@dataclass
+class InstanceMetricHandler:
+    metric_handler_data: MetricHandlerData
+    function: CustomMultipleReturnCallableInterfaceMultiArgs
+
+
 @dataclass
 class RawInputsForHeatmap:
     raw_input_by_vizualizer_arg_name: Dict[str, npt.NDArray[np.float32]]
@@ -258,6 +264,7 @@ class DatasetIntegrationSetup:
     prediction_types: List[PredictionTypeHandler] = field(default_factory=list)
     custom_loss_handlers: List[CustomLossHandler] = field(default_factory=list)
     metrics: List[MetricHandler] = field(default_factory=list)
+    instance_metrics: List[MetricHandler] = field(default_factory=list)
     custom_layers: Dict[str, CustomLayerHandler] = field(default_factory=dict)
     custom_latent_space: Optional[CustomLatentSpaceHandler] = None
 

{code_loader-1.0.173.dev2 → code_loader-1.0.174.dev1}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -877,6 +877,215 @@ def tensorleap_custom_metric(name: str,
     return decorating_function
 
 
+def tensorleap_custom_instances_metric(name: str,
+                             direction: Union[MetricDirection, Dict[str, MetricDirection]] = _UNSET,
+                             compute_insights: Optional[Union[bool, Dict[str, bool]]] = None,
+                             connects_to=None):
+    name_to_unique_name = defaultdict(set)
+
+    def decorating_function(
+            user_function: CustomMultipleReturnCallableInterfaceMultiArgs):
+        nonlocal direction
+
+        direction_was_provided = direction is not _UNSET
+
+        def _validate_decorators_signature():
+            err_message = f"{user_function.__name__} validation failed.\n"
+            if not isinstance(name, str):
+                raise TypeError(err_message + f"`name` must be a string, got type {type(name).__name__}.")
+            valid_directions = {MetricDirection.Upward, MetricDirection.Downward}
+            if direction is _UNSET:
+                pass
+            elif isinstance(direction, MetricDirection):
+                if direction not in valid_directions:
+                    raise ValueError(
+                        err_message +
+                        f"Invalid MetricDirection: {direction}. Must be one of {valid_directions}, "
+                        f"got type {type(direction).__name__}."
+                    )
+            elif isinstance(direction, dict):
+                if not all(isinstance(k, str) for k in direction.keys()):
+                    invalid_keys = {k: type(k).__name__ for k in direction.keys() if not isinstance(k, str)}
+                    raise TypeError(
+                        err_message +
+                        f"All keys in `direction` must be strings, got invalid key types: {invalid_keys}."
+                    )
+                for k, v in direction.items():
+                    if v not in valid_directions:
+                        raise ValueError(
+                            err_message +
+                            f"Invalid direction for key '{k}': {v}. Must be one of {valid_directions}, "
+                            f"got type {type(v).__name__}."
+                        )
+            else:
+                raise TypeError(
+                    err_message +
+                    f"`direction` must be a MetricDirection or a Dict[str, MetricDirection], "
+                    f"got type {type(direction).__name__}."
+                )
+            if compute_insights is not None:
+                if not isinstance(compute_insights, (bool, dict)):
+                    raise TypeError(
+                        err_message +
+                        f"`compute_insights` must be a bool or a Dict[str, bool], "
+                        f"got type {type(compute_insights).__name__}."
+                    )
+                if isinstance(compute_insights, dict):
+                    if not all(isinstance(k, str) for k in compute_insights.keys()):
+                        invalid_keys = {k: type(k).__name__ for k in compute_insights.keys() if not isinstance(k, str)}
+                        raise TypeError(
+                            err_message +
+                            f"All keys in `compute_insights` must be strings, got invalid key types: {invalid_keys}."
+                        )
+                    for k, v in compute_insights.items():
+                        if not isinstance(v, bool):
+                            raise TypeError(
+                                err_message +
+                                f"Invalid type for compute_insights['{k}']: expected bool, got type {type(v).__name__}."
+                            )
+            if connects_to is not None:
+                valid_types = (str, list, tuple, set)
+                if not isinstance(connects_to, valid_types):
+                    raise TypeError(
+                        err_message +
+                        f"`connects_to` must be one of {valid_types}, got type {type(connects_to).__name__}."
+                    )
+                if isinstance(connects_to, (list, tuple, set)):
+                    invalid_elems = [f"{type(e).__name__}" for e in connects_to if not isinstance(e, str)]
+                    if invalid_elems:
+                        raise TypeError(
+                            err_message +
+                            f"All elements in `connects_to` must be strings, "
+                            f"but found element types: {invalid_elems}."
+                        )
+
+        _validate_decorators_signature()
+
+        for metric_handler in leap_binder.setup_container.instance_metrics:
+            if metric_handler.metric_handler_data.name == name:
+                raise Exception(f'Metric with name {name} already exists. '
+                                f'Please choose another')
+
+        def _validate_input_args(*args, **kwargs) -> None:
+            assert len(args) + len(kwargs) > 0, (
+                f"{user_function.__name__}() validation failed: "
+                f"Expected at least one positional|key-word argument of type np.ndarray, "
+                f"but received none. "
+                f"Correct usage example: tensorleap_custom_metric(input_array: np.ndarray, ...)"
+            )
+            for i, arg in enumerate(args):
+                assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
+                    f'{user_function.__name__}() validation failed: '
+                    f'Argument #{i} should be a numpy array. Got {type(arg)}.')
+                if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
+                    assert arg.shape[0] == leap_binder.batch_size_to_validate, \
+                        (f'{user_function.__name__}() validation failed: Argument #{i} '
+                         f'first dim should be as the batch size. Got {arg.shape[0]} '
+                         f'instead of {leap_binder.batch_size_to_validate}')
+
+            for _arg_name, arg in kwargs.items():
+                assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
+                    f'{user_function.__name__}() validation failed: '
+                    f'Argument {_arg_name} should be a numpy array. Got {type(arg)}.')
+                if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
+                    assert arg.shape[0] == leap_binder.batch_size_to_validate, \
+                        (f'{user_function.__name__}() validation failed: Argument {_arg_name} '
+                         f'first dim should be as the batch size. Got {arg.shape[0]} '
+                         f'instead of {leap_binder.batch_size_to_validate}')
+
+        def _validate_result(result) -> None:
+            nonlocal direction
+            supported_types_message = (f'{user_function.__name__}() validation failed: '
+                                       f'{user_function.__name__}() has returned unsupported type.\nSupported type is Dict[List[NDArray[np.float32]]]')
+
+            def _validate_single_metric(single_metric_result, key=None):
+                assert isinstance(single_metric_result,
+                                  np.ndarray), f'{supported_types_message}\nGot {type(single_metric_result)}.'
+                assert len(single_metric_result.shape) == 1, (f'{user_function.__name__}() validation failed: '
+                                                              f'The return shape should be 1D. Got {len(single_metric_result.shape)}D.')
+
+                if leap_binder.batch_size_to_validate:
+                    assert len(single_metric_result) == leap_binder.batch_size_to_validate, \
+                        f'{user_function.__name__}() validation failed: The return len {f"of srt{key} value" if key is not None else ""} should be as the batch size.'
+
+            assert isinstance(result, dict), f'{supported_types_message}\nGot {type(result)}.'
+
+            result_keys = set(result.keys())
+            for key, value in result.items():
+                _validate_single_metric(value, key)
+
+                assert isinstance(key, int), \
+                    (f'{user_function.__name__}() validation failed: '
+                     f'Keys in the return dict should be of type int (instance number). Got {type(key)}.')
+
+            leap_binder.setup_container.instance_metrics[-1].metric_handler_data.direction = direction
+
+        @functools.wraps(user_function)
+        def inner_without_validate(*args, **kwargs):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+
+            try:
+                result = user_function(*args, **kwargs)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+
+            return result
+
+        try:
+            inner_without_validate.__signature__ = inspect.signature(user_function)
+        except (TypeError, ValueError):
+            pass
+
+        leap_binder.add_custom_metric(inner_without_validate, name, direction, compute_insights)
+
+        if connects_to is not None:
+            arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
+            _add_mapping_connections(connects_to, arg_names, NodeMappingType.Metric, name)
+
+        def inner(*args, **kwargs):
+            if not _call_from_tl_platform:
+                set_current('tensorleap_custom_instances_metric')
+            _validate_input_args(*args, **kwargs)
+
+            result = inner_without_validate(*args, **kwargs)
+
+            _validate_result(result)
+            if not _call_from_tl_platform:
+                update_env_params_func("tensorleap_custom_instances_metric", "v")
+            return result
+
+        def mapping_inner(*args, **kwargs):
+            user_unique_name = mapping_inner.name
+            if 'user_unique_name' in kwargs:
+                user_unique_name = kwargs['user_unique_name']
+
+            ordered_connections = [kwargs[n] for n in mapping_inner.arg_names if n in kwargs]
+            ordered_connections = list(args) + ordered_connections
+
+            if user_unique_name in name_to_unique_name[mapping_inner.name]:
+                user_unique_name = f'{user_unique_name}_{len(name_to_unique_name[mapping_inner.name])}'
+            name_to_unique_name[mapping_inner.name].add(user_unique_name)
+
+            _add_mapping_connection(user_unique_name, ordered_connections, mapping_inner.arg_names,
+                                    mapping_inner.name, NodeMappingType.Metric)
+
+            return None
+
+        mapping_inner.arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
+        mapping_inner.name = name
+
+        def final_inner(*args, **kwargs):
+            if os.environ.get(mapping_runtime_mode_env_var_mame):
+                return mapping_inner(*args, **kwargs)
+            else:
+                return inner(*args, **kwargs)
+
+        return final_inner
+
+    return decorating_function
+
+
 def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
                                  heatmap_function: Optional[Callable[..., npt.NDArray[np.float32]]] = None,
                                  connects_to=None):

{code_loader-1.0.173.dev2 → code_loader-1.0.174.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.173.dev2"
+version = "1.0.174.dev1"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"