code-loader 1.0.164.dev1__tar.gz → 1.0.165.dev2__tar.gz

{code_loader-1.0.164.dev1 → code_loader-1.0.165.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.164.dev1
+Version: 1.0.165.dev2
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.164.dev1 → code_loader-1.0.165.dev2}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -362,7 +362,7 @@ def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]
 
             class ModelPlaceholder:
                 def __init__(self, prediction_types):
-                    self.model = load_model_func()  # TODO- check why this fails on onnx model
+                    self.model = load_model_func()
                     self.prediction_types = prediction_types
                     _validate_result(self.model)
 
@@ -532,23 +532,14 @@ def tensorleap_custom_metric(name: str,
 
         direction_was_provided = direction is not _UNSET
 
-        if not direction_was_provided:
-            direction = MetricDirection.Downward
-            if not _call_from_tl_platform:
-                store_warning_by_param(
-                    param_name="direction",
-                    user_func_name=user_function.__name__,
-                    default_value=direction,
-                    link_to_docs="https://docs.tensorleap.ai/tensorleap-integration/writing-integration-code/custom-metrics"
-
-                )
-
         def _validate_decorators_signature():
             err_message = f"{user_function.__name__} validation failed.\n"
             if not isinstance(name, str):
                 raise TypeError(err_message + f"`name` must be a string, got type {type(name).__name__}.")
             valid_directions = {MetricDirection.Upward, MetricDirection.Downward}
-            if isinstance(direction, MetricDirection):
+            if direction is _UNSET:
+                pass
+            elif isinstance(direction, MetricDirection):
                 if direction not in valid_directions:
                     raise ValueError(
                         err_message +
@@ -646,6 +637,7 @@ def tensorleap_custom_metric(name: str,
                          f'instead of {leap_binder.batch_size_to_validate}')
 
         def _validate_result(result) -> None:
+            nonlocal direction
             validate_output_structure(result, func_name=user_function.__name__,
                                       expected_type_name="List[float | int | None | List[ConfusionMatrixElement] ]  | NDArray[np.float32]  or dictonary with one of these types as its values types")
             supported_types_message = (f'{user_function.__name__}() validation failed: '
@@ -678,13 +670,17 @@ def tensorleap_custom_metric(name: str,
                         f'{user_function.__name__}() validation failed: The return len {f"of srt{key} value" if key is not None else ""} should be as the batch size.'
 
             if isinstance(result, dict):
+                result_keys = set(result.keys())
                 for key, value in result.items():
                     _validate_single_metric(value, key)
 
                     assert isinstance(key, str), \
                         (f'{user_function.__name__}() validation failed: '
                          f'Keys in the return dict should be of type str. Got {type(key)}.')
-                    _validate_single_metric(value)
+
+                assert direction is _UNSET or isinstance(direction, dict), \
+                    (f'{user_function.__name__}() validation failed: '
+                     f'direction should be a dict when using dict results. Got {type(direction)}.') #TODO - remove this assertion so a single value would controll all directions
 
                 if isinstance(direction, dict):
                     for direction_key in direction:
@@ -693,9 +689,9 @@ def tensorleap_custom_metric(name: str,
                              f'Keys in the direction mapping should be part of result keys. Got key {direction_key}.')
 
                 if compute_insights is not None:
-                    assert isinstance(compute_insights, (bool, dict)), \
+                    assert isinstance(compute_insights, dict), \
                         (f'{user_function.__name__}() validation failed: '
-                         f'compute_insights should be bool or dict if using the dict results. Got {type(compute_insights)}.')
+                         f'compute_insights should be dict if using the dict results. Got {type(compute_insights)}.') #TODO - remove this assertion so a single value would controll all compute_insights
 
                     if isinstance(compute_insights, dict):
                         for ci_key in compute_insights:
@@ -703,6 +699,25 @@ def tensorleap_custom_metric(name: str,
                                 (f'{user_function.__name__}() validation failed: '
                                  f'Keys in the compute_insights mapping should be part of result keys. Got key {ci_key}.')
 
+                effective_direction = {} if direction is _UNSET else direction
+                defaulted_direction_keys = result_keys.difference(effective_direction.keys())
+
+                if defaulted_direction_keys:
+                    effective_direction = {key: effective_direction.get(key, MetricDirection.Downward) for key in result}
+
+                leap_binder.setup_container.metrics[-1].metric_handler_data.direction = effective_direction
+
+                if defaulted_direction_keys and not _call_from_tl_platform:
+                    effective_compute_insights = compute_insights or {}
+                    warning_keys = {key for key in defaulted_direction_keys if effective_compute_insights.get(key, True)}
+                    if warning_keys:
+                        store_warning_by_param(
+                            param_name=f"direction[{warning_keys}]",
+                            user_func_name=user_function.__name__,
+                            default_value=MetricDirection.Downward,
+                            link_to_docs="https://docs.tensorleap.ai/tensorleap-integration/writing-integration-code/custom-metrics"
+                        )
+
             else:
                 _validate_single_metric(result)
 
@@ -710,6 +725,21 @@ def tensorleap_custom_metric(name: str,
                     assert isinstance(compute_insights, bool), \
                         (f'{user_function.__name__}() validation failed: '
                          f'compute_insights should be boolean. Got {type(compute_insights)}.')
+                if direction is not _UNSET:
+                    assert isinstance(direction, MetricDirection), \
+                        (f'{user_function.__name__}() validation failed: '
+                         f'direction should be a MetricDirection when using a single result. Got {type(direction)}.')
+
+                else:
+                    direction = MetricDirection.Downward
+                    leap_binder.setup_container.metrics[-1].metric_handler_data.direction = direction
+                    if not _call_from_tl_platform and compute_insights is not False:
+                        store_warning_by_param(
+                            param_name="direction",
+                            user_func_name=user_function.__name__,
+                            default_value=direction,
+                            link_to_docs="https://docs.tensorleap.ai/tensorleap-integration/writing-integration-code/custom-metrics"
+                        )
 
         @functools.wraps(user_function)
         def inner_without_validate(*args, **kwargs):
@@ -1770,5 +1800,3 @@ if not _call_from_tl_platform:
 
 
 
-
-

{code_loader-1.0.164.dev1 → code_loader-1.0.165.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.164.dev1"
+version = "1.0.165.dev2"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"