code-loader 1.0.174.dev4__tar.gz → 1.0.175.dev1__tar.gz

{code_loader-1.0.174.dev4 → code_loader-1.0.175.dev1}/PKG-INFO

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: code-loader
-Version: 1.0.174.dev4
+Version: 1.0.175.dev1
 Summary: 
-Home-page: https://github.com/tensorleap/code-loader
 License: MIT
 Author: dorhar
 Author-email: doron.harnoy@tensorleap.ai
@@ -20,6 +19,7 @@ Requires-Dist: numpy (>=2.3.2,<3.0.0) ; python_version >= "3.11" and python_vers
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
+Project-URL: Homepage, https://github.com/tensorleap/code-loader
 Project-URL: Repository, https://github.com/tensorleap/code-loader
 Description-Content-Type: text/markdown
 

{code_loader-1.0.174.dev4 → code_loader-1.0.175.dev1}/code_loader/contract/datasetclasses.py

@@ -171,12 +171,6 @@ class MetricHandler:
     function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]
 
 
-@dataclass
-class InstanceMetricHandler:
-    metric_handler_data: MetricHandlerData
-    function: CustomMultipleReturnCallableInterfaceMultiArgs
-
-
 @dataclass
 class RawInputsForHeatmap:
     raw_input_by_vizualizer_arg_name: Dict[str, npt.NDArray[np.float32]]
@@ -264,7 +258,6 @@ class DatasetIntegrationSetup:
     prediction_types: List[PredictionTypeHandler] = field(default_factory=list)
     custom_loss_handlers: List[CustomLossHandler] = field(default_factory=list)
     metrics: List[MetricHandler] = field(default_factory=list)
-    instance_metrics: List[InstanceMetricHandler] = field(default_factory=list)
     custom_layers: Dict[str, CustomLayerHandler] = field(default_factory=dict)
     custom_latent_space: Optional[CustomLatentSpaceHandler] = None
 

{code_loader-1.0.174.dev4 → code_loader-1.0.175.dev1}/code_loader/contract/responsedataclasses.py

@@ -128,6 +128,7 @@ class LeapAnalysisConfiguration:
     domain_gap_metadata: Optional[List[str]] = None
     feature_flags: Optional[List[str]] = None
     deterministic_results: Optional[bool] = None  # if true, results will be deterministic but will increase memory usage and runtime
+    ignore_latent_spaces: Optional[List[str]] = None  # names of latent spaces to skip during extraction; see LeapBinder.check for validation rules
 
 
 @dataclass

{code_loader-1.0.174.dev4 → code_loader-1.0.175.dev1}/code_loader/inner_leap_binder/leapbinder.py

@@ -12,7 +12,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
-    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler, InstanceMetricHandler
+    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload, LeapAnalysisConfiguration
@@ -335,49 +335,6 @@ class LeapBinder:
         metric_handler_data = MetricHandlerData(name, regular_arg_names, direction, compute_insights)
         self.setup_container.metrics.append(MetricHandler(metric_handler_data, function))
 
-    def add_custom_instance_metric(self,
-                          function: Union[CustomCallableInterfaceMultiArgs,
-                          CustomMultipleReturnCallableInterfaceMultiArgs,
-                          ConfusionMatrixCallableInterfaceMultiArgs],
-                          name: str,
-                          direction: Optional[
-                              Union[MetricDirection, Dict[str, MetricDirection]]] = MetricDirection.Downward,
-                          compute_insights: Optional[Union[bool, Dict[str, bool]]] = None) -> None:
-        """
-        Add a custom metric to the setup.
-
-        Args:
-        function (Union[CustomCallableInterfaceMultiArgs, CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]): The custom metric function.
-        name (str): The name of the custom metric.
-        direction (Optional[Union[MetricDirection, Dict[str, MetricDirection]]]): The direction of the metric, either
-        MetricDirection.Upward or MetricDirection.Downward, in case custom metric return a dictionary of metrics we can
-        supply a dictionary of directions correspondingly.
-            - MetricDirection.Upward: Indicates that higher values of the metric are better and should be maximized.
-            - MetricDirection.Downward: Indicates that lower values of the metric are better and should be minimized.
-        compute_insights (Union[bool, Dict[str, bool]]): Whether to compute insights or not. in case custom metric
-        return a dictionary of metrics we can supply a dictionary of values correspondingly
-
-
-
-        Example:
-            def custom_metric_function(y_true, y_pred):
-                return np.mean(np.abs(y_true - y_pred))
-
-            leap_binder.add_custom_metric(custom_metric_function, name='custom_metric', direction=MetricDirection.Downward)
-        """
-
-        regular_arg_names = inspect.getfullargspec(function)[0]
-        preprocess_response_arg_name = None
-        for arg_name, arg_type in inspect.getfullargspec(function).annotations.items():
-            if arg_type == SamplePreprocessResponse:
-                if preprocess_response_arg_name is not None:
-                    raise Exception("only one argument can be of type SamplePreprocessResponse")
-                preprocess_response_arg_name = arg_name
-                regular_arg_names.remove(arg_name)
-
-        metric_handler_data = MetricHandlerData(name, regular_arg_names, direction, compute_insights)
-        self.setup_container.instance_metrics.append(InstanceMetricHandler(metric_handler_data, function))
-
     def add_prediction(self, name: str, labels: List[str], channel_dim: int = -1, prediction_index: Optional[int]=None) -> None:
         """
         Add prediction labels to the setup.
@@ -659,7 +616,51 @@ class LeapBinder:
         preprocess_result = self.get_preprocess_result()
         self.check_preprocess(preprocess_result)
         self.check_handlers(preprocess_result)
+        self.validate_ignore_latent_spaces()
         print("Successful!")
 
+    def validate_ignore_latent_spaces(self) -> None:
+        """Validate leap_analysis_configuration.ignore_latent_spaces against this binder.
+
+        Each entry must be one of:
+          - "foreground" (the friendly name for the max-pooled default LS)
+          - An input name registered via set_input(name=...)
+          - A prediction name registered via add_prediction(name=...)
+
+        Forbidden tokens (raise with a clear message):
+          - "balanced" — the friendly name for the universal fallback default
+            LS; cannot be disabled.
+          - "user_custom" — populated by set_custom_latent_space; if the user
+            registered a custom LS they should use it, not ignore it.
+
+        Order-independent: this runs after all set_input / add_prediction calls
+        because it consults setup_container.inputs and prediction_types.
+        """
+        if not self.leap_analysis_configuration:
+            return
+        names = self.leap_analysis_configuration.ignore_latent_spaces
+        if not names:
+            return
+
+        forbidden = {"balanced", "user_custom"}
+        input_names = {h.name for h in self.setup_container.inputs}
+        prediction_names = {h.name for h in self.setup_container.prediction_types}
+        allowed_tokens = {"foreground"} | input_names | prediction_names
+
+        for name in names:
+            if name in forbidden:
+                raise Exception(
+                    f"Latent space '{name}' cannot be ignored. "
+                    f"'balanced' is the universal fallback default LS and 'user_custom' is "
+                    f"reserved for set_custom_latent_space."
+                )
+            if name not in allowed_tokens:
+                raise Exception(
+                    f"Latent space name '{name}' in ignore_latent_spaces did not match "
+                    f"any registered input or prediction. Available names: "
+                    f"inputs={sorted(input_names)}, predictions={sorted(prediction_names)}, "
+                    f"tokens=['foreground']."
+                )
+
     def set_batch_size_to_validate(self, batch_size: int) -> None:
         self.batch_size_to_validate = batch_size

{code_loader-1.0.174.dev4 → code_loader-1.0.175.dev1}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -877,215 +877,6 @@ def tensorleap_custom_metric(name: str,
     return decorating_function
 
 
-def tensorleap_custom_instances_metric(name: str,
-                             direction: Union[MetricDirection, Dict[str, MetricDirection]] = _UNSET,
-                             compute_insights: Optional[Union[bool, Dict[str, bool]]] = None,
-                             connects_to=None):
-    name_to_unique_name = defaultdict(set)
-
-    def decorating_function(
-            user_function: CustomMultipleReturnCallableInterfaceMultiArgs):
-        nonlocal direction
-
-        direction_was_provided = direction is not _UNSET
-
-        def _validate_decorators_signature():
-            err_message = f"{user_function.__name__} validation failed.\n"
-            if not isinstance(name, str):
-                raise TypeError(err_message + f"`name` must be a string, got type {type(name).__name__}.")
-            valid_directions = {MetricDirection.Upward, MetricDirection.Downward}
-            if direction is _UNSET:
-                pass
-            elif isinstance(direction, MetricDirection):
-                if direction not in valid_directions:
-                    raise ValueError(
-                        err_message +
-                        f"Invalid MetricDirection: {direction}. Must be one of {valid_directions}, "
-                        f"got type {type(direction).__name__}."
-                    )
-            elif isinstance(direction, dict):
-                if not all(isinstance(k, str) for k in direction.keys()):
-                    invalid_keys = {k: type(k).__name__ for k in direction.keys() if not isinstance(k, str)}
-                    raise TypeError(
-                        err_message +
-                        f"All keys in `direction` must be strings, got invalid key types: {invalid_keys}."
-                    )
-                for k, v in direction.items():
-                    if v not in valid_directions:
-                        raise ValueError(
-                            err_message +
-                            f"Invalid direction for key '{k}': {v}. Must be one of {valid_directions}, "
-                            f"got type {type(v).__name__}."
-                        )
-            else:
-                raise TypeError(
-                    err_message +
-                    f"`direction` must be a MetricDirection or a Dict[str, MetricDirection], "
-                    f"got type {type(direction).__name__}."
-                )
-            if compute_insights is not None:
-                if not isinstance(compute_insights, (bool, dict)):
-                    raise TypeError(
-                        err_message +
-                        f"`compute_insights` must be a bool or a Dict[str, bool], "
-                        f"got type {type(compute_insights).__name__}."
-                    )
-                if isinstance(compute_insights, dict):
-                    if not all(isinstance(k, str) for k in compute_insights.keys()):
-                        invalid_keys = {k: type(k).__name__ for k in compute_insights.keys() if not isinstance(k, str)}
-                        raise TypeError(
-                            err_message +
-                            f"All keys in `compute_insights` must be strings, got invalid key types: {invalid_keys}."
-                        )
-                    for k, v in compute_insights.items():
-                        if not isinstance(v, bool):
-                            raise TypeError(
-                                err_message +
-                                f"Invalid type for compute_insights['{k}']: expected bool, got type {type(v).__name__}."
-                            )
-            if connects_to is not None:
-                valid_types = (str, list, tuple, set)
-                if not isinstance(connects_to, valid_types):
-                    raise TypeError(
-                        err_message +
-                        f"`connects_to` must be one of {valid_types}, got type {type(connects_to).__name__}."
-                    )
-                if isinstance(connects_to, (list, tuple, set)):
-                    invalid_elems = [f"{type(e).__name__}" for e in connects_to if not isinstance(e, str)]
-                    if invalid_elems:
-                        raise TypeError(
-                            err_message +
-                            f"All elements in `connects_to` must be strings, "
-                            f"but found element types: {invalid_elems}."
-                        )
-
-        _validate_decorators_signature()
-
-        for metric_handler in leap_binder.setup_container.instance_metrics:
-            if metric_handler.metric_handler_data.name == name:
-                raise Exception(f'Metric with name {name} already exists. '
-                                f'Please choose another')
-
-        def _validate_input_args(*args, **kwargs) -> None:
-            assert len(args) + len(kwargs) > 0, (
-                f"{user_function.__name__}() validation failed: "
-                f"Expected at least one positional|key-word argument of type np.ndarray, "
-                f"but received none. "
-                f"Correct usage example: tensorleap_custom_metric(input_array: np.ndarray, ...)"
-            )
-            for i, arg in enumerate(args):
-                assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
-                    f'{user_function.__name__}() validation failed: '
-                    f'Argument #{i} should be a numpy array. Got {type(arg)}.')
-                if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
-                    assert arg.shape[0] == leap_binder.batch_size_to_validate, \
-                        (f'{user_function.__name__}() validation failed: Argument #{i} '
-                         f'first dim should be as the batch size. Got {arg.shape[0]} '
-                         f'instead of {leap_binder.batch_size_to_validate}')
-
-            for _arg_name, arg in kwargs.items():
-                assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
-                    f'{user_function.__name__}() validation failed: '
-                    f'Argument {_arg_name} should be a numpy array. Got {type(arg)}.')
-                if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
-                    assert arg.shape[0] == leap_binder.batch_size_to_validate, \
-                        (f'{user_function.__name__}() validation failed: Argument {_arg_name} '
-                         f'first dim should be as the batch size. Got {arg.shape[0]} '
-                         f'instead of {leap_binder.batch_size_to_validate}')
-
-        def _validate_result(result) -> None:
-            nonlocal direction
-            supported_types_message = (f'{user_function.__name__}() validation failed: '
-                                       f'{user_function.__name__}() has returned unsupported type.\nSupported type is Dict[List[NDArray[np.float32]]]')
-
-            def _validate_single_metric(single_metric_result, key=None):
-                assert isinstance(single_metric_result,
-                                  np.ndarray), f'{supported_types_message}\nGot {type(single_metric_result)}.'
-                assert len(single_metric_result.shape) == 1, (f'{user_function.__name__}() validation failed: '
-                                                              f'The return shape should be 1D. Got {len(single_metric_result.shape)}D.')
-
-                if leap_binder.batch_size_to_validate:
-                    assert len(single_metric_result) == leap_binder.batch_size_to_validate, \
-                        f'{user_function.__name__}() validation failed: The return len {f"of srt{key} value" if key is not None else ""} should be as the batch size.'
-
-            assert isinstance(result, dict), f'{supported_types_message}\nGot {type(result)}.'
-
-            result_keys = set(result.keys())
-            for key, value in result.items():
-                _validate_single_metric(value, key)
-
-                assert isinstance(key, int), \
-                    (f'{user_function.__name__}() validation failed: '
-                     f'Keys in the return dict should be of type int (instance number). Got {type(key)}.')
-
-            leap_binder.setup_container.instance_metrics[-1].metric_handler_data.direction = direction
-
-        @functools.wraps(user_function)
-        def inner_without_validate(*args, **kwargs):
-            global _called_from_inside_tl_decorator
-            _called_from_inside_tl_decorator += 1
-
-            try:
-                result = user_function(*args, **kwargs)
-            finally:
-                _called_from_inside_tl_decorator -= 1
-
-            return result
-
-        try:
-            inner_without_validate.__signature__ = inspect.signature(user_function)
-        except (TypeError, ValueError):
-            pass
-
-        leap_binder.add_custom_instance_metric(inner_without_validate, name, direction, compute_insights)
-
-        if connects_to is not None:
-            arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
-            _add_mapping_connections(connects_to, arg_names, NodeMappingType.Metric, name)
-
-        def inner(*args, **kwargs):
-            if not _call_from_tl_platform:
-                set_current('tensorleap_custom_instances_metric')
-            _validate_input_args(*args, **kwargs)
-
-            result = inner_without_validate(*args, **kwargs)
-
-            _validate_result(result)
-            if not _call_from_tl_platform:
-                update_env_params_func("tensorleap_custom_instances_metric", "v")
-            return result
-
-        def mapping_inner(*args, **kwargs):
-            user_unique_name = mapping_inner.name
-            if 'user_unique_name' in kwargs:
-                user_unique_name = kwargs['user_unique_name']
-
-            ordered_connections = [kwargs[n] for n in mapping_inner.arg_names if n in kwargs]
-            ordered_connections = list(args) + ordered_connections
-
-            if user_unique_name in name_to_unique_name[mapping_inner.name]:
-                user_unique_name = f'{user_unique_name}_{len(name_to_unique_name[mapping_inner.name])}'
-            name_to_unique_name[mapping_inner.name].add(user_unique_name)
-
-            _add_mapping_connection(user_unique_name, ordered_connections, mapping_inner.arg_names,
-                                    mapping_inner.name, NodeMappingType.Metric)
-
-            return None
-
-        mapping_inner.arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
-        mapping_inner.name = name
-
-        def final_inner(*args, **kwargs):
-            if os.environ.get(mapping_runtime_mode_env_var_mame):
-                return mapping_inner(*args, **kwargs)
-            else:
-                return inner(*args, **kwargs)
-
-        return final_inner
-
-    return decorating_function
-
-
 def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
                                  heatmap_function: Optional[Callable[..., npt.NDArray[np.float32]]] = None,
                                  connects_to=None):

{code_loader-1.0.174.dev4 → code_loader-1.0.175.dev1}/code_loader/leaploader.py

@@ -205,6 +205,7 @@ class LeapLoader(LeapLoaderBase):
                         for metadata_name in global_leap_binder.leap_analysis_configuration.domain_gap_metadata:
                             if metadata_name not in m_names:
                                 raise Exception(f"Domain gap metadata '{metadata_name}' is not found in dataset metadata list.")
+                    global_leap_binder.validate_ignore_latent_spaces()
             except DatasetScriptException as e:
                 line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
                 general_error = f"Something went wrong. {repr(e.__cause__)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"

{code_loader-1.0.174.dev4 → code_loader-1.0.175.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.174.dev4"
+version = "1.0.175.dev1"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"