code-loader 1.0.139.dev4__tar.gz → 1.0.139.dev6__tar.gz

{code_loader-1.0.139.dev4 → code_loader-1.0.139.dev6}/PKG-INFO

@@ -1,9 +1,9 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.139.dev4
+Version: 1.0.139.dev6
 Summary: 
+Home-page: https://github.com/tensorleap/code-loader
 License: MIT
-License-File: LICENSE
 Author: dorhar
 Author-email: doron.harnoy@tensorleap.ai
 Requires-Python: >=3.8,<3.13
@@ -20,7 +20,6 @@ Requires-Dist: numpy (>=2.3.2,<3.0.0) ; python_version >= "3.11" and python_vers
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
-Project-URL: Homepage, https://github.com/tensorleap/code-loader
 Project-URL: Repository, https://github.com/tensorleap/code-loader
 Description-Content-Type: text/markdown
 

{code_loader-1.0.139.dev4 → code_loader-1.0.139.dev6}/code_loader/contract/datasetclasses.py

@@ -1,3 +1,4 @@
+import warnings
 from dataclasses import dataclass, field
 from typing import Any, Callable, List, Optional, Dict, Union, Type
 import re
@@ -56,7 +57,14 @@ class PreprocessResponse:
                 for sample_id in self.sample_ids:
                     assert isinstance(sample_id, str), f"Sample id should be of type str. Got: {type(sample_id)}"
         else:
-            raise Exception("length is deprecated.")
+            raise Exception("length is deprecated, please use sample_ids instead.")
+
+        if self.state is None:
+            warnings.warn(
+                "PreprocessResponse.state is not set. For best practice, assign a unique `state` value to each PreprocessResponse instance."
+            )
+        else:
+            assert isinstance(self.state, DataStateType), f"PreprocessResponse.state must be of type {DataStateType.__name__} but got {type(self.state)}"
 
     def __hash__(self) -> int:
         return id(self)

{code_loader-1.0.139.dev4 → code_loader-1.0.139.dev6}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1,9 +1,10 @@
 # mypy: ignore-errors
 import os
+import warnings
 from collections import defaultdict
 from functools import lru_cache
 from pathlib import Path
-from typing import Optional, Union, Callable, List, Dict, Set, Any
+from typing import Optional, Union, Callable, List, Dict, get_args, get_origin
 
 import numpy as np
 import numpy.typing as npt
@@ -25,9 +26,116 @@ import functools
 
 _called_from_inside_tl_decorator = 0
 _called_from_inside_tl_integration_test_decorator = False
-_integration_events_emitted: Set[str] = set()
-
-
+_update_env_status = None
+
+
+
+
+def validate_args_structure(*args, types_order, func_name, expected_names, **kwargs):
+    def _type_to_str(t):
+        origin = get_origin(t)
+        if origin is Union:
+            return " | ".join(tt.__name__ for tt in get_args(t))
+        elif hasattr(t, "__name__"):
+            return t.__name__
+        else:
+            return str(t)
+
+    def _format_types(types, names=None):
+        return ", ".join(
+            f"{(names[i] + ': ') if names else f'arg{i}: '}{_type_to_str(ty)}"
+            for i, ty in enumerate(types)
+        )
+
+    if expected_names:
+        normalized_args = []
+        for i, name in enumerate(expected_names):
+            if i < len(args):
+                normalized_args.append(args[i])
+            elif name in kwargs:
+                normalized_args.append(kwargs[name])
+            else:
+                raise AssertionError(
+                    f"{func_name} validation failed: "
+                    f"Missing required argument '{name}'. "
+                    f"Expected arguments: {expected_names}."
+                )
+    else:
+        normalized_args = list(args)
+    if len(normalized_args) != len(types_order):
+        expected = _format_types(types_order, expected_names)
+        got_types = ", ".join(type(arg).__name__ for arg in normalized_args)
+        raise AssertionError(
+            f"{func_name} validation failed: "
+            f"Expected exactly {len(types_order)} arguments ({expected}), "
+            f"but got {len(normalized_args)} argument(s) of type(s): ({got_types}). "
+            f"Correct usage example: {func_name}({expected})"
+        )
+
+    for i, (arg, expected_type) in enumerate(zip(normalized_args, types_order)):
+        origin = get_origin(expected_type)
+        if origin is Union:
+            allowed_types = get_args(expected_type)
+        else:
+            allowed_types = (expected_type,)
+
+        if not isinstance(arg, allowed_types):
+            allowed_str = " | ".join(t.__name__ for t in allowed_types)
+            raise AssertionError(
+                f"{func_name} validation failed: "
+                f"Argument '{expected_names[i] if expected_names else f'arg{i}'}' "
+                f"expected type {allowed_str}, but got {type(arg).__name__}. "
+                f"Correct usage example: {func_name}({_format_types(types_order, expected_names)})"
+            )
+
+
+def validate_output_structure(result, func_name: str, expected_type_name="np.ndarray",gt_flag=False):
+    if result is None or (isinstance(result, float) and np.isnan(result)):
+        if gt_flag:
+            raise AssertionError(
+                f"{func_name} validation failed: "
+                f"The function returned {result!r}. "
+                f"If you are working with an unlabeled dataset and no ground truth is available, "
+                f"use 'return np.array([], dtype=np.float32)' instead. "
+                f"Otherwise, {func_name} expected a single {expected_type_name} object. "
+                f"Make sure the function ends with 'return <{expected_type_name}>'."
+            )
+
+        raise AssertionError(
+            f"{func_name} validation failed: "
+            f"The function returned None. "
+            f"Expected a single {expected_type_name} object. "
+            f"Make sure the function ends with 'return <{expected_type_name}>'."
+        )
+    if isinstance(result, tuple):
+        element_descriptions = [
+            f"[{i}] type: {type(r).__name__}"
+            for i, r in enumerate(result)
+        ]
+        element_summary = "\n    ".join(element_descriptions)
+
+        raise AssertionError(
+            f"{func_name} validation failed: "
+            f"The function returned multiple outputs ({len(result)} values), "
+            f"but only a single {expected_type_name} is allowed.\n\n"
+            f"Returned elements:\n"
+            f"    {element_summary}\n\n"
+            f"Correct usage example:\n"
+            f"    def {func_name}(...):\n"
+            f"        return <{expected_type_name}>\n\n"
+            f"If you intended to return multiple values, combine them into a single "
+            f"{expected_type_name} (e.g., by concatenation or stacking)."
+        )
+
+def batch_warning(result, func_name):
+    if result.shape[0] == 1:
+        warnings.warn(
+            f"{func_name} warning: Tensorleap will add a batch dimension at axis 0 to the output of {func_name}, "
+            f"although the detected size of axis 0 is already 1. "
+            f"This may lead to an extra batch dimension (e.g., shape (1, 1, ...)). "
+            f"Please ensure that the output of '{func_name}' is not already batched "
+            f"to avoid computation errors."
+        )
 def _add_mapping_connection(user_unique_name, connection_destinations, arg_names, name, node_mapping_type):
     connection_destinations = [connection_destination for connection_destination in connection_destinations
                                if not isinstance(connection_destination, SamplePreprocessResponse)]
@@ -50,12 +158,24 @@ def tensorleap_integration_test():
     def decorating_function(integration_test_function: Callable):
         leap_binder.integration_test_func = integration_test_function
 
+        def _validate_input_args(*args, **kwargs):
+            sample_id,preprocess_response=args
+            assert type(sample_id) == preprocess_response.sample_id_type, (
+                f"tensorleap_integration_test validation failed: "
+                f"sample_id type ({type(sample_id).__name__}) does not match the expected "
+                f"type ({preprocess_response.sample_id_type}) from the PreprocessResponse."
+            )
+
         def inner(*args, **kwargs):
+            validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
+                                    func_name='integration_test',expected_names=["idx", "preprocess"],**kwargs)
+            _validate_input_args(*args, **kwargs)
+
             global _called_from_inside_tl_integration_test_decorator
-            global _integration_events_emitted
-            _integration_events_emitted.clear()  # Clear events for new test
             try:
                 _called_from_inside_tl_integration_test_decorator = True
+                if not _update_env_status is None:
+                    _update_env_status("tensorleap_integration_test", "v")#put here because otherwise it will become  v only if it finishes all the script
                 ret = integration_test_function(*args, **kwargs)
 
                 try:
@@ -68,7 +188,7 @@ def tensorleap_integration_test():
                     line_number = first_tb.lineno
                     if isinstance(e, TypeError) and 'is not subscriptable' in str(e):
                         print(f'Invalid integration code. File {file_name}, line {line_number}: '
-                              f'Please remove this indexing operation usage from the integration test code.')
+                              f"indexing is supported only on the model's predictions inside the integration test. Please remove this indexing operation usage from the integration test code.")
                     else:
                         print(f'Invalid integration code. File {file_name}, line {line_number}: '
                               f'Integration test is only allowed to call Tensorleap decorators. '
@@ -80,9 +200,9 @@ def tensorleap_integration_test():
                 _called_from_inside_tl_integration_test_decorator = False
 
             leap_binder.check()
-
         return inner
 
+
     return decorating_function
 
 def _safe_get_item(key):
@@ -91,31 +211,59 @@ def _safe_get_item(key):
     except ValueError:
         raise Exception(f'Tensorleap currently supports models with no more then 10 inputs')
 
-def _emit_integration_event_once(event_name: str, props: Dict[str, Any]) -> None:
-    """Emit an integration test event only once per test run."""
-    if event_name in _integration_events_emitted:
-        return
-    
-    try:
-        from code_loader.mixpanel_tracker import track_integration_test_event
-        track_integration_test_event(event_name, props)
-        _integration_events_emitted.add(event_name)
-    except Exception:
-        pass
-
 def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]] = []):
+    assert isinstance(prediction_types, list),(
+        f"tensorleap_load_model validation failed: "
+                f" prediction_types is an optional argument of type List[PredictionTypeHandler]] but got {type(prediction_types).__name__}."
+    )
     for i, prediction_type in enumerate(prediction_types):
+        assert isinstance(prediction_type, PredictionTypeHandler),(f"tensorleap_load_model validation failed: "
+                f" prediction_types at position {i} must be of type PredictionTypeHandler but got {type(prediction_types[i]).__name__}.")
         leap_binder.add_prediction(prediction_type.name, prediction_type.labels, prediction_type.channel_dim, i)
 
+    def _validate_result(result) -> None:
+        valid_types=["onnxruntime","keras"]
+        err_message=f"tensorleap_load_model validation failed:\nSupported models are Keras and onnxruntime only and non of them was returned."
+        validate_output_structure(result, func_name="tensorleap_load_model", expected_type_name= [" | ".join(t for t in valid_types)][0])
+        try:
+            import keras
+        except ImportError:
+            keras = None
+        try:
+            import tensorflow as tf
+        except ImportError:
+            tf = None
+        try:
+            import onnxruntime
+        except ImportError:
+            onnxruntime = None
+
+        if not keras and not onnxruntime:
+            raise AssertionError(err_message)
+
+        is_keras_model = (
+                bool(keras and isinstance(result, getattr(keras, "Model", tuple())))
+                or bool(tf and isinstance(result, getattr(tf.keras, "Model", tuple())))
+        )
+        is_onnx_model = bool(onnxruntime and isinstance(result, onnxruntime.InferenceSession))
+
+        if not any([is_keras_model, is_onnx_model]):
+            raise AssertionError( err_message)
+
+
+
     def decorating_function(load_model_func):
         class TempMapping:
             pass
 
         @lru_cache()
-        def inner():
+        def inner(*args, **kwargs):
+            validate_args_structure(*args, types_order=[],
+                                    func_name='tensorleap_load_model',expected_names=[],**kwargs)
             class ModelPlaceholder:
                 def __init__(self):
-                    self.model = load_model_func()
+                    self.model = load_model_func() #TODO- check why this fails on onnx model
+                    _validate_result(self.model)
 
                 # keras interface
                 def __call__(self, arg):
@@ -179,8 +327,10 @@ def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]
 
                 def get_inputs(self):
                     return self.model.get_inputs()
-
-            return ModelPlaceholder()
+            model_placeholder=ModelPlaceholder()
+            if not _update_env_status is None:
+                _update_env_status("tensorleap_load_model", "v")
+            return model_placeholder
 
         def mapping_inner():
             class ModelOutputPlaceholder:
@@ -243,12 +393,11 @@ def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]
 
             return ModelPlaceholder()
 
-        def final_inner():
+        def final_inner(*args, **kwargs):
             if os.environ.get(mapping_runtime_mode_env_var_mame):
                 return mapping_inner()
             else:
-                return inner()
-
+                return inner(*args, **kwargs)
         return final_inner
 
     return decorating_function
@@ -259,81 +408,168 @@ def tensorleap_custom_metric(name: str,
                              compute_insights: Optional[Union[bool, Dict[str, bool]]] = None,
                              connects_to=None):
     name_to_unique_name = defaultdict(set)
-
     def decorating_function(
             user_function: Union[CustomCallableInterfaceMultiArgs, CustomMultipleReturnCallableInterfaceMultiArgs,
             ConfusionMatrixCallableInterfaceMultiArgs]):
+
+        def _validate_decorators_signature():
+            err_message = f"{user_function.__name__} validation failed.\n"
+            if not isinstance(name, str):
+                raise TypeError(err_message + f"`name` must be a string, got type {type(name).__name__}.")
+            valid_directions = {MetricDirection.Upward, MetricDirection.Downward}
+            if isinstance(direction, MetricDirection):
+                if direction not in valid_directions:
+                    raise ValueError(
+                        err_message +
+                        f"Invalid MetricDirection: {direction}. Must be one of {valid_directions}, "
+                        f"got type {type(direction).__name__}."
+                    )
+            elif isinstance(direction, dict):
+                if not all(isinstance(k, str) for k in direction.keys()):
+                    invalid_keys = {k: type(k).__name__ for k in direction.keys() if not isinstance(k, str)}
+                    raise TypeError(
+                        err_message +
+                        f"All keys in `direction` must be strings, got invalid key types: {invalid_keys}."
+                    )
+                for k, v in direction.items():
+                    if v not in valid_directions:
+                        raise ValueError(
+                            err_message +
+                            f"Invalid direction for key '{k}': {v}. Must be one of {valid_directions}, "
+                            f"got type {type(v).__name__}."
+                        )
+            else:
+                raise TypeError(
+                    err_message +
+                    f"`direction` must be a MetricDirection or a Dict[str, MetricDirection], "
+                    f"got type {type(direction).__name__}."
+                )
+            if compute_insights is not None:
+                if not isinstance(compute_insights, (bool, dict)):
+                    raise TypeError(
+                        err_message +
+                        f"`compute_insights` must be a bool or a Dict[str, bool], "
+                        f"got type {type(compute_insights).__name__}."
+                    )
+                if isinstance(compute_insights, dict):
+                    if not all(isinstance(k, str) for k in compute_insights.keys()):
+                        invalid_keys = {k: type(k).__name__ for k in compute_insights.keys() if not isinstance(k, str)}
+                        raise TypeError(
+                            err_message +
+                            f"All keys in `compute_insights` must be strings, got invalid key types: {invalid_keys}."
+                        )
+                    for k, v in compute_insights.items():
+                        if not isinstance(v, bool):
+                            raise TypeError(
+                                err_message +
+                                f"Invalid type for compute_insights['{k}']: expected bool, got type {type(v).__name__}."
+                            )
+            if connects_to is not None:
+                valid_types = (str, list, tuple, set)
+                if not isinstance(connects_to, valid_types):
+                    raise TypeError(
+                        err_message +
+                        f"`connects_to` must be one of {valid_types}, got type {type(connects_to).__name__}."
+                    )
+                if isinstance(connects_to, (list, tuple, set)):
+                    invalid_elems = [f"{type(e).__name__}" for e in connects_to if not isinstance(e, str)]
+                    if invalid_elems:
+                        raise TypeError(
+                            err_message +
+                            f"All elements in `connects_to` must be strings, "
+                            f"but found element types: {invalid_elems}."
+                        )
+
+
+        _validate_decorators_signature()
+
         for metric_handler in leap_binder.setup_container.metrics:
             if metric_handler.metric_handler_data.name == name:
                 raise Exception(f'Metric with name {name} already exists. '
                                 f'Please choose another')
 
         def _validate_input_args(*args, **kwargs) -> None:
+            assert len(args) > 0, (
+                f"{user_function.__name__}() validation failed: "
+                f"Expected at least one positional|key-word argument of type np.ndarray, "
+                f"but received none. "
+                f"Correct usage example: tensorleap_custom_metric(input_array: np.ndarray, ...)"
+            )
             for i, arg in enumerate(args):
                 assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
-                    f'tensorleap_custom_metric validation failed: '
+                    f'{user_function.__name__}() validation failed: '
                     f'Argument #{i} should be a numpy array. Got {type(arg)}.')
                 if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
                     assert arg.shape[0] == leap_binder.batch_size_to_validate, \
-                        (f'tensorleap_custom_metric validation failed: Argument #{i} '
+                        (f'{user_function.__name__}() validation failed: Argument #{i} '
                          f'first dim should be as the batch size. Got {arg.shape[0]} '
                          f'instead of {leap_binder.batch_size_to_validate}')
 
             for _arg_name, arg in kwargs.items():
                 assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
-                    f'tensorleap_custom_metric validation failed: '
+                    f'{user_function.__name__}() validation failed: '
                     f'Argument {_arg_name} should be a numpy array. Got {type(arg)}.')
                 if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
                     assert arg.shape[0] == leap_binder.batch_size_to_validate, \
-                        (f'tensorleap_custom_metric validation failed: Argument {_arg_name} '
+                        (f'{user_function.__name__}() validation failed: Argument {_arg_name} '
                          f'first dim should be as the batch size. Got {arg.shape[0]} '
                          f'instead of {leap_binder.batch_size_to_validate}')
 
         def _validate_result(result) -> None:
-            supported_types_message = (f'tensorleap_custom_metric validation failed: '
-                                       f'Metric has returned unsupported type. Supported types are List[float], '
-                                       f'List[List[ConfusionMatrixElement]], NDArray[np.float32]. ')
+            validate_output_structure(result, func_name=user_function.__name__,
+                                      expected_type_name="List[float | int | None | List[ConfusionMatrixElement] ]  | NDArray[np.float32]  or dictonary with one of these types as its values types")
+            supported_types_message = (f'{user_function.__name__}() validation failed: '
+                                       f'{user_function.__name__}() has returned unsupported type.\nSupported types are List[float|int|None], '
+                                       f'List[List[ConfusionMatrixElement]], NDArray[np.float32] or dictonary with one of these types as its values types. ')
 
-            def _validate_single_metric(single_metric_result):
+            def _validate_single_metric(single_metric_result,key=None):
                 if isinstance(single_metric_result, list):
                     if isinstance(single_metric_result[0], list):
-                        assert isinstance(single_metric_result[0][0], ConfusionMatrixElement), \
-                            f'{supported_types_message}Got List[List[{type(single_metric_result[0][0])}]].'
+                        assert all(isinstance(cm, ConfusionMatrixElement) for cm in single_metric_result[0]), (
+                            f"{supported_types_message} "
+                            f"Got {'a dict where the value of ' + str(key) + ' is of type ' if key is not None else ''}"
+                            f"List[List[{', '.join(type(cm).__name__ for cm in single_metric_result[0])}]]."
+                        )
+
                     else:
-                        assert isinstance(single_metric_result[0], (
-                            float, int,
-                            type(None))), f'{supported_types_message}Got List[{type(single_metric_result[0])}].'
+                        assert all(isinstance(v, (float,int,type(None),np.float32)) for v in single_metric_result), (
+                            f"{supported_types_message}\n"
+                            f"Got {'a dict where the value of ' + str(key) + ' is of type ' if key is not None else ''}"
+                            f"List[{', '.join(type(v).__name__ for v in single_metric_result)}]."
+                        )
                 else:
                     assert isinstance(single_metric_result,
-                                      np.ndarray), f'{supported_types_message}Got {type(single_metric_result)}.'
-                    assert len(single_metric_result.shape) == 1, (f'tensorleap_custom_metric validation failed: '
+                                      np.ndarray), f'{supported_types_message}\nGot {type(single_metric_result)}.'
+                    assert len(single_metric_result.shape) == 1, (f'{user_function.__name__}() validation failed: '
                                                                   f'The return shape should be 1D. Got {len(single_metric_result.shape)}D.')
 
                 if leap_binder.batch_size_to_validate:
                     assert len(single_metric_result) == leap_binder.batch_size_to_validate, \
-                        f'tensorleap_custom_metrix validation failed: The return len should be as the batch size.'
+                        f'{user_function.__name__}() validation failed: The return len {f"of srt{key} value" if key is not None else ""} should be as the batch size.'
 
             if isinstance(result, dict):
                 for key, value in result.items():
+                    _validate_single_metric(value,key)
+
                     assert isinstance(key, str), \
-                        (f'tensorleap_custom_metric validation failed: '
+                        (f'{user_function.__name__}() validation failed: '
                          f'Keys in the return dict should be of type str. Got {type(key)}.')
                     _validate_single_metric(value)
 
                 if isinstance(direction, dict):
                     for direction_key in direction:
                         assert direction_key in result, \
-                            (f'tensorleap_custom_metric validation failed: '
+                            (f'{user_function.__name__}() validation failed: '
                              f'Keys in the direction mapping should be part of result keys. Got key {direction_key}.')
 
                 if compute_insights is not None:
                     assert isinstance(compute_insights, dict), \
-                        (f'tensorleap_custom_metric validation failed: '
+                        (f'{user_function.__name__}() validation failed: '
                          f'compute_insights should be dict if using the dict results. Got {type(compute_insights)}.')
 
                     for ci_key in compute_insights:
                         assert ci_key in result, \
-                            (f'tensorleap_custom_metric validation failed: '
+                            (f'{user_function.__name__}() validation failed: '
                              f'Keys in the compute_insights mapping should be part of result keys. Got key {ci_key}.')
 
             else:
@@ -341,7 +577,7 @@ def tensorleap_custom_metric(name: str,
 
                 if compute_insights is not None:
                     assert isinstance(compute_insights, bool), \
-                        (f'tensorleap_custom_metric validation failed: '
+                        (f'{user_function.__name__}() validation failed: '
                          f'compute_insights should be boolean. Got {type(compute_insights)}.')
 
         @functools.wraps(user_function)
@@ -373,6 +609,8 @@ def tensorleap_custom_metric(name: str,
             result = inner_without_validate(*args, **kwargs)
 
             _validate_result(result)
+            if not _update_env_status is None:
+                _update_env_status("tensorleap_custom_metric","v")
             return result
 
         def mapping_inner(*args, **kwargs):
@@ -412,28 +650,38 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
     name_to_unique_name = defaultdict(set)
 
     def decorating_function(user_function: VisualizerCallableInterface):
+        assert isinstance(visualizer_type,LeapDataType),(f"{user_function.__name__} validation failed: "
+                f"visualizer_type should be of type {LeapDataType.__name__} but got {type(visualizer_type)}"
+                                                         )
+
         for viz_handler in leap_binder.setup_container.visualizers:
             if viz_handler.visualizer_handler_data.name == name:
                 raise Exception(f'Visualizer with name {name} already exists. '
                                 f'Please choose another')
 
         def _validate_input_args(*args, **kwargs):
+            assert len(args) > 0, (
+                f"{user_function.__name__}() validation failed: "
+                f"Expected at least one positional|key-word argument of type np.ndarray, "
+                f"but received none. "
+                f"Correct usage example: {user_function.__name__}(input_array: np.ndarray, ...)"
+            )
             for i, arg in enumerate(args):
                 assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
-                    f'tensorleap_custom_visualizer validation failed: '
+                    f'{user_function.__name__}() validation failed: '
                     f'Argument #{i} should be a numpy array. Got {type(arg)}.')
                 if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
                     assert arg.shape[0] != leap_binder.batch_size_to_validate, \
-                        (f'tensorleap_custom_visualizer validation failed: '
+                        (f'{user_function.__name__}() validation failed: '
                          f'Argument #{i} should be without batch dimension. ')
 
             for _arg_name, arg in kwargs.items():
                 assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
-                    f'tensorleap_custom_visualizer validation failed: '
+                    f'{user_function.__name__}() validation failed: '
                     f'Argument {_arg_name} should be a numpy array. Got {type(arg)}.')
                 if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
                     assert arg.shape[0] != leap_binder.batch_size_to_validate, \
-                        (f'tensorleap_custom_visualizer validation failed: Argument {_arg_name} '
+                        (f'{user_function.__name__}() validation failed: Argument {_arg_name} '
                          f'should be without batch dimension. ')
 
         def _validate_result(result):
@@ -447,8 +695,11 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
                 LeapDataType.ImageWithBBox: LeapImageWithBBox,
                 LeapDataType.ImageWithHeatmap: LeapImageWithHeatmap
             }
+            validate_output_structure(result, func_name=user_function.__name__,
+                                      expected_type_name=result_type_map[visualizer_type])
+
             assert isinstance(result, result_type_map[visualizer_type]), \
-                (f'tensorleap_custom_visualizer validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'The return type should be {result_type_map[visualizer_type]}. Got {type(result)}.')
 
         @functools.wraps(user_function)
@@ -480,6 +731,8 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
             result = inner_without_validate(*args, **kwargs)
 
             _validate_result(result)
+            if not _update_env_status is None:
+                _update_env_status("tensorleap_custom_visualizer","v")
             return result
 
         def mapping_inner(*args, **kwargs):
@@ -521,30 +774,26 @@ def tensorleap_metadata(
                                 f'Please choose another')
 
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
-            assert isinstance(sample_id, (int, str)), \
-                (f'tensorleap_metadata validation failed: '
-                 f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
-            assert isinstance(preprocess_response, PreprocessResponse), \
-                (f'tensorleap_metadata validation failed: '
-                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
             assert type(sample_id) == preprocess_response.sample_id_type, \
-                (f'tensorleap_metadata validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
 
         def _validate_result(result):
             supported_result_types = (type(None), int, str, bool, float, dict, np.floating,
                                       np.bool_, np.unsignedinteger, np.signedinteger, np.integer)
+            validate_output_structure(result, func_name=user_function.__name__,
+                                      expected_type_name=supported_result_types)
             assert isinstance(result, supported_result_types), \
-                (f'tensorleap_metadata validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'Unsupported return type. Got {type(result)}. should be any of {str(supported_result_types)}')
             if isinstance(result, dict):
                 for key, value in result.items():
                     assert isinstance(key, str), \
-                        (f'tensorleap_metadata validation failed: '
+                        (f'{user_function.__name__}() validation failed: '
                          f'Keys in the return dict should be of type str. Got {type(key)}.')
                     assert isinstance(value, supported_result_types), \
-                        (f'tensorleap_metadata validation failed: '
+                        (f'{user_function.__name__}() validation failed: '
                          f'Values in the return dict should be of type {str(supported_result_types)}. Got {type(value)}.')
 
         def inner_without_validate(sample_id, preprocess_response):
@@ -561,15 +810,19 @@ def tensorleap_metadata(
 
         leap_binder.set_metadata(inner_without_validate, name, metadata_type)
 
-        def inner(sample_id, preprocess_response):
+        def inner(*args,**kwargs):
             if os.environ.get(mapping_runtime_mode_env_var_mame):
                 return None
-
+            validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
+                                    func_name=user_function.__name__, expected_names=["idx", "preprocess"],**kwargs)
+            sample_id, preprocess_response = args if len(args)!=0 else  kwargs.values()
             _validate_input_args(sample_id, preprocess_response)
 
             result = inner_without_validate(sample_id, preprocess_response)
 
             _validate_result(result)
+            if not _update_env_status is None:
+                _update_env_status("tensorleap_metadata","v")
             return result
 
         return inner
@@ -632,19 +885,23 @@ def tensorleap_preprocess():
 
         def _validate_input_args(*args, **kwargs):
             assert len(args) == 0 and len(kwargs) == 0, \
-                (f'tensorleap_preprocess validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'The function should not take any arguments. Got {args} and {kwargs}.')
 
         def _validate_result(result):
-            assert isinstance(result, list), \
-                (f'tensorleap_preprocess validation failed: '
-                 f'The return type should be a list. Got {type(result)}.')
+            assert isinstance(result, list), (
+                f"{user_function.__name__}() validation failed: expected return type list[{PreprocessResponse.__name__}]"
+                f"(e.g., [PreprocessResponse1, PreprocessResponse2, ...]), but returned type is {type(result).__name__}."
+                if not isinstance(result, tuple)
+                else f"{user_function.__name__}() validation failed: expected to return a single list[{PreprocessResponse.__name__}] object, "
+                     f"but returned {len(result)} objects instead."
+            )
             for i, response in enumerate(result):
                 assert isinstance(response, PreprocessResponse), \
-                    (f'tensorleap_preprocess validation failed: '
+                    (f'{user_function.__name__}() validation failed: '
                      f'Element #{i} in the return list should be a PreprocessResponse. Got {type(response)}.')
             assert len(set(result)) == len(result), \
-                (f'tensorleap_preprocess validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'The return list should not contain duplicate PreprocessResponse objects.')
 
         def inner(*args, **kwargs):
@@ -652,9 +909,10 @@ def tensorleap_preprocess():
                 return [None, None, None, None]
 
             _validate_input_args(*args, **kwargs)
-
             result = user_function()
             _validate_result(result)
+            if not _update_env_status is None:
+                _update_env_status("tensorleap_preprocess", "v")
             return result
 
         return inner
@@ -853,29 +1111,23 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
             raise Exception(f"Channel dim for input {name} is expected to be either -1 or positive")
 
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
-            assert isinstance(sample_id, (int, str)), \
-                (f'tensorleap_input_encoder validation failed: '
-                 f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
-            assert isinstance(preprocess_response, PreprocessResponse), \
-                (f'tensorleap_input_encoder validation failed: '
-                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
             assert type(sample_id) == preprocess_response.sample_id_type, \
-                (f'tensorleap_input_encoder validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
 
         def _validate_result(result):
+            validate_output_structure(result, func_name=user_function.__name__, expected_type_name = "np.ndarray")
             assert isinstance(result, np.ndarray), \
-                (f'tensorleap_input_encoder validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
             assert result.dtype == np.float32, \
-                (f'tensorleap_input_encoder validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'The return type should be a numpy array of type float32. Got {result.dtype}.')
-            assert channel_dim - 1 <= len(result.shape), (f'tensorleap_input_encoder validation failed: '
+            assert channel_dim - 1 <= len(result.shape), (f'{user_function.__name__}() validation failed: '
                                                           f'The channel_dim ({channel_dim}) should be <= to the rank of the resulting input rank ({len(result.shape)}).')
 
         def inner_without_validate(sample_id, preprocess_response):
-
             global _called_from_inside_tl_decorator
             _called_from_inside_tl_decorator += 1
 
@@ -889,7 +1141,10 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
         leap_binder.set_input(inner_without_validate, name, channel_dim=channel_dim)
 
 
-        def inner(sample_id, preprocess_response):
+        def inner(*args, **kwargs):
+            validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
+                                    func_name=user_function.__name__,    expected_names=["idx", "preprocess"], **kwargs)
+            sample_id, preprocess_response = args if len(args)!=0 else  kwargs.values()
             _validate_input_args(sample_id, preprocess_response)
 
             result = inner_without_validate(sample_id, preprocess_response)
@@ -897,24 +1152,20 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
             _validate_result(result)
 
             if _called_from_inside_tl_decorator == 0 and _called_from_inside_tl_integration_test_decorator:
+                batch_warning(result,user_function.__name__)
                 result = np.expand_dims(result, axis=0)
-                # Emit integration test event once per test
-                _emit_integration_event_once('input_encoder_integration_test', {
-                    'encoder_name': name,
-                    'channel_dim': channel_dim,
-                    'model_input_index': model_input_index
-                })
+            if not _update_env_status is None:
+                _update_env_status("tensorleap_input_encoder", "v")
 
             return result
 
 
-
         node_mapping_type = NodeMappingType.Input
         if model_input_index is not None:
             node_mapping_type = NodeMappingType(f'Input{str(model_input_index)}')
         inner.node_mapping = NodeMapping(name, node_mapping_type)
 
-        def mapping_inner(sample_id, preprocess_response):
+        def mapping_inner(*args, **kwargs):
             class TempMapping:
                 pass
 
@@ -926,11 +1177,11 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
 
         mapping_inner.node_mapping = NodeMapping(name, node_mapping_type)
 
-        def final_inner(sample_id, preprocess_response):
+        def final_inner(*args, **kwargs):
             if os.environ.get(mapping_runtime_mode_env_var_mame):
-                return mapping_inner(sample_id, preprocess_response)
+                return mapping_inner(*args, **kwargs)
             else:
-                return inner(sample_id, preprocess_response)
+                return inner(*args, **kwargs)
 
         final_inner.node_mapping = NodeMapping(name, node_mapping_type)
 
@@ -947,23 +1198,18 @@ def tensorleap_gt_encoder(name: str):
                                 f'Please choose another')
 
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
-            assert isinstance(sample_id, (int, str)), \
-                (f'tensorleap_gt_encoder validation failed: '
-                 f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
-            assert isinstance(preprocess_response, PreprocessResponse), \
-                (f'tensorleap_gt_encoder validation failed: '
-                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
             assert type(sample_id) == preprocess_response.sample_id_type, \
-                (f'tensorleap_gt_encoder validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
 
         def _validate_result(result):
+            validate_output_structure(result, func_name=user_function.__name__, expected_type_name = "np.ndarray",gt_flag=True)
             assert isinstance(result, np.ndarray), \
-                (f'tensorleap_gt_encoder validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
             assert result.dtype == np.float32, \
-                (f'tensorleap_gt_encoder validation failed: '
+                (f'{user_function.__name__}() validation failed: '
                  f'The return type should be a numpy array of type float32. Got {result.dtype}.')
 
         def inner_without_validate(sample_id, preprocess_response):
@@ -980,7 +1226,10 @@ def tensorleap_gt_encoder(name: str):
         leap_binder.set_ground_truth(inner_without_validate, name)
 
 
-        def inner(sample_id, preprocess_response):
+        def inner(*args, **kwargs):
+            validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
+                                    func_name=user_function.__name__, expected_names=["idx", "preprocess"], **kwargs)
+            sample_id, preprocess_response = args
             _validate_input_args(sample_id, preprocess_response)
 
             result = inner_without_validate(sample_id, preprocess_response)
@@ -988,13 +1237,15 @@ def tensorleap_gt_encoder(name: str):
             _validate_result(result)
 
             if _called_from_inside_tl_decorator == 0 and _called_from_inside_tl_integration_test_decorator:
+                batch_warning(result, user_function.__name__)
                 result = np.expand_dims(result, axis=0)
-
+            if not _update_env_status is None:
+                _update_env_status("tensorleap_gt_encoder", "v")
             return result
 
         inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
 
-        def mapping_inner(sample_id, preprocess_response):
+        def mapping_inner(*args, **kwargs):
             class TempMapping:
                 pass
 
@@ -1005,11 +1256,11 @@ def tensorleap_gt_encoder(name: str):
 
         mapping_inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
 
-        def final_inner(sample_id, preprocess_response):
+        def final_inner(*args, **kwargs):
             if os.environ.get(mapping_runtime_mode_env_var_mame):
-                return mapping_inner(sample_id, preprocess_response)
+                return mapping_inner(*args, **kwargs)
             else:
-                return inner(sample_id, preprocess_response)
+                return inner(*args, **kwargs)
 
         final_inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
 
@@ -1030,28 +1281,37 @@ def tensorleap_custom_loss(name: str, connects_to=None):
         valid_types = (np.ndarray, SamplePreprocessResponse)
 
         def _validate_input_args(*args, **kwargs):
+            assert len(args) > 0 and len(kwargs)==0, (
+                f"{user_function.__name__}() validation failed: "
+                f"Expected at least one positional|key-word argument of the allowed types (np.ndarray|SamplePreprocessResponse|list(np.ndarray|SamplePreprocessResponse)). "
+                f"but received none. "
+                f"Correct usage example: {user_function.__name__}(input_array: np.ndarray, ...)"
+            )
             for i, arg in enumerate(args):
                 if isinstance(arg, list):
                     for y, elem in enumerate(arg):
-                        assert isinstance(elem, valid_types), (f'tensorleap_custom_loss validation failed: '
+                        assert isinstance(elem, valid_types), (f'{user_function.__name__}() validation failed: '
                                                                f'Element #{y} of list should be a numpy array. Got {type(elem)}.')
                 else:
-                    assert isinstance(arg, valid_types), (f'tensorleap_custom_loss validation failed: '
+                    assert isinstance(arg, valid_types), (f'{user_function.__name__}() validation failed: '
                                                           f'Argument #{i} should be a numpy array. Got {type(arg)}.')
             for _arg_name, arg in kwargs.items():
                 if isinstance(arg, list):
                     for y, elem in enumerate(arg):
-                        assert isinstance(elem, valid_types), (f'tensorleap_custom_loss validation failed: '
+                        assert isinstance(elem, valid_types), (f'{user_function.__name__}() validation failed: '
                                                                f'Element #{y} of list should be a numpy array. Got {type(elem)}.')
                 else:
-                    assert isinstance(arg, valid_types), (f'tensorleap_custom_loss validation failed: '
+                    assert isinstance(arg, valid_types), (f'{user_function.__name__}() validation failed: '
                                                           f'Argument #{_arg_name} should be a numpy array. Got {type(arg)}.')
 
         def _validate_result(result):
+            validate_output_structure(result, func_name=user_function.__name__,
+                                      expected_type_name="np.ndarray")
             assert isinstance(result, np.ndarray), \
-                (f'tensorleap_custom_loss validation failed: '
+                (f'{user_function.__name__} validation failed: '
                  f'The return type should be a numpy array. Got {type(result)}.')
-
+            assert  result.ndim<2 ,(f'{user_function.__name__}  validation failed: '
+                 f'The return type should be a 1Dim numpy array but got {result.ndim}Dim.')
 
         @functools.wraps(user_function)
         def inner_without_validate(*args, **kwargs):
@@ -1082,6 +1342,9 @@ def tensorleap_custom_loss(name: str, connects_to=None):
             result = inner_without_validate(*args, **kwargs)
 
             _validate_result(result)
+            if not _update_env_status is None:
+                _update_env_status("tensorleap_custom_loss", "v")
+
             return result
 
         def mapping_inner(*args, **kwargs):
@@ -1138,3 +1401,88 @@ def tensorleap_custom_layer(name: str):
         return custom_layer
 
     return decorating_function
+
+
+def tensorleap_status_table():
+    '''
+    Usage example:
+    ###################
+    leap_integration.py
+    ###################
+    from code_loader.inner_leap_binder.leapbinder_decorators import tensorleap_status_table
+    ...
+    ...
+    ...
+    if __name__ == '__main__':
+        tensorleap_status_table()
+        ...
+    '''
+    import atexit
+    import sys
+    import traceback
+    CHECK = "✅"
+    CROSS = "❌"
+
+    table = [
+        {"name": "tensorleap_preprocess", "Added to integration": CROSS},
+        {"name": "tensorleap_integration_test", "Added to integration": CROSS},
+        {"name": "tensorleap_input_encoder", "Added to integration": CROSS},
+        {"name": "tensorleap_gt_encoder", "Added to integration": CROSS},
+        {"name": "tensorleap_load_model", "Added to integration": CROSS},
+        {"name": "tensorleap_custom_loss", "Added to integration": CROSS},
+        {"name": "tensorleap_custom_metric (optional)", "Added to integration": CROSS},
+        {"name": "tensorleap_metadata (optional)", "Added to integration": CROSS},
+        {"name": "tensorleap_custom_visualizer (optional)", "Added to integration": CROSS},
+
+    ]
+
+    _finalizer_called = {"done": False}
+
+    def _print_table():
+        ready_mess = "\nAll parts have been successfully set. If no errors accured, you can now push the project to the Tensorleap system."
+        not_ready_mess = "\nSome mandatory components have not yet been added to the Integration test. Recommended next interface to add is: "
+        mandatory_ready_mess = "\nAll mandatory parts have been successfully set. If no errors accured, you can now push the project to the Tensorleap system or continue to the next optional reccomeded interface,adding: "
+
+        name_width = max(len(row["name"]) for row in table)
+        status_width = max(len(row["Added to integration"]) for row in table)
+        header = f"{'Decorator Name'.ljust(name_width)} | {'Added to integration'.ljust(status_width)}"
+        sep = "-" * len(header)
+        print("\n" + header)
+        print(sep)
+        ready=True
+        for row in table:
+            print(f"{row['name'].ljust(name_width)} | {row['Added to integration'].ljust(status_width)}")
+            if row['Added to integration']==CROSS and ready:
+                ready=False
+                next_step=row['name']
+
+
+        print(ready_mess) if ready else print(mandatory_ready_mess+next_step) if "optional" in next_step else print(not_ready_mess+next_step)
+    def update_env_params(name: str, status: str = "✓"):
+        for row in table:
+            if row["name"].removesuffix(" (optional)") == name:
+                row["Added to integration"] = CHECK if status=="v" else CROSS
+                break
+    def run_on_exit():
+        if _finalizer_called["done"]:
+            return
+        _finalizer_called["done"] = True
+        _print_table()
+    def handle_exception(exc_type, exc_value, exc_traceback):
+        traceback.print_exception(exc_type, exc_value, exc_traceback)
+        run_on_exit()
+    atexit.register(run_on_exit)
+    sys.excepthook = handle_exception
+    global _update_env_status
+    _update_env_status = update_env_params
+    return update_env_params
+
+
+
+
+
+
+
+
+
+

{code_loader-1.0.139.dev4 → code_loader-1.0.139.dev6}/code_loader/mixpanel_tracker.py

@@ -126,43 +126,6 @@ class MixpanelTracker:
         except Exception as e:
             pass
 
-    def track_integration_test_event(self, event_name: str, event_properties: Optional[Dict[str, Any]] = None) -> None:
-        """Track an integration test event with device identification.
-        
-        Args:
-            event_name: The name of the event to track
-            event_properties: Optional additional properties to include in the event
-        """
-        # Skip tracking if IS_TENSORLEAP_PLATFORM environment variable is set to 'true'
-        if os.environ.get('IS_TENSORLEAP_PLATFORM') == 'true':
-            return
-            
-        try:
-            distinct_id = self._get_distinct_id()
-            
-            tensorleap_user_id = self._get_tensorleap_user_id()
-            whoami = self._get_whoami()
-            device_id = self._get_or_create_device_id()
-            
-            properties = {
-                'tracking_version': TRACKING_VERSION,
-                'service': 'code-loader',
-                'whoami': whoami,
-                '$device_id': device_id,  # Always use device_id for $device_id
-                'python_version': f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
-                'platform': os.name,
-            }
-            
-            if tensorleap_user_id:
-                properties['user_id'] = tensorleap_user_id
-            
-            if event_properties:
-                properties.update(event_properties)
-            
-            self.mp.track(distinct_id, event_name, properties)
-        except Exception as e:
-            pass
-
 
 # Global tracker instance
 _tracker = None
@@ -177,7 +140,3 @@ def get_tracker() -> MixpanelTracker:
 
 def track_code_loader_loaded(event_properties: Optional[Dict[str, Any]] = None) -> None:
     get_tracker().track_code_loader_loaded(event_properties)
-
-
-def track_integration_test_event(event_name: str, event_properties: Optional[Dict[str, Any]] = None) -> None:
-    get_tracker().track_integration_test_event(event_name, event_properties)

{code_loader-1.0.139.dev4 → code_loader-1.0.139.dev6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.139.dev4"
+version = "1.0.139.dev6"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"