code-loader 1.0.139.dev3__tar.gz → 1.0.139.dev4__tar.gz

{code_loader-1.0.139.dev3 → code_loader-1.0.139.dev4}/PKG-INFO

@@ -1,9 +1,9 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: code-loader
-Version: 1.0.139.dev3
+Version: 1.0.139.dev4
 Summary: 
-Home-page: https://github.com/tensorleap/code-loader
 License: MIT
+License-File: LICENSE
 Author: dorhar
 Author-email: doron.harnoy@tensorleap.ai
 Requires-Python: >=3.8,<3.13
@@ -20,6 +20,7 @@ Requires-Dist: numpy (>=2.3.2,<3.0.0) ; python_version >= "3.11" and python_vers
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
+Project-URL: Homepage, https://github.com/tensorleap/code-loader
 Project-URL: Repository, https://github.com/tensorleap/code-loader
 Description-Content-Type: text/markdown
 

{code_loader-1.0.139.dev3 → code_loader-1.0.139.dev4}/code_loader/contract/datasetclasses.py

@@ -1,4 +1,3 @@
-import warnings
 from dataclasses import dataclass, field
 from typing import Any, Callable, List, Optional, Dict, Union, Type
 import re
@@ -57,14 +56,7 @@ class PreprocessResponse:
                 for sample_id in self.sample_ids:
                     assert isinstance(sample_id, str), f"Sample id should be of type str. Got: {type(sample_id)}"
         else:
-            raise Exception("length is deprecated, please use sample_ids instead.")
-
-        if self.state is None:
-            warnings.warn(
-                "PreprocessResponse.state is not set. For best practice, assign a unique `state` value to each PreprocessResponse instance."
-            )
-        else:
-            assert isinstance(self.state, DataStateType), f"PreprocessResponse.state must be of type {DataStateType.__name__} but got {type(self.state)}"
+            raise Exception("length is deprecated.")
 
     def __hash__(self) -> int:
         return id(self)

{code_loader-1.0.139.dev3 → code_loader-1.0.139.dev4}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1,10 +1,9 @@
 # mypy: ignore-errors
 import os
-import warnings
 from collections import defaultdict
 from functools import lru_cache
 from pathlib import Path
-from typing import Optional, Union, Callable, List, Dict, get_args, get_origin
+from typing import Optional, Union, Callable, List, Dict, Set, Any
 
 import numpy as np
 import numpy.typing as npt
@@ -26,116 +25,9 @@ import functools
 
 _called_from_inside_tl_decorator = 0
 _called_from_inside_tl_integration_test_decorator = False
-_update_env_status = None
-
-
-
-
-def validate_args_structure(*args, types_order, func_name, expected_names, **kwargs):
-    def _type_to_str(t):
-        origin = get_origin(t)
-        if origin is Union:
-            return " | ".join(tt.__name__ for tt in get_args(t))
-        elif hasattr(t, "__name__"):
-            return t.__name__
-        else:
-            return str(t)
-
-    def _format_types(types, names=None):
-        return ", ".join(
-            f"{(names[i] + ': ') if names else f'arg{i}: '}{_type_to_str(ty)}"
-            for i, ty in enumerate(types)
-        )
-
-    if expected_names:
-        normalized_args = []
-        for i, name in enumerate(expected_names):
-            if i < len(args):
-                normalized_args.append(args[i])
-            elif name in kwargs:
-                normalized_args.append(kwargs[name])
-            else:
-                raise AssertionError(
-                    f"{func_name} validation failed: "
-                    f"Missing required argument '{name}'. "
-                    f"Expected arguments: {expected_names}."
-                )
-    else:
-        normalized_args = list(args)
-    if len(normalized_args) != len(types_order):
-        expected = _format_types(types_order, expected_names)
-        got_types = ", ".join(type(arg).__name__ for arg in normalized_args)
-        raise AssertionError(
-            f"{func_name} validation failed: "
-            f"Expected exactly {len(types_order)} arguments ({expected}), "
-            f"but got {len(normalized_args)} argument(s) of type(s): ({got_types}). "
-            f"Correct usage example: {func_name}({expected})"
-        )
-
-    for i, (arg, expected_type) in enumerate(zip(normalized_args, types_order)):
-        origin = get_origin(expected_type)
-        if origin is Union:
-            allowed_types = get_args(expected_type)
-        else:
-            allowed_types = (expected_type,)
-
-        if not isinstance(arg, allowed_types):
-            allowed_str = " | ".join(t.__name__ for t in allowed_types)
-            raise AssertionError(
-                f"{func_name} validation failed: "
-                f"Argument '{expected_names[i] if expected_names else f'arg{i}'}' "
-                f"expected type {allowed_str}, but got {type(arg).__name__}. "
-                f"Correct usage example: {func_name}({_format_types(types_order, expected_names)})"
-            )
-
-
-def validate_output_structure(result, func_name: str, expected_type_name="np.ndarray",gt_flag=False):
-    if result is None or (isinstance(result, float) and np.isnan(result)):
-        if gt_flag:
-            raise AssertionError(
-                f"{func_name} validation failed: "
-                f"The function returned {result!r}. "
-                f"If you are working with an unlabeled dataset and no ground truth is available, "
-                f"use 'return np.array([], dtype=np.float32)' instead. "
-                f"Otherwise, {func_name} expected a single {expected_type_name} object. "
-                f"Make sure the function ends with 'return <{expected_type_name}>'."
-            )
-
-        raise AssertionError(
-            f"{func_name} validation failed: "
-            f"The function returned None. "
-            f"Expected a single {expected_type_name} object. "
-            f"Make sure the function ends with 'return <{expected_type_name}>'."
-        )
-    if isinstance(result, tuple):
-        element_descriptions = [
-            f"[{i}] type: {type(r).__name__}"
-            for i, r in enumerate(result)
-        ]
-        element_summary = "\n    ".join(element_descriptions)
-
-        raise AssertionError(
-            f"{func_name} validation failed: "
-            f"The function returned multiple outputs ({len(result)} values), "
-            f"but only a single {expected_type_name} is allowed.\n\n"
-            f"Returned elements:\n"
-            f"    {element_summary}\n\n"
-            f"Correct usage example:\n"
-            f"    def {func_name}(...):\n"
-            f"        return <{expected_type_name}>\n\n"
-            f"If you intended to return multiple values, combine them into a single "
-            f"{expected_type_name} (e.g., by concatenation or stacking)."
-        )
-
-def batch_warning(result, func_name):
-    if result.shape[0] == 1:
-        warnings.warn(
-            f"{func_name} warning: Tensorleap will add a batch dimension at axis 0 to the output of {func_name}, "
-            f"although the detected size of axis 0 is already 1. "
-            f"This may lead to an extra batch dimension (e.g., shape (1, 1, ...)). "
-            f"Please ensure that the output of '{func_name}' is not already batched "
-            f"to avoid computation errors."
-        )
+_integration_events_emitted: Set[str] = set()
+
+
 def _add_mapping_connection(user_unique_name, connection_destinations, arg_names, name, node_mapping_type):
     connection_destinations = [connection_destination for connection_destination in connection_destinations
                                if not isinstance(connection_destination, SamplePreprocessResponse)]
@@ -158,24 +50,12 @@ def tensorleap_integration_test():
     def decorating_function(integration_test_function: Callable):
         leap_binder.integration_test_func = integration_test_function
 
-        def _validate_input_args(*args, **kwargs):
-            sample_id,preprocess_response=args
-            assert type(sample_id) == preprocess_response.sample_id_type, (
-                f"tensorleap_integration_test validation failed: "
-                f"sample_id type ({type(sample_id).__name__}) does not match the expected "
-                f"type ({preprocess_response.sample_id_type}) from the PreprocessResponse."
-            )
-
         def inner(*args, **kwargs):
-            validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
-                                    func_name='integration_test',expected_names=["idx", "preprocess"],**kwargs)
-            _validate_input_args(*args, **kwargs)
-
             global _called_from_inside_tl_integration_test_decorator
+            global _integration_events_emitted
+            _integration_events_emitted.clear()  # Clear events for new test
             try:
                 _called_from_inside_tl_integration_test_decorator = True
-                if not _update_env_status is None:
-                    _update_env_status("tensorleap_integration_test", "v")#put here because otherwise it will become  v only if it finishes all the script
                 ret = integration_test_function(*args, **kwargs)
 
                 try:
@@ -188,7 +68,7 @@ def tensorleap_integration_test():
                     line_number = first_tb.lineno
                     if isinstance(e, TypeError) and 'is not subscriptable' in str(e):
                         print(f'Invalid integration code. File {file_name}, line {line_number}: '
-                              f"indexing is supported only on the model's predictions inside the integration test. Please remove this indexing operation usage from the integration test code.")
+                              f'Please remove this indexing operation usage from the integration test code.')
                     else:
                         print(f'Invalid integration code. File {file_name}, line {line_number}: '
                               f'Integration test is only allowed to call Tensorleap decorators. '
@@ -200,8 +80,8 @@ def tensorleap_integration_test():
                 _called_from_inside_tl_integration_test_decorator = False
 
             leap_binder.check()
-        return inner
 
+        return inner
 
     return decorating_function
 
@@ -211,52 +91,31 @@ def _safe_get_item(key):
     except ValueError:
         raise Exception(f'Tensorleap currently supports models with no more then 10 inputs')
 
+def _emit_integration_event_once(event_name: str, props: Dict[str, Any]) -> None:
+    """Emit an integration test event only once per test run."""
+    if event_name in _integration_events_emitted:
+        return
+    
+    try:
+        from code_loader.mixpanel_tracker import track_integration_test_event
+        track_integration_test_event(event_name, props)
+        _integration_events_emitted.add(event_name)
+    except Exception:
+        pass
+
 def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]] = []):
-    assert isinstance(prediction_types, list),(
-        f"tensorleap_load_model validation failed: "
-                f" prediction_types is an optional argument of type List[PredictionTypeHandler]] but got {type(prediction_types).__name__}."
-    )
     for i, prediction_type in enumerate(prediction_types):
-        assert isinstance(prediction_type, PredictionTypeHandler),(f"tensorleap_load_model validation failed: "
-                f" prediction_types at position {i} must be of type PredictionTypeHandler but got {type(prediction_types[i]).__name__}.")
         leap_binder.add_prediction(prediction_type.name, prediction_type.labels, prediction_type.channel_dim, i)
 
-    def _validate_result(result) -> None:
-        valid_types=["onnxruntime","keras"]
-        err_message=f"tensorleap_load_model validation failed:\nSupported models are Keras and onnxruntime only and non of them was returned."
-        validate_output_structure(result, func_name="tensorleap_load_model", expected_type_name= [" | ".join(t for t in valid_types)][0])
-        try:
-            import keras
-        except ImportError:
-            keras = None
-        try:
-            import onnxruntime
-        except ImportError:
-            onnxruntime = None
-
-        if not keras and not onnxruntime:
-            raise AssertionError(err_message)
-
-        is_keras_model = bool(keras and isinstance(result, getattr(keras, "Model", tuple())))
-        is_onnx_model = bool(onnxruntime and isinstance(result, onnxruntime.InferenceSession))
-
-        if not any([is_keras_model, is_onnx_model]):
-            raise AssertionError( err_message)
-
-
-
     def decorating_function(load_model_func):
         class TempMapping:
             pass
 
         @lru_cache()
-        def inner(*args, **kwargs):
-            validate_args_structure(*args, types_order=[],
-                                    func_name='tensorleap_load_model',expected_names=[],**kwargs)
+        def inner():
             class ModelPlaceholder:
                 def __init__(self):
-                    self.model = load_model_func() #TODO- check why this fails on onnx model
-                    _validate_result(self.model)
+                    self.model = load_model_func()
 
                 # keras interface
                 def __call__(self, arg):
@@ -320,10 +179,8 @@ def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]
 
                 def get_inputs(self):
                     return self.model.get_inputs()
-            model_placeholder=ModelPlaceholder()
-            if not _update_env_status is None:
-                _update_env_status("tensorleap_load_model", "v")
-            return model_placeholder
+
+            return ModelPlaceholder()
 
         def mapping_inner():
             class ModelOutputPlaceholder:
@@ -386,11 +243,12 @@ def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]
 
             return ModelPlaceholder()
 
-        def final_inner(*args, **kwargs):
+        def final_inner():
             if os.environ.get(mapping_runtime_mode_env_var_mame):
                 return mapping_inner()
             else:
-                return inner(*args, **kwargs)
+                return inner()
+
         return final_inner
 
     return decorating_function
@@ -401,168 +259,81 @@ def tensorleap_custom_metric(name: str,
                              compute_insights: Optional[Union[bool, Dict[str, bool]]] = None,
                              connects_to=None):
     name_to_unique_name = defaultdict(set)
+
     def decorating_function(
             user_function: Union[CustomCallableInterfaceMultiArgs, CustomMultipleReturnCallableInterfaceMultiArgs,
             ConfusionMatrixCallableInterfaceMultiArgs]):
-
-        def _validate_decorators_signature():
-            err_message = f"{user_function.__name__} validation failed.\n"
-            if not isinstance(name, str):
-                raise TypeError(err_message + f"`name` must be a string, got type {type(name).__name__}.")
-            valid_directions = {MetricDirection.Upward, MetricDirection.Downward}
-            if isinstance(direction, MetricDirection):
-                if direction not in valid_directions:
-                    raise ValueError(
-                        err_message +
-                        f"Invalid MetricDirection: {direction}. Must be one of {valid_directions}, "
-                        f"got type {type(direction).__name__}."
-                    )
-            elif isinstance(direction, dict):
-                if not all(isinstance(k, str) for k in direction.keys()):
-                    invalid_keys = {k: type(k).__name__ for k in direction.keys() if not isinstance(k, str)}
-                    raise TypeError(
-                        err_message +
-                        f"All keys in `direction` must be strings, got invalid key types: {invalid_keys}."
-                    )
-                for k, v in direction.items():
-                    if v not in valid_directions:
-                        raise ValueError(
-                            err_message +
-                            f"Invalid direction for key '{k}': {v}. Must be one of {valid_directions}, "
-                            f"got type {type(v).__name__}."
-                        )
-            else:
-                raise TypeError(
-                    err_message +
-                    f"`direction` must be a MetricDirection or a Dict[str, MetricDirection], "
-                    f"got type {type(direction).__name__}."
-                )
-            if compute_insights is not None:
-                if not isinstance(compute_insights, (bool, dict)):
-                    raise TypeError(
-                        err_message +
-                        f"`compute_insights` must be a bool or a Dict[str, bool], "
-                        f"got type {type(compute_insights).__name__}."
-                    )
-                if isinstance(compute_insights, dict):
-                    if not all(isinstance(k, str) for k in compute_insights.keys()):
-                        invalid_keys = {k: type(k).__name__ for k in compute_insights.keys() if not isinstance(k, str)}
-                        raise TypeError(
-                            err_message +
-                            f"All keys in `compute_insights` must be strings, got invalid key types: {invalid_keys}."
-                        )
-                    for k, v in compute_insights.items():
-                        if not isinstance(v, bool):
-                            raise TypeError(
-                                err_message +
-                                f"Invalid type for compute_insights['{k}']: expected bool, got type {type(v).__name__}."
-                            )
-            if connects_to is not None:
-                valid_types = (str, list, tuple, set)
-                if not isinstance(connects_to, valid_types):
-                    raise TypeError(
-                        err_message +
-                        f"`connects_to` must be one of {valid_types}, got type {type(connects_to).__name__}."
-                    )
-                if isinstance(connects_to, (list, tuple, set)):
-                    invalid_elems = [f"{type(e).__name__}" for e in connects_to if not isinstance(e, str)]
-                    if invalid_elems:
-                        raise TypeError(
-                            err_message +
-                            f"All elements in `connects_to` must be strings, "
-                            f"but found element types: {invalid_elems}."
-                        )
-
-
-        _validate_decorators_signature()
-
         for metric_handler in leap_binder.setup_container.metrics:
             if metric_handler.metric_handler_data.name == name:
                 raise Exception(f'Metric with name {name} already exists. '
                                 f'Please choose another')
 
         def _validate_input_args(*args, **kwargs) -> None:
-            assert len(args) > 0, (
-                f"{user_function.__name__}() validation failed: "
-                f"Expected at least one positional|key-word argument of type np.ndarray, "
-                f"but received none. "
-                f"Correct usage example: tensorleap_custom_metric(input_array: np.ndarray, ...)"
-            )
             for i, arg in enumerate(args):
                 assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
-                    f'{user_function.__name__}() validation failed: '
+                    f'tensorleap_custom_metric validation failed: '
                     f'Argument #{i} should be a numpy array. Got {type(arg)}.')
                 if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
                     assert arg.shape[0] == leap_binder.batch_size_to_validate, \
-                        (f'{user_function.__name__}() validation failed: Argument #{i} '
+                        (f'tensorleap_custom_metric validation failed: Argument #{i} '
                          f'first dim should be as the batch size. Got {arg.shape[0]} '
                          f'instead of {leap_binder.batch_size_to_validate}')
 
             for _arg_name, arg in kwargs.items():
                 assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
-                    f'{user_function.__name__}() validation failed: '
+                    f'tensorleap_custom_metric validation failed: '
                     f'Argument {_arg_name} should be a numpy array. Got {type(arg)}.')
                 if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
                     assert arg.shape[0] == leap_binder.batch_size_to_validate, \
-                        (f'{user_function.__name__}() validation failed: Argument {_arg_name} '
+                        (f'tensorleap_custom_metric validation failed: Argument {_arg_name} '
                          f'first dim should be as the batch size. Got {arg.shape[0]} '
                          f'instead of {leap_binder.batch_size_to_validate}')
 
         def _validate_result(result) -> None:
-            validate_output_structure(result, func_name=user_function.__name__,
-                                      expected_type_name="List[float | int | None | List[ConfusionMatrixElement] ]  | NDArray[np.float32]  or dictonary with one of these types as its values types")
-            supported_types_message = (f'{user_function.__name__}() validation failed: '
-                                       f'{user_function.__name__}() has returned unsupported type.\nSupported types are List[float|int|None], '
-                                       f'List[List[ConfusionMatrixElement]], NDArray[np.float32] or dictonary with one of these types as its values types. ')
+            supported_types_message = (f'tensorleap_custom_metric validation failed: '
+                                       f'Metric has returned unsupported type. Supported types are List[float], '
+                                       f'List[List[ConfusionMatrixElement]], NDArray[np.float32]. ')
 
-            def _validate_single_metric(single_metric_result,key=None):
+            def _validate_single_metric(single_metric_result):
                 if isinstance(single_metric_result, list):
                     if isinstance(single_metric_result[0], list):
-                        assert all(isinstance(cm, ConfusionMatrixElement) for cm in single_metric_result[0]), (
-                            f"{supported_types_message} "
-                            f"Got {'a dict where the value of ' + str(key) + ' is of type ' if key is not None else ''}"
-                            f"List[List[{', '.join(type(cm).__name__ for cm in single_metric_result[0])}]]."
-                        )
-
+                        assert isinstance(single_metric_result[0][0], ConfusionMatrixElement), \
+                            f'{supported_types_message}Got List[List[{type(single_metric_result[0][0])}]].'
                     else:
-                        assert all(isinstance(v, (float,int,type(None),np.float32)) for v in single_metric_result), (
-                            f"{supported_types_message}\n"
-                            f"Got {'a dict where the value of ' + str(key) + ' is of type ' if key is not None else ''}"
-                            f"List[{', '.join(type(v).__name__ for v in single_metric_result)}]."
-                        )
+                        assert isinstance(single_metric_result[0], (
+                            float, int,
+                            type(None))), f'{supported_types_message}Got List[{type(single_metric_result[0])}].'
                 else:
                     assert isinstance(single_metric_result,
-                                      np.ndarray), f'{supported_types_message}\nGot {type(single_metric_result)}.'
-                    assert len(single_metric_result.shape) == 1, (f'{user_function.__name__}() validation failed: '
+                                      np.ndarray), f'{supported_types_message}Got {type(single_metric_result)}.'
+                    assert len(single_metric_result.shape) == 1, (f'tensorleap_custom_metric validation failed: '
                                                                   f'The return shape should be 1D. Got {len(single_metric_result.shape)}D.')
 
                 if leap_binder.batch_size_to_validate:
                     assert len(single_metric_result) == leap_binder.batch_size_to_validate, \
-                        f'{user_function.__name__}() validation failed: The return len {f"of srt{key} value" if key is not None else ""} should be as the batch size.'
+                        f'tensorleap_custom_metrix validation failed: The return len should be as the batch size.'
 
             if isinstance(result, dict):
                 for key, value in result.items():
-                    _validate_single_metric(value,key)
-
                     assert isinstance(key, str), \
-                        (f'{user_function.__name__}() validation failed: '
+                        (f'tensorleap_custom_metric validation failed: '
                          f'Keys in the return dict should be of type str. Got {type(key)}.')
                     _validate_single_metric(value)
 
                 if isinstance(direction, dict):
                     for direction_key in direction:
                         assert direction_key in result, \
-                            (f'{user_function.__name__}() validation failed: '
+                            (f'tensorleap_custom_metric validation failed: '
                              f'Keys in the direction mapping should be part of result keys. Got key {direction_key}.')
 
                 if compute_insights is not None:
                     assert isinstance(compute_insights, dict), \
-                        (f'{user_function.__name__}() validation failed: '
+                        (f'tensorleap_custom_metric validation failed: '
                          f'compute_insights should be dict if using the dict results. Got {type(compute_insights)}.')
 
                     for ci_key in compute_insights:
                         assert ci_key in result, \
-                            (f'{user_function.__name__}() validation failed: '
+                            (f'tensorleap_custom_metric validation failed: '
                              f'Keys in the compute_insights mapping should be part of result keys. Got key {ci_key}.')
 
             else:
@@ -570,7 +341,7 @@ def tensorleap_custom_metric(name: str,
 
                 if compute_insights is not None:
                     assert isinstance(compute_insights, bool), \
-                        (f'{user_function.__name__}() validation failed: '
+                        (f'tensorleap_custom_metric validation failed: '
                          f'compute_insights should be boolean. Got {type(compute_insights)}.')
 
         @functools.wraps(user_function)
@@ -602,8 +373,6 @@ def tensorleap_custom_metric(name: str,
             result = inner_without_validate(*args, **kwargs)
 
             _validate_result(result)
-            if not _update_env_status is None:
-                _update_env_status("tensorleap_custom_metric","v")
             return result
 
         def mapping_inner(*args, **kwargs):
@@ -643,38 +412,28 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
     name_to_unique_name = defaultdict(set)
 
     def decorating_function(user_function: VisualizerCallableInterface):
-        assert isinstance(visualizer_type,LeapDataType),(f"{user_function.__name__} validation failed: "
-                f"visualizer_type should be of type {LeapDataType.__name__} but got {type(visualizer_type)}"
-                                                         )
-
         for viz_handler in leap_binder.setup_container.visualizers:
             if viz_handler.visualizer_handler_data.name == name:
                 raise Exception(f'Visualizer with name {name} already exists. '
                                 f'Please choose another')
 
         def _validate_input_args(*args, **kwargs):
-            assert len(args) > 0, (
-                f"{user_function.__name__}() validation failed: "
-                f"Expected at least one positional|key-word argument of type np.ndarray, "
-                f"but received none. "
-                f"Correct usage example: {user_function.__name__}(input_array: np.ndarray, ...)"
-            )
             for i, arg in enumerate(args):
                 assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
-                    f'{user_function.__name__}() validation failed: '
+                    f'tensorleap_custom_visualizer validation failed: '
                     f'Argument #{i} should be a numpy array. Got {type(arg)}.')
                 if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
                     assert arg.shape[0] != leap_binder.batch_size_to_validate, \
-                        (f'{user_function.__name__}() validation failed: '
+                        (f'tensorleap_custom_visualizer validation failed: '
                          f'Argument #{i} should be without batch dimension. ')
 
             for _arg_name, arg in kwargs.items():
                 assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
-                    f'{user_function.__name__}() validation failed: '
+                    f'tensorleap_custom_visualizer validation failed: '
                     f'Argument {_arg_name} should be a numpy array. Got {type(arg)}.')
                 if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
                     assert arg.shape[0] != leap_binder.batch_size_to_validate, \
-                        (f'{user_function.__name__}() validation failed: Argument {_arg_name} '
+                        (f'tensorleap_custom_visualizer validation failed: Argument {_arg_name} '
                          f'should be without batch dimension. ')
 
         def _validate_result(result):
@@ -688,11 +447,8 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
                 LeapDataType.ImageWithBBox: LeapImageWithBBox,
                 LeapDataType.ImageWithHeatmap: LeapImageWithHeatmap
             }
-            validate_output_structure(result, func_name=user_function.__name__,
-                                      expected_type_name=result_type_map[visualizer_type])
-
             assert isinstance(result, result_type_map[visualizer_type]), \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_custom_visualizer validation failed: '
                  f'The return type should be {result_type_map[visualizer_type]}. Got {type(result)}.')
 
         @functools.wraps(user_function)
@@ -724,8 +480,6 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
             result = inner_without_validate(*args, **kwargs)
 
             _validate_result(result)
-            if not _update_env_status is None:
-                _update_env_status("tensorleap_custom_visualizer","v")
             return result
 
         def mapping_inner(*args, **kwargs):
@@ -767,26 +521,30 @@ def tensorleap_metadata(
                                 f'Please choose another')
 
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
+            assert isinstance(sample_id, (int, str)), \
+                (f'tensorleap_metadata validation failed: '
+                 f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_metadata validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
             assert type(sample_id) == preprocess_response.sample_id_type, \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_metadata validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
 
         def _validate_result(result):
             supported_result_types = (type(None), int, str, bool, float, dict, np.floating,
                                       np.bool_, np.unsignedinteger, np.signedinteger, np.integer)
-            validate_output_structure(result, func_name=user_function.__name__,
-                                      expected_type_name=supported_result_types)
             assert isinstance(result, supported_result_types), \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_metadata validation failed: '
                  f'Unsupported return type. Got {type(result)}. should be any of {str(supported_result_types)}')
             if isinstance(result, dict):
                 for key, value in result.items():
                     assert isinstance(key, str), \
-                        (f'{user_function.__name__}() validation failed: '
+                        (f'tensorleap_metadata validation failed: '
                          f'Keys in the return dict should be of type str. Got {type(key)}.')
                     assert isinstance(value, supported_result_types), \
-                        (f'{user_function.__name__}() validation failed: '
+                        (f'tensorleap_metadata validation failed: '
                          f'Values in the return dict should be of type {str(supported_result_types)}. Got {type(value)}.')
 
         def inner_without_validate(sample_id, preprocess_response):
@@ -803,19 +561,15 @@ def tensorleap_metadata(
 
         leap_binder.set_metadata(inner_without_validate, name, metadata_type)
 
-        def inner(*args,**kwargs):
+        def inner(sample_id, preprocess_response):
             if os.environ.get(mapping_runtime_mode_env_var_mame):
                 return None
-            validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
-                                    func_name=user_function.__name__, expected_names=["idx", "preprocess"],**kwargs)
-            sample_id, preprocess_response = args if len(args)!=0 else  kwargs.values()
+
             _validate_input_args(sample_id, preprocess_response)
 
             result = inner_without_validate(sample_id, preprocess_response)
 
             _validate_result(result)
-            if not _update_env_status is None:
-                _update_env_status("tensorleap_metadata","v")
             return result
 
         return inner
@@ -878,23 +632,19 @@ def tensorleap_preprocess():
 
         def _validate_input_args(*args, **kwargs):
             assert len(args) == 0 and len(kwargs) == 0, \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_preprocess validation failed: '
                  f'The function should not take any arguments. Got {args} and {kwargs}.')
 
         def _validate_result(result):
-            assert isinstance(result, list), (
-                f"{user_function.__name__}() validation failed: expected return type list[{PreprocessResponse.__name__}]"
-                f"(e.g., [PreprocessResponse1, PreprocessResponse2, ...]), but returned type is {type(result).__name__}."
-                if not isinstance(result, tuple)
-                else f"{user_function.__name__}() validation failed: expected to return a single list[{PreprocessResponse.__name__}] object, "
-                     f"but returned {len(result)} objects instead."
-            )
+            assert isinstance(result, list), \
+                (f'tensorleap_preprocess validation failed: '
+                 f'The return type should be a list. Got {type(result)}.')
             for i, response in enumerate(result):
                 assert isinstance(response, PreprocessResponse), \
-                    (f'{user_function.__name__}() validation failed: '
+                    (f'tensorleap_preprocess validation failed: '
                      f'Element #{i} in the return list should be a PreprocessResponse. Got {type(response)}.')
             assert len(set(result)) == len(result), \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_preprocess validation failed: '
                  f'The return list should not contain duplicate PreprocessResponse objects.')
 
         def inner(*args, **kwargs):
@@ -902,10 +652,9 @@ def tensorleap_preprocess():
                 return [None, None, None, None]
 
             _validate_input_args(*args, **kwargs)
+
             result = user_function()
             _validate_result(result)
-            if not _update_env_status is None:
-                _update_env_status("tensorleap_preprocess", "v")
             return result
 
         return inner
@@ -1104,23 +853,29 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
             raise Exception(f"Channel dim for input {name} is expected to be either -1 or positive")
 
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
+            assert isinstance(sample_id, (int, str)), \
+                (f'tensorleap_input_encoder validation failed: '
+                 f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_input_encoder validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
             assert type(sample_id) == preprocess_response.sample_id_type, \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_input_encoder validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
 
         def _validate_result(result):
-            validate_output_structure(result, func_name=user_function.__name__, expected_type_name = "np.ndarray")
             assert isinstance(result, np.ndarray), \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_input_encoder validation failed: '
                  f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
             assert result.dtype == np.float32, \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_input_encoder validation failed: '
                  f'The return type should be a numpy array of type float32. Got {result.dtype}.')
-            assert channel_dim - 1 <= len(result.shape), (f'{user_function.__name__}() validation failed: '
+            assert channel_dim - 1 <= len(result.shape), (f'tensorleap_input_encoder validation failed: '
                                                           f'The channel_dim ({channel_dim}) should be <= to the rank of the resulting input rank ({len(result.shape)}).')
 
         def inner_without_validate(sample_id, preprocess_response):
+
             global _called_from_inside_tl_decorator
             _called_from_inside_tl_decorator += 1
 
@@ -1134,10 +889,7 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
         leap_binder.set_input(inner_without_validate, name, channel_dim=channel_dim)
 
 
-        def inner(*args, **kwargs):
-            validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
-                                    func_name=user_function.__name__,    expected_names=["idx", "preprocess"], **kwargs)
-            sample_id, preprocess_response = args if len(args)!=0 else  kwargs.values()
+        def inner(sample_id, preprocess_response):
             _validate_input_args(sample_id, preprocess_response)
 
             result = inner_without_validate(sample_id, preprocess_response)
@@ -1145,20 +897,24 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
             _validate_result(result)
 
             if _called_from_inside_tl_decorator == 0 and _called_from_inside_tl_integration_test_decorator:
-                batch_warning(result,user_function.__name__)
                 result = np.expand_dims(result, axis=0)
-            if not _update_env_status is None:
-                _update_env_status("tensorleap_input_encoder", "v")
+                # Emit integration test event once per test
+                _emit_integration_event_once('input_encoder_integration_test', {
+                    'encoder_name': name,
+                    'channel_dim': channel_dim,
+                    'model_input_index': model_input_index
+                })
 
             return result
 
 
+
         node_mapping_type = NodeMappingType.Input
         if model_input_index is not None:
             node_mapping_type = NodeMappingType(f'Input{str(model_input_index)}')
         inner.node_mapping = NodeMapping(name, node_mapping_type)
 
-        def mapping_inner(*args, **kwargs):
+        def mapping_inner(sample_id, preprocess_response):
             class TempMapping:
                 pass
 
@@ -1170,11 +926,11 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
 
         mapping_inner.node_mapping = NodeMapping(name, node_mapping_type)
 
-        def final_inner(*args, **kwargs):
+        def final_inner(sample_id, preprocess_response):
             if os.environ.get(mapping_runtime_mode_env_var_mame):
-                return mapping_inner(*args, **kwargs)
+                return mapping_inner(sample_id, preprocess_response)
             else:
-                return inner(*args, **kwargs)
+                return inner(sample_id, preprocess_response)
 
         final_inner.node_mapping = NodeMapping(name, node_mapping_type)
 
@@ -1191,18 +947,23 @@ def tensorleap_gt_encoder(name: str):
                                 f'Please choose another')
 
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
+            assert isinstance(sample_id, (int, str)), \
+                (f'tensorleap_gt_encoder validation failed: '
+                 f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_gt_encoder validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
             assert type(sample_id) == preprocess_response.sample_id_type, \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_gt_encoder validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
 
         def _validate_result(result):
-            validate_output_structure(result, func_name=user_function.__name__, expected_type_name = "np.ndarray",gt_flag=True)
             assert isinstance(result, np.ndarray), \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_gt_encoder validation failed: '
                  f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
             assert result.dtype == np.float32, \
-                (f'{user_function.__name__}() validation failed: '
+                (f'tensorleap_gt_encoder validation failed: '
                  f'The return type should be a numpy array of type float32. Got {result.dtype}.')
 
         def inner_without_validate(sample_id, preprocess_response):
@@ -1219,10 +980,7 @@ def tensorleap_gt_encoder(name: str):
         leap_binder.set_ground_truth(inner_without_validate, name)
 
 
-        def inner(*args, **kwargs):
-            validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
-                                    func_name=user_function.__name__, expected_names=["idx", "preprocess"], **kwargs)
-            sample_id, preprocess_response = args
+        def inner(sample_id, preprocess_response):
             _validate_input_args(sample_id, preprocess_response)
 
             result = inner_without_validate(sample_id, preprocess_response)
@@ -1230,15 +988,13 @@ def tensorleap_gt_encoder(name: str):
             _validate_result(result)
 
             if _called_from_inside_tl_decorator == 0 and _called_from_inside_tl_integration_test_decorator:
-                batch_warning(result, user_function.__name__)
                 result = np.expand_dims(result, axis=0)
-            if not _update_env_status is None:
-                _update_env_status("tensorleap_gt_encoder", "v")
+
             return result
 
         inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
 
-        def mapping_inner(*args, **kwargs):
+        def mapping_inner(sample_id, preprocess_response):
             class TempMapping:
                 pass
 
@@ -1249,11 +1005,11 @@ def tensorleap_gt_encoder(name: str):
 
         mapping_inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
 
-        def final_inner(*args, **kwargs):
+        def final_inner(sample_id, preprocess_response):
             if os.environ.get(mapping_runtime_mode_env_var_mame):
-                return mapping_inner(*args, **kwargs)
+                return mapping_inner(sample_id, preprocess_response)
             else:
-                return inner(*args, **kwargs)
+                return inner(sample_id, preprocess_response)
 
         final_inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
 
@@ -1274,37 +1030,28 @@ def tensorleap_custom_loss(name: str, connects_to=None):
         valid_types = (np.ndarray, SamplePreprocessResponse)
 
         def _validate_input_args(*args, **kwargs):
-            assert len(args) > 0 and len(kwargs)==0, (
-                f"{user_function.__name__}() validation failed: "
-                f"Expected at least one positional|key-word argument of the allowed types (np.ndarray|SamplePreprocessResponse|list(np.ndarray|SamplePreprocessResponse)). "
-                f"but received none. "
-                f"Correct usage example: {user_function.__name__}(input_array: np.ndarray, ...)"
-            )
             for i, arg in enumerate(args):
                 if isinstance(arg, list):
                     for y, elem in enumerate(arg):
-                        assert isinstance(elem, valid_types), (f'{user_function.__name__}() validation failed: '
+                        assert isinstance(elem, valid_types), (f'tensorleap_custom_loss validation failed: '
                                                                f'Element #{y} of list should be a numpy array. Got {type(elem)}.')
                 else:
-                    assert isinstance(arg, valid_types), (f'{user_function.__name__}() validation failed: '
+                    assert isinstance(arg, valid_types), (f'tensorleap_custom_loss validation failed: '
                                                           f'Argument #{i} should be a numpy array. Got {type(arg)}.')
             for _arg_name, arg in kwargs.items():
                 if isinstance(arg, list):
                     for y, elem in enumerate(arg):
-                        assert isinstance(elem, valid_types), (f'{user_function.__name__}() validation failed: '
+                        assert isinstance(elem, valid_types), (f'tensorleap_custom_loss validation failed: '
                                                                f'Element #{y} of list should be a numpy array. Got {type(elem)}.')
                 else:
-                    assert isinstance(arg, valid_types), (f'{user_function.__name__}() validation failed: '
+                    assert isinstance(arg, valid_types), (f'tensorleap_custom_loss validation failed: '
                                                           f'Argument #{_arg_name} should be a numpy array. Got {type(arg)}.')
 
         def _validate_result(result):
-            validate_output_structure(result, func_name=user_function.__name__,
-                                      expected_type_name="np.ndarray")
             assert isinstance(result, np.ndarray), \
-                (f'{user_function.__name__} validation failed: '
+                (f'tensorleap_custom_loss validation failed: '
                  f'The return type should be a numpy array. Got {type(result)}.')
-            assert  result.ndim<2 ,(f'{user_function.__name__}  validation failed: '
-                 f'The return type should be a 1Dim numpy array but got {result.ndim}Dim.')
+
 
         @functools.wraps(user_function)
         def inner_without_validate(*args, **kwargs):
@@ -1335,9 +1082,6 @@ def tensorleap_custom_loss(name: str, connects_to=None):
             result = inner_without_validate(*args, **kwargs)
 
             _validate_result(result)
-            if not _update_env_status is None:
-                _update_env_status("tensorleap_custom_loss", "v")
-
             return result
 
         def mapping_inner(*args, **kwargs):
@@ -1394,87 +1138,3 @@ def tensorleap_custom_layer(name: str):
         return custom_layer
 
     return decorating_function
-
-
-def tensorleap_status_table():
-    '''
-    Usage example:
-    ###################
-    leap_integration.py
-    ###################
-    from code_loader.inner_leap_binder.leapbinder_decorators import tensorleap_status_table
-    ...
-    ...
-    ...
-    if __name__ == '__main__':
-        tensorleap_status_table()
-        ...
-    '''
-    import atexit
-    import sys
-    import traceback
-
-
-    table = [
-        {"name": "tensorleap_preprocess", "status": "x"},
-        {"name": "tensorleap_integration_test", "status": "x"},
-        {"name": "tensorleap_input_encoder", "status": "x"},
-        {"name": "tensorleap_gt_encoder", "status": "x"},
-        {"name": "tensorleap_load_model", "status": "x"},
-        {"name": "tensorleap_custom_loss", "status": "x"},
-        {"name": "tensorleap_custom_metric (optional)", "status": "x"},
-        {"name": "tensorleap_metadata (optional)", "status": "x"},
-        {"name": "tensorleap_custom_visualizer (optional)", "status": "x"},
-
-    ]
-
-    _finalizer_called = {"done": False}
-
-    def _print_table():
-        ready_mess = "\nAll parts have been successfully set. If no errors accured, you can now push the project to the Tensorleap system."
-        not_ready_mess = "\nSome mandatory components have not been set yet. Recommended next implementation step: "
-        mandatory_ready_mess = "\nAll mandatory parts have been successfully set. If no errors accured, you can now push the project to the Tensorleap system or continue to the next optional implementation step: "
-
-        name_width = max(len(row["name"]) for row in table)
-        status_width = max(len(row["status"]) for row in table)
-        header = f"{'Function Name'.ljust(name_width)} | {'Status'.ljust(status_width)}"
-        sep = "-" * len(header)
-        print("\n" + header)
-        print(sep)
-        ready=True
-        for row in table:
-            print(f"{row['name'].ljust(name_width)} | {row['status'].ljust(status_width)}")
-            if row['status']=='x' and ready:
-                ready=False
-                next_step=row['name']
-
-
-        print(ready_mess) if ready else print(mandatory_ready_mess+next_step) if "optional" in next_step else print(not_ready_mess+next_step)
-    def update_env_params(name: str, status: str = "✓"):
-        for row in table:
-            if row["name"].removesuffix(" (optional)") == name:
-                row["status"] = status
-                break
-    def run_on_exit():
-        if _finalizer_called["done"]:
-            return
-        _finalizer_called["done"] = True
-        _print_table()
-    def handle_exception(exc_type, exc_value, exc_traceback):
-        traceback.print_exception(exc_type, exc_value, exc_traceback)
-        run_on_exit()
-    atexit.register(run_on_exit)
-    sys.excepthook = handle_exception
-    global _update_env_status
-    _update_env_status = update_env_params
-    return update_env_params
-
-
-
-
-
-
-
-
-
-

{code_loader-1.0.139.dev3 → code_loader-1.0.139.dev4}/code_loader/mixpanel_tracker.py

@@ -126,6 +126,43 @@ class MixpanelTracker:
         except Exception as e:
             pass
 
+    def track_integration_test_event(self, event_name: str, event_properties: Optional[Dict[str, Any]] = None) -> None:
+        """Track an integration test event with device identification.
+        
+        Args:
+            event_name: The name of the event to track
+            event_properties: Optional additional properties to include in the event
+        """
+        # Skip tracking if IS_TENSORLEAP_PLATFORM environment variable is set to 'true'
+        if os.environ.get('IS_TENSORLEAP_PLATFORM') == 'true':
+            return
+            
+        try:
+            distinct_id = self._get_distinct_id()
+            
+            tensorleap_user_id = self._get_tensorleap_user_id()
+            whoami = self._get_whoami()
+            device_id = self._get_or_create_device_id()
+            
+            properties = {
+                'tracking_version': TRACKING_VERSION,
+                'service': 'code-loader',
+                'whoami': whoami,
+                '$device_id': device_id,  # Always use device_id for $device_id
+                'python_version': f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+                'platform': os.name,
+            }
+            
+            if tensorleap_user_id:
+                properties['user_id'] = tensorleap_user_id
+            
+            if event_properties:
+                properties.update(event_properties)
+            
+            self.mp.track(distinct_id, event_name, properties)
+        except Exception as e:
+            pass
+
 
 # Global tracker instance
 _tracker = None
@@ -140,3 +177,7 @@ def get_tracker() -> MixpanelTracker:
 
 def track_code_loader_loaded(event_properties: Optional[Dict[str, Any]] = None) -> None:
     get_tracker().track_code_loader_loaded(event_properties)
+
+
+def track_integration_test_event(event_name: str, event_properties: Optional[Dict[str, Any]] = None) -> None:
+    get_tracker().track_integration_test_event(event_name, event_properties)

{code_loader-1.0.139.dev3 → code_loader-1.0.139.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.139.dev3"
+version = "1.0.139.dev4"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"