code-loader 1.0.179.dev2__tar.gz → 1.0.180__tar.gz

{code_loader-1.0.179.dev2 → code_loader-1.0.180}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.179.dev2
+Version: 1.0.180
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.179.dev2 → code_loader-1.0.180}/code_loader/contract/datasetclasses.py

@@ -9,9 +9,12 @@ from code_loader.contract.enums import DataStateType, DataStateEnum, LeapDataTyp
     MetricDirection, DatasetMetadataType
 from code_loader.contract.visualizer_classes import LeapImage, LeapText, LeapGraph, LeapHorizontalBar, \
     LeapTextMask, LeapImageMask, LeapImageWithBBox, LeapImageWithHeatmap, LeapVideo
+from code_loader.contract.sim_config import SimConfig
 
 custom_latent_space_attribute = "custom_latent_space"
 
+_simulation_context: dict = {"active": False}
+
 
 @dataclass
 class PreprocessResponse:
@@ -41,6 +44,7 @@ class PreprocessResponse:
     sample_id_type: Optional[Union[Type[str], Type[int]]] = None
     sample_ids_to_instance_mappings: Optional[Dict[str, List[str]]] = None # in use only for element instance
     instance_to_sample_ids_mappings: Optional[Dict[str, str]] = None # in use only for element instance
+    tl_generated: bool = False
 
     def __post_init__(self) -> None:
         assert self.sample_ids_to_instance_mappings is None, f"Keep sample_ids_to_instance_mappings None when initializing PreprocessResponse"
@@ -60,14 +64,14 @@ class PreprocessResponse:
             raise Exception("length is deprecated, please use sample_ids instead.")
 
         if self.state is None:
-            from code_loader.inner_leap_binder.leapbinder_decorators import store_warning_by_param
-            store_warning_by_param(
-                param_name="PreprocessResponse.state",
-                user_func_name="tensorleap_preprocess",
-                default_value=str("specific order"),
-                link_to_docs="https://docs.tensorleap.ai/tensorleap-integration/writing-integration-code/preprocess-function",
-            )
-
+            if not _simulation_context["active"]:
+                from code_loader.inner_leap_binder.leapbinder_decorators import store_warning_by_param
+                store_warning_by_param(
+                    param_name="PreprocessResponse.state",
+                    user_func_name="tensorleap_preprocess",
+                    default_value=str("specific order"),
+                    link_to_docs="https://docs.tensorleap.ai/tensorleap-integration/writing-integration-code/preprocess-function",
+                )
         else:
             assert isinstance(self.state, DataStateType), f"PreprocessResponse.state must be of type {DataStateType.__name__} but got {type(self.state)}"
 
@@ -252,6 +256,13 @@ class CustomLayerHandler:
     use_custom_latent_space: bool = False
 
 
+@dataclass
+class SimulationHandler:
+    name: str
+    function: Callable[..., PreprocessResponse]
+    sim_config: SimConfig
+
+
 @dataclass
 class DatasetIntegrationSetup:
     preprocess: Optional[PreprocessHandler] = None
@@ -267,6 +278,7 @@ class DatasetIntegrationSetup:
     instance_metrics: List[InstanceMetricHandler] = field(default_factory=list)
     custom_layers: Dict[str, CustomLayerHandler] = field(default_factory=dict)
     custom_latent_space: Optional[CustomLatentSpaceHandler] = None
+    simulations: List[SimulationHandler] = field(default_factory=list)
 
 
 @dataclass

{code_loader-1.0.179.dev2 → code_loader-1.0.180}/code_loader/contract/responsedataclasses.py

@@ -69,6 +69,12 @@ class PredictionTypeInstance:
     channel_dim: int
 
 
+@dataclass
+class SimulationInstance:
+    name: str
+    sim_config: Dict[str, Any]
+
+
 @dataclass
 class DatasetSetup:
     preprocess: DatasetPreprocess
@@ -79,6 +85,7 @@ class DatasetSetup:
     prediction_types: List[PredictionTypeInstance]
     custom_losses: List[CustomLossInstance]
     metrics: List[MetricInstance] = field(default_factory=list)
+    simulations: List[SimulationInstance] = field(default_factory=list)
 
 
 @dataclass

code_loader-1.0.180/code_loader/contract/sim_config.py

@@ -0,0 +1,92 @@
+from dataclasses import dataclass
+from typing import Any, Dict, List, Union
+
+from code_loader.contract.enums import DatasetMetadataType
+
+
+@dataclass
+class FloatBounds:
+    min: float
+    max: float
+
+
+@dataclass
+class IntBounds:
+    min: int
+    max: int
+
+
+@dataclass
+class CategoricalBounds:
+    values: List[str]
+
+
+@dataclass
+class SimParamConfig:
+    type: DatasetMetadataType
+    bounds: Union[FloatBounds, IntBounds, CategoricalBounds]
+
+
+SimConfig = Dict[str, SimParamConfig]
+
+PARAM_TYPE_MAP = {
+    float:    DatasetMetadataType.float,
+    int:      DatasetMetadataType.int,
+    str:      DatasetMetadataType.string,
+    "float":  DatasetMetadataType.float,
+    "int":    DatasetMetadataType.int,
+    "str":    DatasetMetadataType.string,
+    "string": DatasetMetadataType.string,
+}
+
+
+def _parse_bounds(name: str, metadata_type: DatasetMetadataType, bounds_raw: Dict[str, Any]) -> Union[FloatBounds, IntBounds, CategoricalBounds]:
+    if metadata_type in (DatasetMetadataType.float, DatasetMetadataType.int):
+        if "min" not in bounds_raw or "max" not in bounds_raw:
+            raise ValueError(
+                f"Parameter '{name}' bounds must have 'min' and 'max' for numeric type."
+            )
+        if bounds_raw["min"] >= bounds_raw["max"]:
+            raise ValueError(
+                f"Parameter '{name}': min ({bounds_raw['min']}) must be < max ({bounds_raw['max']})."
+            )
+        if metadata_type == DatasetMetadataType.float:
+            return FloatBounds(min=float(bounds_raw["min"]), max=float(bounds_raw["max"]))
+        return IntBounds(min=int(bounds_raw["min"]), max=int(bounds_raw["max"]))
+
+    if metadata_type == DatasetMetadataType.string:
+        if "values" not in bounds_raw:
+            raise ValueError(
+                f"Parameter '{name}' bounds must have 'values' for string type."
+            )
+        if not bounds_raw["values"]:
+            raise ValueError(f"Parameter '{name}' 'values' must not be empty.")
+        return CategoricalBounds(values=list(bounds_raw["values"]))
+
+    raise ValueError(f"Parameter '{name}' has unsupported metadata type: {metadata_type}.")
+
+
+def parse_sim_config(raw: Dict[str, Any]) -> SimConfig:
+    if not raw:
+        raise ValueError("sim_config must have at least one parameter.")
+
+    result: SimConfig = {}
+    for name, param in raw.items():
+        if "type" not in param:
+            raise ValueError(f"Parameter '{name}' missing 'type'.")
+        if param["type"] not in PARAM_TYPE_MAP:
+            raise ValueError(
+                f"Parameter '{name}' has unsupported type '{param['type']}'. "
+                f"Supported: float, int, str (or string)."
+            )
+        metadata_type = PARAM_TYPE_MAP[param["type"]]
+
+        if "bounds" not in param:
+            raise ValueError(f"Parameter '{name}' missing 'bounds'.")
+
+        result[name] = SimParamConfig(
+            type=metadata_type,
+            bounds=_parse_bounds(name, metadata_type, param["bounds"]),
+        )
+
+    return result

{code_loader-1.0.179.dev2 → code_loader-1.0.180}/code_loader/inner_leap_binder/leapbinder.py

@@ -12,7 +12,8 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
-    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler, InstanceMetricHandler
+    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler, InstanceMetricHandler, \
+    SimulationHandler
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload, LeapAnalysisConfiguration
@@ -20,6 +21,7 @@ from code_loader.contract.visualizer_classes import map_leap_data_type_to_visual
 from code_loader.default_losses import loss_name_to_function
 from code_loader.default_metrics import metrics_names_to_functions_and_direction
 from code_loader.utils import to_numpy_return_wrapper, get_shape, to_numpy_return_masks_wrapper
+from code_loader.contract.sim_config import parse_sim_config
 from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
     default_graph_visualizer, \
     default_image_visualizer, default_horizontal_bar_visualizer, default_word_visualizer, \
@@ -219,6 +221,17 @@ class LeapBinder:
         """
         self.setup_container.unlabeled_data_preprocess = UnlabeledDataPreprocessHandler(function)
 
+    def set_simulation(self, function: Callable[..., PreprocessResponse], name: str, sim_config_raw: Dict[str, Any]) -> None:
+        for sim in self.setup_container.simulations:
+            if sim.name == name:
+                raise Exception(
+                    f"Simulation with name '{name}' already exists. Please choose another."
+                )
+        sim_config = parse_sim_config(sim_config_raw)
+        self.setup_container.simulations.append(
+            SimulationHandler(name=name, function=function, sim_config=sim_config)
+        )
+
     def set_input(self, function: SectionCallableInterface, name: str, channel_dim: int = -1) -> None:
         """
         Set the input handler function.

{code_loader-1.0.179.dev2 → code_loader-1.0.180}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1,8 +1,10 @@
 # mypy: ignore-errors
+import inspect
 import os
 import warnings
 import logging
 from collections import defaultdict
+import functools
 from functools import lru_cache
 from pathlib import Path
 from typing import Optional, Union, Callable, List, Dict, Set, Any
@@ -28,10 +30,6 @@ from code_loader.contract.visualizer_classes import LeapImage, LeapImageMask, Le
 from code_loader.inner_leap_binder.leapbinder import mapping_runtime_mode_env_var_mame
 from code_loader.mixpanel_tracker import clear_integration_events, AnalyticsEvent, emit_integration_event_once
 
-import inspect
-import functools
-from pathlib import Path
-
 _called_from_inside_tl_decorator = 0
 _called_from_inside_tl_integration_test_decorator = False
 _call_from_tl_platform = os.environ.get('IS_TENSORLEAP_PLATFORM') == 'true'
@@ -1233,8 +1231,9 @@ def tensorleap_metadata(
         def _validate_result(result):
             supported_result_types = (type(None), int, str, bool, float, dict, np.floating,
                                       np.bool_, np.unsignedinteger, np.signedinteger, np.integer)
-            validate_output_structure(result, func_name=user_function.__name__,
-                                      expected_type_name=supported_result_types)
+            if isinstance(result, tuple):
+                validate_output_structure(result, func_name=user_function.__name__,
+                                          expected_type_name=supported_result_types)
             assert isinstance(result, supported_result_types), \
                 (f'{user_function.__name__}() validation failed: '
                  f'Unsupported return type. Got {type(result)}. should be any of {str(supported_result_types)}')
@@ -1400,6 +1399,89 @@ def tensorleap_preprocess():
     return decorating_function
 
 
+def tensorleap_simulation(name: str, sim_params: dict):
+    def decorating_function(user_function: Callable[..., PreprocessResponse]):
+        sig = inspect.signature(user_function)
+        func_params = set(sig.parameters.keys())
+        expected_params = set(sim_params.keys()) | {"N"}
+
+        missing = expected_params - func_params
+        if missing:
+            raise Exception(
+                f"{user_function.__name__}() registration failed: "
+                f"Missing required parameters: {missing}. "
+                f"Function must accept all sim_params params plus 'N'."
+            )
+
+        extra = func_params - expected_params - {"seed"}
+        if extra:
+            raise Exception(
+                f"{user_function.__name__}() registration failed: "
+                f"Unexpected parameters: {extra}. "
+                f"Function must only accept sim_params params plus 'N' and optionally 'seed'."
+            )
+
+        leap_binder.set_simulation(user_function, name, sim_params)
+
+        if not _call_from_tl_platform:
+            register_sim(name)
+
+        def _validate_input_args(*args, **kwargs):
+            assert len(args) == 0, (
+                f"{user_function.__name__}() validation failed: "
+                f"Simulation functions must be called with keyword arguments only. Got positional args: {args}."
+            )
+            missing = expected_params - set(kwargs.keys())
+            assert not missing, (
+                f"{user_function.__name__}() validation failed: "
+                f"Missing required keyword arguments: {missing}."
+            )
+
+        def _validate_result(result):
+            assert isinstance(result, PreprocessResponse), (
+                f"{user_function.__name__}() validation failed: "
+                f"Expected return type PreprocessResponse. Got {type(result).__name__}."
+            )
+
+        def inner(*args, **kwargs):
+            if not _call_from_tl_platform:
+                set_current('tensorleap simulation')
+            try:
+                from code_loader.contract.datasetclasses import _simulation_context
+                _validate_input_args(*args, **kwargs)
+                _simulation_context["active"] = True
+                try:
+                    result = user_function(*args, **kwargs)
+                finally:
+                    _simulation_context["active"] = False
+                _validate_result(result)
+                if result.state is not None and result.state != DataStateType.additional:
+                    logger.warning(
+                        f"{user_function.__name__}() returned state={result.state!r}; "
+                        f"overriding to DataStateType.additional."
+                    )
+                result.state = DataStateType.additional
+                result.tl_generated = True
+                try:
+                    emit_integration_event_once(AnalyticsEvent.SIMULATION_INTEGRATION_TEST, {
+                        'simulation_name': name,
+                        'sim_params_count': len(sim_params)
+                    })
+                except Exception as e:
+                    logger.debug(f"Failed to emit simulation integration test event: {e}")
+                if not _call_from_tl_platform:
+                    mark_sim_result(name, True)
+                return result
+            except Exception:
+                if not _call_from_tl_platform:
+                    mark_sim_result(name, False)
+                raise
+
+        return inner
+
+    return decorating_function
+
+
 def tensorleap_element_instance_preprocess(
         instance_length_encoder: InstanceLengthCallableInterface, instance_mask_encoder: InstanceCallableInterface):
     def decorating_function(user_function: Callable[[], List[PreprocessResponse]]):
@@ -2021,6 +2103,9 @@ def tensorleap_status_table():
     def _print_table():
         _print_param_default_warnings()
 
+        for msg in _sim_messages:
+            print(f"\n⚠️  {msg}")
+
         if not started_from("leap_integration.py"):
             return
 
@@ -2073,12 +2158,41 @@ def tensorleap_status_table():
         else:
             _set_status(name, CROSS)
 
+    _sim_tracking: dict = {}
+    _sim_messages: list = []
+
+    def register_sim(sim_name: str):
+        if not _sim_tracking:
+            table.append({"name": "tensorleap simulation (optional)", "Added to integration": UNKNOWN})
+        _sim_tracking[sim_name] = "registered"
+
+    def mark_sim_result(sim_name: str, passed: bool):
+        _sim_tracking[sim_name] = "passed" if passed else "failed"
+
+    def _resolve_sim_row():
+        if not _sim_tracking:
+            return
+        row = _find_row("tensorleap simulation")
+        if not row:
+            return
+        failed = [n for n, s in _sim_tracking.items() if s == "failed"]
+        not_called = [n for n, s in _sim_tracking.items() if s == "registered"]
+        if failed:
+            row["Added to integration"] = CROSS
+            _sim_messages.append(f"Simulation(s) failed validation: {', '.join(sorted(failed))}")
+        elif not_called:
+            row["Added to integration"] = UNKNOWN
+            _sim_messages.append(f"Simulation(s) not called in integration test: {', '.join(sorted(not_called))}")
+        else:
+            row["Added to integration"] = CHECK
+
     def run_on_exit():
         if _finalizer_called["done"]:
             return
         _finalizer_called["done"] = True
         if not _crashed["value"]:
             _mark_unknowns_as_cross()
+        _resolve_sim_row()
 
         _print_table()
 
@@ -2096,11 +2210,11 @@ def tensorleap_status_table():
     atexit.register(run_on_exit)
     sys.excepthook = handle_exception
 
-    return set_current, update_env_params
+    return set_current, update_env_params, register_sim, mark_sim_result
 
 
 if not _call_from_tl_platform:
-    set_current, update_env_params_func = tensorleap_status_table()
+    set_current, update_env_params_func, register_sim, mark_sim_result = tensorleap_status_table()
 
 
 

{code_loader-1.0.179.dev2 → code_loader-1.0.180}/code_loader/leaploader.py

@@ -22,13 +22,22 @@ from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance, \
-    EngineFileContract
+    EngineFileContract, SimulationInstance
+from code_loader.contract.sim_config import FloatBounds, IntBounds, CategoricalBounds
 from code_loader.inner_leap_binder import global_leap_binder
 from code_loader.inner_leap_binder.leapbinder import mapping_runtime_mode_env_var_mame
 from code_loader.leaploaderbase import LeapLoaderBase
 from code_loader.utils import get_root_exception_file_and_line_number, get_metadata_type_from_variable
 
 
+def _serialize_sim_bounds(bounds) -> dict:
+    if isinstance(bounds, (FloatBounds, IntBounds)):
+        return {"min": bounds.min, "max": bounds.max}
+    if isinstance(bounds, CategoricalBounds):
+        return {"values": bounds.values}
+    raise ValueError(f"Unknown bounds type: {type(bounds)}")
+
+
 class LeapLoader(LeapLoaderBase):
     def __init__(self, code_path: str, code_entry_name: str):
         super().__init__(code_path, code_entry_name)
@@ -454,9 +463,20 @@ class LeapLoader(LeapLoaderBase):
             metric_inst = MetricInstance(metric.metric_handler_data.name, metric.metric_handler_data.arg_names)
             metrics.append(metric_inst)
 
+        simulations = []
+        for sim in setup.simulations:
+            sim_config_serialized = {
+                name: {
+                    "type": param.type.value,
+                    "bounds": _serialize_sim_bounds(param.bounds),
+                }
+                for name, param in sim.sim_config.items()
+            }
+            simulations.append(SimulationInstance(name=sim.name, sim_config=sim_config_serialized))
+
         return DatasetSetup(preprocess=dataset_preprocess, inputs=inputs, outputs=ground_truths,
                             metadata=metadata_instances, visualizers=visualizers, prediction_types=prediction_types,
-                            custom_losses=custom_losses, metrics=metrics)
+                            custom_losses=custom_losses, metrics=metrics, simulations=simulations)
 
     def get_model_setup_response(self) -> ModelSetup:
         setup = global_leap_binder.setup_container

{code_loader-1.0.179.dev2 → code_loader-1.0.180}/code_loader/mixpanel_tracker.py

@@ -23,6 +23,7 @@ class AnalyticsEvent(str, Enum):
     PREPROCESS_INTEGRATION_TEST = "preprocess_integration_test"
     INPUT_ENCODER_INTEGRATION_TEST = "input_encoder_integration_test"
     GT_ENCODER_INTEGRATION_TEST = "gt_encoder_integration_test"
+    SIMULATION_INTEGRATION_TEST = "simulation_integration_test"
 
 
 class CodeLoaderLoadedProps(TypedDict, total=False):

{code_loader-1.0.179.dev2 → code_loader-1.0.180}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.179.dev2"
+version = "1.0.180"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"