code-loader 1.0.180.dev2__tar.gz → 1.0.180.dev4__tar.gz

{code_loader-1.0.180.dev2 → code_loader-1.0.180.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.180.dev2
+Version: 1.0.180.dev4
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.180.dev2 → code_loader-1.0.180.dev4}/code_loader/contract/datasetclasses.py

@@ -42,6 +42,7 @@ class PreprocessResponse:
     sample_id_type: Optional[Union[Type[str], Type[int]]] = None
     sample_ids_to_instance_mappings: Optional[Dict[str, List[str]]] = None # in use only for element instance
     instance_to_sample_ids_mappings: Optional[Dict[str, str]] = None # in use only for element instance
+    tl_generated: bool = False
 
     def __post_init__(self) -> None:
         assert self.sample_ids_to_instance_mappings is None, f"Keep sample_ids_to_instance_mappings None when initializing PreprocessResponse"
@@ -254,9 +255,9 @@ class CustomLayerHandler:
 
 
 @dataclass
-class SimulationInstance:
+class SimulationHandler:
     name: str
-    function: Callable[..., Any]
+    function: Callable[..., PreprocessResponse]
     sim_config: SimConfig
 
 
@@ -275,7 +276,7 @@ class DatasetIntegrationSetup:
     instance_metrics: List[InstanceMetricHandler] = field(default_factory=list)
     custom_layers: Dict[str, CustomLayerHandler] = field(default_factory=dict)
     custom_latent_space: Optional[CustomLatentSpaceHandler] = None
-    simulations: List[SimulationInstance] = field(default_factory=list)
+    simulations: List[SimulationHandler] = field(default_factory=list)
 
 
 @dataclass

{code_loader-1.0.180.dev2 → code_loader-1.0.180.dev4}/code_loader/contract/responsedataclasses.py

@@ -70,7 +70,7 @@ class PredictionTypeInstance:
 
 
 @dataclass
-class SimulationSetupInstance:
+class SimulationInstance:
     name: str
     sim_config: Dict[str, Any]
 
@@ -85,7 +85,7 @@ class DatasetSetup:
     prediction_types: List[PredictionTypeInstance]
     custom_losses: List[CustomLossInstance]
     metrics: List[MetricInstance] = field(default_factory=list)
-    simulations: List[SimulationSetupInstance] = field(default_factory=list)
+    simulations: List[SimulationInstance] = field(default_factory=list)
 
 
 @dataclass

{code_loader-1.0.180.dev2 → code_loader-1.0.180.dev4}/code_loader/inner_leap_binder/leapbinder.py

@@ -13,7 +13,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
     ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler, InstanceMetricHandler, \
-    SimulationInstance
+    SimulationHandler
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload, LeapAnalysisConfiguration
@@ -221,7 +221,7 @@ class LeapBinder:
         """
         self.setup_container.unlabeled_data_preprocess = UnlabeledDataPreprocessHandler(function)
 
-    def set_simulation(self, function: Callable[..., Any], name: str, sim_config_raw: Dict[str, Any]) -> None:
+    def set_simulation(self, function: Callable[..., PreprocessResponse], name: str, sim_config_raw: Dict[str, Any]) -> None:
         for sim in self.setup_container.simulations:
             if sim.name == name:
                 raise Exception(
@@ -229,7 +229,7 @@ class LeapBinder:
                 )
         sim_config = parse_sim_config(sim_config_raw)
         self.setup_container.simulations.append(
-            SimulationInstance(name=name, function=function, sim_config=sim_config)
+            SimulationHandler(name=name, function=function, sim_config=sim_config)
         )
 
     def set_input(self, function: SectionCallableInterface, name: str, channel_dim: int = -1) -> None:

{code_loader-1.0.180.dev2 → code_loader-1.0.180.dev4}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1400,7 +1400,7 @@ def tensorleap_preprocess():
 
 
 def tensorleap_simulation(name: str, sim_params: dict):
-    def decorating_function(user_function: Callable):
+    def decorating_function(user_function: Callable[..., PreprocessResponse]):
         sig = inspect.signature(user_function)
         func_params = set(sig.parameters.keys())
         expected_params = set(sim_params.keys()) | {"N"}
@@ -1423,13 +1423,54 @@ def tensorleap_simulation(name: str, sim_params: dict):
 
         leap_binder.set_simulation(user_function, name, sim_params)
 
-        def inner(*args, **kwargs):
-            result = user_function(*args, **kwargs)
+        if not _call_from_tl_platform:
+            register_sim(name)
+
+        def _validate_input_args(*args, **kwargs):
+            assert len(args) == 0, (
+                f"{user_function.__name__}() validation failed: "
+                f"Simulation functions must be called with keyword arguments only. Got positional args: {args}."
+            )
+            missing = expected_params - set(kwargs.keys())
+            assert not missing, (
+                f"{user_function.__name__}() validation failed: "
+                f"Missing required keyword arguments: {missing}."
+            )
+
+        def _validate_result(result):
             assert isinstance(result, PreprocessResponse), (
                 f"{user_function.__name__}() validation failed: "
                 f"Expected return type PreprocessResponse. Got {type(result).__name__}."
             )
-            return result
+
+        def inner(*args, **kwargs):
+            if not _call_from_tl_platform:
+                set_current('tensorleap simulation')
+            try:
+                _validate_input_args(*args, **kwargs)
+                result = user_function(*args, **kwargs)
+                _validate_result(result)
+                if result.state != DataStateType.additional:
+                    logger.warning(
+                        f"{user_function.__name__}() returned state={result.state!r}; "
+                        f"overriding to DataStateType.additional."
+                    )
+                result.state = DataStateType.additional
+                result.tl_generated = True
+                try:
+                    emit_integration_event_once(AnalyticsEvent.SIMULATION_INTEGRATION_TEST, {
+                        'simulation_name': name,
+                        'sim_params_count': len(sim_params)
+                    })
+                except Exception as e:
+                    logger.debug(f"Failed to emit simulation integration test event: {e}")
+                if not _call_from_tl_platform:
+                    mark_sim_result(name, True)
+                return result
+            except Exception:
+                if not _call_from_tl_platform:
+                    mark_sim_result(name, False)
+                raise
 
         return inner
 
@@ -2057,6 +2098,9 @@ def tensorleap_status_table():
     def _print_table():
         _print_param_default_warnings()
 
+        for msg in _sim_messages:
+            print(f"\n⚠️  {msg}")
+
         if not started_from("leap_integration.py"):
             return
 
@@ -2109,12 +2153,41 @@ def tensorleap_status_table():
         else:
             _set_status(name, CROSS)
 
+    _sim_tracking: dict = {}
+    _sim_messages: list = []
+
+    def register_sim(sim_name: str):
+        if not _sim_tracking:
+            table.append({"name": "tensorleap simulation (optional)", "Added to integration": UNKNOWN})
+        _sim_tracking[sim_name] = "registered"
+
+    def mark_sim_result(sim_name: str, passed: bool):
+        _sim_tracking[sim_name] = "passed" if passed else "failed"
+
+    def _resolve_sim_row():
+        if not _sim_tracking:
+            return
+        row = _find_row("tensorleap simulation")
+        if not row:
+            return
+        failed = [n for n, s in _sim_tracking.items() if s == "failed"]
+        not_called = [n for n, s in _sim_tracking.items() if s == "registered"]
+        if failed:
+            row["Added to integration"] = CROSS
+            _sim_messages.append(f"Simulation(s) failed validation: {', '.join(sorted(failed))}")
+        elif not_called:
+            row["Added to integration"] = UNKNOWN
+            _sim_messages.append(f"Simulation(s) not called in integration test: {', '.join(sorted(not_called))}")
+        else:
+            row["Added to integration"] = CHECK
+
     def run_on_exit():
         if _finalizer_called["done"]:
             return
         _finalizer_called["done"] = True
         if not _crashed["value"]:
             _mark_unknowns_as_cross()
+        _resolve_sim_row()
 
         _print_table()
 
@@ -2132,11 +2205,11 @@ def tensorleap_status_table():
     atexit.register(run_on_exit)
     sys.excepthook = handle_exception
 
-    return set_current, update_env_params
+    return set_current, update_env_params, register_sim, mark_sim_result
 
 
 if not _call_from_tl_platform:
-    set_current, update_env_params_func = tensorleap_status_table()
+    set_current, update_env_params_func, register_sim, mark_sim_result = tensorleap_status_table()
 
 
 

{code_loader-1.0.180.dev2 → code_loader-1.0.180.dev4}/code_loader/leaploader.py

@@ -22,7 +22,7 @@ from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance, \
-    EngineFileContract, SimulationSetupInstance
+    EngineFileContract, SimulationInstance
 from code_loader.contract.sim_config import FloatBounds, IntBounds, CategoricalBounds
 from code_loader.inner_leap_binder import global_leap_binder
 from code_loader.inner_leap_binder.leapbinder import mapping_runtime_mode_env_var_mame
@@ -38,20 +38,6 @@ def _serialize_sim_bounds(bounds) -> dict:
     raise ValueError(f"Unknown bounds type: {type(bounds)}")
 
 
-def _sample_sim_params(sim_config):
-    # type: (Dict[str, Any]) -> Dict[str, Any]
-    params = {}
-    for name, param_config in sim_config.items():
-        bounds = param_config.bounds
-        if isinstance(bounds, FloatBounds):
-            params[name] = (bounds.min + bounds.max) / 2.0
-        elif isinstance(bounds, IntBounds):
-            params[name] = int((bounds.min + bounds.max) / 2)
-        else:  # CategoricalBounds
-            params[name] = bounds.values[0]
-    return params
-
-
 class LeapLoader(LeapLoaderBase):
     def __init__(self, code_path: str, code_entry_name: str):
         super().__init__(code_path, code_entry_name)
@@ -237,8 +223,6 @@ class LeapLoader(LeapLoaderBase):
                 test_payloads.append(preprocess_test_payload)
                 handlers_test_payloads = self._check_handlers()
                 test_payloads.extend(handlers_test_payloads)
-                simulation_test_payloads = self._check_simulations()
-                test_payloads.extend(simulation_test_payloads)
                 is_valid = all([payload.is_passed for payload in test_payloads])
                 setup_response = self.get_dataset_setup_response(handlers_test_payloads)
 
@@ -315,54 +299,6 @@ class LeapLoader(LeapLoaderBase):
             result_payloads.extend(test_result)
         return result_payloads
 
-    def _check_simulations(self):
-        # type: () -> List[DatasetTestResultPayload]
-        result_payloads = []
-        for sim in global_leap_binder.setup_container.simulations:
-            test_result = DatasetTestResultPayload(sim.name)
-            try:
-                kwargs = _sample_sim_params(sim.sim_config)
-                kwargs["N"] = 1
-                if "seed" in inspect.signature(sim.function).parameters:
-                    kwargs["seed"] = 0
-                preprocess_response = sim.function(**kwargs)
-                if not isinstance(preprocess_response, PreprocessResponse):
-                    raise TypeError(
-                        "Expected PreprocessResponse, got {}".format(type(preprocess_response).__name__)
-                    )
-                if preprocess_response.length < 1:
-                    raise ValueError("Simulation returned PreprocessResponse with length < 1")
-            except Exception as e:
-                line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
-                test_result.display[TestingSectionEnum.Errors.name] = (
-                    "{} in file {}, line_number: {}\nStacktrace:\n{}".format(
-                        repr(e), file_name, line_number, stacktrace
-                    )
-                )
-                test_result.is_passed = False
-            result_payloads.append(test_result)
-        return result_payloads
-
-    def run_simulation_check(self, sim_name):
-        # type: (str) -> Dict[str, npt.NDArray[np.float32]]
-        self.exec_script()
-        sim = next(
-            (s for s in global_leap_binder.setup_container.simulations if s.name == sim_name),
-            None,
-        )
-        if sim is None:
-            raise ValueError("No simulation registered with name '{}'".format(sim_name))
-        kwargs = _sample_sim_params(sim.sim_config)
-        kwargs["N"] = 1
-        if "seed" in inspect.signature(sim.function).parameters:
-            kwargs["seed"] = 0
-        sim_preprocess = sim.function(**kwargs)
-        sim_preprocess.state = DataStateType.additional
-        result = {}
-        for handler in global_leap_binder.setup_container.inputs:
-            result[handler.name] = handler.function(0, sim_preprocess)
-        return result
-
     @staticmethod
     def _get_all_dataset_base_handlers() -> List[Union[DatasetBaseHandler, MetadataHandler]]:
         all_dataset_base_handlers: List[Union[DatasetBaseHandler, MetadataHandler]] = []
@@ -536,7 +472,7 @@ class LeapLoader(LeapLoaderBase):
                 }
                 for name, param in sim.sim_config.items()
             }
-            simulations.append(SimulationSetupInstance(name=sim.name, sim_config=sim_config_serialized))
+            simulations.append(SimulationInstance(name=sim.name, sim_config=sim_config_serialized))
 
         return DatasetSetup(preprocess=dataset_preprocess, inputs=inputs, outputs=ground_truths,
                             metadata=metadata_instances, visualizers=visualizers, prediction_types=prediction_types,

{code_loader-1.0.180.dev2 → code_loader-1.0.180.dev4}/code_loader/leaploaderbase.py

@@ -2,7 +2,7 @@
 
 from abc import abstractmethod
 
-from typing import Any, Dict, List, Union, Type, Optional, Tuple
+from typing import Dict, List, Union, Type, Optional, Tuple
 
 import numpy as np
 import numpy.typing as npt
@@ -153,11 +153,6 @@ class LeapLoaderBase:
     def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:
         pass
 
-    @abstractmethod
-    def run_simulation_check(self, sim_name):
-        # type: (str) -> Dict[str, Any]
-        pass
-
     def is_custom_latent_space(self) -> bool:
         if not self.code_entry_name or not self.code_path:
             return False

{code_loader-1.0.180.dev2 → code_loader-1.0.180.dev4}/code_loader/mixpanel_tracker.py

@@ -23,6 +23,7 @@ class AnalyticsEvent(str, Enum):
     PREPROCESS_INTEGRATION_TEST = "preprocess_integration_test"
     INPUT_ENCODER_INTEGRATION_TEST = "input_encoder_integration_test"
     GT_ENCODER_INTEGRATION_TEST = "gt_encoder_integration_test"
+    SIMULATION_INTEGRATION_TEST = "simulation_integration_test"
 
 
 class CodeLoaderLoadedProps(TypedDict, total=False):

{code_loader-1.0.180.dev2 → code_loader-1.0.180.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.180.dev2"
+version = "1.0.180.dev4"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"