code-loader 1.0.180.dev4__py3-none-any.whl → 1.0.181__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -13,6 +13,8 @@ from code_loader.contract.sim_config import SimConfig
 
 custom_latent_space_attribute = "custom_latent_space"
 
+_simulation_context: Dict[str, bool] = {"active": False}
+
 
 @dataclass
 class PreprocessResponse:
@@ -62,14 +64,14 @@ class PreprocessResponse:
             raise Exception("length is deprecated, please use sample_ids instead.")
 
         if self.state is None:
-            from code_loader.inner_leap_binder.leapbinder_decorators import store_warning_by_param
-            store_warning_by_param(
-                param_name="PreprocessResponse.state",
-                user_func_name="tensorleap_preprocess",
-                default_value=str("specific order"),
-                link_to_docs="https://docs.tensorleap.ai/tensorleap-integration/writing-integration-code/preprocess-function",
-            )
-
+            if not _simulation_context["active"]:
+                from code_loader.inner_leap_binder.leapbinder_decorators import store_warning_by_param
+                store_warning_by_param(
+                    param_name="PreprocessResponse.state",
+                    user_func_name="tensorleap_preprocess",
+                    default_value=str("specific order"),
+                    link_to_docs="https://docs.tensorleap.ai/tensorleap-integration/writing-integration-code/preprocess-function",
+                )
         else:
             assert isinstance(self.state, DataStateType), f"PreprocessResponse.state must be of type {DataStateType.__name__} but got {type(self.state)}"
 

code_loader/contract/sim_config.py

@@ -66,6 +66,19 @@ def _parse_bounds(name: str, metadata_type: DatasetMetadataType, bounds_raw: Dic
     raise ValueError(f"Parameter '{name}' has unsupported metadata type: {metadata_type}.")
 
 
+def sample_sim_params(sim_config: SimConfig) -> Dict[str, Any]:
+    params: Dict[str, Any] = {}
+    for name, param_config in sim_config.items():
+        bounds = param_config.bounds
+        if isinstance(bounds, FloatBounds):
+            params[name] = (bounds.min + bounds.max) / 2.0
+        elif isinstance(bounds, IntBounds):
+            params[name] = int((bounds.min + bounds.max) / 2)
+        else:  # CategoricalBounds
+            params[name] = bounds.values[0]
+    return params
+
+
 def parse_sim_config(raw: Dict[str, Any]) -> SimConfig:
     if not raw:
         raise ValueError("sim_config must have at least one parameter.")

code_loader/inner_leap_binder/leapbinder.py

@@ -13,7 +13,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
     ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler, InstanceMetricHandler, \
-    SimulationHandler
+    SimulationHandler, _simulation_context
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload, LeapAnalysisConfiguration
@@ -21,7 +21,7 @@ from code_loader.contract.visualizer_classes import map_leap_data_type_to_visual
 from code_loader.default_losses import loss_name_to_function
 from code_loader.default_metrics import metrics_names_to_functions_and_direction
 from code_loader.utils import to_numpy_return_wrapper, get_shape, to_numpy_return_masks_wrapper
-from code_loader.contract.sim_config import parse_sim_config
+from code_loader.contract.sim_config import parse_sim_config, sample_sim_params
 from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
     default_graph_visualizer, \
     default_image_visualizer, default_horizontal_bar_visualizer, default_word_visualizer, \
@@ -666,10 +666,46 @@ class LeapBinder:
                     continue
                 test_result = self.check_handler(preprocess_response, test_result, dataset_base_handler, state)
 
+    def check_simulations(self) -> None:
+        from code_loader.inner_leap_binder.leapbinder_decorators import _call_from_tl_platform
+        if not _call_from_tl_platform:
+            from code_loader.inner_leap_binder.leapbinder_decorators import set_current
+            set_current("tensorleap simulation")
+        for sim in self.setup_container.simulations:
+            kwargs = sample_sim_params(sim.sim_config)
+            kwargs["N"] = 1
+            kwargs["seed"] = 0
+            _simulation_context["active"] = True
+            try:
+                preprocess_response = sim.function(**kwargs)
+            finally:
+                _simulation_context["active"] = False
+            if not isinstance(preprocess_response, PreprocessResponse):
+                raise Exception(
+                    "Simulation '{}' returned {} instead of PreprocessResponse".format(
+                        sim.name, type(preprocess_response).__name__
+                    )
+                )
+            preprocess_response.state = DataStateType.additional
+            preprocess_response.tl_generated = True
+            if not preprocess_response.length or preprocess_response.length < 1:
+                raise Exception("Simulation '{}' returned PreprocessResponse with length < 1".format(sim.name))
+            preprocess_response.sample_ids = [0]
+            for handler in self.setup_container.inputs:
+                out1 = handler.function(preprocess_response.sample_ids[0], preprocess_response)
+                out2 = handler.function(preprocess_response.sample_ids[0], preprocess_response)
+                if not np.array_equal(out1, out2):
+                    raise Exception(
+                        "Simulation '{}': encoder '{}' is non-deterministic — consecutive calls with seed=0 returned different outputs".format(
+                            sim.name, handler.name
+                        )
+                    )
+
     def check(self) -> None:
         preprocess_result = self.get_preprocess_result()
         self.check_preprocess(preprocess_result)
         self.check_handlers(preprocess_result)
+        self.check_simulations()
         self.validate_ignore_latent_spaces()
         print("Successful!")
 

code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1403,22 +1403,22 @@ def tensorleap_simulation(name: str, sim_params: dict):
     def decorating_function(user_function: Callable[..., PreprocessResponse]):
         sig = inspect.signature(user_function)
         func_params = set(sig.parameters.keys())
-        expected_params = set(sim_params.keys()) | {"N"}
+        expected_params = set(sim_params.keys()) | {"N", "seed"}
 
         missing = expected_params - func_params
         if missing:
             raise Exception(
                 f"{user_function.__name__}() registration failed: "
                 f"Missing required parameters: {missing}. "
-                f"Function must accept all sim_params params plus 'N'."
+                f"Function must accept all sim_params params plus 'N' and 'seed'."
             )
 
-        extra = func_params - expected_params - {"seed"}
+        extra = func_params - expected_params
         if extra:
             raise Exception(
                 f"{user_function.__name__}() registration failed: "
                 f"Unexpected parameters: {extra}. "
-                f"Function must only accept sim_params params plus 'N' and optionally 'seed'."
+                f"Function must only accept sim_params params plus 'N' and 'seed'."
             )
 
         leap_binder.set_simulation(user_function, name, sim_params)
@@ -1447,10 +1447,15 @@ def tensorleap_simulation(name: str, sim_params: dict):
             if not _call_from_tl_platform:
                 set_current('tensorleap simulation')
             try:
+                from code_loader.contract.datasetclasses import _simulation_context
                 _validate_input_args(*args, **kwargs)
-                result = user_function(*args, **kwargs)
+                _simulation_context["active"] = True
+                try:
+                    result = user_function(*args, **kwargs)
+                finally:
+                    _simulation_context["active"] = False
                 _validate_result(result)
-                if result.state != DataStateType.additional:
+                if result.state is not None and result.state != DataStateType.additional:
                     logger.warning(
                         f"{user_function.__name__}() returned state={result.state!r}; "
                         f"overriding to DataStateType.additional."
@@ -2170,6 +2175,8 @@ def tensorleap_status_table():
         row = _find_row("tensorleap simulation")
         if not row:
             return
+        if _crashed["value"]:
+            return
         failed = [n for n, s in _sim_tracking.items() if s == "failed"]
         not_called = [n for n, s in _sim_tracking.items() if s == "registered"]
         if failed:

code_loader/leaploader.py

@@ -16,14 +16,14 @@ from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandle
     PreprocessResponse, VisualizerHandler, LeapData, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
     MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse, \
-    ElementInstance, custom_latent_space_attribute, DatasetIntegrationSetup, InstanceMetricHandler
+    ElementInstance, custom_latent_space_attribute, DatasetIntegrationSetup, InstanceMetricHandler, _simulation_context
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance, \
     EngineFileContract, SimulationInstance
-from code_loader.contract.sim_config import FloatBounds, IntBounds, CategoricalBounds
+from code_loader.contract.sim_config import FloatBounds, IntBounds, CategoricalBounds, sample_sim_params
 from code_loader.inner_leap_binder import global_leap_binder
 from code_loader.inner_leap_binder.leapbinder import mapping_runtime_mode_env_var_mame
 from code_loader.leaploaderbase import LeapLoaderBase
@@ -38,6 +38,7 @@ def _serialize_sim_bounds(bounds) -> dict:
     raise ValueError(f"Unknown bounds type: {type(bounds)}")
 
 
+
 class LeapLoader(LeapLoaderBase):
     def __init__(self, code_path: str, code_entry_name: str):
         super().__init__(code_path, code_entry_name)
@@ -223,6 +224,8 @@ class LeapLoader(LeapLoaderBase):
                 test_payloads.append(preprocess_test_payload)
                 handlers_test_payloads = self._check_handlers()
                 test_payloads.extend(handlers_test_payloads)
+                simulation_test_payloads = self._check_simulations()
+                test_payloads.extend(simulation_test_payloads)
                 is_valid = all([payload.is_passed for payload in test_payloads])
                 setup_response = self.get_dataset_setup_response(handlers_test_payloads)
 
@@ -299,6 +302,74 @@ class LeapLoader(LeapLoaderBase):
             result_payloads.extend(test_result)
         return result_payloads
 
+    def _check_simulations(self):
+        # type: () -> List[DatasetTestResultPayload]
+        result_payloads = []
+        for sim in global_leap_binder.setup_container.simulations:
+            test_result = DatasetTestResultPayload(sim.name)
+            try:
+                kwargs = sample_sim_params(sim.sim_config)
+                kwargs["N"] = 1
+                kwargs["seed"] = 0
+                _simulation_context["active"] = True
+                try:
+                    preprocess_response = sim.function(**kwargs)
+                finally:
+                    _simulation_context["active"] = False
+                if not isinstance(preprocess_response, PreprocessResponse):
+                    raise TypeError(
+                        "Expected PreprocessResponse, got {}".format(type(preprocess_response).__name__)
+                    )
+                preprocess_response.state = DataStateType.additional
+                preprocess_response.tl_generated = True
+                if preprocess_response.length < 1:
+                    raise ValueError("Simulation returned PreprocessResponse with length < 1")
+                preprocess_response.sample_ids = [0]
+                for handler in global_leap_binder.setup_container.inputs:
+                    out1 = handler.function(preprocess_response.sample_ids[0], preprocess_response)
+                    out2 = handler.function(preprocess_response.sample_ids[0], preprocess_response)
+                    if not np.array_equal(out1, out2):
+                        raise ValueError(
+                            "Encoder '{}' is non-deterministic: consecutive calls with seed=0 returned different outputs".format(
+                                handler.name
+                            )
+                        )
+            except Exception as e:
+                line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
+                test_result.display[TestingSectionEnum.Errors.name] = (
+                    "{} in file {}, line_number: {}\nStacktrace:\n{}".format(
+                        repr(e), file_name, line_number, stacktrace
+                    )
+                )
+                test_result.is_passed = False
+            result_payloads.append(test_result)
+        return result_payloads
+
+    def run_simulation_check(self, sim_name):
+        # type: (str) -> Dict[str, npt.NDArray[np.float32]]
+        self.exec_script()
+        sim = next(
+            (s for s in global_leap_binder.setup_container.simulations if s.name == sim_name),
+            None,
+        )
+        if sim is None:
+            raise ValueError("No simulation registered with name '{}'".format(sim_name))
+        kwargs = sample_sim_params(sim.sim_config)
+        kwargs["N"] = 1
+        kwargs["seed"] = 0
+        _simulation_context["active"] = True
+        try:
+            sim_preprocess = sim.function(**kwargs)
+        finally:
+            _simulation_context["active"] = False
+        sim_preprocess.state = DataStateType.additional
+        sim_preprocess.tl_generated = True
+        sim_preprocess.sample_ids = [0]
+        result = {}
+        for handler in global_leap_binder.setup_container.inputs:
+            result[handler.name] = handler.function(sim_preprocess.sample_ids[0], sim_preprocess)
+        return result
+
     @staticmethod
     def _get_all_dataset_base_handlers() -> List[Union[DatasetBaseHandler, MetadataHandler]]:
         all_dataset_base_handlers: List[Union[DatasetBaseHandler, MetadataHandler]] = []

code_loader/leaploaderbase.py

@@ -2,7 +2,7 @@
 
 from abc import abstractmethod
 
-from typing import Dict, List, Union, Type, Optional, Tuple
+from typing import Any, Dict, List, Union, Type, Optional, Tuple
 
 import numpy as np
 import numpy.typing as npt
@@ -153,6 +153,11 @@ class LeapLoaderBase:
     def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:
         pass
 
+    @abstractmethod
+    def run_simulation_check(self, sim_name):
+        # type: (str) -> Dict[str, Any]
+        pass
+
     def is_custom_latent_space(self) -> bool:
         if not self.code_entry_name or not self.code_path:
             return False

{code_loader-1.0.180.dev4.dist-info → code_loader-1.0.181.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.180.dev4
+Version: 1.0.181
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.180.dev4.dist-info → code_loader-1.0.181.dist-info}/RECORD

@@ -1,12 +1,12 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=outxRQ0M-zMfV0QGVJmAed5qWfRmyD0TV6-goEGAzBw,406
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=KefWGK04SDGQe4KjwBCnP14LO7sO7esS5vYmevFZW_A,10321
+code_loader/contract/datasetclasses.py,sha256=RYZcnX7vLMYhyQtCbX2PscVFNAYULxgAA9U4DRRfLqA,10456
 code_loader/contract/enums.py,sha256=2q-IV_5g9lLE306DIbWA1c0tn5IhDtxsKxyV1x_Lreg,1671
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/mapping.py,sha256=sWJhpng-IkOzQnWQdMT5w2ZZ3X1Z_OOzSwCLXIS7oxE,1446
 code_loader/contract/responsedataclasses.py,sha256=5VFgGjRubMW8ItMPils3rkBNejunCGLaa192AIi-xko,4925
-code_loader/contract/sim_config.py,sha256=tWgM3n2_UTP8HnWBUFYLY928ya1ezFWhMOOU9c3WsHU,3002
+code_loader/contract/sim_config.py,sha256=le8KMALZiP0WU4UcuKnTOSWBW2rNjpnWYfII502NqDM,3493
 code_loader/contract/visualizer_classes.py,sha256=Wz9eItmoRaKEHa3p0aW0Ypxx4_xUmaZyLBznnTuxwi0,15425
 code_loader/default_losses.py,sha256=NoOQym1106bDN5dcIk56Elr7ZG5quUHArqfP5-Nyxyo,1139
 code_loader/default_metrics.py,sha256=2XSlyNw_XLDGSJDoz5W_Evi5wbL0dhwq24pPr15vSPc,5025
@@ -21,10 +21,10 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=Avt4lQv4mehoEOlH7UyF7Wna_GGJWSAF6r55qUk1MYE,38497
-code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=iu5m72-ugEze3UBfNXL_QrL151FXI94JICykEyN3hTM,105461
-code_loader/leaploader.py,sha256=vz812i2a0-9UTqsxNx9z0lqJlLt4BWGsPwl4E4lWjZ4,33022
-code_loader/leaploaderbase.py,sha256=NJGaas8S6JeHYSsKkMSutyfcSKdK9jXTic7BcjC5uNc,6303
+code_loader/inner_leap_binder/leapbinder.py,sha256=cbw1zSAYeaRFTvcPPs5cNszs3LK8I-AiP9oJsK5A-pc,40494
+code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=YlK51b5Ryo396f7tOA7Ole3vYCLs3f5ZLm2qDQ9K1NE,105781
+code_loader/leaploader.py,sha256=V1GXbUwY1a8Cxo6AY1h2BLutzFPTLE7iLxX4wyMsVJQ,36498
+code_loader/leaploaderbase.py,sha256=0JMwslRs6w7psIkj4pTjYHd_pSio3vt5H-X0Lar2nBI,6428
 code_loader/mixpanel_tracker.py,sha256=rNwRmFifNbdUoqLQvvhhgpKczWpWiEmd8MfyJe27sxw,9131
 code_loader/plot_functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/plot_functions/plot_functions.py,sha256=6Q7VWGxetL2W0EK2QeCdObVATvBuHs3YBA09H4uoIk0,14996
@@ -32,7 +32,7 @@ code_loader/plot_functions/visualize.py,sha256=gsBAYYkwMh7jIpJeDMPS8G4CW-pxwx6Lz
 code_loader/utils.py,sha256=YecipkdTA-VcE9F0RQcY9cFnY8P3AksPnHM2Db7xUSk,3972
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=onRnLE_TXfgLN4o52hQIOOhUcFexGlqJ3xSpQDVLuZM,2604
-code_loader-1.0.180.dev4.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.180.dev4.dist-info/METADATA,sha256=rXrzhwdLrT5gn1PUab_L0ee4z1Wx-aH8TwWq54EG7Hw,1095
-code_loader-1.0.180.dev4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.180.dev4.dist-info/RECORD,,
+code_loader-1.0.181.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.181.dist-info/METADATA,sha256=QOCmvNOXjYgtVk2-N5Q-G47Z6IchT1_rxlprzM68_kM,1090
+code_loader-1.0.181.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.181.dist-info/RECORD,,