code-loader 1.0.208.dev3__tar.gz → 1.0.208.dev5__tar.gz

{code_loader-1.0.208.dev3 → code_loader-1.0.208.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.208.dev3
+Version: 1.0.208.dev5
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.208.dev3 → code_loader-1.0.208.dev5}/code_loader/contract/datasetclasses.py

@@ -6,7 +6,7 @@ import numpy as np
 import numpy.typing as npt
 
 from code_loader.contract.enums import DataStateType, DataStateEnum, LeapDataType, ConfusionMatrixValue, \
-    MetricDirection, DatasetMetadataType, LatentSpaceReduction
+    MetricDirection, DatasetMetadataType, LatentSpaceReduction, CustomLatentSpaceComputedAt
 from code_loader.contract.visualizer_classes import LeapImage, LeapText, LeapGraph, LeapHorizontalBar, \
     LeapTextMask, LeapImageMask, LeapImageWithBBox, LeapImageWithHeatmap, LeapVideo, LeapAudio
 from code_loader.contract.sim_config import SimConfig
@@ -325,7 +325,7 @@ class CustomLatentSpaceHandler:
     name: str = 'custom_latent_space'
     use_ls_for_analysis: bool = False
     instance_aware: bool = False
-    computed_at: str = 'dataset'
+    computed_at: CustomLatentSpaceComputedAt = CustomLatentSpaceComputedAt.DATASET
     arg_names: Optional[List[str]] = None
     reduce: Optional[LatentSpaceReduction] = None
     n_components: int = 512

{code_loader-1.0.208.dev3 → code_loader-1.0.208.dev5}/code_loader/contract/enums.py

@@ -73,3 +73,8 @@ class TestingSectionEnum(Enum):
 class LatentSpaceReduction(Enum):
     MEAN_POOL = 'MEAN_POOL'
     RANDOM_PROJECTION = 'RANDOM_PROJECTION'
+
+
+class CustomLatentSpaceComputedAt(Enum):
+    DATASET = 'dataset'
+    MODEL = 'model'

{code_loader-1.0.208.dev3 → code_loader-1.0.208.dev5}/code_loader/inner_leap_binder/leapbinder.py

@@ -22,7 +22,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     AUTOREGRESSIVE_LATENT_SPACE_AGGREGATIONS, AUTOREGRESSIVE_IMPLICIT_ARG_NAMES, \
     AutoregressiveMetricHandler, AutoregressiveLossHandler, AutoregressiveVisualizerHandler
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType, \
-    TestingSectionEnum, LatentSpaceReduction
+    TestingSectionEnum, LatentSpaceReduction, CustomLatentSpaceComputedAt
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload, LeapAnalysisConfiguration
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
@@ -561,7 +561,7 @@ class LeapBinder:
                                 name: Optional[str] = None,
                                 use_ls_for_analysis: bool = False,
                                 instance_aware: bool = False,
-                                computed_at: str = 'dataset',
+                                computed_at: CustomLatentSpaceComputedAt = CustomLatentSpaceComputedAt.DATASET,
                                 arg_names: Optional[List[str]] = None,
                                 reduce: Optional[LatentSpaceReduction] = None,
                                 n_components: int = 512,

{code_loader-1.0.208.dev3 → code_loader-1.0.208.dev5}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -18,7 +18,7 @@ import numpy.typing as npt
 from code_loader.utils import map_dict_to_metadata_types, is_absent_metadata_value, \
     validate_autoregressive_state_types, autoregressive_nests_equal, \
     simulate_engine_float16_downcast_on_call_args, ENGINE_STORAGE_DTYPE, \
-    TL_DISABLE_ENGINE_FLOAT16_SIMULATION_ENV_VAR
+    TL_DISABLE_ENGINE_FLOAT16_SIMULATION_ENV_VAR, sample_preprocess_response_arg_name
 
 logger = logging.getLogger(__name__)
 
@@ -29,7 +29,7 @@ from code_loader.contract.datasetclasses import CustomCallableInterfaceMultiArgs
     InstanceLengthCallableInterface, InstanceSectionCallableInterface, AutoregressiveStepCallableInterface, \
     MAX_CUSTOM_LATENT_SPACE_DIM, CUSTOM_LATENT_SPACE_WARN_DIM
 from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType, DataStateType, \
-    DataStateEnum, LatentSpaceReduction
+    DataStateEnum, LatentSpaceReduction, CustomLatentSpaceComputedAt
 from code_loader import leap_binder, LeapLoader
 from code_loader.contract.mapping import NodeMapping, NodeMappingType, NodeConnection
 from code_loader.contract.visualizer_classes import LeapImage, LeapImageMask, LeapTextMask, LeapText, LeapGraph, \
@@ -291,11 +291,7 @@ def _require_sample_preprocess_response_supplied(user_function: Callable, args:
     """A SamplePreprocessResponse argument is auto-injected by the platform / check_dataset
     but NOT inside integration_test, where the author calls the function directly. Fail fast
     with an actionable message instead of a raw 'missing argument' TypeError."""
-    spr_arg_name = None
-    for arg_name, arg_type in inspect.getfullargspec(user_function).annotations.items():
-        if arg_type == SamplePreprocessResponse:
-            spr_arg_name = arg_name
-            break
+    spr_arg_name = sample_preprocess_response_arg_name(user_function)
     if spr_arg_name is None:
         return
     signature = inspect.signature(user_function)
@@ -1786,16 +1782,25 @@ def _classify_custom_latent_space_signature(user_function) -> str:
 def _model_latent_space_arg_names(user_function) -> List[str]:
     argspec = inspect.getfullargspec(user_function)
     arg_names = list(argspec.args)
-    preprocess_response_arg_name = None
+    spr_count = 0
     for arg_name, arg_type in argspec.annotations.items():
+        if arg_name == 'return':
+            continue
         _reject_stringized_sample_preprocess_response(user_function, arg_name, arg_type)
         if arg_type == SamplePreprocessResponse:
-            if preprocess_response_arg_name is not None:
-                raise Exception(
-                    f"tensorleap_custom_latent_space validation failed: only one argument of "
-                    f"'{user_function.__name__}' can be of type SamplePreprocessResponse.")
-            preprocess_response_arg_name = arg_name
-            arg_names.remove(arg_name)
+            spr_count += 1
+    if spr_count > 1:
+        raise Exception(
+            f"tensorleap_custom_latent_space validation failed: only one argument of "
+            f"'{user_function.__name__}' can be of type SamplePreprocessResponse.")
+    spr_arg_name = sample_preprocess_response_arg_name(user_function)
+    if spr_arg_name is not None:
+        arg_names.remove(spr_arg_name)
+    if not arg_names:
+        raise Exception(
+            f"tensorleap_custom_latent_space validation failed: '{user_function.__name__}' is "
+            f"model-computed and expects at least one np.ndarray argument, but its signature "
+            f"declares none.")
     return arg_names
 
 
@@ -1911,7 +1916,7 @@ def tensorleap_custom_latent_space(name: Optional[str] = None, use_ls_for_analys
 
         leap_binder.set_custom_latent_space(inner_without_validate, ls_name,
                                             use_ls_for_analysis=use_ls_for_analysis,
-                                            computed_at='dataset', reduce=reduce,
+                                            computed_at=CustomLatentSpaceComputedAt.DATASET, reduce=reduce,
                                             n_components=n_components, channel_axis=channel_axis)
 
         def inner(sample_id, preprocess_response):
@@ -1937,6 +1942,9 @@ def _decorate_model_latent_space(user_function, ls_name, use_ls_for_analysis, re
     arg_names = _model_latent_space_arg_names(user_function)
 
     def _validate_input_args(*args, **kwargs):
+        # Every bound argument must be an array: a ground truth is never passed as None, so a
+        # model-computed LS cannot fall back from ground truth to predictions on unlabeled rows.
+        # The engine skips the LS for those rows instead (see leaploader._check_model_latent_spaces).
         assert len(args) + len(kwargs) > 0, (
             f"tensorleap_custom_latent_space validation failed: '{user_function.__name__}' is "
             f"model-computed and expects at least one np.ndarray argument, but received none.")
@@ -1972,6 +1980,7 @@ def _decorate_model_latent_space(user_function, ls_name, use_ls_for_analysis, re
         _called_from_inside_tl_decorator += 1
 
         try:
+            _require_sample_preprocess_response_supplied(user_function, args, kwargs)
             result = user_function(*args, **kwargs)
         finally:
             _called_from_inside_tl_decorator -= 1
@@ -1982,7 +1991,7 @@ def _decorate_model_latent_space(user_function, ls_name, use_ls_for_analysis, re
 
     leap_binder.set_custom_latent_space(inner_without_validate, ls_name,
                                         use_ls_for_analysis=use_ls_for_analysis,
-                                        computed_at='model', arg_names=arg_names, reduce=reduce,
+                                        computed_at=CustomLatentSpaceComputedAt.MODEL, arg_names=arg_names, reduce=reduce,
                                         n_components=n_components, channel_axis=channel_axis)
 
     def inner(*args, **kwargs):

{code_loader-1.0.208.dev3 → code_loader-1.0.208.dev5}/code_loader/leaploader.py

@@ -17,7 +17,9 @@ from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandle
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
     MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse, \
     ElementInstance, custom_latent_space_attribute, DatasetIntegrationSetup, InstanceMetricHandler, _simulation_context
-from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
+from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType, \
+    CustomLatentSpaceComputedAt
+from code_loader.contract.mapping import NodeMappingType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
@@ -28,7 +30,8 @@ from code_loader.inner_leap_binder import global_leap_binder
 from code_loader.inner_leap_binder.leapbinder import mapping_runtime_mode_env_var_mame
 from code_loader.leaploaderbase import LeapLoaderBase
 from code_loader.utils import get_root_exception_file_and_line_number, get_metadata_type_from_variable, \
-    validate_autoregressive_state_types, autoregressive_nests_equal, is_absent_metadata_value
+    validate_autoregressive_state_types, autoregressive_nests_equal, is_absent_metadata_value, \
+    sample_preprocess_response_arg_name
 
 
 def _serialize_sim_bounds(bounds) -> dict:
@@ -433,7 +436,7 @@ class LeapLoader(LeapLoaderBase):
     def _check_model_latent_spaces(self) -> Optional[DatasetTestResultPayload]:
         model_names = [
             name for name, handler in global_leap_binder.setup_container.custom_latent_spaces.items()
-            if handler.computed_at == 'model'
+            if handler.computed_at == CustomLatentSpaceComputedAt.MODEL
         ]
         if not model_names:
             return None
@@ -450,6 +453,19 @@ class LeapLoader(LeapLoaderBase):
                 f"grouped preprocess response (grouped: {grouped_states}). Use the "
                 f"(sample_id, preprocess: PreprocessResponse) form instead."
             )
+        if global_leap_binder.setup_container.unlabeled_data_preprocess is not None:
+            gt_bound_names = [
+                connection.node.name for connection in global_leap_binder.latent_space_connections
+                if any(node_input.type == NodeMappingType.GroundTruth
+                       for node_input in (connection.node_inputs or {}).values())
+            ]
+            if gt_bound_names:
+                test_result.display[TestingSectionEnum.Warnings.name] = (
+                    f"Model-computed custom latent space(s) {gt_bound_names} read a ground truth, so "
+                    f"they are skipped for unlabeled samples, which will have no vector in them. A "
+                    f"latent space cannot fall back from ground truth to predictions; to cover "
+                    f"unlabeled samples, add one bound only to model predictions."
+                )
         return test_result
 
     def _check_instance_custom_latent_spaces(self) -> Optional[DatasetTestResultPayload]:
@@ -800,10 +816,7 @@ class LeapLoader(LeapLoaderBase):
     @staticmethod
     def _get_preprocess_response_arg_name(
             func: Callable) -> Optional[str]:
-        for arg_name, arg_type in inspect.getfullargspec(func).annotations.items():
-            if arg_type == SamplePreprocessResponse:
-                return arg_name
-        return None
+        return sample_preprocess_response_arg_name(func)
 
     def run_custom_loss(self, custom_loss_name: str, sample_ids: np.array, state: DataStateEnum,
                         input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
@@ -1419,7 +1432,7 @@ class LeapLoader(LeapLoaderBase):
             instance_id: Optional[int] = None) -> Optional[Dict[str, npt.NDArray[np.float32]]]:
         handlers = {handler_name: handler for handler_name, handler
                     in global_leap_binder.setup_container.custom_latent_spaces.items()
-                    if handler.computed_at != 'model'}
+                    if handler.computed_at != CustomLatentSpaceComputedAt.MODEL}
         if not handlers:
             return None
         if preprocess.is_grouped:
@@ -1461,20 +1474,14 @@ class LeapLoader(LeapLoaderBase):
     def get_dataset_custom_latent_space_names(self) -> Tuple[str, ...]:
         self.exec_script()
         return tuple(name for name, handler in global_leap_binder.setup_container.custom_latent_spaces.items()
-                     if handler.computed_at != 'model')
-
-    @lru_cache()
-    def get_model_custom_latent_space_names(self) -> Tuple[str, ...]:
-        self.exec_script()
-        return tuple(name for name, handler in global_leap_binder.setup_container.custom_latent_spaces.items()
-                     if handler.computed_at == 'model')
+                     if handler.computed_at != CustomLatentSpaceComputedAt.MODEL)
 
     @lru_cache()
     def get_custom_latent_space_specs(self) -> Dict[str, Dict[str, Any]]:
         self.exec_script()
         return {
             name: {
-                'computed_at': handler.computed_at,
+                'computed_at': handler.computed_at.value,
                 'arg_names': list(handler.arg_names or []),
                 'reduce': handler.reduce.value if handler.reduce is not None else None,
                 'n_components': handler.n_components,
@@ -1484,18 +1491,19 @@ class LeapLoader(LeapLoaderBase):
         }
 
     @lru_cache()
-    def get_model_latent_space_specs(self) -> Dict[str, Dict[str, Any]]:
+    def get_model_custom_latent_space_specs(self) -> Dict[str, Dict[str, Any]]:
         return {name: spec for name, spec in self.get_custom_latent_space_specs().items()
                 if spec['computed_at'] == 'model'}
 
     def run_model_latent_space(self, ls_name: str, sample_ids: np.array, state: DataStateEnum,
                                input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
                                ) -> npt.NDArray[np.float32]:
-        self._preprocess_result()
-
+        self.exec_script()
         handler = global_leap_binder.setup_container.custom_latent_spaces[ls_name]
         preprocess_response_arg_name = self._get_preprocess_response_arg_name(handler.function)
 
+        # Preprocess runs only when the function asks for a SamplePreprocessResponse; the metrics
+        # pod that calls this has no other reason to pay for it.
         if preprocess_response_arg_name is not None:
             input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(
                 sample_ids, self._preprocess_result()[state])

{code_loader-1.0.208.dev3 → code_loader-1.0.208.dev5}/code_loader/leaploaderbase.py

@@ -234,20 +234,15 @@ class LeapLoaderBase:
         raise NotImplementedError(f'{type(self).__name__} does not implement '
                                   'get_dataset_custom_latent_space_names.')
 
-    @abstractmethod
-    def get_model_custom_latent_space_names(self) -> Tuple[str, ...]:
-        raise NotImplementedError(f'{type(self).__name__} does not implement '
-                                  'get_model_custom_latent_space_names.')
-
     @abstractmethod
     def get_custom_latent_space_specs(self) -> Dict[str, Dict[str, Any]]:
         raise NotImplementedError(f'{type(self).__name__} does not implement '
                                   'get_custom_latent_space_specs.')
 
     @abstractmethod
-    def get_model_latent_space_specs(self) -> Dict[str, Dict[str, Any]]:
+    def get_model_custom_latent_space_specs(self) -> Dict[str, Dict[str, Any]]:
         raise NotImplementedError(f'{type(self).__name__} does not implement '
-                                  'get_model_latent_space_specs.')
+                                  'get_model_custom_latent_space_specs.')
 
     @abstractmethod
     def run_model_latent_space(self, ls_name: str, sample_ids: np.array, state: DataStateEnum,

{code_loader-1.0.208.dev3 → code_loader-1.0.208.dev5}/code_loader/utils.py

@@ -1,3 +1,4 @@
+import inspect
 import io
 import math
 import os
@@ -11,7 +12,7 @@ import numpy as np
 import numpy.typing as npt
 
 from code_loader.contract.datasetclasses import SectionCallableInterface, PreprocessResponse, \
-    InstanceCallableInterface, ElementInstance
+    InstanceCallableInterface, ElementInstance, SamplePreprocessResponse
 from code_loader.contract.enums import DatasetMetadataType
 
 
@@ -246,3 +247,11 @@ def autoregressive_nests_equal(a: Any, b: Any) -> bool:
             math.isnan(a) and math.isnan(b):
         return True
     return bool(a == b)
+
+
+def sample_preprocess_response_arg_name(func: Callable[..., Any]) -> Optional[str]:
+    # 'return' lives in annotations too and must never be mistaken for a parameter.
+    for arg_name, arg_type in inspect.getfullargspec(func).annotations.items():
+        if arg_name != 'return' and arg_type == SamplePreprocessResponse:
+            return arg_name
+    return None

{code_loader-1.0.208.dev3 → code_loader-1.0.208.dev5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.208.dev3"
+version = "1.0.208.dev5"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"