code-loader 1.0.208.dev5__tar.gz → 1.0.208.dev7__tar.gz

{code_loader-1.0.208.dev5 → code_loader-1.0.208.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.208.dev5
+Version: 1.0.208.dev7
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.208.dev5 → code_loader-1.0.208.dev7}/code_loader/contract/responsedataclasses.py

@@ -144,7 +144,6 @@ class LeapAnalysisConfiguration:
 class EngineFileContract:
     node_connections: Optional[List[NodeConnection]] = None
     leap_analysis_configuration: Optional[LeapAnalysisConfiguration] = None
-    latent_space_connections: Optional[List[NodeConnection]] = None
 
 
 @dataclass

{code_loader-1.0.208.dev5 → code_loader-1.0.208.dev7}/code_loader/inner_leap_binder/leapbinder.py

@@ -115,7 +115,6 @@ class LeapBinder:
         self._extend_with_default_losses()
 
         self.mapping_connections: List[NodeConnection] = []
-        self.latent_space_connections: List[NodeConnection] = []
         self.integration_test_func: Optional[Callable[[str, PreprocessResponse], Any]] = None
 
         self.batch_size_to_validate: Optional[int] = None

{code_loader-1.0.208.dev5 → code_loader-1.0.208.dev7}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1843,6 +1843,48 @@ def _check_custom_latent_space_width(result, ls_name, reduce, n_components, chan
                 f"reduce=LatentSpaceReduction.RANDOM_PROJECTION."))
 
 
+_FLOAT16_MAX = float(np.finfo(np.float16).max)
+
+
+def _check_custom_latent_space_values(result: np.ndarray, ls_name: str, has_batch_axis: bool) -> None:
+    """Reject what no sample could store and warn about values the engine handles per sample."""
+    if isinstance(result, np.ma.MaskedArray):
+        raise Exception(
+            f"tensorleap_custom_latent_space validation failed: '{ls_name}' returned a numpy masked "
+            f"array; the mask would be ignored. Return a plain ndarray, with np.nan for a sample "
+            f"you cannot compute.")
+    if result.dtype.kind not in 'biuf':
+        raise Exception(
+            f"tensorleap_custom_latent_space validation failed: '{ls_name}' returned dtype "
+            f"{result.dtype}. Return a real-valued numeric array (bool, int or float), with np.nan "
+            f"for a sample you cannot compute.")
+    dims = result.shape[1:] if has_batch_axis else result.shape
+    if int(np.prod(dims)) == 0:
+        raise Exception(
+            f"tensorleap_custom_latent_space validation failed: '{ls_name}' returned shape "
+            f"{tuple(result.shape)}, which has no features per sample.")
+    finite = np.isfinite(result)
+    if not finite.all():
+        store_general_warning(
+            key=("tensorleap_custom_latent_space_nonfinite", ls_name),
+            message=(
+                f"Custom latent space '{ls_name}' returned NaN or inf. Those samples are left out of "
+                f"this latent space; if they exceed 5% of the evaluated samples the latent space is "
+                f"dropped."))
+    peak = 0.0
+    if result.size and result.dtype.kind == 'f':
+        peak = float(np.max(np.abs(result), where=finite, initial=0.0))
+    elif result.size and result.dtype.kind in 'iu':
+        peak = float(max(abs(int(result.min())), abs(int(result.max()))))
+    if peak > _FLOAT16_MAX:
+        store_general_warning(
+            key=("tensorleap_custom_latent_space_clipped", ls_name),
+            message=(
+                f"Custom latent space '{ls_name}' returned values beyond +-{_FLOAT16_MAX:.0f}. Latent "
+                f"spaces are stored as float16, so they are clipped to that range; scale or normalize "
+                f"them inside the function to keep their relative size."))
+
+
 def tensorleap_custom_latent_space(name: Optional[str] = None, use_ls_for_analysis: bool = False,
                                    reduce: Optional[LatentSpaceReduction] = None,
                                    n_components: int = 512, channel_axis: int = -1):
@@ -1864,6 +1906,7 @@ def tensorleap_custom_latent_space(name: Optional[str] = None, use_ls_for_analys
             assert isinstance(single_result, np.ndarray), \
                 (f'tensorleap_custom_latent_space validation failed: '
                  f'The return type should be a numpy array. Got {type(single_result)}.')
+            _check_custom_latent_space_values(single_result, ls_name, has_batch_axis=False)
             if single_result.ndim > 1:
                 flat_dim = int(np.prod(single_result.shape))
                 store_general_warning(
@@ -1966,6 +2009,7 @@ def _decorate_model_latent_space(user_function, ls_name, use_ls_for_analysis, re
             f"tensorleap_custom_latent_space validation failed: '{user_function.__name__}' "
             f"returned shape {tuple(result.shape)}. A model-computed latent space returns "
             f"(batch, d), so the result needs a batch axis and at least one feature axis.")
+        _check_custom_latent_space_values(result, ls_name, has_batch_axis=True)
         if leap_binder.batch_size_to_validate:
             assert result.shape[0] == leap_binder.batch_size_to_validate, (
                 f"tensorleap_custom_latent_space validation failed: '{user_function.__name__}' "
@@ -2011,14 +2055,15 @@ def _decorate_model_latent_space(user_function, ls_name, use_ls_for_analysis, re
         ordered_connections = [kwargs[arg_name] for arg_name in arg_names if arg_name in kwargs]
         ordered_connections = list(args) + ordered_connections
 
-        leap_binder.latent_space_connections[:] = [
-            connection for connection in leap_binder.latent_space_connections
-            if connection.node.name != ls_name]
+        leap_binder.mapping_connections[:] = [
+            connection for connection in leap_binder.mapping_connections
+            if not (connection.node.type is NodeMappingType.CustomLatentSpace
+                    and connection.node.name == ls_name)]
         _add_mapping_connection(ls_name, ordered_connections, arg_names, ls_name,
-                                NodeMappingType.CustomLatentSpace,
-                                target_list=leap_binder.latent_space_connections)
+                                NodeMappingType.CustomLatentSpace)
         return None
 
+    @functools.wraps(user_function)
     def final_inner(*args, **kwargs):
         if os.environ.get(mapping_runtime_mode_env_var_mame):
             return mapping_inner(*args, **kwargs)
@@ -2052,6 +2097,7 @@ def tensorleap_instance_custom_latent_space(name: Optional[str] = None, use_ls_f
             assert isinstance(result, np.ndarray), \
                 (f'tensorleap_instance_custom_latent_space validation failed: '
                  f'The return type should be a numpy array. Got {type(result)}.')
+            _check_custom_latent_space_values(result, ls_name, has_batch_axis=False)
             if result.ndim > 1:
                 flat_dim = int(np.prod(result.shape))
                 store_general_warning(

{code_loader-1.0.208.dev5 → code_loader-1.0.208.dev7}/code_loader/leaploader.py

@@ -55,6 +55,8 @@ class LeapLoader(LeapLoaderBase):
         self._preprocess_result_cached = None
         self._synthetic_lookup: Dict[str, Tuple[PreprocessResponse, Any]] = {}
         self._synthetic_populator: Optional[Callable[[str], None]] = None
+        # Dataset-computed custom LS the engine dropped for this job; get_sample stops computing them.
+        self._skipped_custom_latent_spaces: FrozenSet[str] = frozenset()
 
         try:
             from code_loader.mixpanel_tracker import track_code_loader_loaded
@@ -394,8 +396,7 @@ class LeapLoader(LeapLoaderBase):
                                        print_log=print_log,
                                        engine_file_contract=EngineFileContract(
                                            global_leap_binder.mapping_connections,
-                                           global_leap_binder.leap_analysis_configuration,
-                                           global_leap_binder.latent_space_connections))
+                                           global_leap_binder.leap_analysis_configuration))
 
     def _check_integration_test_exists(self) -> DatasetTestResultPayload:
         test_result = DatasetTestResultPayload('integration_test')
@@ -455,9 +456,10 @@ class LeapLoader(LeapLoaderBase):
             )
         if global_leap_binder.setup_container.unlabeled_data_preprocess is not None:
             gt_bound_names = [
-                connection.node.name for connection in global_leap_binder.latent_space_connections
-                if any(node_input.type == NodeMappingType.GroundTruth
-                       for node_input in (connection.node_inputs or {}).values())
+                connection.node.name for connection in global_leap_binder.mapping_connections
+                if connection.node.type is NodeMappingType.CustomLatentSpace
+                and any(node_input.type == NodeMappingType.GroundTruth
+                        for node_input in (connection.node_inputs or {}).values())
             ]
             if gt_bound_names:
                 test_result.display[TestingSectionEnum.Warnings.name] = (
@@ -1209,6 +1211,7 @@ class LeapLoader(LeapLoaderBase):
             custom_latent_spaces = {
                 name: self._to_grouped_list(handler.function(group_ids, preprocess_state))
                 for name, handler in latent_handlers.items()
+                if name not in self._skipped_custom_latent_spaces
             }
 
         return DatasetSample(inputs=inputs, gt=gt, metadata=metadata, metadata_is_none=metadata_is_none,
@@ -1432,7 +1435,8 @@ class LeapLoader(LeapLoaderBase):
             instance_id: Optional[int] = None) -> Optional[Dict[str, npt.NDArray[np.float32]]]:
         handlers = {handler_name: handler for handler_name, handler
                     in global_leap_binder.setup_container.custom_latent_spaces.items()
-                    if handler.computed_at != CustomLatentSpaceComputedAt.MODEL}
+                    if handler.computed_at != CustomLatentSpaceComputedAt.MODEL
+                    and handler_name not in self._skipped_custom_latent_spaces}
         if not handlers:
             return None
         if preprocess.is_grouped:
@@ -1459,6 +1463,10 @@ class LeapLoader(LeapLoaderBase):
         self.exec_script()
         return len(global_leap_binder.setup_container.custom_latent_spaces) > 0
 
+    def set_skipped_custom_latent_spaces(self, names: Iterable[str]) -> None:
+        """Stop computing these dataset-computed custom latent spaces in get_sample(s)."""
+        self._skipped_custom_latent_spaces = frozenset(names)
+
     @lru_cache()
     def get_custom_latent_space_names(self) -> Tuple[str, ...]:
         """Names of all registered custom latent spaces, in registration order.

{code_loader-1.0.208.dev5 → code_loader-1.0.208.dev7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.208.dev5"
+version = "1.0.208.dev7"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"