code-loader 1.0.208.dev5__tar.gz → 1.0.208.dev6__tar.gz

{code_loader-1.0.208.dev5 → code_loader-1.0.208.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.208.dev5
+Version: 1.0.208.dev6
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.208.dev5 → code_loader-1.0.208.dev6}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1843,6 +1843,48 @@ def _check_custom_latent_space_width(result, ls_name, reduce, n_components, chan
                 f"reduce=LatentSpaceReduction.RANDOM_PROJECTION."))
 
 
+_FLOAT16_MAX = float(np.finfo(np.float16).max)
+
+
+def _check_custom_latent_space_values(result: np.ndarray, ls_name: str, has_batch_axis: bool) -> None:
+    """Reject what no sample could store and warn about values the engine handles per sample."""
+    if isinstance(result, np.ma.MaskedArray):
+        raise Exception(
+            f"tensorleap_custom_latent_space validation failed: '{ls_name}' returned a numpy masked "
+            f"array; the mask would be ignored. Return a plain ndarray, with np.nan for a sample "
+            f"you cannot compute.")
+    if result.dtype.kind not in 'biuf':
+        raise Exception(
+            f"tensorleap_custom_latent_space validation failed: '{ls_name}' returned dtype "
+            f"{result.dtype}. Return a real-valued numeric array (bool, int or float), with np.nan "
+            f"for a sample you cannot compute.")
+    dims = result.shape[1:] if has_batch_axis else result.shape
+    if int(np.prod(dims)) == 0:
+        raise Exception(
+            f"tensorleap_custom_latent_space validation failed: '{ls_name}' returned shape "
+            f"{tuple(result.shape)}, which has no features per sample.")
+    finite = np.isfinite(result)
+    if not finite.all():
+        store_general_warning(
+            key=("tensorleap_custom_latent_space_nonfinite", ls_name),
+            message=(
+                f"Custom latent space '{ls_name}' returned NaN or inf. Those samples are left out of "
+                f"this latent space; if they exceed 5% of the evaluated samples the latent space is "
+                f"dropped."))
+    peak = 0.0
+    if result.size and result.dtype.kind == 'f':
+        peak = float(np.max(np.abs(result), where=finite, initial=0.0))
+    elif result.size and result.dtype.kind in 'iu':
+        peak = float(max(abs(int(result.min())), abs(int(result.max()))))
+    if peak > _FLOAT16_MAX:
+        store_general_warning(
+            key=("tensorleap_custom_latent_space_clipped", ls_name),
+            message=(
+                f"Custom latent space '{ls_name}' returned values beyond +-{_FLOAT16_MAX:.0f}. Latent "
+                f"spaces are stored as float16, so they are clipped to that range; scale or normalize "
+                f"them inside the function to keep their relative size."))
+
+
 def tensorleap_custom_latent_space(name: Optional[str] = None, use_ls_for_analysis: bool = False,
                                    reduce: Optional[LatentSpaceReduction] = None,
                                    n_components: int = 512, channel_axis: int = -1):
@@ -1864,6 +1906,7 @@ def tensorleap_custom_latent_space(name: Optional[str] = None, use_ls_for_analys
             assert isinstance(single_result, np.ndarray), \
                 (f'tensorleap_custom_latent_space validation failed: '
                  f'The return type should be a numpy array. Got {type(single_result)}.')
+            _check_custom_latent_space_values(single_result, ls_name, has_batch_axis=False)
             if single_result.ndim > 1:
                 flat_dim = int(np.prod(single_result.shape))
                 store_general_warning(
@@ -1966,6 +2009,7 @@ def _decorate_model_latent_space(user_function, ls_name, use_ls_for_analysis, re
             f"tensorleap_custom_latent_space validation failed: '{user_function.__name__}' "
             f"returned shape {tuple(result.shape)}. A model-computed latent space returns "
             f"(batch, d), so the result needs a batch axis and at least one feature axis.")
+        _check_custom_latent_space_values(result, ls_name, has_batch_axis=True)
         if leap_binder.batch_size_to_validate:
             assert result.shape[0] == leap_binder.batch_size_to_validate, (
                 f"tensorleap_custom_latent_space validation failed: '{user_function.__name__}' "
@@ -2052,6 +2096,7 @@ def tensorleap_instance_custom_latent_space(name: Optional[str] = None, use_ls_f
             assert isinstance(result, np.ndarray), \
                 (f'tensorleap_instance_custom_latent_space validation failed: '
                  f'The return type should be a numpy array. Got {type(result)}.')
+            _check_custom_latent_space_values(result, ls_name, has_batch_axis=False)
             if result.ndim > 1:
                 flat_dim = int(np.prod(result.shape))
                 store_general_warning(

{code_loader-1.0.208.dev5 → code_loader-1.0.208.dev6}/code_loader/leaploader.py

@@ -55,6 +55,8 @@ class LeapLoader(LeapLoaderBase):
         self._preprocess_result_cached = None
         self._synthetic_lookup: Dict[str, Tuple[PreprocessResponse, Any]] = {}
         self._synthetic_populator: Optional[Callable[[str], None]] = None
+        # Dataset-computed custom LS the engine dropped for this job; get_sample stops computing them.
+        self._skipped_custom_latent_spaces: FrozenSet[str] = frozenset()
 
         try:
             from code_loader.mixpanel_tracker import track_code_loader_loaded
@@ -1209,6 +1211,7 @@ class LeapLoader(LeapLoaderBase):
             custom_latent_spaces = {
                 name: self._to_grouped_list(handler.function(group_ids, preprocess_state))
                 for name, handler in latent_handlers.items()
+                if name not in self._skipped_custom_latent_spaces
             }
 
         return DatasetSample(inputs=inputs, gt=gt, metadata=metadata, metadata_is_none=metadata_is_none,
@@ -1432,7 +1435,8 @@ class LeapLoader(LeapLoaderBase):
             instance_id: Optional[int] = None) -> Optional[Dict[str, npt.NDArray[np.float32]]]:
         handlers = {handler_name: handler for handler_name, handler
                     in global_leap_binder.setup_container.custom_latent_spaces.items()
-                    if handler.computed_at != CustomLatentSpaceComputedAt.MODEL}
+                    if handler.computed_at != CustomLatentSpaceComputedAt.MODEL
+                    and handler_name not in self._skipped_custom_latent_spaces}
         if not handlers:
             return None
         if preprocess.is_grouped:
@@ -1459,6 +1463,10 @@ class LeapLoader(LeapLoaderBase):
         self.exec_script()
         return len(global_leap_binder.setup_container.custom_latent_spaces) > 0
 
+    def set_skipped_custom_latent_spaces(self, names: Iterable[str]) -> None:
+        """Stop computing these dataset-computed custom latent spaces in get_sample(s)."""
+        self._skipped_custom_latent_spaces = frozenset(names)
+
     @lru_cache()
     def get_custom_latent_space_names(self) -> Tuple[str, ...]:
         """Names of all registered custom latent spaces, in registration order.

{code_loader-1.0.208.dev5 → code_loader-1.0.208.dev6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.208.dev5"
+version = "1.0.208.dev6"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"