code-loader 1.0.208.dev7__tar.gz → 1.0.209__tar.gz

{code_loader-1.0.208.dev7 → code_loader-1.0.209}/PKG-INFO

@@ -1,9 +1,9 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: code-loader
-Version: 1.0.208.dev7
+Version: 1.0.209
 Summary: 
-Home-page: https://github.com/tensorleap/code-loader
 License: MIT
+License-File: LICENSE
 Author: dorhar
 Author-email: doron.harnoy@tensorleap.ai
 Requires-Python: >=3.8,<3.13
@@ -20,6 +20,7 @@ Requires-Dist: numpy (>=2.3.2,<3.0.0) ; python_version >= "3.11" and python_vers
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
+Project-URL: Homepage, https://github.com/tensorleap/code-loader
 Project-URL: Repository, https://github.com/tensorleap/code-loader
 Description-Content-Type: text/markdown
 

{code_loader-1.0.208.dev7 → code_loader-1.0.209}/code_loader/inner_leap_binder/leapbinder.py

@@ -85,11 +85,6 @@ def _reject_stringized_sample_preprocess_response(function: Callable[..., Any],
         "This breaks Tensorleap type detection, so it must be referenced as a real type.")
 
 
-def _reject_stringized_preprocess_response(function: Callable[..., Any], arg_name: str, annotation: Any) -> None:
-    _reject_stringized_annotation(
-        function, arg_name, annotation, PreprocessResponse,
-        "Tensorleap uses that annotation to tell a dataset-computed custom latent space from a "
-        "model-computed one, so it must be a real type.")
 
 
 
@@ -116,6 +111,10 @@ class LeapBinder:
 
         self.mapping_connections: List[NodeConnection] = []
         self.integration_test_func: Optional[Callable[[str, PreprocessResponse], Any]] = None
+        # Set by LeapLoader.exec_script when running the dataset script failed, because that
+        # also resets setup_container: an empty container after a failure means "the script
+        # crashed", not "the user forgot to register". See get_preprocess_result.
+        self.previous_script_failure: Optional[BaseException] = None
 
         self.batch_size_to_validate: Optional[int] = None
         self.leap_analysis_configuration = LeapAnalysisConfiguration()
@@ -808,6 +807,13 @@ class LeapBinder:
     def get_preprocess_result(self) -> Dict[DataStateEnum, PreprocessResponse]:
         preprocess = self.setup_container.preprocess
         if preprocess is None:
+            if self.previous_script_failure is not None:
+                # Running the dataset script already failed once, and that reset the
+                # registrations (LeapLoader.exec_script). Blaming a missing
+                # set_preprocess call here would hide the actual crash.
+                raise Exception(
+                    f"The dataset script failed before registering its handlers: "
+                    f"{self.previous_script_failure}") from self.previous_script_failure
             raise Exception("Please make sure you call the leap_binder.set_preprocess method")
         preprocess_results = preprocess.function()
         preprocess_result_dict = {}

{code_loader-1.0.208.dev7 → code_loader-1.0.209}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -7,6 +7,7 @@ import warnings
 import logging
 from collections import defaultdict
 import functools
+import typing
 from functools import lru_cache
 from pathlib import Path
 from typing import Optional, Union, Callable, List, Dict, Set, Any
@@ -36,7 +37,7 @@ from code_loader.contract.visualizer_classes import LeapImage, LeapImageMask, Le
     LeapHorizontalBar, LeapImageWithBBox, LeapImageWithHeatmap, LeapVideo, LeapAudio, LeapValidationError, \
     map_leap_data_type_to_visualizer_class
 from code_loader.inner_leap_binder.leapbinder import mapping_runtime_mode_env_var_mame, \
-    _reject_stringized_preprocess_response, _reject_stringized_sample_preprocess_response
+    _reject_stringized_sample_preprocess_response
 from code_loader.mixpanel_tracker import clear_integration_events, AnalyticsEvent, emit_integration_event_once
 
 _called_from_inside_tl_decorator = 0
@@ -1751,15 +1752,41 @@ def tensorleap_metadata(
     return decorating_function
 
 
+def _is_preprocess_response_type(annotation: Any) -> bool:
+    if isinstance(annotation, type):
+        return issubclass(annotation, PreprocessResponse)
+    return any(_is_preprocess_response_type(arg) for arg in get_args(annotation))
+
+
 def _classify_custom_latent_space_signature(user_function) -> str:
-    argspec = inspect.getfullargspec(user_function)
-    params = argspec.args
-    if len(params) != 2:
-        return 'model'
-
-    second = params[1]
-    first = params[0]
-    if second not in argspec.annotations:
+    """Dataset-computed when a parameter is typed PreprocessResponse (a subclass or an
+    Optional[...] of it included), model-computed otherwise."""
+    params = list(inspect.signature(user_function).parameters.values())
+    try:
+        hints = typing.get_type_hints(user_function)
+    except NameError as e:
+        raise Exception(
+            f"tensorleap_custom_latent_space validation failed: could not resolve the type "
+            f"annotations of '{user_function.__name__}' ({e}). Tensorleap reads them to tell a "
+            f"dataset-computed latent space from a model-computed one, so every annotation, the "
+            f"return type included, must be resolvable when the function is decorated. Import the "
+            f"annotated types at module level (not only under TYPE_CHECKING) or remove the "
+            f"annotation.") from e
+    preprocess_params = [p.name for p in params if _is_preprocess_response_type(hints.get(p.name))]
+    if preprocess_params:
+        positional = [p.name for p in params
+                      if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)]
+        if positional[1:2] != preprocess_params[:1] or len(preprocess_params) > 1:
+            raise Exception(
+                f"tensorleap_custom_latent_space validation failed: '{user_function.__name__}' is "
+                f"dataset-computed (it takes a PreprocessResponse), so it is called as "
+                f"{user_function.__name__}(sample_id, preprocess) and its PreprocessResponse must be "
+                f"the second positional parameter, and the only one. Got parameters "
+                f"{[p.name for p in params]} with PreprocessResponse on {preprocess_params}.")
+        return 'dataset'
+
+    if len(params) == 2 and params[1].name not in hints:
+        first, second = params[0].name, params[1].name
         raise Exception(
             f"tensorleap_custom_latent_space validation failed: '{user_function.__name__}' has "
             f"exactly two parameters ('{first}', '{second}') and '{second}' has no type "
@@ -1771,23 +1798,29 @@ def _classify_custom_latent_space_signature(user_function) -> str:
             f"If you are upgrading an existing project, this signature used to be accepted "
             f"unannotated as dataset-computed; add ': PreprocessResponse' to '{second}' to keep "
             f"the previous behavior.")
-
-    annotation = argspec.annotations[second]
-    _reject_stringized_preprocess_response(user_function, second, annotation)
-    if annotation is PreprocessResponse:
-        return 'dataset'
+    if len(params) > 2 and not any(p.name in hints for p in params):
+        names = [p.name for p in params]
+        raise Exception(
+            f"tensorleap_custom_latent_space validation failed: '{user_function.__name__}' has "
+            f"parameters {names} and none of them has a type annotation, so Tensorleap cannot "
+            f"tell whether this is a dataset-computed latent space (one sample at a time) or a "
+            f"model-computed one (a batch of model tensors). Please annotate them:\n"
+            f"  dataset-computed: def {user_function.__name__}({names[0]}, {names[1]}: PreprocessResponse, ...) -> (d,)\n"
+            f"  model-computed:   def {user_function.__name__}({names[0]}: np.ndarray, {names[1]}: np.ndarray, ...) -> (batch, d)")
     return 'model'
 
 
 def _model_latent_space_arg_names(user_function) -> List[str]:
-    argspec = inspect.getfullargspec(user_function)
-    arg_names = list(argspec.args)
+    # inspect.signature follows functools.wraps' __wrapped__, as the classifier does.
+    params = inspect.signature(user_function).parameters.values()
+    arg_names = [p.name for p in params
+                 if p.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)]
     spr_count = 0
-    for arg_name, arg_type in argspec.annotations.items():
-        if arg_name == 'return':
+    for param in params:
+        if param.annotation is inspect.Parameter.empty:
             continue
-        _reject_stringized_sample_preprocess_response(user_function, arg_name, arg_type)
-        if arg_type == SamplePreprocessResponse:
+        _reject_stringized_sample_preprocess_response(user_function, param.name, param.annotation)
+        if param.annotation == SamplePreprocessResponse:
             spr_count += 1
     if spr_count > 1:
         raise Exception(
@@ -1869,7 +1902,7 @@ def _check_custom_latent_space_values(result: np.ndarray, ls_name: str, has_batc
             key=("tensorleap_custom_latent_space_nonfinite", ls_name),
             message=(
                 f"Custom latent space '{ls_name}' returned NaN or inf. Those samples are left out of "
-                f"this latent space; if they exceed 5% of the evaluated samples the latent space is "
+                f"this latent space; if they exceed 15% of the evaluated samples the latent space is "
                 f"dropped."))
     peak = 0.0
     if result.size and result.dtype.kind == 'f':

{code_loader-1.0.208.dev7 → code_loader-1.0.209}/code_loader/leaploader.py

@@ -1,4 +1,5 @@
 # mypy: ignore-errors
+import copy
 import importlib.util
 import inspect
 import io
@@ -7,6 +8,7 @@ import sys
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
+from types import TracebackType
 from typing import Dict, List, Iterable, Set, FrozenSet, Union, Any, Type, Optional, Callable, Tuple
 
 import numpy as np
@@ -33,6 +35,44 @@ from code_loader.utils import get_root_exception_file_and_line_number, get_metad
     validate_autoregressive_state_types, autoregressive_nests_equal, is_absent_metadata_value, \
     sample_preprocess_response_arg_name
 
+_code_loader_dir = str(Path(__file__).parent)
+
+
+def _innermost_frame_outside_code_loader(e: BaseException) -> Optional[TracebackType]:
+    """The deepest traceback frame that is not code_loader's own code, if any.
+
+    That is the line the user can go and look at. The deepest frame overall is often
+    ours — a validation that raises, or a builtin we wrapped — and "raised at
+    leapbinder.py" tells the user nothing about their script.
+    """
+    frame_in_user_code = None
+    _traceback = e.__traceback__
+    while _traceback is not None:
+        if not _traceback.tb_frame.f_code.co_filename.startswith(_code_loader_dir):
+            frame_in_user_code = _traceback
+        _traceback = _traceback.tb_next
+    return frame_in_user_code
+
+
+def describe_script_exception(e: BaseException) -> str:
+    """One line describing a failure raised by user code: its type, its message and where it happened.
+
+    ``repr(e)`` on its own throws away what the user needs most: ``repr`` of an OSError
+    drops the filename (``FileNotFoundError(2, 'No such file or directory')`` — but not
+    *which* file), and no ``repr`` says which line of the integration script raised. Both
+    are what turns "the dataset script crashed" into an error the user can act on.
+    """
+    message = getattr(e, 'message', None)
+    if not isinstance(message, str) or not message:
+        message = str(e)
+    description = f'{type(e).__name__}: {message}' if message else repr(e)
+
+    frame_in_user_code = _innermost_frame_outside_code_loader(e)
+    if frame_in_user_code is not None:
+        file_name = Path(frame_in_user_code.tb_frame.f_code.co_filename).name
+        description = f'{description} (raised at {file_name}, line {frame_in_user_code.tb_lineno})'
+    return description
+
 
 def _serialize_sim_bounds(bounds) -> dict:
     if isinstance(bounds, (FloatBounds, IntBounds)):
@@ -55,8 +95,8 @@ class LeapLoader(LeapLoaderBase):
         self._preprocess_result_cached = None
         self._synthetic_lookup: Dict[str, Tuple[PreprocessResponse, Any]] = {}
         self._synthetic_populator: Optional[Callable[[str], None]] = None
-        # Dataset-computed custom LS the engine dropped for this job; get_sample stops computing them.
-        self._skipped_custom_latent_spaces: FrozenSet[str] = frozenset()
+        # The first exec_script failure, re-raised by every later call. See exec_script.
+        self._exec_script_error: Optional[BaseException] = None
 
         try:
             from code_loader.mixpanel_tracker import track_code_loader_loaded
@@ -68,9 +108,33 @@ class LeapLoader(LeapLoaderBase):
         except Exception:
             pass
 
+    def _remember_exec_script_error(self, error: BaseException) -> BaseException:
+        # Also recorded on the binder: the failure handlers below wipe its setup_container,
+        # so anything that reaches the binder afterwards (another LeapLoader in the same
+        # process, say) can report the real failure instead of an empty container.
+        global_leap_binder.previous_script_failure = error
+        self._exec_script_error = error
+        return error
+
     @lru_cache()
     def exec_script(self) -> None:
         from code_loader.inner_leap_binder import leapbinder_decorators as _leap_dec
+
+        # A failed exec_script is sticky. lru_cache does not cache exceptions, so without
+        # this every later caller re-ran the script — and the re-run reports the WRONG
+        # error: the handlers below reset global_leap_binder.setup_container, while
+        # re-importing the entry file does not re-register handlers that live in modules
+        # already in sys.modules. The second run therefore raised "Please make sure you
+        # call the leap_binder.set_preprocess method", burying the real cause (a missing
+        # file, a bad path, a typo'd import). Callers do retry — the engine's samples
+        # generator calls this once per loop iteration — so the real error was logged once
+        # and then drowned in generic repeats.
+        if self._exec_script_error is not None:
+            # A fresh copy each time: re-raising the stored object would append this call's
+            # frames to its __traceback__, so a caller retrying in a loop grows it (and keeps
+            # every retry's frames alive) without bound.
+            error = self._exec_script_error
+            raise copy.copy(error) from error.__cause__
         try:
             os.environ[mapping_runtime_mode_env_var_mame] = 'TRUE'
             self.evaluate_module()
@@ -100,16 +164,19 @@ class LeapLoader(LeapLoaderBase):
                     if is_grouped else
                     PreprocessResponse(state=DataStateType.training, length=0))
                 global_leap_binder.integration_test_func(None, mapping_preprocess)
+            global_leap_binder.previous_script_failure = None
         except TypeError as e:
             import traceback
             global_leap_binder.setup_container = DatasetIntegrationSetup()
             if "leap_binder.set_metadata(" in traceback.format_exc(5):
-                raise DeprecationWarning(
-                    "Please remove the metadata_type on leap_binder.set_metadata in your dataset script")
-            raise DatasetScriptException(getattr(e, 'message', repr(e))) from e
+                raise self._remember_exec_script_error(DeprecationWarning(
+                    "Please remove the metadata_type on leap_binder.set_metadata in your dataset script"))
+            raise self._remember_exec_script_error(
+                DatasetScriptException(describe_script_exception(e))) from e
         except Exception as e:
             global_leap_binder.setup_container = DatasetIntegrationSetup()
-            raise DatasetScriptException(getattr(e, 'message', repr(e))) from e
+            raise self._remember_exec_script_error(
+                DatasetScriptException(describe_script_exception(e))) from e
         finally:
             # ensure that the environment variable is removed after the script execution
             _leap_dec._mapping_dataset_is_grouped = False
@@ -1208,10 +1275,17 @@ class LeapLoader(LeapLoaderBase):
             if any(handler.instance_aware for handler in latent_handlers.values()):
                 raise Exception(
                     "instance-aware custom latent spaces are not supported with grouped preprocess responses")
+            # TODO: support model-computed custom latent spaces on grouped datasets (skip them here,
+            # as the per-sample path does, once the grouped metrics path is verified to compute them).
+            model_ls_names = [name for name, handler in latent_handlers.items()
+                              if handler.computed_at == CustomLatentSpaceComputedAt.MODEL]
+            if model_ls_names:
+                raise Exception(
+                    f"model-computed custom latent spaces {model_ls_names} are not supported with "
+                    f"grouped preprocess responses yet")
             custom_latent_spaces = {
                 name: self._to_grouped_list(handler.function(group_ids, preprocess_state))
                 for name, handler in latent_handlers.items()
-                if name not in self._skipped_custom_latent_spaces
             }
 
         return DatasetSample(inputs=inputs, gt=gt, metadata=metadata, metadata_is_none=metadata_is_none,
@@ -1435,8 +1509,7 @@ class LeapLoader(LeapLoaderBase):
             instance_id: Optional[int] = None) -> Optional[Dict[str, npt.NDArray[np.float32]]]:
         handlers = {handler_name: handler for handler_name, handler
                     in global_leap_binder.setup_container.custom_latent_spaces.items()
-                    if handler.computed_at != CustomLatentSpaceComputedAt.MODEL
-                    and handler_name not in self._skipped_custom_latent_spaces}
+                    if handler.computed_at != CustomLatentSpaceComputedAt.MODEL}
         if not handlers:
             return None
         if preprocess.is_grouped:
@@ -1463,10 +1536,6 @@ class LeapLoader(LeapLoaderBase):
         self.exec_script()
         return len(global_leap_binder.setup_container.custom_latent_spaces) > 0
 
-    def set_skipped_custom_latent_spaces(self, names: Iterable[str]) -> None:
-        """Stop computing these dataset-computed custom latent spaces in get_sample(s)."""
-        self._skipped_custom_latent_spaces = frozenset(names)
-
     @lru_cache()
     def get_custom_latent_space_names(self) -> Tuple[str, ...]:
         """Names of all registered custom latent spaces, in registration order.
@@ -1513,8 +1582,9 @@ class LeapLoader(LeapLoaderBase):
         # Preprocess runs only when the function asks for a SamplePreprocessResponse; the metrics
         # pod that calls this has no other reason to pay for it.
         if preprocess_response_arg_name is not None:
-            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(
-                sample_ids, self._preprocess_result()[state])
+            input_tensors_by_arg_name = {
+                **input_tensors_by_arg_name,
+                preprocess_response_arg_name: SamplePreprocessResponse(sample_ids, self._preprocess_result()[state])}
 
         return handler.function(**input_tensors_by_arg_name)
 

{code_loader-1.0.208.dev7 → code_loader-1.0.209}/code_loader/utils.py

@@ -250,8 +250,8 @@ def autoregressive_nests_equal(a: Any, b: Any) -> bool:
 
 
 def sample_preprocess_response_arg_name(func: Callable[..., Any]) -> Optional[str]:
-    # 'return' lives in annotations too and must never be mistaken for a parameter.
-    for arg_name, arg_type in inspect.getfullargspec(func).annotations.items():
-        if arg_name != 'return' and arg_type == SamplePreprocessResponse:
-            return arg_name
+    # inspect.signature follows functools.wraps' __wrapped__; getfullargspec does not.
+    for param in inspect.signature(func).parameters.values():
+        if param.annotation == SamplePreprocessResponse:
+            return param.name
     return None

{code_loader-1.0.208.dev7 → code_loader-1.0.209}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.208.dev7"
+version = "1.0.209"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"