code-loader 1.0.196.dev0__py3-none-any.whl → 1.0.196.dev1__py3-none-any.whl

code_loader/inner_leap_binder/leapbinder.py

@@ -1,6 +1,9 @@
 
+import builtins
 import inspect
-from typing import Callable, List, Optional, Dict, Any, Type, Union, get_args, cast
+import os
+from contextlib import contextmanager
+from typing import Callable, List, Optional, Dict, Any, Type, Union, get_args, cast, Iterator, Set
 
 import numpy as np
 import numpy.typing as npt
@@ -16,7 +19,8 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     SimulationHandler, _simulation_context, AutoregressiveStepHandler, AutoregressiveStepCallableInterface, \
     AUTOREGRESSIVE_LATENT_SPACE_AGGREGATIONS, AUTOREGRESSIVE_IMPLICIT_ARG_NAMES, \
     AutoregressiveMetricHandler, AutoregressiveLossHandler, AutoregressiveVisualizerHandler
-from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
+from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType, \
+    TestingSectionEnum
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload, LeapAnalysisConfiguration
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
@@ -33,6 +37,25 @@ from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
 mapping_runtime_mode_env_var_mame = '__MAPPING_RUNTIME_MODE__'
 
 
+@contextmanager
+def _track_opened_files() -> Iterator[Set[str]]:
+    opened: Set[str] = set()
+    original_open = builtins.open
+
+    def tracking_open(file: Any, *args: Any, **kwargs: Any) -> Any:
+        mode = args[0] if args else kwargs.get('mode', 'r')
+        is_read_mode = isinstance(mode, str) and not any(c in mode for c in ('a', 'w', 'x'))
+        if is_read_mode and isinstance(file, (str, os.PathLike)):
+            opened.add(os.fspath(file))
+        return original_open(file, *args, **kwargs)
+
+    builtins.open = tracking_open
+    try:
+        yield opened
+    finally:
+        builtins.open = original_open
+
+
 def _stringized_annotation_type_name(annotation: Any) -> Optional[str]:
     """Bare type name if ``annotation`` is a stringized annotation (from ``from __future__
     import annotations`` or a quoted hint), else ``None``. code_loader inspects raw
@@ -744,15 +767,34 @@ class LeapBinder:
             preprocess_response: PreprocessResponse, test_result: List[DatasetTestResultPayload],
             dataset_base_handler: Union[DatasetBaseHandler, MetadataHandler], state: DataStateEnum) -> List[DatasetTestResultPayload]:
         assert preprocess_response.sample_ids is not None
+        opened_files: Optional[Set[str]] = None
         if preprocess_response.is_grouped:
             # Grouped: probe with the first group, then reduce to a single sample's result so the
             # recorded shape/type is per-sample (matching the flat case), not the (B, *) batch.
             # (casts: sample_ids[0] is a group list here, and the grouped result is a per-sample list.)
             group = preprocess_response.sample_ids[0]
-            raw_result = cast(Any, dataset_base_handler.function(cast(Any, group), preprocess_response))[0]
+            with _track_opened_files() as opened_files:
+                raw_result = cast(Any, dataset_base_handler.function(cast(Any, group), preprocess_response))[0]
         else:
             raw_result = dataset_base_handler.function(
                 cast(Union[int, str], preprocess_response.sample_ids[0]), preprocess_response)
+
+        def _warn_if_group_spans_multiple_files(payloads: List[DatasetTestResultPayload]) -> None:
+            if opened_files is None or len(opened_files) <= 1:
+                return
+            shown = sorted(opened_files)[:5]
+            suffix = f" (+{len(opened_files) - 5} more)" if len(opened_files) > 5 else ""
+            warning = (
+                f"Encoding declared group 0 (size {len(cast(List[Any], group))}) for '{dataset_base_handler.name}' opened "
+                f"{len(opened_files)} distinct files: {shown}{suffix}. If this group is meant to be one "
+                f"physical source (e.g. one Parquet file), it may be grouped incorrectly - check your "
+                f"preprocess grouping logic. (Only files opened via Python's open() are tracked, so this "
+                f"can miss other I/O paths or be a false positive if grouping isn't meant to reflect file "
+                f"locality.)")
+            for payload in payloads:
+                existing = payload.display.get(TestingSectionEnum.Errors.name, '')
+                payload.display[TestingSectionEnum.Errors.name] = (existing + '\n' if existing else '') + warning
+
         handler_type = 'metadata' if isinstance(dataset_base_handler, MetadataHandler) else None
         if isinstance(dataset_base_handler, MetadataHandler):
             if isinstance(raw_result, dict):
@@ -792,6 +834,7 @@ class LeapBinder:
             else:
                 if raw_result is None:
                     if state != DataStateEnum.training:
+                        _warn_if_group_spans_multiple_files(test_result)
                         return test_result
 
                     if dataset_base_handler.metadata_type is None:
@@ -813,6 +856,7 @@ class LeapBinder:
             # setting shape in setup for all encoders
             if isinstance(dataset_base_handler, (InputHandler, GroundTruthHandler)):
                 dataset_base_handler.shape = result_shape
+        _warn_if_group_spans_multiple_files(test_result)
         return test_result
 
     def check_handlers(self, preprocess_result: Dict[DataStateEnum, PreprocessResponse]) -> None:

code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -2010,9 +2010,10 @@ def tensorleap_autoregressive_step(latent_space_aggregation: str = 'last_step'):
     state returned by the previous call, and return the next step's full input dict together with
     the updated state; returning (None, state) ends the chain — the terminating call still
     returns state, so the final step participates in state aggregation. State is never fed to
-    the model; it may be any nest of dicts/lists holding numpy arrays, numbers, strings or bools,
-    and it rides the platform queue on every step — accumulate reductions, not raw per-step
-    tensors. The hook must be stateless and deterministically seeded (seed stochasticity from
+    the model; any picklable nest of builtin/numpy values works (dicts, lists, tuples, sets,
+    numbers, strings, arrays — not user-defined classes, lambdas or open resources, since the
+    platform deserializes state without the integration code), and it rides the platform queue
+    on every step — accumulate reductions, not raw per-step tensors. The hook must be stateless and deterministically seeded (seed stochasticity from
     sample_id) — the engine replays steps after crash recovery and relies on identical results.
 
     latent_space_aggregation ('last_step' | 'mean') declares how the chain's latent-space
@@ -3114,10 +3115,12 @@ def tensorleap_input_encoder(name: str, channel_dim=_UNSET, model_input_index=No
             if _called_from_inside_tl_decorator == 0 and _called_from_inside_tl_integration_test_decorator:
                 if grouped:
                     # A grouped result is a list of per-sample arrays (never stacked, so a
-                    # B=1-only model can be fed one sample at a time). Add a batch dim to each
-                    # sample, mirroring the flat path's single-sample expand_dims.
-                    if isinstance(result, list):
-                        result = [np.expand_dims(r, axis=0) for r in result]
+                    # B=1-only model can be fed one sample at a time). A user may also return a
+                    # supported (B, *) array; split it back into per-sample rows first (mirrors
+                    # LeapLoader._to_grouped_list) so both forms match the runtime path. Then add
+                    # a batch dim to each sample, mirroring the flat path's single-sample expand_dims.
+                    rows = result if isinstance(result, list) else [row for row in np.asarray(result)]
+                    result = [np.expand_dims(r, axis=0) for r in rows]
                 else:
                     batch_warning(result, user_function.__name__)
                     result = np.expand_dims(result, axis=0)
@@ -3228,10 +3231,12 @@ def tensorleap_gt_encoder(name: str):
             if _called_from_inside_tl_decorator == 0 and _called_from_inside_tl_integration_test_decorator:
                 if grouped:
                     # A grouped result is a list of per-sample arrays (never stacked, so a
-                    # B=1-only model can be fed one sample at a time). Add a batch dim to each
-                    # sample, mirroring the flat path's single-sample expand_dims.
-                    if isinstance(result, list):
-                        result = [np.expand_dims(r, axis=0) for r in result]
+                    # B=1-only model can be fed one sample at a time). A user may also return a
+                    # supported (B, *) array; split it back into per-sample rows first (mirrors
+                    # LeapLoader._to_grouped_list) so both forms match the runtime path. Then add
+                    # a batch dim to each sample, mirroring the flat path's single-sample expand_dims.
+                    rows = result if isinstance(result, list) else [row for row in np.asarray(result)]
+                    result = [np.expand_dims(r, axis=0) for r in rows]
                 else:
                     batch_warning(result, user_function.__name__)
                     result = np.expand_dims(result, axis=0)

code_loader/leaploader.py

@@ -77,8 +77,6 @@ class LeapLoader(LeapLoaderBase):
                     preprocess_result = global_leap_binder.get_preprocess_result()
                     self._preprocess_result_cached = preprocess_result
                     is_grouped = any(r.is_grouped for r in preprocess_result.values())
-                except Exception:
-                    is_grouped = False
                 finally:
                     os.environ[mapping_runtime_mode_env_var_mame] = 'TRUE'
 

code_loader/utils.py

@@ -1,4 +1,6 @@
+import io
 import math
+import pickle
 import sys
 from pathlib import Path
 from types import TracebackType
@@ -15,10 +17,12 @@ from code_loader.contract.enums import DatasetMetadataType
 def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> SectionCallableInterface:
     def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse) -> npt.NDArray[np.float32]:
         result = encoder_function(idx, samples)
-        if isinstance(result, list) and len(result) > 0 and all(isinstance(sample, np.ndarray) for sample in result):
-            # A list of per-sample arrays only comes from a grouped encoder (a flat encoder returns
-            # a single array). Never stack it into a (B, *) batch — grouping is only a storage
-            # read-unit; batching the group to the model's B is the engine's single responsibility.
+        if isinstance(idx, list) and isinstance(result, list) and len(result) > 0 \
+                and all(isinstance(sample, np.ndarray) for sample in result):
+            # Grouped call (idx is a list of sample ids): keep the per-sample list, never stack
+            # it into a (B, *) batch — grouping is only a storage read-unit; batching the group to
+            # the model's B is the engine's single responsibility. Gate on the call shape (idx is a
+            # list), not the result shape, so a flat encoder returning a list of arrays still stacks.
             return result
         numpy_result: npt.NDArray[np.float32] = np.array(result)
         return numpy_result
@@ -109,27 +113,31 @@ def get_metadata_type_from_variable(variable: Union[Optional[str], int, bool, Op
 
 
 
-AUTOREGRESSIVE_STATE_LEAF_TYPES = (np.ndarray, np.generic, int, float, str, bool, type(None))
+# The platform queue transports state as pickled payloads which the engine deserializes
+# WITHOUT the integration code loaded, so a class defined in the user's modules round-trips
+# locally but crashes on the engine. Only modules importable on the engine may resolve.
+_STATE_SAFE_PICKLE_MODULES = ('builtins', 'copyreg', 'collections', 'numpy', 'datetime')
+
+
+class _EngineSafeUnpickler(pickle.Unpickler):
+    def find_class(self, module: str, name: str) -> Any:
+        if module.split('.')[0] in _STATE_SAFE_PICKLE_MODULES:
+            return super().find_class(module, name)
+        raise pickle.UnpicklingError(
+            f'{module}.{name} is defined in integration code, which is not importable on the '
+            f'platform engine')
 
 
 def validate_autoregressive_state_types(value: Any, path: str = 'state') -> None:
-    if isinstance(value, dict):
-        for key, item in value.items():
-            if not isinstance(key, str):
-                raise AssertionError(
-                    f'autoregressive state validation failed: dict key {key!r} at {path} must be '
-                    f'a string. Got {type(key).__name__}.')
-            validate_autoregressive_state_types(item, f'{path}[{key!r}]')
-        return
-    if isinstance(value, (list, tuple)):
-        for i, item in enumerate(value):
-            validate_autoregressive_state_types(item, f'{path}[{i}]')
-        return
-    if not isinstance(value, AUTOREGRESSIVE_STATE_LEAF_TYPES):
+    try:
+        _EngineSafeUnpickler(io.BytesIO(pickle.dumps(value))).load()
+    except Exception as e:
         raise AssertionError(
-            f'autoregressive state validation failed: {path} holds a {type(value).__name__}. '
-            f'State must be built only from numpy arrays, numbers, strings, bools and nests of '
-            f'dicts/lists of those — it is serialized to the platform queue on every chain step.')
+            f'autoregressive state validation failed: {path} is not serializable ({e}). State '
+            'is serialized to the platform queue on every chain step and deserialized without '
+            'the integration code — any picklable nest of builtin/numpy values works (dicts, '
+            'lists, tuples, sets, numbers, strings, arrays), but user-defined classes, lambdas '
+            'and open resources do not.') from e
 
 
 def autoregressive_nests_equal(a: Any, b: Any) -> bool:

{code_loader-1.0.196.dev0.dist-info → code_loader-1.0.196.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.196.dev0
+Version: 1.0.196.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.196.dev0.dist-info → code_loader-1.0.196.dev1.dist-info}/RECORD

@@ -21,18 +21,18 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=hTkOL-JLtIQSaCqg8EVxnWWaHMbChrnTkgrT1Tj5AxE,54488
-code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=1zKGCW-TgC9AYv8oc_jEplIVlnpJEwqM9o2IpGxOwTI,183070
-code_loader/leaploader.py,sha256=k1yfKV6SBqxggUp3Yw1sNGfd50xsCaNb8-CXBsgsVY8,83459
+code_loader/inner_leap_binder/leapbinder.py,sha256=yW9EBA4E7YwWoWaiVELWoSP9uWTQjnPl5PfLbmX4cFk,56617
+code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=GyuyffnSmkELj63q_NBAxa6Z0szP6MsaNxDodwCo-pE,183746
+code_loader/leaploader.py,sha256=yWzSc-ynm_ykfup_tk8mwZX3TFq1ZW-fN18fEDXOrfc,83386
 code_loader/leaploaderbase.py,sha256=PvBU_2OQqBfJLDMsbK1s372954eYr4Y3O4r18y5S7uY,10739
 code_loader/mixpanel_tracker.py,sha256=rNwRmFifNbdUoqLQvvhhgpKczWpWiEmd8MfyJe27sxw,9131
 code_loader/plot_functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/plot_functions/plot_functions.py,sha256=2DC-zlVaN13P4VNx5d8csgs80C6SisaeP1-Kq2LW7iM,16075
 code_loader/plot_functions/visualize.py,sha256=gsBAYYkwMh7jIpJeDMPS8G4CW-pxwx6LznoQIvi4vpo,657
-code_loader/utils.py,sha256=mt1V191TL-6goWPnsqjW76wffGFGNdx8xdQ0J4ZT9bU,6779
+code_loader/utils.py,sha256=Grrp8OLagrVAOYjt83DDmE0CPfjE22DuOQftgqDASyU,7215
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=onRnLE_TXfgLN4o52hQIOOhUcFexGlqJ3xSpQDVLuZM,2604
-code_loader-1.0.196.dev0.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.196.dev0.dist-info/METADATA,sha256=okTBjK87MBiyLc7eEgcuXgtl2XjCMCk_FSOtBTAnXQw,1095
-code_loader-1.0.196.dev0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.196.dev0.dist-info/RECORD,,
+code_loader-1.0.196.dev1.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.196.dev1.dist-info/METADATA,sha256=qjJUAz8M-nUFhOQa1jAFKQXk2Qm-pdh4zns9jWNEglY,1095
+code_loader-1.0.196.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.196.dev1.dist-info/RECORD,,