code-loader 1.0.195.dev1__py3-none-any.whl → 1.0.197.dev0__py3-none-any.whl

code_loader/leaploaderbase.py

@@ -149,10 +149,78 @@ class LeapLoaderBase:
     def has_custom_latent_space_decorator(self) -> bool:
         pass
 
+    # The autoregressive entry points raise instead of `pass`-ing like the abstract methods
+    # above: this class has no ABCMeta, so @abstractmethod is not enforced at runtime and an
+    # un-overridden `pass` body would return None — has_autoregressive_step would silently
+    # report False for an AR integration and the engine would skip the chain rollout with no
+    # error. A stale loader subclass keeps working for non-AR flows and fails loudly here the
+    # moment an AR flow reaches it.
+    @abstractmethod
     def has_autoregressive_step(self) -> bool:
-        # Default False so existing LeapLoaderBase implementations stay valid; the concrete
-        # loaders that can answer (LeapLoader in-process, LeapLoaderWithRedis via RPC) override.
-        return False
+        raise NotImplementedError(f'{type(self).__name__} does not implement '
+                                  'has_autoregressive_step — it must be overridden '
+                                  '(LeapLoader in-process, LeapLoaderWithRedis via RPC).')
+
+    @abstractmethod
+    def get_gt(self, state: DataStateEnum, sample_id: Union[int, str]
+               ) -> Dict[str, npt.NDArray[np.float32]]:
+        raise NotImplementedError(f'{type(self).__name__} does not implement get_gt.')
+
+    @abstractmethod
+    def get_prediction_names_in_order(self) -> List[str]:
+        raise NotImplementedError(f'{type(self).__name__} does not implement '
+                                  'get_prediction_names_in_order.')
+
+    @abstractmethod
+    def get_autoregressive_latent_space_aggregation(self) -> str:
+        raise NotImplementedError(f'{type(self).__name__} does not implement '
+                                  'get_autoregressive_latent_space_aggregation.')
+
+    @abstractmethod
+    def get_autoregressive_decorator_names(self) -> Dict[str, List[str]]:
+        raise NotImplementedError(f'{type(self).__name__} does not implement '
+                                  'get_autoregressive_decorator_names.')
+
+    @abstractmethod
+    def run_autoregressive_step(self, sample_id: Union[int, str],
+                                prev_inputs: Optional[Dict[str, npt.NDArray[np.float32]]],
+                                prev_outputs: Optional[Dict[str, npt.NDArray[np.float32]]],
+                                chain_state: Any,
+                                state: DataStateEnum
+                                ) -> Tuple[Optional[Dict[str, npt.NDArray[np.float32]]], Any]:
+        raise NotImplementedError(f'{type(self).__name__} does not implement '
+                                  'run_autoregressive_step.')
+
+    @abstractmethod
+    def run_autoregressive_metric(self, name: str, inputs: Dict[str, npt.NDArray[np.float32]],
+                                  outputs: Dict[str, npt.NDArray[np.float32]], chain_state: Any,
+                                  wired_args: Dict[str, npt.NDArray[np.float32]]) -> Any:
+        raise NotImplementedError(f'{type(self).__name__} does not implement '
+                                  'run_autoregressive_metric.')
+
+    @abstractmethod
+    def run_autoregressive_loss(self, name: str, inputs: Dict[str, npt.NDArray[np.float32]],
+                                outputs: Dict[str, npt.NDArray[np.float32]], chain_state: Any,
+                                wired_args: Dict[str, npt.NDArray[np.float32]]) -> Any:
+        raise NotImplementedError(f'{type(self).__name__} does not implement '
+                                  'run_autoregressive_loss.')
+
+    @abstractmethod
+    def run_autoregressive_visualizer(self, name: str, inputs: Dict[str, npt.NDArray[np.float32]],
+                                      outputs: Dict[str, npt.NDArray[np.float32]],
+                                      chain_state: Any,
+                                      wired_args: Dict[str, npt.NDArray[np.float32]]) -> Any:
+        raise NotImplementedError(f'{type(self).__name__} does not implement '
+                                  'run_autoregressive_visualizer.')
+
+    @abstractmethod
+    def autoregressive_visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
+        raise NotImplementedError(f'{type(self).__name__} does not implement '
+                                  'autoregressive_visualizer_by_name.')
+
+    @abstractmethod
+    def get_custom_latent_space_names(self) -> Tuple[str, ...]:
+        pass
 
     @abstractmethod
     def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:

code_loader/utils.py

@@ -1,3 +1,6 @@
+import io
+import math
+import pickle
 import sys
 from pathlib import Path
 from types import TracebackType
@@ -14,11 +17,6 @@ from code_loader.contract.enums import DatasetMetadataType
 def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> SectionCallableInterface:
     def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse) -> npt.NDArray[np.float32]:
         result = encoder_function(idx, samples)
-        if isinstance(result, list) and len(result) > 0 and all(isinstance(sample, np.ndarray) for sample in result):
-            # A list of per-sample arrays only comes from a grouped encoder (a flat encoder returns
-            # a single array). Never stack it into a (B, *) batch — grouping is only a storage
-            # read-unit; batching the group to the model's B is the engine's single responsibility.
-            return result
         numpy_result: npt.NDArray[np.float32] = np.array(result)
         return numpy_result
 
@@ -106,3 +104,51 @@ def get_metadata_type_from_variable(variable: Union[Optional[str], int, bool, Op
                         f"The return type should be one of [int, float, str, bool]. Got {metadata_type}")
     return dataset_metadata_type
 
+
+
+# The platform queue transports state as pickled payloads which the engine deserializes
+# WITHOUT the integration code loaded, so a class defined in the user's modules round-trips
+# locally but crashes on the engine. Only modules importable on the engine may resolve.
+_STATE_SAFE_PICKLE_MODULES = ('builtins', 'copyreg', 'collections', 'numpy', 'datetime')
+
+
+class _EngineSafeUnpickler(pickle.Unpickler):
+    def find_class(self, module: str, name: str) -> Any:
+        if module.split('.')[0] in _STATE_SAFE_PICKLE_MODULES:
+            return super().find_class(module, name)
+        raise pickle.UnpicklingError(
+            f'{module}.{name} is defined in integration code, which is not importable on the '
+            f'platform engine')
+
+
+def validate_autoregressive_state_types(value: Any, path: str = 'state') -> None:
+    try:
+        _EngineSafeUnpickler(io.BytesIO(pickle.dumps(value))).load()
+    except Exception as e:
+        raise AssertionError(
+            f'autoregressive state validation failed: {path} is not serializable ({e}). State '
+            'is serialized to the platform queue on every chain step and deserialized without '
+            'the integration code — any picklable nest of builtin/numpy values works (dicts, '
+            'lists, tuples, sets, numbers, strings, arrays), but user-defined classes, lambdas '
+            'and open resources do not.') from e
+
+
+def autoregressive_nests_equal(a: Any, b: Any) -> bool:
+    if isinstance(a, dict) or isinstance(b, dict):
+        return isinstance(a, dict) and isinstance(b, dict) and set(a) == set(b) and \
+            all(autoregressive_nests_equal(a[key], b[key]) for key in a)
+    if isinstance(a, (list, tuple)) or isinstance(b, (list, tuple)):
+        return isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)) and \
+            len(a) == len(b) and all(autoregressive_nests_equal(x, y) for x, y in zip(a, b))
+    if isinstance(a, np.ndarray) or isinstance(b, np.ndarray):
+        if not (isinstance(a, np.ndarray) and isinstance(b, np.ndarray) and a.shape == b.shape):
+            return False
+        # NaN leaves must equal themselves: this comparison backs the determinism checks, where
+        # NaN padding / running stats in a perfectly deterministic hook are legitimate.
+        if a.dtype.kind in 'fc' and b.dtype.kind in 'fc':
+            return bool(np.array_equal(a, b, equal_nan=True))
+        return bool(np.array_equal(a, b))
+    if isinstance(a, (float, np.floating)) and isinstance(b, (float, np.floating)) and \
+            math.isnan(a) and math.isnan(b):
+        return True
+    return bool(a == b)

{code_loader-1.0.195.dev1.dist-info → code_loader-1.0.197.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.195.dev1
+Version: 1.0.197.dev0
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.195.dev1.dist-info → code_loader-1.0.197.dev0.dist-info}/RECORD

@@ -1,7 +1,7 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=outxRQ0M-zMfV0QGVJmAed5qWfRmyD0TV6-goEGAzBw,406
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=USxsP1ax2knqO1DXNMsmzzNCxWVdp5PtSdL1axVT330,14382
+code_loader/contract/datasetclasses.py,sha256=rKitPlVyI9PZdAZHpH1q-0_s4Vs7oPg8Z659XeuAujM,13788
 code_loader/contract/enums.py,sha256=2q-IV_5g9lLE306DIbWA1c0tn5IhDtxsKxyV1x_Lreg,1671
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/mapping.py,sha256=sWJhpng-IkOzQnWQdMT5w2ZZ3X1Z_OOzSwCLXIS7oxE,1446
@@ -21,18 +21,18 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=D-ahDW-1ZiB9wqPkiIMz8eOD6p_Z3spm5b_Mi8y0af0,48340
-code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=7hUDsG2YrRh_HgLmVidTvwi8xr4eH70r2MWmSWfY6aY,143870
-code_loader/leaploader.py,sha256=kYD2jousMGb0wEG9HdhzFYjRAekVYXMfZxX1S28tdns,71025
-code_loader/leaploaderbase.py,sha256=f6LsJeCAktTFU9U_c3NXddwOZjjchLtvbxl4eKMdwGc,6966
+code_loader/inner_leap_binder/leapbinder.py,sha256=52nLuWLbOU7ljlABRRuS-e-hKscVlR5zxJTm5FPBdk8,54073
+code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=tW8QmjzkloQU37xryr958aVwyrVOm2FoSFL1LH0S8EU,176272
+code_loader/leaploader.py,sha256=0YZhh2dCouQMCQAakqaPa-nRNrNn65q4rrLTwC8yjE0,68695
+code_loader/leaploaderbase.py,sha256=MMuMv2014d9VM5FQgTsbUBL-9HOBJ4h6L67uF_7jhR4,10837
 code_loader/mixpanel_tracker.py,sha256=rNwRmFifNbdUoqLQvvhhgpKczWpWiEmd8MfyJe27sxw,9131
 code_loader/plot_functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/plot_functions/plot_functions.py,sha256=2DC-zlVaN13P4VNx5d8csgs80C6SisaeP1-Kq2LW7iM,16075
 code_loader/plot_functions/visualize.py,sha256=gsBAYYkwMh7jIpJeDMPS8G4CW-pxwx6LznoQIvi4vpo,657
-code_loader/utils.py,sha256=SPDQ4_f67hOIOgTfaTGlwBfbnusPrzH1OBYDLEndyr8,4413
+code_loader/utils.py,sha256=gEJCKpDWf6p9_KoEz-0EGiJqxD6QfPgZN6zdKdtSAz8,6628
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=onRnLE_TXfgLN4o52hQIOOhUcFexGlqJ3xSpQDVLuZM,2604
-code_loader-1.0.195.dev1.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.195.dev1.dist-info/METADATA,sha256=FwRv4x1Z8Ymh7WQmu9sK_jQ6682oG2vay6K2AHbLSpU,1095
-code_loader-1.0.195.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.195.dev1.dist-info/RECORD,,
+code_loader-1.0.197.dev0.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.197.dev0.dist-info/METADATA,sha256=WZ4kCsTIDPch7g1xrPJ5fy1EHEQoRGz9q_iygXHTM4s,1095
+code_loader-1.0.197.dev0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.197.dev0.dist-info/RECORD,,