code-loader 1.0.206.dev2__tar.gz → 1.0.207.dev0__tar.gz

{code_loader-1.0.206.dev2 → code_loader-1.0.207.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.206.dev2
+Version: 1.0.207.dev0
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.206.dev2 → code_loader-1.0.207.dev0}/code_loader/contract/responsedataclasses.py

@@ -86,6 +86,8 @@ class DatasetSetup:
     custom_losses: List[CustomLossInstance]
     metrics: List[MetricInstance] = field(default_factory=list)
     simulations: List[SimulationInstance] = field(default_factory=list)
+    # Structural: the integration registered an element-instance mask encoder.
+    has_element_instances: bool = False
 
 
 @dataclass

{code_loader-1.0.206.dev2 → code_loader-1.0.207.dev0}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -15,7 +15,7 @@ from typing import Optional, Union, Callable, List, Dict, get_args, get_origin,
 import numpy as np
 import numpy.typing as npt
 
-from code_loader.utils import get_metadata_type_from_variable, is_absent_metadata_value, \
+from code_loader.utils import map_dict_to_metadata_types, is_absent_metadata_value, \
     validate_autoregressive_state_types, autoregressive_nests_equal, \
     simulate_engine_float16_downcast_on_call_args, ENGINE_STORAGE_DTYPE, \
     TL_DISABLE_ENGINE_FLOAT16_SIMULATION_ENV_VAR
@@ -3006,39 +3006,47 @@ def tensorleap_element_instance_preprocess(
 
         def resolve_instance_metadata_types(sample_id: Union[int, str], preprocess_response: PreprocessResponse,
                                             idx: int) -> Optional[Dict[str, DatasetMetadataType]]:
-            # The probe enumerates the KEYS; the declaration, where given, supplies the TYPE. A key
-            # that is absent on the probed instance and undeclared has no type available from either
-            # side -- fail with mechanism (a)'s own message so both paths read the same.
+            # The probe enumerates the KEYS; the declaration, where given, supplies the TYPE.
             element_instance = instance_mask_encoder(sample_id, preprocess_response, idx)
-            element_instance_metadata = getattr(element_instance, 'instance_metadata', None)
-            if element_instance_metadata is None:
+            if element_instance is None or element_instance.instance_metadata is None:
                 return dict(instance_metadata_types) if instance_metadata_types else None
+            metadata = element_instance.instance_metadata
             declared = instance_metadata_types or {}
-            resolved = {}
-            for name, value in element_instance_metadata.items():
-                if name in declared:
-                    continue
-                if is_absent_metadata_value(value):
-                    raise Exception(
-                        f"Metadata {name} is None and no metadata type is provided. It is absent on "
-                        f"instance {idx} of sample {sample_id!r}, so the type cannot come from the "
-                        f"value -- declare it: tensorleap_element_instance_preprocess(..., "
-                        f"instance_metadata_types={{'{name}': DatasetMetadataType.float}}).")
-                resolved[name] = get_metadata_type_from_variable(value)
-            # Declared wins, and a declared key gets a column even when the probed instance lacks it.
-            resolved.update(declared)
-            return resolved
+            # A key absent here and undeclared has no type from either side. Mechanism (a)'s own
+            # message, so both paths read the same, plus where to declare it.
+            absent = [name for name, value in metadata.items()
+                      if name not in declared and is_absent_metadata_value(value)]
+            if absent:
+                raise Exception(
+                    f"Metadata {absent[0]} is None and no metadata type is provided. It is absent "
+                    f"on instance {idx} of sample {sample_id!r}, so the type cannot come from the "
+                    f"value -- declare it: tensorleap_element_instance_preprocess(..., "
+                    f"instance_metadata_types={{'{absent[0]}': DatasetMetadataType.float}}).")
+            # Every present value goes through the same helper the sample path uses, which is also
+            # what rejects an unsupported type -- so declaring a key cannot smuggle a list past
+            # validation. The declaration then wins on type, and a declared key gets a column even
+            # when the probed instance lacks it.
+            inferred = map_dict_to_metadata_types(
+                {name: value for name, value in metadata.items()
+                 if not is_absent_metadata_value(value)})
+            return {**(inferred or {}), **declared}
 
         def user_function_instance() -> List[PreprocessResponse]:
             result = user_function()
             found_instance_metadata = False
-            # Resolve types on training, like mechanism (a) does (leapbinder.get_preprocess_result):
-            # only training is allowed to fail on an unresolvable absence. `state` is still None here
-            # whenever the integration did not set it, so fall back to list position -- which is what
-            # get_preprocess_result would assign, and position 0 is training.
-            probe_order = sorted(
-                range(len(result)),
-                key=lambda i: DataStateEnum[result[i].state.name] if result[i].state is not None else i)
+            # Resolve types on training, as mechanism (a) does: only training is allowed to fail
+            # on an unresolvable absence (leapbinder.check_handler). `state` is still None here when
+            # the integration left it to position, which is how get_preprocess_result assigns it,
+            # and position 0 is training. Rank training against the rest rather than sorting on the
+            # state value: mixing enum values with positions in one key lets validation tie with
+            # position 1 and stay ahead of training.
+            def _is_training(i: int) -> bool:
+                response = result[i]
+                if response.state is not None:
+                    return response.state == DataStateType.training
+                return i == 0
+
+            probe_order = sorted(range(len(result)), key=lambda i: (0 if _is_training(i) else 1, i))
             for preprocess_response in (result[i] for i in probe_order):
                 if preprocess_response.is_grouped:
                     raise Exception(

{code_loader-1.0.206.dev2 → code_loader-1.0.207.dev0}/code_loader/leaploader.py

@@ -935,7 +935,29 @@ class LeapLoader(LeapLoaderBase):
 
         return DatasetSetup(preprocess=dataset_preprocess, inputs=inputs, outputs=ground_truths,
                             metadata=metadata_instances, visualizers=visualizers, prediction_types=prediction_types,
-                            custom_losses=custom_losses, metrics=metrics, simulations=simulations)
+                            custom_losses=custom_losses, metrics=metrics, simulations=simulations,
+                            has_element_instances=self._has_element_instances())
+
+    def _has_element_instances(self) -> bool:
+        """Does this dataset have element instances? Both halves, because either alone lies.
+
+        A registered mask encoder without instance mappings is an integration that declares
+        instances and produces none: the engine would take the pooling path with nothing to pool,
+        and ask the UI to visualize an instance population that does not exist. Mappings without an
+        encoder cannot be masked. Preprocessing has already run by the time this is called (the
+        lengths above come from it), so reading the mappings costs nothing extra.
+        """
+        setup = global_leap_binder.setup_container
+        if not setup.instance_masks:
+            return False
+        try:
+            return any(response.instance_to_sample_ids_mappings
+                       for response in self._preprocess_result().values())
+        except Exception:
+            # Preprocessing is broken, so this push is already failing. Re-raising here would replace
+            # the specific preprocess error in the exit table with a generic one from check_dataset's
+            # outer handler. False is the safe answer and the real error survives.
+            return False
 
     def get_model_setup_response(self) -> ModelSetup:
         setup = global_leap_binder.setup_container

{code_loader-1.0.206.dev2 → code_loader-1.0.207.dev0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.206.dev2"
+version = "1.0.207.dev0"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"