code-loader 1.0.205.dev2__tar.gz → 1.0.206__tar.gz

{code_loader-1.0.205.dev2 → code_loader-1.0.206}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.205.dev2
+Version: 1.0.206
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.205.dev2 → code_loader-1.0.206}/code_loader/contract/datasetclasses.py

@@ -143,6 +143,10 @@ class ElementInstance:
     name: str
     mask: npt.NDArray[np.float32]
     instance_metadata: Optional[Dict[str, Union[Optional[str], int, bool, Optional[float]]]] = None
+    # Per-key "this value is absent", the same pair shape DatasetSample carries. Populated at the
+    # masks funnel (utils.to_numpy_return_masks_wrapper) so the engine copies flags rather than
+    # deriving them, exactly as it does for sample-level metadata.
+    instance_metadata_is_none: Optional[Dict[str, bool]] = None
 
 SectionCallableInterface = Callable[[Union[int, str], PreprocessResponse], npt.NDArray[np.float32]]
 InstanceCallableInterface = Callable[[Union[int, str], PreprocessResponse, int], Optional[ElementInstance]]

{code_loader-1.0.205.dev2 → code_loader-1.0.206}/code_loader/contract/responsedataclasses.py

@@ -86,8 +86,6 @@ class DatasetSetup:
     custom_losses: List[CustomLossInstance]
     metrics: List[MetricInstance] = field(default_factory=list)
     simulations: List[SimulationInstance] = field(default_factory=list)
-    # Structural: the integration registered an element-instance mask encoder.
-    has_element_instances: bool = False
 
 
 @dataclass

{code_loader-1.0.205.dev2 → code_loader-1.0.206}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -15,7 +15,7 @@ from typing import Optional, Union, Callable, List, Dict, get_args, get_origin,
 import numpy as np
 import numpy.typing as npt
 
-from code_loader.utils import get_metadata_type_from_variable, map_dict_to_metadata_types, \
+from code_loader.utils import map_dict_to_metadata_types, is_absent_metadata_value, \
     validate_autoregressive_state_types, autoregressive_nests_equal, \
     simulate_engine_float16_downcast_on_call_args, ENGINE_STORAGE_DTYPE, \
     TL_DISABLE_ENGINE_FLOAT16_SIMULATION_ENV_VAR
@@ -27,7 +27,8 @@ from code_loader.contract.datasetclasses import CustomCallableInterfaceMultiArgs
     VisualizerCallableInterface, MetadataSectionCallableInterface, PreprocessResponse, SectionCallableInterface, \
     ConfusionMatrixElement, SamplePreprocessResponse, PredictionTypeHandler, InstanceCallableInterface, ElementInstance, \
     InstanceLengthCallableInterface, InstanceSectionCallableInterface, AutoregressiveStepCallableInterface
-from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType, DataStateType
+from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType, DataStateType, \
+    DataStateEnum
 from code_loader import leap_binder, LeapLoader
 from code_loader.contract.mapping import NodeMapping, NodeMappingType, NodeConnection
 from code_loader.contract.visualizer_classes import LeapImage, LeapImageMask, LeapTextMask, LeapText, LeapGraph, \
@@ -2981,12 +2982,72 @@ def tensorleap_simulation(name: str, sim_params: dict):
 
 
 def tensorleap_element_instance_preprocess(
-        instance_length_encoder: InstanceLengthCallableInterface, instance_mask_encoder: InstanceCallableInterface):
+        instance_length_encoder: InstanceLengthCallableInterface,
+        instance_mask_encoder: InstanceCallableInterface,
+        instance_metadata_types: Optional[Dict[str, DatasetMetadataType]] = None):
     def decorating_function(user_function: Callable[[], List[PreprocessResponse]]):
+        def register_instance_extra_metadata(metadata_types: Dict[str, DatasetMetadataType]) -> None:
+            # Idempotent: set_metadata appends without deduping, and the probe path runs inside
+            # preprocess, so a second preprocess call would register a duplicate handler.
+            if any(handler.name == "builtin_instance_extra_metadata"
+                   for handler in leap_binder.setup_container.metadata):
+                return
+            names = list(metadata_types)
+
+            def builtin_instance_extra_metadata(idx: str, preprocess: PreprocessResponse) -> Dict[str, None]:
+                # Sample rows declare the columns without claiming values; the engine injects the
+                # real per-instance ones. A fresh dict, because nulling the probed
+                # ElementInstance's own dict in place destroyed any encoder that returned a
+                # reference into preprocess.data.
+                return {name: None for name in names}
+
+            leap_binder.set_metadata(builtin_instance_extra_metadata,
+                                     "builtin_instance_extra_metadata", metadata_types)
+
+        def resolve_instance_metadata_types(sample_id: Union[int, str], preprocess_response: PreprocessResponse,
+                                            idx: int) -> Optional[Dict[str, DatasetMetadataType]]:
+            # The probe enumerates the KEYS; the declaration, where given, supplies the TYPE.
+            element_instance = instance_mask_encoder(sample_id, preprocess_response, idx)
+            if element_instance is None or element_instance.instance_metadata is None:
+                return dict(instance_metadata_types) if instance_metadata_types else None
+            metadata = element_instance.instance_metadata
+            declared = instance_metadata_types or {}
+            # A key absent here and undeclared has no type from either side. Mechanism (a)'s own
+            # message, so both paths read the same, plus where to declare it.
+            absent = [name for name, value in metadata.items()
+                      if name not in declared and is_absent_metadata_value(value)]
+            if absent:
+                raise Exception(
+                    f"Metadata {absent[0]} is None and no metadata type is provided. It is absent "
+                    f"on instance {idx} of sample {sample_id!r}, so the type cannot come from the "
+                    f"value -- declare it: tensorleap_element_instance_preprocess(..., "
+                    f"instance_metadata_types={{'{absent[0]}': DatasetMetadataType.float}}).")
+            # Every present value goes through the same helper the sample path uses, which is also
+            # what rejects an unsupported type -- so declaring a key cannot smuggle a list past
+            # validation. The declaration then wins on type, and a declared key gets a column even
+            # when the probed instance lacks it.
+            inferred = map_dict_to_metadata_types(
+                {name: value for name, value in metadata.items()
+                 if not is_absent_metadata_value(value)})
+            return {**(inferred or {}), **declared}
+
         def user_function_instance() -> List[PreprocessResponse]:
             result = user_function()
             found_instance_metadata = False
-            for preprocess_response in result:
+            # Resolve types on training, as mechanism (a) does: only training is allowed to fail
+            # on an unresolvable absence (leapbinder.check_handler). `state` is still None here when
+            # the integration left it to position, which is how get_preprocess_result assigns it,
+            # and position 0 is training. Rank training against the rest rather than sorting on the
+            # state value: mixing enum values with positions in one key lets validation tie with
+            # position 1 and stay ahead of training.
+            def _is_training(i: int) -> bool:
+                response = result[i]
+                if response.state is not None:
+                    return response.state == DataStateType.training
+                return i == 0
+
+            probe_order = sorted(range(len(result)), key=lambda i: (0 if _is_training(i) else 1, i))
+            for preprocess_response in (result[i] for i in probe_order):
                 if preprocess_response.is_grouped:
                     raise Exception(
                         "tensorleap_element_instance_preprocess validation failed: a grouped "
@@ -3009,15 +3070,10 @@ def tensorleap_element_instance_preprocess(
                     # "Index <id> with sample_id: <id> cannot be found!".
                     for idx, instance_id in enumerate(instances_ids):
                         if not found_instance_metadata:
-                            element_instance = instance_mask_encoder(sample_id, preprocess_response, idx)
-                            element_instance_metadata = element_instance.instance_metadata
-                            if element_instance_metadata is not None:
-                                element_instance_metadata_types = map_dict_to_metadata_types(element_instance_metadata)
-                                def builtin_instance_extra_metadata(idx: str, preprocess: PreprocessResponse) -> Dict[str, str]:
-                                    for k, v in element_instance_metadata.items():
-                                        element_instance_metadata[k] = None
-                                    return element_instance_metadata
-                                leap_binder.set_metadata(builtin_instance_extra_metadata,"builtin_instance_extra_metadata", element_instance_metadata_types)
+                            resolved_types = resolve_instance_metadata_types(
+                                sample_id, preprocess_response, idx)
+                            if resolved_types:
+                                register_instance_extra_metadata(resolved_types)
                             found_instance_metadata = True
 
                         instance_to_sample_ids_mappings[instance_id] = sample_id

{code_loader-1.0.205.dev2 → code_loader-1.0.206}/code_loader/leaploader.py

@@ -28,7 +28,7 @@ from code_loader.inner_leap_binder import global_leap_binder
 from code_loader.inner_leap_binder.leapbinder import mapping_runtime_mode_env_var_mame
 from code_loader.leaploaderbase import LeapLoaderBase
 from code_loader.utils import get_root_exception_file_and_line_number, get_metadata_type_from_variable, \
-    validate_autoregressive_state_types, autoregressive_nests_equal
+    validate_autoregressive_state_types, autoregressive_nests_equal, is_absent_metadata_value
 
 
 def _serialize_sim_bounds(bounds) -> dict:
@@ -935,23 +935,7 @@ class LeapLoader(LeapLoaderBase):
 
         return DatasetSetup(preprocess=dataset_preprocess, inputs=inputs, outputs=ground_truths,
                             metadata=metadata_instances, visualizers=visualizers, prediction_types=prediction_types,
-                            custom_losses=custom_losses, metrics=metrics, simulations=simulations,
-                            has_element_instances=self._has_element_instances())
-
-    def _has_element_instances(self) -> bool:
-        """Does this dataset have element instances? Both halves, because either alone lies.
-
-        A registered mask encoder without instance mappings is an integration that declares
-        instances and produces none: the engine would take the pooling path with nothing to pool,
-        and ask the UI to visualize an instance population that does not exist. Mappings without an
-        encoder cannot be masked. Preprocessing has already run by the time this is called (the
-        lengths above come from it), so reading the mappings costs nothing extra.
-        """
-        setup = global_leap_binder.setup_container
-        if not setup.instance_masks:
-            return False
-        return any(response.instance_to_sample_ids_mappings
-                   for response in self._preprocess_result().values())
+                            custom_losses=custom_losses, metrics=metrics, simulations=simulations)
 
     def get_model_setup_response(self) -> ModelSetup:
         setup = global_leap_binder.setup_container
@@ -1244,7 +1228,7 @@ class LeapLoader(LeapLoaderBase):
 
         try:
             is_none = False
-            if value is None or (isinstance(value, (int, float, np.integer, np.floating)) and not np.isfinite(value)):
+            if is_absent_metadata_value(value):
                 raise ValueError()
             converted_value = metadata_type_to_python_type[metadata_name_to_type[metadata_name]](value)
         except ValueError:

{code_loader-1.0.205.dev2 → code_loader-1.0.206}/code_loader/utils.py

@@ -15,6 +15,12 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, Prepro
 from code_loader.contract.enums import DatasetMetadataType
 
 
+def is_absent_metadata_value(value: Any) -> bool:
+    """The one definition of an absent metadata value, shared by every mechanism."""
+    return value is None or (isinstance(value, (int, float, np.integer, np.floating))
+                             and not np.isfinite(value))
+
+
 def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> SectionCallableInterface:
     def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse) -> npt.NDArray[np.float32]:
         result = encoder_function(idx, samples)
@@ -39,6 +45,9 @@ def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -
         if result is None:
             return None
         result.mask = np.array(result.mask)
+        if result.instance_metadata is not None and result.instance_metadata_is_none is None:
+            result.instance_metadata_is_none = {
+                k: is_absent_metadata_value(v) for k, v in result.instance_metadata.items()}
         return result
     return numpy_encoder_function
 

{code_loader-1.0.205.dev2 → code_loader-1.0.206}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.205.dev2"
+version = "1.0.206"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"