code-loader 1.0.86.dev0__py3-none-any.whl → 1.0.87__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -40,6 +40,9 @@ class PreprocessResponse:
     sample_id_type: Optional[Union[Type[str], Type[int]]] = None
 
     def __post_init__(self) -> None:
+        def is_valid_string(s: str) -> bool:
+            return bool(re.match(r'^[A-Za-z0-9_]+$', s))
+
         if self.length is not None and self.sample_ids is None:
             self.sample_ids = [i for i in range(self.length)]
             self.sample_id_type = int
@@ -50,6 +53,8 @@ class PreprocessResponse:
             if self.sample_id_type == str:
                 for sample_id in self.sample_ids:
                     assert isinstance(sample_id, str), f"Sample id should be of type str. Got: {type(sample_id)}"
+                    if not is_valid_string(sample_id):
+                        raise Exception(f"Sample id should contain only letters (A-Z, a-z), numbers or '_'. Got: {sample_id}")
         else:
             raise Exception("length is deprecated.")
 

code_loader/leaploader.py

@@ -22,7 +22,7 @@ from code_loader.contract.responsedataclasses import DatasetIntegParseResult, Da
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance
 from code_loader.inner_leap_binder import global_leap_binder
 from code_loader.leaploaderbase import LeapLoaderBase
-from code_loader.utils import get_root_exception_file_and_line_number, flatten
+from code_loader.utils import get_root_exception_file_and_line_number
 
 
 class LeapLoader(LeapLoaderBase):
@@ -204,6 +204,8 @@ class LeapLoader(LeapLoaderBase):
         preprocess_result = self._preprocess_result()
         result_payloads: List[DatasetTestResultPayload] = []
         dataset_base_handlers: List[Union[DatasetBaseHandler, MetadataHandler]] = self._get_all_dataset_base_handlers()
+        assert sum(isinstance(x, MetadataHandler) for x in dataset_base_handlers) < 100, "Metadata handles must be less than 100"
+        metadata_length = 0
         for dataset_base_handler in dataset_base_handlers:
             test_result = [DatasetTestResultPayload(dataset_base_handler.name)]
             for state, preprocess_response in preprocess_result.items():
@@ -218,9 +220,10 @@ class LeapLoader(LeapLoaderBase):
                     test_result[0].display[
                         state_name] = f"{repr(e)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
                     test_result[0].is_passed = False
-
+            if isinstance(dataset_base_handler, MetadataHandler):
+                metadata_length += len(test_result)
+                assert metadata_length < 800, "Total number of metadata columns handles must be less than 800"
             result_payloads.extend(test_result)
-
         return result_payloads
 
     @staticmethod
@@ -471,18 +474,22 @@ class LeapLoader(LeapLoaderBase):
 
         return converted_value, is_none
 
-    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[
-        Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
+    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
         result_agg = {}
         is_none = {}
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
         for handler in global_leap_binder.setup_container.metadata:
             handler_result = handler.function(sample_id, preprocess_state)
-
-            for flat_name, flat_result in flatten(handler_result, prefix=handler.name):
-                result_agg[flat_name], is_none[flat_name] = self._convert_metadata_to_correct_type(
-                    flat_name, flat_result)
+            if isinstance(handler_result, dict):
+                for single_metadata_name, single_metadata_result in handler_result.items():
+                    handler_name = f'{handler.name}_{single_metadata_name}'
+                    result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
+                        handler_name, single_metadata_result)
+            else:
+                handler_name = handler.name
+                result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
+                    handler_name, handler_result)
 
         return result_agg, is_none
 

code_loader/utils.py

@@ -1,7 +1,7 @@
 import sys
 from pathlib import Path
 from types import TracebackType
-from typing import List, Union, Tuple, Any, Iterator
+from typing import List, Union, Tuple, Any
 import traceback
 import numpy as np
 import numpy.typing as npt
@@ -66,28 +66,3 @@ def rescale_min_max(image: npt.NDArray[np.float32]) -> npt.NDArray[np.float32]:
     return image
 
 
-def flatten(
-    value: Any,
-    *,
-    prefix: str = "",
-    list_token: str = "e",
-) -> Iterator[Tuple[str, Any]]:
-    """
-    Recursively walk `value` and yield (flat_key, leaf_value) pairs.
-
-    • Dicts → descend with   new_prefix = f"{prefix}_{key}" (or just key if top level)
-    • Sequences → descend with new_prefix = f"{prefix}_{list_token}{idx}"
-    • Leaf scalars → yield the accumulated flat key and the scalar itself
-    """
-    if isinstance(value, dict):
-        for k, v in value.items():
-            new_prefix = f"{prefix}_{k}" if prefix else k
-            yield from flatten(v, prefix=new_prefix, list_token=list_token)
-
-    elif isinstance(value, (list, tuple)):
-        for idx, v in enumerate(value):
-            new_prefix = f"{prefix}_{list_token}{idx}"
-            yield from flatten(v, prefix=new_prefix, list_token=list_token)
-
-    else:  # primitive leaf (str, int, float, bool, None…)
-        yield prefix, value

{code_loader-1.0.86.dev0.dist-info → code_loader-1.0.87.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.86.dev0
+Version: 1.0.87
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.86.dev0.dist-info → code_loader-1.0.87.dist-info}/RECORD

@@ -1,7 +1,7 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=3BWSCHaKtNWlKucMkPKSMiuvZosnnQgXFq2R-GbVOgg,7679
+code_loader/contract/datasetclasses.py,sha256=ZN6XC-tg0qaT-GF4YiV2SsM2dUQW4fc1akLF-J4YT0w,7964
 code_loader/contract/enums.py,sha256=GEFkvUMXnCNt-GOoz7NJ9ecQZ2PPDettJNOsxsiM0wk,1622
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/responsedataclasses.py,sha256=RSx9m_R3LawhK5o1nAcO3hfp2F9oJYtxZr_bpP3bTmw,4005
@@ -21,12 +21,12 @@ code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaS
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
 code_loader/inner_leap_binder/leapbinder.py,sha256=-LaQNJ2EVJA7HnxaTJyEYthZgOE7kvaq60OXcDwWgJE,30815
 code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=qcYyekpIN1hgbmSqTrN7o6IMpfu8ZWp2J_lhMEoV8I8,22598
-code_loader/leaploader.py,sha256=t9ITSzECPg5sGXHxsYkcr45tDipd79hJw6lhwo4yEIE,25412
+code_loader/leaploader.py,sha256=s768FDDiTPaLOcemT3nJbdgk4ScbTx1tneL-3wz_jyU,26153
 code_loader/leaploaderbase.py,sha256=VH0vddRmkqLtcDlYPCO7hfz1_VbKo43lUdHDAbd4iJc,4198
-code_loader/utils.py,sha256=4tXLum2AT3Z1ldD6BeYScNg0ATyE4oM8cuIGQxrXyjM,3163
+code_loader/utils.py,sha256=aw2i_fqW_ADjLB66FWZd9DfpCQ7mPdMyauROC5Nd51I,2197
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=f95WzLYO2KU8endWxrmtwjHx7diDfm8AaWjucYHT7GI,2439
-code_loader-1.0.86.dev0.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.86.dev0.dist-info/METADATA,sha256=cF-s4EglKkisMEpQdK5UVTWTuMT7sO06aAPWVY0hPH4,854
-code_loader-1.0.86.dev0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.86.dev0.dist-info/RECORD,,
+code_loader-1.0.87.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.87.dist-info/METADATA,sha256=5V-qSOWz_VAFA9VrblQJsJxYtB4ogGkk0OJ8-sCy7wQ,849
+code_loader-1.0.87.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.87.dist-info/RECORD,,