code-loader 1.0.93.dev0__tar.gz → 1.0.93.dev3__tar.gz

{code_loader-1.0.93.dev0 → code_loader-1.0.93.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.93.dev0
+Version: 1.0.93.dev3
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.93.dev0 → code_loader-1.0.93.dev3}/code_loader/contract/datasetclasses.py

@@ -40,6 +40,9 @@ class PreprocessResponse:
     sample_id_type: Optional[Union[Type[str], Type[int]]] = None
 
     def __post_init__(self) -> None:
+        def is_valid_string(s: str) -> bool:
+            return bool(re.match(r'^[A-Za-z0-9_]+$', s))
+
         if self.length is not None and self.sample_ids is None:
             self.sample_ids = [i for i in range(self.length)]
             self.sample_id_type = int
@@ -50,6 +53,8 @@ class PreprocessResponse:
             if self.sample_id_type == str:
                 for sample_id in self.sample_ids:
                     assert isinstance(sample_id, str), f"Sample id should be of type str. Got: {type(sample_id)}"
+                    if not is_valid_string(sample_id):
+                        raise Exception(f"Sample id should contain only letters (A-Z, a-z), numbers or '_'. Got: {sample_id}")
         else:
             raise Exception("length is deprecated.")
 
@@ -200,7 +205,7 @@ class MetadataHandler:
 class PredictionTypeHandler:
     name: str
     labels: List[str]
-    channel_dim: int
+    channel_dim: int = -1
 
 
 @dataclass

{code_loader-1.0.93.dev0 → code_loader-1.0.93.dev3}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1,5 +1,5 @@
 # mypy: ignore-errors
-
+import os
 from typing import Optional, Union, Callable, List, Dict
 
 import numpy as np
@@ -8,7 +8,7 @@ import numpy.typing as npt
 from code_loader.contract.datasetclasses import CustomCallableInterfaceMultiArgs, \
     CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, CustomCallableInterface, \
     VisualizerCallableInterface, MetadataSectionCallableInterface, PreprocessResponse, SectionCallableInterface, \
-    ConfusionMatrixElement, SamplePreprocessResponse
+    ConfusionMatrixElement, SamplePreprocessResponse, PredictionTypeHandler
 from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType
 from code_loader import leap_binder
 from code_loader.contract.mapping import NodeMapping, NodeMappingType, NodeConnection
@@ -16,14 +16,77 @@ from code_loader.contract.visualizer_classes import LeapImage, LeapImageMask, Le
     LeapHorizontalBar, LeapImageWithBBox, LeapImageWithHeatmap
 
 
+mapping_runtime_mode_env_var_mame = '__mapping_runtime_mode__'
+
+
+def _add_mapping_connection(user_unique_name, connection_destinations, arg_names, name, node_mapping_type):
+    main_node_mapping = NodeMapping(name, node_mapping_type, user_unique_name, arg_names=arg_names)
+    node_inputs = {}
+    for arg_name, destination in zip(arg_names, connection_destinations):
+        node_inputs[arg_name] = destination.node_mapping
+
+    leap_binder.mapping_connections.append(NodeConnection(main_node_mapping, node_inputs))
+
+
 def _add_mapping_connections(connects_to, arg_names, node_mapping_type, name):
     for user_unique_name, connection_destinations in connects_to.items():
-        main_node_mapping = NodeMapping(name, node_mapping_type, user_unique_name, arg_names=arg_names)
-        node_inputs = {}
-        for arg_name, destination in zip(arg_names, connection_destinations):
-            node_inputs[arg_name] = destination.node_mapping
+        _add_mapping_connection(user_unique_name, connection_destinations, arg_names, name, node_mapping_type)
+
+
+
+
+def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]] = None):
+    for i, prediction_type in enumerate(prediction_types):
+        leap_binder.add_prediction(prediction_type.name, prediction_type.labels, prediction_type.channel_dim, i)
+
+    def decorating_function(load_model_func):
+        class TempMapping:
+            pass
+
+        def mapping_inner():
+            class ModelOutputPlaceholder:
+                def __init__(self):
+                    self.node_mapping = NodeMapping('', NodeMappingType.Prediction0)
+
+                def __getitem__(self, key):
+                    assert isinstance(key, int), \
+                        f'Expected key to be an int, got {type(key)} instead.'
+
+                    ret = TempMapping()
+                    ret.node_mapping = NodeMapping('', NodeMappingType(f'Prediction{str(key)}'))
+                    return ret
+
+            class ModelPlaceholder:
+                #keras interface
+                def __call__(self, arg):
+                    if isinstance(arg, list):
+                        for i, elem in enumerate(arg):
+                            elem.node_mapping.type = NodeMappingType[f'Input{str(i)}']
+                    else:
+                        arg.node_mapping.type = NodeMappingType.Input0
+
+                    return ModelOutputPlaceholder()
+
+                # onnx runtime interface
+                def run(self, output_names, input_dict):
+                    assert output_names is None
+                    assert isinstance(input_dict, dict), \
+                        f'Expected input_dict to be a dict, got {type(input_dict)} instead.'
+                    for i, elem in enumerate(input_dict.values()):
+                        elem.node_mapping.type = NodeMappingType[f'Input{str(i)}']
+
+                    return ModelOutputPlaceholder()
+
+            return ModelPlaceholder()
+
+
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return load_model_func
+
+    return decorating_function
 
-        leap_binder.mapping_connections.append(NodeConnection(main_node_mapping, node_inputs))
 
 
 def tensorleap_custom_metric(name: str,
@@ -31,8 +94,8 @@ def tensorleap_custom_metric(name: str,
                              compute_insights: Optional[Union[bool, Dict[str, bool]]] = None,
                              connects_to=None):
     def decorating_function(user_function: Union[CustomCallableInterfaceMultiArgs,
-    CustomMultipleReturnCallableInterfaceMultiArgs,
-    ConfusionMatrixCallableInterfaceMultiArgs]):
+            CustomMultipleReturnCallableInterfaceMultiArgs,
+            ConfusionMatrixCallableInterfaceMultiArgs]):
         for metric_handler in leap_binder.setup_container.metrics:
             if metric_handler.metric_handler_data.name == name:
                 raise Exception(f'Metric with name {name} already exists. '
@@ -119,14 +182,31 @@ def tensorleap_custom_metric(name: str,
                         (f'tensorleap_custom_metric validation failed: '
                          f'compute_insights should be boolean. Got {type(compute_insights)}.')
 
-
         def inner(*args, **kwargs):
             _validate_input_args(*args, **kwargs)
             result = user_function(*args, **kwargs)
             _validate_result(result)
             return result
 
-        return inner
+        def mapping_inner(*args, **kwargs):
+            user_unique_name = mapping_inner.name
+            if 'user_unique_name' in kwargs:
+                user_unique_name = kwargs['user_unique_name']
+
+            ordered_connections = [kwargs[n] for n in mapping_inner.arg_names if n in kwargs]
+            ordered_connections = list(args) + ordered_connections
+            _add_mapping_connection(user_unique_name, ordered_connections, mapping_inner.arg_names,
+                                    mapping_inner.name,  NodeMappingType.Metric)
+
+            return None
+
+        mapping_inner.arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
+        mapping_inner.name = name
+
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return inner
 
     return decorating_function
 
@@ -186,7 +266,25 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
             _validate_result(result)
             return result
 
-        return inner
+        def mapping_inner(*args, **kwargs):
+            user_unique_name = mapping_inner.name
+            if 'user_unique_name' in kwargs:
+                user_unique_name = kwargs['user_unique_name']
+
+            ordered_connections = [kwargs[n] for n in mapping_inner.arg_names if n in kwargs]
+            ordered_connections = list(args) + ordered_connections
+            _add_mapping_connection(user_unique_name, ordered_connections, mapping_inner.arg_names,
+                                    mapping_inner.name, NodeMappingType.Visualizer)
+
+            return None
+
+        mapping_inner.arg_names = leap_binder.setup_container.visualizers[-1].visualizer_handler_data.arg_names
+        mapping_inner.name = name
+
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return inner
 
     return decorating_function
 
@@ -340,7 +438,21 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
             node_mapping_type = NodeMappingType(f'Input{str(model_input_index)}')
         inner.node_mapping = NodeMapping(name, node_mapping_type)
 
-        return inner
+
+        def mapping_inner(*args, **kwargs):
+            class TempMapping:
+                pass
+            ret = TempMapping()
+            ret.node_mapping = mapping_inner.node_mapping
+
+            return ret
+
+        mapping_inner.node_mapping = NodeMapping(name, node_mapping_type)
+
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return inner
 
     return decorating_function
 
@@ -382,9 +494,20 @@ def tensorleap_gt_encoder(name: str):
 
         inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
 
-        return inner
+        def mapping_inner(*args, **kwargs):
+            class TempMapping:
+                pass
+            ret = TempMapping()
+            ret.node_mapping = mapping_inner.node_mapping
+
+            return ret
 
+        mapping_inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
 
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return inner
 
     return decorating_function
 
@@ -440,7 +563,25 @@ def tensorleap_custom_loss(name: str, connects_to=None):
             _validate_result(result)
             return result
 
-        return inner
+        def mapping_inner(*args, **kwargs):
+            user_unique_name = mapping_inner.name
+            if 'user_unique_name' in kwargs:
+                user_unique_name = kwargs['user_unique_name']
+
+            ordered_connections = [kwargs[n] for n in mapping_inner.arg_names if n in kwargs]
+            ordered_connections = list(args) + ordered_connections
+            _add_mapping_connection(user_unique_name, ordered_connections, mapping_inner.arg_names,
+                                    mapping_inner.name, NodeMappingType.CustomLoss)
+
+            return None
+
+        mapping_inner.arg_names = leap_binder.setup_container.custom_loss_handlers[-1].custom_loss_handler_data.arg_names
+        mapping_inner.name = name
+
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return inner
 
     return decorating_function
 

{code_loader-1.0.93.dev0 → code_loader-1.0.93.dev3}/code_loader/leaploader.py

@@ -23,7 +23,7 @@ from code_loader.contract.responsedataclasses import DatasetIntegParseResult, Da
     EngineFileContract
 from code_loader.inner_leap_binder import global_leap_binder
 from code_loader.leaploaderbase import LeapLoaderBase
-from code_loader.utils import get_root_exception_file_and_line_number, flatten
+from code_loader.utils import get_root_exception_file_and_line_number
 
 
 class LeapLoader(LeapLoaderBase):
@@ -477,18 +477,22 @@ class LeapLoader(LeapLoaderBase):
 
         return converted_value, is_none
 
-    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[
-        Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
+    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
         result_agg = {}
         is_none = {}
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
         for handler in global_leap_binder.setup_container.metadata:
             handler_result = handler.function(sample_id, preprocess_state)
-
-            for flat_name, flat_result in flatten(handler_result, prefix=handler.name):
-                result_agg[flat_name], is_none[flat_name] = self._convert_metadata_to_correct_type(
-                    flat_name, flat_result)
+            if isinstance(handler_result, dict):
+                for single_metadata_name, single_metadata_result in handler_result.items():
+                    handler_name = f'{handler.name}_{single_metadata_name}'
+                    result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
+                        handler_name, single_metadata_result)
+            else:
+                handler_name = handler.name
+                result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
+                    handler_name, handler_result)
 
         return result_agg, is_none
 

{code_loader-1.0.93.dev0 → code_loader-1.0.93.dev3}/code_loader/utils.py

@@ -1,7 +1,7 @@
 import sys
 from pathlib import Path
 from types import TracebackType
-from typing import List, Union, Tuple, Any, Iterator
+from typing import List, Union, Tuple, Any
 import traceback
 import numpy as np
 import numpy.typing as npt
@@ -66,28 +66,3 @@ def rescale_min_max(image: npt.NDArray[np.float32]) -> npt.NDArray[np.float32]:
     return image
 
 
-def flatten(
-    value: Any,
-    *,
-    prefix: str = "",
-    list_token: str = "e",
-) -> Iterator[Tuple[str, Any]]:
-    """
-    Recursively walk `value` and yield (flat_key, leaf_value) pairs.
-
-    • Dicts → descend with   new_prefix = f"{prefix}_{key}" (or just key if top level)
-    • Sequences → descend with new_prefix = f"{prefix}_{list_token}{idx}"
-    • Leaf scalars → yield the accumulated flat key and the scalar itself
-    """
-    if isinstance(value, dict):
-        for k, v in value.items():
-            new_prefix = f"{prefix}_{k}" if prefix else k
-            yield from flatten(v, prefix=new_prefix, list_token=list_token)
-
-    elif isinstance(value, (list, tuple)):
-        for idx, v in enumerate(value):
-            new_prefix = f"{prefix}_{list_token}{idx}"
-            yield from flatten(v, prefix=new_prefix, list_token=list_token)
-
-    else:  # primitive leaf (str, int, float, bool, None…)
-        yield prefix, value

{code_loader-1.0.93.dev0 → code_loader-1.0.93.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.93.dev0"
+version = "1.0.93.dev3"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"