code-loader 1.0.41__py3-none-any.whl → 1.0.42a0__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -4,7 +4,7 @@ from typing import Any, Callable, List, Optional, Dict, Union, Type
 import numpy as np
 import numpy.typing as npt
 
-from code_loader.contract.enums import DataStateType, DataStateEnum, LeapDataType, ConfusionMatrixValue, MetricDirection
+from code_loader.contract.enums import DataStateType, DataStateEnum, LeapDataType, ConfusionMatrixValue, MetricDirection, InstanceAnalysisType
 from code_loader.contract.visualizer_classes import LeapImage, LeapText, LeapGraph, LeapHorizontalBar, \
     LeapTextMask, LeapImageMask, LeapImageWithBBox, LeapImageWithHeatmap
 
@@ -13,12 +13,39 @@ custom_latent_space_attribute = "custom_latent_space"
 
 @dataclass
 class PreprocessResponse:
+    """
+    An object that holds the preprocessed data for use within the Tensorleap platform.
+
+    This class is used to encapsulate the results of data preprocessing, including inputs, metadata, labels, and other relevant information.
+    It facilitates handling and integration of the processed data within Tensorleap.
+
+    Attributes:
+    length (int): The length of the preprocessed data.
+    data (Any): The preprocessed data itself. This can be any data type depending on the preprocessing logic.
+
+    Example:
+        # Example usage of PreprocessResponse
+        preprocessed_data = {
+            'images': ['path/to/image1.jpg', 'path/to/image2.jpg'],
+            'labels': ['SUV', 'truck'],
+            'metadata': [{'id': 1, 'source': 'camera1'}, {'id': 2, 'source': 'camera2'}]
+        }
+        response = PreprocessResponse(length=len(preprocessed_data), data=preprocessed_data)
+    """
     length: int
     data: Any
 
 
+@dataclass
+class ElementInstance:
+    name: str
+    mask: npt.NDArray[np.float32]
+
+
 SectionCallableInterface = Callable[[int, PreprocessResponse], npt.NDArray[np.float32]]
 
+InstanceCallableInterface = Callable[[int, PreprocessResponse], List[ElementInstance]]
+
 MetadataSectionCallableInterface = Union[
     Callable[[int, PreprocessResponse], int],
     Callable[[int, PreprocessResponse], Dict[str, int]],
@@ -90,6 +117,7 @@ class MetricHandler:
     arg_names: List[str]
     direction: Optional[MetricDirection] = MetricDirection.Downward
 
+
 @dataclass
 class RawInputsForHeatmap:
     raw_input_by_vizualizer_arg_name: Dict[str, npt.NDArray[np.float32]]
@@ -115,6 +143,14 @@ class InputHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None
 
 
+@dataclass
+class ElementInstanceHandler:
+    input_name: str
+    instance_function: InstanceCallableInterface
+    analysis_type: InstanceAnalysisType
+
+
+
 @dataclass
 class GroundTruthHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None
@@ -148,6 +184,7 @@ class DatasetIntegrationSetup:
     unlabeled_data_preprocess: Optional[UnlabeledDataPreprocessHandler] = None
     visualizers: List[VisualizerHandler] = field(default_factory=list)
     inputs: List[InputHandler] = field(default_factory=list)
+    element_instances: List[ElementInstanceHandler] = field(default_factory=list)
     ground_truths: List[GroundTruthHandler] = field(default_factory=list)
     metadata: List[MetadataHandler] = field(default_factory=list)
     prediction_types: List[PredictionTypeHandler] = field(default_factory=list)

code_loader/contract/enums.py

@@ -64,3 +64,9 @@ class ConfusionMatrixValue(Enum):
 class TestingSectionEnum(Enum):
     Warnings = "Warnings"
     Errors = "Errors"
+
+
+
+class InstanceAnalysisType(Enum):
+    MaskInput = "MaskInput"
+    MaskLatentSpace = "MaskLatentSpace"

code_loader/contract/responsedataclasses.py

@@ -95,7 +95,19 @@ class DatasetTestResultPayload:
 
 @dataclass
 class BoundingBox:
-    # (x, y) is the center of the bounding box
+    """
+    Represents a bounding box for an object in an image.
+
+    Attributes:
+    x (float): The x-coordinate of the center of the bounding box, a value between [0, 1] representing the percentage according to the image width.
+    y (float): The y-coordinate of the center of the bounding box, a value between [0, 1] representing the percentage according to the image height.
+    width (float): The width of the bounding box, a value between [0, 1] representing the percentage according to the image width.
+    height (float): The height of the bounding box, a value between [0, 1] representing the percentage according to the image height.
+    confidence (float): The confidence score of the bounding box. For predictions, this is a score typically between [0, 1]. For ground truth data, this can be 1.
+    label (str): The label or class name associated with the bounding box.
+    rotation (float): The rotation of the bounding box, a value between [0, 360] representing the degree of rotation. Default is 0.0.
+    metadata (Optional[Dict[str, Union[str, int, float]]]): Optional metadata associated with the bounding box.
+    """
     x: float  # value between [0, 1], represent the percentage according to the image size.
     y: float  # value between [0, 1], represent the percentage according to the image size.
 

code_loader/contract/visualizer_classes.py

@@ -28,6 +28,13 @@ def validate_type(actual: Any, expected: Any, prefix_message: str = '') -> None:
 
 @dataclass
 class LeapImage:
+    """
+    Visualizer representing an image for Tensorleap.
+
+    Attributes:
+    data (npt.NDArray[np.float32] | npt.NDArray[np.uint8]): The image data.
+    type (LeapDataType): The data type, default is LeapDataType.Image.
+    """
     data: Union[npt.NDArray[np.float32], npt.NDArray[np.uint8]]
     type: LeapDataType = LeapDataType.Image
 
@@ -41,6 +48,14 @@ class LeapImage:
 
 @dataclass
 class LeapImageWithBBox:
+    """
+    Visualizer representing an image with bounding boxes for Tensorleap, used for object detection tasks.
+
+    Attributes:
+    data (npt.NDArray[np.float32] | npt.NDArray[np.uint8]): The image data, shaped [H, W, 3] or [H, W, 1].
+    bounding_boxes (List[BoundingBox]): List of Tensorleap bounding boxes objects in relative size to image size.
+    type (LeapDataType): The data type, default is LeapDataType.ImageWithBBox.
+    """
     data: Union[npt.NDArray[np.float32], npt.NDArray[np.uint8]]
     bounding_boxes: List[BoundingBox]
     type: LeapDataType = LeapDataType.ImageWithBBox
@@ -55,6 +70,13 @@ class LeapImageWithBBox:
 
 @dataclass
 class LeapGraph:
+    """
+    Visualizer representing a line chart data for Tensorleap.
+
+    Attributes:
+    data (npt.NDArray[np.float32]): The array data, shaped [M, N] where M is the number of data points and N is the number of variables.
+    type (LeapDataType): The data type, default is LeapDataType.Graph.
+    """
     data: npt.NDArray[np.float32]
     type: LeapDataType = LeapDataType.Graph
 
@@ -67,6 +89,14 @@ class LeapGraph:
 
 @dataclass
 class LeapText:
+    """
+    Visualizer representing text data for Tensorleap.
+
+    Attributes:
+    data (List[str]): The text data, consisting of a list of text tokens. If the model requires fixed-length inputs,
+    it is recommended to maintain the fixed length, using empty strings ('') instead of padding tokens ('PAD') e.g., ['I', 'ate', 'a', 'banana', '', '', '', ...]
+    type (LeapDataType): The data type, default is LeapDataType.Text.
+    """
     data: List[str]
     type: LeapDataType = LeapDataType.Text
 
@@ -79,6 +109,16 @@ class LeapText:
 
 @dataclass
 class LeapHorizontalBar:
+    """
+    Visualizer representing horizontal bar data for Tensorleap.
+    For example, this can be used to visualize the model's prediction scores in a classification problem.
+
+    Attributes:
+    body (npt.NDArray[np.float32]): The data for the bar, shaped [C], where C is the number of data points.
+    labels (List[str]): Labels for the horizontal bar; e.g., when visualizing the model's classification output, labels are the class names.
+    Length of `body` should match the length of `labels`, C.
+    type (LeapDataType): The data type, default is LeapDataType.HorizontalBar.
+    """
     body: npt.NDArray[np.float32]
     labels: List[str]
     type: LeapDataType = LeapDataType.HorizontalBar
@@ -96,6 +136,16 @@ class LeapHorizontalBar:
 
 @dataclass
 class LeapImageMask:
+    """
+    Visualizer representing an image with a mask for Tensorleap.
+    This can be used for tasks such as segmentation, and other applications where it is important to highlight specific regions within an image.
+
+    Attributes:
+    mask (npt.NDArray[np.uint8]): The mask data, shaped [H, W].
+    image (npt.NDArray[np.float32] | npt.NDArray[np.uint8]): The image data, shaped [H, W, 3] or shaped [H, W, 1].
+    labels (List[str]): Labels associated with the mask regions; e.g., class names for segmented objects. The length of `labels` should match the number of unique values in `mask`.
+    type (LeapDataType): The data type, default is LeapDataType.ImageMask.
+    """
     mask: npt.NDArray[np.uint8]
     image: Union[npt.NDArray[np.float32], npt.NDArray[np.uint8]]
     labels: List[str]
@@ -117,6 +167,16 @@ class LeapImageMask:
 
 @dataclass
 class LeapTextMask:
+    """
+    Visualizer representing text data with a mask for Tensorleap.
+    This can be used for tasks such as named entity recognition (NER), sentiment analysis, and other applications where it is important to highlight specific tokens or parts of the text.
+
+    Attributes:
+    mask (npt.NDArray[np.uint8]): The mask data, shaped [L].
+    text (List[str]): The text data, consisting of a list of text tokens, length of L.
+    labels (List[str]): Labels associated with the masked tokens; e.g., named entities or sentiment categories. The length of `labels` should match the number of unique values in `mask`.
+    type (LeapDataType): The data type, default is LeapDataType.TextMask.
+    """
     mask: npt.NDArray[np.uint8]
     text: List[str]
     labels: List[str]
@@ -137,6 +197,16 @@ class LeapTextMask:
 
 @dataclass
 class LeapImageWithHeatmap:
+    """
+    Visualizer representing an image with heatmaps for Tensorleap.
+    This can be used for tasks such as highlighting important regions in an image, visualizing attention maps, and other applications where it is important to overlay heatmaps on images.
+
+    Attributes:
+    image (npt.NDArray[np.float32]): The image data, shaped [H, W, C], where C is the number of channels.
+    heatmaps (npt.NDArray[np.float32]): The heatmap data, shaped [N, H, W], where N is the number of heatmaps.
+    labels (List[str]): Labels associated with the heatmaps; e.g., feature names or attention regions. The length of `labels` should match the number of heatmaps, N.
+    type (LeapDataType): The data type, default is LeapDataType.ImageWithHeatmap.
+    """
     image: npt.NDArray[np.float32]
     heatmaps: npt.NDArray[np.float32]
     labels: List[str]

code_loader/inner_leap_binder/leapbinder.py

@@ -9,8 +9,9 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     PreprocessHandler, VisualizerCallableInterface, CustomLossHandler, CustomCallableInterface, PredictionTypeHandler, \
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, VisualizerCallableReturnType, \
-    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, RawInputsForHeatmap
-from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
+    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
+    RawInputsForHeatmap, InstanceCallableInterface, ElementInstanceHandler
+from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, InstanceAnalysisType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
 from code_loader.utils import to_numpy_return_wrapper, get_shape
@@ -21,6 +22,15 @@ from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
 
 
 class LeapBinder:
+    """
+    Interface to the Tensorleap platform. Provides methods to set up preprocessing,
+    visualization, custom loss functions, metrics, and other essential components for integrating the dataset and model
+    with Tensorleap.
+
+    Attributes:
+    setup_container (DatasetIntegrationSetup): Container to hold setup configurations.
+    cache_container (Dict[str, Any]): Cache container to store intermediate data.
+    """
     def __init__(self) -> None:
         self.setup_container = DatasetIntegrationSetup()
         self.cache_container: Dict[str, Any] = {"word_to_index": {}}
@@ -48,6 +58,36 @@ class LeapBinder:
                        name: str,
                        visualizer_type: LeapDataType,
                        heatmap_visualizer: Optional[Callable[..., npt.NDArray[np.float32]]] = None) -> None:
+        """
+        Set a visualizer for a specific data type.
+
+        Args:
+        function (VisualizerCallableInterface): The visualizer function to be used for visualizing the data.
+        name (str): The name of the visualizer.
+        visualizer_type (LeapDataType): The type of data the visualizer handles (e.g., LeapDataType.Image, LeapDataType.Graph, LeapDataType.Text).
+        heatmap_visualizer (Optional[Callable[..., npt.NDArray[np.float32]]]): An optional heatmap visualizer function.
+        This is used when a heatmap must be reshaped to overlay correctly on the transformed data within the visualizer
+        function i.e., if the visualizer changes the shape or scale of the input data, the heatmap visualizer
+        adjusts the heatmap accordingly to ensure it aligns properly with the visualized data.
+
+        Example:
+        def image_resize_visualizer(data: np.ndarray) -> LeapImage:
+            # Resize the image to a fixed size
+            resized_image = resize_image(data, (224, 224))
+            return LeapImage(data=resized_image)
+
+        def image_resize_heatmap_visualizer(heatmap: RawInputsForHeatmap) -> np.ndarray:
+            # Resize the heatmap to match the resized image
+            resized_heatmap = resize_heatmap(heatmap, (224, 224))
+            return resized_heatmap
+
+        leap_binder.set_visualizer(
+            function=image_resize_visualizer,
+            name='image_resize_visualizer',
+            visualizer_type=LeapDataType.Image,
+            heatmap_visualizer=image_resize_heatmap_visualizer
+        )
+        """
         arg_names = inspect.getfullargspec(function)[0]
         if heatmap_visualizer:
             visualizer_arg_names_set = set(arg_names)
@@ -90,18 +130,103 @@ class LeapBinder:
         self._visualizer_names.append(name)
 
     def set_preprocess(self, function: Callable[[], List[PreprocessResponse]]) -> None:
+        """
+        Set the preprocessing function for the dataset. That is the function that returns a list of PreprocessResponse objects for use within the Tensorleap platform.
+
+        Args:
+        function (Callable[[], List[PreprocessResponse]]): The preprocessing function.
+
+        Example:
+            def preprocess_func() -> List[PreprocessResponse]:
+                # Preprocess the dataset
+                train_data = {
+                'subset': 'train',
+                'images': ['path/to/train/image1.jpg', 'path/to/train/image2.jpg'],
+                'labels': ['SUV', 'truck'],
+                'metadata': [{'id': 1, 'source': 'camera1'}, {'id': 2, 'source': 'camera2'}]}
+
+                val_data = {
+                'subset': 'val',
+                'images': ['path/to/val/image1.jpg', 'path/to/va;/image2.jpg'],
+                'labels': ['truck', 'truck'],
+                'metadata': [{'id': 1, 'source': 'camera1'}, {'id': 2, 'source': 'camera2'}]}
+
+                return [PreprocessResponse(length=len(train_data['images']), data=train_data),
+                        PreprocessResponse(length=len(val_data['images']), data=val_data)]
+
+            leap_binder.set_preprocess(preprocess_func)
+        """
         self.setup_container.preprocess = PreprocessHandler(function)
 
     def set_unlabeled_data_preprocess(self, function: Callable[[], PreprocessResponse]) -> None:
+        """
+        Set the preprocessing function for unlabeled dataset. This function returns a PreprocessResponse object for use within the Tensorleap platform for sample data that does not contain labels.
+
+        Args:
+        function (Callable[[], PreprocessResponse]): The preprocessing function for unlabeled data.
+
+        Example:
+            def unlabeled_preprocess_func() -> List[PreprocessResponse]:
+
+                # Preprocess the dataset
+                ul_data = {
+                'subset': 'unlabeled',
+                'images': ['path/to/train/image1.jpg', 'path/to/train/image2.jpg'],
+                'metadata': [{'id': 1, 'source': 'camera1'}, {'id': 2, 'source': 'camera2'}]}
+
+                return [PreprocessResponse(length=len(train_data['images']), data=train_data)]
+
+            leap_binder.set_preprocess(unlabeled_preprocess_func)
+        """
         self.setup_container.unlabeled_data_preprocess = UnlabeledDataPreprocessHandler(function)
 
     def set_input(self, function: SectionCallableInterface, name: str) -> None:
+        """
+        Set the input handler function.
+
+        Args:
+        function (SectionCallableInterface): The input handler function.
+        name (str): The name of the input section.
+
+        Example:
+            def input_encoder(subset: PreprocessResponse, index: int) -> np.ndarray:
+                # Return the processed input data for the given index and given subset response
+                img_path = subset.`data["images"][idx]
+                img = read_img(img_path)
+                img = normalize(img)
+                return img
+
+            leap_binder.set_input(input_encoder, name='input_encoder')
+        """
         function = to_numpy_return_wrapper(function)
         self.setup_container.inputs.append(InputHandler(name, function))
 
         self._encoder_names.append(name)
 
+    def set_instance_element(self, input_name: str, instance_function: Optional[InstanceCallableInterface] = None,
+                             analysis_type: InstanceAnalysisType = InstanceAnalysisType.MaskInput) -> None:
+        self.setup_container.element_instances.append(ElementInstanceHandler(input_name, instance_function, analysis_type))
+
     def add_custom_loss(self, function: CustomCallableInterface, name: str) -> None:
+        """
+        Add a custom loss function to the setup.
+
+        Args:
+        function (CustomCallableInterface): The custom loss function.
+            This function receives:
+                - y_true: The true labels or values.
+                - y_pred: The predicted labels or values.
+            This function should return:
+                - A numeric value representing the loss.
+        name (str): The name of the custom loss function.
+
+        Example:
+            def custom_loss_function(y_true, y_pred):
+                # Calculate mean squared error as custom loss
+                return np.mean(np.square(y_true - y_pred))
+
+            leap_binder.add_custom_loss(custom_loss_function, name='custom_loss')
+        """
         arg_names = inspect.getfullargspec(function)[0]
         self.setup_container.custom_loss_handlers.append(CustomLossHandler(name, function, arg_names))
 
@@ -111,28 +236,138 @@ class LeapBinder:
                           ConfusionMatrixCallableInterfaceMultiArgs],
                           name: str,
                           direction: Optional[MetricDirection] = MetricDirection.Downward) -> None:
+        """
+        Add a custom metric to the setup.
+
+        Args:
+        function (Union[CustomCallableInterfaceMultiArgs, CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]): The custom metric function.
+        name (str): The name of the custom metric.
+        direction (Optional[MetricDirection]): The direction of the metric, either MetricDirection.Upward or MetricDirection.Downward.
+            - MetricDirection.Upward: Indicates that higher values of the metric are better and should be maximized.
+            - MetricDirection.Downward: Indicates that lower values of the metric are better and should be minimized.
+
+
+        Example:
+            def custom_metric_function(y_true, y_pred):
+                return np.mean(np.abs(y_true - y_pred))
+
+            leap_binder.add_custom_metric(custom_metric_function, name='custom_metric', direction=MetricDirection.Downward)
+        """
         arg_names = inspect.getfullargspec(function)[0]
         self.setup_container.metrics.append(MetricHandler(name, function, arg_names, direction))
 
     def add_prediction(self, name: str, labels: List[str], channel_dim: int = -1) -> None:
+        """
+        Add prediction labels to the setup.
+
+        Args:
+        name (str): The name of the prediction.
+        labels (List[str]): The list of labels for the prediction.
+        channel_dim (int): The axis along which the prediction scores are located, default is -1.
+
+        Must satisfy len(labels) == len(output[channel_dim]).
+
+        Example:
+            leap_binder.add_prediction(name='class_labels', labels=['cat', 'dog'])
+        """
         self.setup_container.prediction_types.append(PredictionTypeHandler(name, labels, channel_dim))
 
     def set_ground_truth(self, function: SectionCallableInterface, name: str) -> None:
+        """
+        Set the ground truth handler function.
+
+        Args:
+        function: The ground truth handler function.
+            This function receives two parameters:
+                - subset: A `PreprocessResponse` object that contains the preprocessed data.
+                - index: The index of the sample within the subset.
+            This function should return:
+                - A numpy array representing the ground truth for the given sample.
+
+        name (str): The name of the ground truth section.
+
+        Example:
+            def ground_truth_handler(subset, index):
+                label = subset.data['labels'][index]
+                # Assuming labels are integers starting from 0
+                num_classes = 10  # Example number of classes
+                one_hot_label = np.zeros(num_classes)
+                one_hot_label[label] = 1
+                return one_hot_label
+
+            leap_binder.set_ground_truth(ground_truth_handler, name='ground_truth')
+        """
+
         function = to_numpy_return_wrapper(function)
         self.setup_container.ground_truths.append(GroundTruthHandler(name, function))
 
         self._encoder_names.append(name)
 
     def set_metadata(self, function: MetadataSectionCallableInterface, name: str) -> None:
+        """
+        Set the metadata handler function. This function is used for measuring and analyzing external variable values per sample, which is recommended for analysis within the Tensorleap platform.
+
+        Args:
+        function (MetadataSectionCallableInterface): The metadata handler function.
+            This function receives:
+                subset (PreprocessResponse): The subset of the data.
+                index (int): The index of the sample within the subset.
+            This function should return one of the following:
+                int: A single integer value.
+                Dict[str, int]: A dictionary with string keys and integer values.
+                str: A single string value.
+                Dict[str, str]: A dictionary with string keys and string values.
+                bool: A single boolean value.
+                Dict[str, bool]: A dictionary with string keys and boolean values.
+                float: A single float value.
+                Dict[str, float]: A dictionary with string keys and float values.
+
+        name (str): The name of the metadata section.
+
+        Example:
+            def metadata_handler_index(subset: PreprocessResponse, index: int) -> int:
+                return subset.data['metadata'][index]
+
+
+            def metadata_handler_image_mean(subset: PreprocessResponse, index: int) -> float:
+                fpath = subset.data['images'][index]
+                image = load_image(fpath)
+                mean_value = np.mean(image)
+                return mean_value
+
+            leap_binder.set_metadata(metadata_handler_index, name='metadata_index')
+            leap_binder.set_metadata(metadata_handler_image_mean, name='metadata_image_mean')
+        """
         self.setup_container.metadata.append(MetadataHandler(name, function))
 
     def set_custom_layer(self, custom_layer: Type[Any], name: str, inspect_layer: bool = False,
                          kernel_index: Optional[int] = None, use_custom_latent_space: bool = False) -> None:
+        """
+        Set a custom layer for the model.
+
+        Args:
+        custom_layer (Type[Any]): The custom layer class.
+        name (str): The name of the custom layer.
+        inspect_layer (bool): Whether to inspect the layer, default is False.
+        kernel_index (Optional[int]): The index of the kernel to inspect, if inspect_layer is True.
+        use_custom_latent_space (bool): Whether to use a custom latent space, default is False.
+
+        Example:
+            class CustomLayer:
+                def __init__(self, units: int):
+                    self.units = units
+
+                def call(self, inputs):
+                    return inputs * self.units
+
+            leap_binder.set_custom_layer(CustomLayer, name='custom_layer', inspect_layer=True, kernel_index=0)
+        """
         if inspect_layer and kernel_index is not None:
             custom_layer.kernel_index = kernel_index
 
         if use_custom_latent_space and not hasattr(custom_layer, custom_latent_space_attribute):
-            raise Exception(f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
+            raise Exception(
+                f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
 
         init_args = inspect.getfullargspec(custom_layer.__init__)[0][1:]
         call_args = inspect.getfullargspec(custom_layer.call)[0][1:]

code_loader/leaploader.py

@@ -5,15 +5,16 @@ import sys
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, Iterable, Union, Any
+from typing import Dict, List, Iterable, Union, Any, Optional, Tuple
 
 import numpy as np
 import numpy.typing as npt
 
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
     PreprocessResponse, VisualizerHandler, VisualizerCallableReturnType, CustomLossHandler, \
-    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler
-from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
+    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, ElementInstance
+from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType, \
+    InstanceAnalysisType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
@@ -30,7 +31,6 @@ class LeapLoader:
     @lru_cache()
     def exec_script(self) -> None:
         try:
-            print("executing script")
             self.evaluate_module()
         except TypeError as e:
             import traceback
@@ -42,34 +42,25 @@ class LeapLoader:
             raise DatasetScriptException(getattr(e, 'message', repr(e))) from e
 
     def evaluate_module(self) -> None:
-        print("evaluate_module")
-
         def append_path_recursively(full_path: str) -> None:
             if '/' not in full_path or full_path == '/':
                 return
 
             parent_path = str(Path(full_path).parent)
-            print(f"evaluate_module.append_path_recursively full_path: {full_path}, parent_path: {parent_path}")
             append_path_recursively(parent_path)
-
             sys.path.append(parent_path)
 
         file_path = Path(self.code_path, self.code_entry_name)
-        print(f"evaluate_module: self.code_path: {self.code_path} self.code_entry_name: {self.code_entry_name}, "
-              f"file_path for append_path_recursively: {str(file_path)}")
         append_path_recursively(str(file_path))
 
         spec = importlib.util.spec_from_file_location(self.code_path, file_path)
-        print(f"evaluate_module: spec: {str(spec)}")
         if spec is None or spec.loader is None:
             raise DatasetScriptException(f'Something is went wrong with spec file from: {file_path}')
 
         file = importlib.util.module_from_spec(spec)
-        print(f"evaluate_module: file module_from_spec: {str(file)}")
         if file is None:
             raise DatasetScriptException(f'Something is went wrong with import module from: {file_path}')
 
-        print(f"evaluate_module: spec.loader.exec_module")
         spec.loader.exec_module(file)
 
     @lru_cache()
@@ -181,8 +172,7 @@ class LeapLoader:
                         preprocess_response, test_result, dataset_base_handler)
                 except Exception as e:
                     line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
-                    test_result[0].display[
-                        state_name] = f"{repr(e)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
+                    test_result[0].display[state_name] = f"{repr(e)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
                     test_result[0].is_passed = False
 
             result_payloads.extend(test_result)
@@ -309,6 +299,17 @@ class LeapLoader:
     def _get_inputs(self, state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:
         return self._get_dataset_handlers(global_leap_binder.setup_container.inputs, state, idx)
 
+    def get_instance_elements(self, state: DataStateEnum, idx: int, input_name: str) \
+            -> Tuple[Optional[List[ElementInstance]], Optional[InstanceAnalysisType]]:
+        preprocess_result = self._preprocess_result()
+        preprocess_state = preprocess_result[state]
+        for element_instance in global_leap_binder.setup_container.element_instances:
+            if element_instance.input_name == input_name:
+                return element_instance.instance_function(idx, preprocess_state), element_instance.analysis_type
+
+        return None, None
+
+
     def _get_gt(self, state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:
         return self._get_dataset_handlers(global_leap_binder.setup_container.ground_truths, state, idx)
 
@@ -354,8 +355,7 @@ class LeapLoader:
             if isinstance(handler_result, dict):
                 for single_metadata_name, single_metadata_result in handler_result.items():
                     handler_name = f'{handler.name}_{single_metadata_name}'
-                    result_agg[handler_name] = self._convert_metadata_to_correct_type(handler_name,
-                                                                                      single_metadata_result)
+                    result_agg[handler_name] = self._convert_metadata_to_correct_type(handler_name, single_metadata_result)
             else:
                 handler_name = handler.name
                 result_agg[handler_name] = self._convert_metadata_to_correct_type(handler_name, handler_result)

{code_loader-1.0.41.dist-info → code_loader-1.0.42a0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.41
+Version: 1.0.42a0
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

code_loader-1.0.42a0.dist-info/RECORD

@@ -0,0 +1,18 @@
+LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader/__init__.py,sha256=V3DEXSN6Ie6PlGeSAbzjp9ufRj0XPJLpD7pDLLYxk6M,122
+code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+code_loader/contract/datasetclasses.py,sha256=eoDxCGZzvogp440dhWZBBm8oTs6XiqcHxE-JXD4txeA,6128
+code_loader/contract/enums.py,sha256=2tKXamabooQjCe3EBkeBcZMA8vBbeGjDAbACvUTGqLE,1707
+code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
+code_loader/contract/responsedataclasses.py,sha256=w7xVOv2S8Hyb5lqyomMGiKAWXDTSOG-FX1YW39bXD3A,3969
+code_loader/contract/visualizer_classes.py,sha256=t0EKxTGFJoFBnKuMmjtFpj-L32xEFF3NV-E5XutVXa0,10118
+code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
+code_loader/inner_leap_binder/leapbinder.py,sha256=7_ffUiGW54Y27f7eKnvpJ2XLOqfGfTCW5cMNaH21iIk,24464
+code_loader/leaploader.py,sha256=gQaEpHyQx50FP-d0LS2PjQdm9aOUf3jok8ZeLbWkGZc,17917
+code_loader/utils.py,sha256=61I4PgSl-ZBIe4DifLxMNlBELE-HQR2pB9efVYPceIU,2230
+code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+code_loader/visualizers/default_visualizers.py,sha256=F2qRT6rNy7SOmr4QfqVxIkLlYEa00CwDkLJuA45lfSI,2254
+code_loader-1.0.42a0.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.42a0.dist-info/METADATA,sha256=HrRgFfW68Nr0KIBzK-C9Fwnxcw9SYotWTzWfwW2PF4I,770
+code_loader-1.0.42a0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.42a0.dist-info/RECORD,,

code_loader-1.0.41.dist-info/RECORD

@@ -1,18 +0,0 @@
-LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader/__init__.py,sha256=V3DEXSN6Ie6PlGeSAbzjp9ufRj0XPJLpD7pDLLYxk6M,122
-code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=e4pEajPiktogtrctf29Q9IqG5VaZz0QE_ioc4Ri1ohs,4796
-code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
-code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
-code_loader/contract/responsedataclasses.py,sha256=JgISOk1fVAD7dBLmwp2IzCkpItwO7YP3SDqzTX5FMDA,2880
-code_loader/contract/visualizer_classes.py,sha256=EKkQ7qWS48LeT3JoJT8-uMdRDxVOkSp0q-rhVMxrEFY,5990
-code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=cayCJxnksr5lrEIX1HngiuPpTupStti4DD6KX3k-PBI,13610
-code_loader/leaploader.py,sha256=SCAJWVTmuAZ4G4SvPljaxDS1zaM_8BOSO45fKhz4L7g,17956
-code_loader/utils.py,sha256=61I4PgSl-ZBIe4DifLxMNlBELE-HQR2pB9efVYPceIU,2230
-code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/visualizers/default_visualizers.py,sha256=F2qRT6rNy7SOmr4QfqVxIkLlYEa00CwDkLJuA45lfSI,2254
-code_loader-1.0.41.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.41.dist-info/METADATA,sha256=QmhzHlZeg1rUYSRKrDAsdJtCYMEZl-9bolsHDtXYuHk,768
-code_loader-1.0.41.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.41.dist-info/RECORD,,