code-loader 1.0.42a0__py3-none-any.whl → 1.0.43a0__py3-none-any.whl

code_loader/__init__.py

@@ -1,2 +1,3 @@
 from code_loader.leaploader import LeapLoader
 from code_loader.inner_leap_binder import global_leap_binder as leap_binder
+from code_loader.rt_api import Epoch, Experiment

code_loader/contract/datasetclasses.py

@@ -4,7 +4,7 @@ from typing import Any, Callable, List, Optional, Dict, Union, Type
 import numpy as np
 import numpy.typing as npt
 
-from code_loader.contract.enums import DataStateType, DataStateEnum, LeapDataType, ConfusionMatrixValue, MetricDirection, InstanceAnalysisType
+from code_loader.contract.enums import DataStateType, DataStateEnum, LeapDataType, ConfusionMatrixValue, MetricDirection
 from code_loader.contract.visualizer_classes import LeapImage, LeapText, LeapGraph, LeapHorizontalBar, \
     LeapTextMask, LeapImageMask, LeapImageWithBBox, LeapImageWithHeatmap
 
@@ -36,16 +36,8 @@ class PreprocessResponse:
     data: Any
 
 
-@dataclass
-class ElementInstance:
-    name: str
-    mask: npt.NDArray[np.float32]
-
-
 SectionCallableInterface = Callable[[int, PreprocessResponse], npt.NDArray[np.float32]]
 
-InstanceCallableInterface = Callable[[int, PreprocessResponse], List[ElementInstance]]
-
 MetadataSectionCallableInterface = Union[
     Callable[[int, PreprocessResponse], int],
     Callable[[int, PreprocessResponse], Dict[str, int]],
@@ -117,7 +109,6 @@ class MetricHandler:
     arg_names: List[str]
     direction: Optional[MetricDirection] = MetricDirection.Downward
 
-
 @dataclass
 class RawInputsForHeatmap:
     raw_input_by_vizualizer_arg_name: Dict[str, npt.NDArray[np.float32]]
@@ -143,14 +134,6 @@ class InputHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None
 
 
-@dataclass
-class ElementInstanceHandler:
-    input_name: str
-    instance_function: InstanceCallableInterface
-    analysis_type: InstanceAnalysisType
-
-
-
 @dataclass
 class GroundTruthHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None
@@ -184,7 +167,6 @@ class DatasetIntegrationSetup:
     unlabeled_data_preprocess: Optional[UnlabeledDataPreprocessHandler] = None
     visualizers: List[VisualizerHandler] = field(default_factory=list)
     inputs: List[InputHandler] = field(default_factory=list)
-    element_instances: List[ElementInstanceHandler] = field(default_factory=list)
     ground_truths: List[GroundTruthHandler] = field(default_factory=list)
     metadata: List[MetadataHandler] = field(default_factory=list)
     prediction_types: List[PredictionTypeHandler] = field(default_factory=list)

code_loader/contract/enums.py

@@ -64,9 +64,3 @@ class ConfusionMatrixValue(Enum):
 class TestingSectionEnum(Enum):
     Warnings = "Warnings"
     Errors = "Errors"
-
-
-
-class InstanceAnalysisType(Enum):
-    MaskInput = "MaskInput"
-    MaskLatentSpace = "MaskLatentSpace"

code_loader/contract/visualizer_classes.py

@@ -4,6 +4,8 @@ import numpy as np
 import numpy.typing as npt
 from dataclasses import dataclass
 
+import matplotlib.pyplot as plt     # type: ignore
+
 from code_loader.contract.enums import LeapDataType
 from code_loader.contract.responsedataclasses import BoundingBox
 
@@ -34,6 +36,10 @@ class LeapImage:
     Attributes:
     data (npt.NDArray[np.float32] | npt.NDArray[np.uint8]): The image data.
     type (LeapDataType): The data type, default is LeapDataType.Image.
+
+    Example:
+        image_data = np.random.rand(100, 100, 3).astype(np.float32)
+        leap_image = LeapImage(data=image_data)
     """
     data: Union[npt.NDArray[np.float32], npt.NDArray[np.uint8]]
     type: LeapDataType = LeapDataType.Image
@@ -45,6 +51,34 @@ class LeapImage:
         validate_type(len(self.data.shape), 3, 'Image must be of shape 3')
         validate_type(self.data.shape[2], [1, 3], 'Image channel must be either 3(rgb) or 1(gray)')
 
+    def plot_visualizer(self) -> None:
+        """
+        Display the image contained in the LeapImage object.
+
+        Returns:
+            None
+
+        Example:
+            image_data = np.random.rand(100, 100, 3).astype(np.float32)
+            leap_image = LeapImage(data=image_data)
+            leap_image.plot_visualizer()
+        """
+        image_data = self.data
+
+        # If the image has one channel, convert it to a 3-channel image for display
+        if image_data.shape[2] == 1:
+            image_data = np.repeat(image_data, 3, axis=2)
+
+        fig, ax = plt.subplots()
+        fig.patch.set_facecolor('black')
+        ax.set_facecolor('black')
+
+        ax.imshow(image_data)
+
+        plt.axis('off')
+        plt.title('Leap Image Visualization', color='white')
+        plt.show()
+
 
 @dataclass
 class LeapImageWithBBox:
@@ -55,6 +89,11 @@ class LeapImageWithBBox:
     data (npt.NDArray[np.float32] | npt.NDArray[np.uint8]): The image data, shaped [H, W, 3] or [H, W, 1].
     bounding_boxes (List[BoundingBox]): List of Tensorleap bounding boxes objects in relative size to image size.
     type (LeapDataType): The data type, default is LeapDataType.ImageWithBBox.
+
+    Example:
+        image_data = np.random.rand(100, 100, 3).astype(np.float32)
+        bbox = BoundingBox(x=0.5, y=0.5, width=0.2, height=0.2, confidence=0.9, label="object")
+        leap_image_with_bbox = LeapImageWithBBox(data=image_data, bounding_boxes=[bbox])
     """
     data: Union[npt.NDArray[np.float32], npt.NDArray[np.uint8]]
     bounding_boxes: List[BoundingBox]
@@ -67,6 +106,60 @@ class LeapImageWithBBox:
         validate_type(len(self.data.shape), 3, 'Image must be of shape 3')
         validate_type(self.data.shape[2], [1, 3], 'Image channel must be either 3(rgb) or 1(gray)')
 
+    def plot_visualizer(self) -> None:
+        """
+        Plot an image with overlaid bounding boxes.
+
+        Returns:
+            None
+
+        Example:
+            image_data = np.random.rand(100, 100, 3).astype(np.float32)
+            bbox = BoundingBox(x=0.5, y=0.5, width=0.2, height=0.2, confidence=0.9, label="object")
+            leap_image_with_bbox = LeapImageWithBBox(data=image_data, bounding_boxes=[bbox])
+            leap_image_with_bbox.plot_visualizer()
+        """
+
+        image = self.data
+        bounding_boxes = self.bounding_boxes
+
+        # Create figure and axes
+        fig, ax = plt.subplots(1)
+        fig.patch.set_facecolor('black')
+        ax.set_facecolor('black')
+
+        # Display the image
+        ax.imshow(image)
+        ax.set_title('Leap Image With BBox Visualization', color='white')
+
+        # Draw bounding boxes on the image
+        for bbox in bounding_boxes:
+            x, y, width, height = bbox.x, bbox.y, bbox.width, bbox.height
+            confidence, label = bbox.confidence, bbox.label
+
+            # Convert relative coordinates to absolute coordinates
+            abs_x = x * image.shape[1]
+            abs_y = y * image.shape[0]
+            abs_width = width * image.shape[1]
+            abs_height = height * image.shape[0]
+
+            # Create a rectangle patch
+            rect = plt.Rectangle(
+                (abs_x - abs_width / 2, abs_y - abs_height / 2),
+                abs_width, abs_height,
+                linewidth=3, edgecolor='r', facecolor='none'
+            )
+
+            # Add the rectangle to the axes
+            ax.add_patch(rect)
+
+            # Display label and confidence
+            ax.text(abs_x - abs_width / 2, abs_y - abs_height / 2 - 5,
+                    f"{label} {confidence:.2f}", color='r', fontsize=8,
+                    bbox=dict(facecolor='white', alpha=0.7, edgecolor='none', boxstyle='round,pad=0.3'))
+
+        # Show the image with bounding boxes
+        plt.show()
 
 @dataclass
 class LeapGraph:
@@ -76,6 +169,10 @@ class LeapGraph:
     Attributes:
     data (npt.NDArray[np.float32]): The array data, shaped [M, N] where M is the number of data points and N is the number of variables.
     type (LeapDataType): The data type, default is LeapDataType.Graph.
+
+    Example:
+        graph_data = np.random.rand(100, 3).astype(np.float32)
+        leap_graph = LeapGraph(data=graph_data)
     """
     data: npt.NDArray[np.float32]
     type: LeapDataType = LeapDataType.Graph
@@ -86,6 +183,40 @@ class LeapGraph:
         validate_type(self.data.dtype, np.float32)
         validate_type(len(self.data.shape), 2, 'Graph must be of shape 2')
 
+    def plot_visualizer(self) -> None:
+        """
+        Display the line chart contained in the LeapGraph object.
+
+        Returns:
+            None
+
+        Example:
+            graph_data = np.random.rand(100, 3).astype(np.float32)
+            leap_graph = LeapGraph(data=graph_data)
+            leap_graph.plot_visualizer()
+        """
+        graph_data = self.data
+        num_variables = graph_data.shape[1]
+
+        fig, ax = plt.subplots(figsize=(10, 6))
+
+        # Set the background color to black
+        fig.patch.set_facecolor('black')
+        ax.set_facecolor('black')
+
+        for i in range(num_variables):
+            plt.plot(graph_data[:, i], label=f'Variable {i + 1}')
+
+        ax.set_xlabel('Data Points', color='white')
+        ax.set_ylabel('Values', color='white')
+        ax.set_title('Leap Graph Visualization', color='white')
+        ax.legend()
+        ax.grid(True, color='white')
+
+        # Change the color of the tick labels to white
+        ax.tick_params(colors='white')
+
+        plt.show()
 
 @dataclass
 class LeapText:
@@ -96,6 +227,11 @@ class LeapText:
     data (List[str]): The text data, consisting of a list of text tokens. If the model requires fixed-length inputs,
     it is recommended to maintain the fixed length, using empty strings ('') instead of padding tokens ('PAD') e.g., ['I', 'ate', 'a', 'banana', '', '', '', ...]
     type (LeapDataType): The data type, default is LeapDataType.Text.
+
+    Example:
+        text_data = ['I', 'ate', 'a', 'banana', '', '', '']
+        leap_text = LeapText(data=text_data)  # Create LeapText object
+        LeapText(leap_text)
     """
     data: List[str]
     type: LeapDataType = LeapDataType.Text
@@ -106,6 +242,41 @@ class LeapText:
         for value in self.data:
             validate_type(type(value), str)
 
+    def plot_visualizer(self) -> None:
+        """
+        Display the text contained in the LeapText object.
+
+        Returns:
+            None
+
+        Example:
+            text_data = ['I', 'ate', 'a', 'banana', '', '', '']
+            leap_text = LeapText(data=text_data)
+            leap_text.plot_visualizer()
+        """
+        text_data = self.data
+        # Join the text tokens into a single string, ignoring empty strings
+        display_text = ' '.join([token for token in text_data if token])
+
+        # Create a black image using Matplotlib
+        fig, ax = plt.subplots(figsize=(10, 5))
+        fig.patch.set_facecolor('black')
+        ax.set_facecolor('black')
+
+        # Hide the axes
+        ax.axis('off')
+
+        # Set the text properties
+        font_size = 20
+        font_color = 'white'
+
+        # Add the text to the image
+        ax.text(0.5, 0.5, display_text, color=font_color, fontsize=font_size, ha='center', va='center')
+        ax.set_title('Leap Text Visualization', color='white')
+
+        # Display the image
+        plt.show()
+
 
 @dataclass
 class LeapHorizontalBar:
@@ -118,6 +289,11 @@ class LeapHorizontalBar:
     labels (List[str]): Labels for the horizontal bar; e.g., when visualizing the model's classification output, labels are the class names.
     Length of `body` should match the length of `labels`, C.
     type (LeapDataType): The data type, default is LeapDataType.HorizontalBar.
+
+    Example:
+        body_data = np.random.rand(5).astype(np.float32)
+        labels = ['Class A', 'Class B', 'Class C', 'Class D', 'Class E']
+        leap_horizontal_bar = LeapHorizontalBar(body=body_data, labels=labels)
     """
     body: npt.NDArray[np.float32]
     labels: List[str]
@@ -133,6 +309,39 @@ class LeapHorizontalBar:
         for label in self.labels:
             validate_type(type(label), str)
 
+    def plot_visualizer(self) -> None:
+        """
+        Display the horizontal bar chart contained in the LeapHorizontalBar object.
+
+        Returns:
+            None
+
+        Example:
+            body_data = np.random.rand(5).astype(np.float32)
+            labels = ['Class A', 'Class B', 'Class C', 'Class D', 'Class E']
+            leap_horizontal_bar = LeapHorizontalBar(body=body_data, labels=labels)
+            leap_horizontal_bar.plot_visualizer()
+        """
+        body_data = self.body
+        labels = self.labels
+
+        fig, ax = plt.subplots()
+
+        fig.patch.set_facecolor('black')
+        ax.set_facecolor('black')
+
+        # Plot horizontal bar chart
+        ax.barh(labels, body_data, color='green')
+
+        # Set the color of the labels and title to white
+        ax.set_xlabel('Scores', color='white')
+        ax.set_title('Leap Horizontal Bar Visualization', color='white')
+
+        # Set the color of the ticks to white
+        ax.tick_params(axis='x', colors='white')
+        ax.tick_params(axis='y', colors='white')
+
+        plt.show()
 
 @dataclass
 class LeapImageMask:
@@ -145,6 +354,12 @@ class LeapImageMask:
     image (npt.NDArray[np.float32] | npt.NDArray[np.uint8]): The image data, shaped [H, W, 3] or shaped [H, W, 1].
     labels (List[str]): Labels associated with the mask regions; e.g., class names for segmented objects. The length of `labels` should match the number of unique values in `mask`.
     type (LeapDataType): The data type, default is LeapDataType.ImageMask.
+
+    Example:
+        image_data = np.random.rand(100, 100, 3).astype(np.float32)
+        mask_data = np.random.randint(0, 2, (100, 100)).astype(np.uint8)
+        labels = ["background", "object"]
+        leap_image_mask = LeapImageMask(image=image_data, mask=mask_data, labels=labels)
     """
     mask: npt.NDArray[np.uint8]
     image: Union[npt.NDArray[np.float32], npt.NDArray[np.uint8]]
@@ -164,6 +379,51 @@ class LeapImageMask:
         for label in self.labels:
             validate_type(type(label), str)
 
+    def plot_visualizer(self) -> None:
+        """
+        Plots an image with overlaid masks given a LeapImageMask visualizer object.
+
+        Returns:
+            None
+
+
+        Example:
+            image_data = np.random.rand(100, 100, 3).astype(np.float32)
+            mask_data = np.random.randint(0, 2, (100, 100)).astype(np.uint8)
+            labels = ["background", "object"]
+            leap_image_mask = LeapImageMask(image=image_data, mask=mask_data, labels=labels)
+            leap_image_mask.plot_visualizer()
+        """
+
+        image = self.image
+        mask = self.mask
+        labels = self.labels
+
+        # Create a color map for each label
+        colors = plt.cm.jet(np.linspace(0, 1, len(labels)))
+
+        # Make a copy of the image to draw on
+        overlayed_image = image.copy()
+
+        # Iterate through the unique values in the mask (excluding 0)
+        for i, label in enumerate(labels):
+            # Extract binary mask for the current instance
+            instance_mask = (mask == (i + 1))
+
+            # fill the instance mask with a translucent color
+            overlayed_image[instance_mask] = (
+                    overlayed_image[instance_mask] * (1 - 0.5) + np.array(colors[i][:3], dtype=np.uint8) * 0.5)
+
+        # Display the result using matplotlib
+        fig, ax = plt.subplots(1)
+        fig.patch.set_facecolor('black')  # Set the figure background to black
+        ax.set_facecolor('black')  # Set the axis background to black
+
+        ax.imshow(overlayed_image)
+        ax.set_title('Leap Image With Mask Visualization', color='white')
+        plt.axis('off')  # Hide the axis
+        plt.show()
+
 
 @dataclass
 class LeapTextMask:
@@ -176,6 +436,13 @@ class LeapTextMask:
     text (List[str]): The text data, consisting of a list of text tokens, length of L.
     labels (List[str]): Labels associated with the masked tokens; e.g., named entities or sentiment categories. The length of `labels` should match the number of unique values in `mask`.
     type (LeapDataType): The data type, default is LeapDataType.TextMask.
+
+    Example:
+        text_data = ['I', 'ate', 'a', 'banana', '', '', '']
+        mask_data = np.array([0, 0, 0, 1, 0, 0, 0]).astype(np.uint8)
+        labels = ["object"]
+        leap_text_mask = LeapTextMask(text=text_data, mask=mask_data, labels=labels)
+        leap_text_mask.plot_visualizer()
     """
     mask: npt.NDArray[np.uint8]
     text: List[str]
@@ -194,6 +461,55 @@ class LeapTextMask:
         for label in self.labels:
             validate_type(type(label), str)
 
+    def plot_visualizer(self) -> None:
+        """
+        Plots text with overlaid masks given a LeapTextMask visualizer object.
+
+        Returns:
+            None
+
+        Example:
+            text_data = ['I', 'ate', 'a', 'banana', '', '', '']
+            mask_data = np.array([0, 0, 0, 1, 0, 0, 0]).astype(np.uint8)
+            labels = ["object"]
+            leap_text_mask = LeapTextMask(text=text_data, mask=mask_data, labels=labels)
+        """
+
+        text_data = self.text
+        mask_data = self.mask
+        labels = self.labels
+
+        # Create a color map for each label
+        colors = plt.cm.jet(np.linspace(0, 1, len(labels)))
+
+        # Create a figure and axis
+        fig, ax = plt.subplots()
+
+        # Set background to black
+        fig.patch.set_facecolor('black')
+        ax.set_facecolor('black')
+        ax.set_title('Leap Text Mask Visualization', color='white')
+        ax.axis('off')
+
+        # Set initial position
+        x_pos, y_pos = 0.01, 0.5  # Adjusted initial position for better visibility
+
+        # Display the text with colors
+        for token, mask_value in zip(text_data, mask_data):
+            if mask_value > 0:
+                color = colors[mask_value % len(colors)]
+                bbox = dict(facecolor=color, edgecolor='none',
+                            boxstyle='round,pad=0.3')  # Background color for masked tokens
+            else:
+                bbox = None
+
+            ax.text(x_pos, y_pos, token, fontsize=12, color='white', ha='left', va='center', bbox=bbox)
+
+            # Update the x position for the next token
+            x_pos += len(token) * 0.03 + 0.02  # Adjust the spacing between tokens
+
+        plt.show()
+
 
 @dataclass
 class LeapImageWithHeatmap:
@@ -206,6 +522,12 @@ class LeapImageWithHeatmap:
     heatmaps (npt.NDArray[np.float32]): The heatmap data, shaped [N, H, W], where N is the number of heatmaps.
     labels (List[str]): Labels associated with the heatmaps; e.g., feature names or attention regions. The length of `labels` should match the number of heatmaps, N.
     type (LeapDataType): The data type, default is LeapDataType.ImageWithHeatmap.
+
+    Example:
+        image_data = np.random.rand(100, 100, 3).astype(np.float32)
+        heatmaps = np.random.rand(3, 100, 100).astype(np.float32)
+        labels = ["heatmap1", "heatmap2", "heatmap3"]
+        leap_image_with_heatmap = LeapImageWithHeatmap(image=image_data, heatmaps=heatmaps, labels=labels)
     """
     image: npt.NDArray[np.float32]
     heatmaps: npt.NDArray[np.float32]
@@ -225,6 +547,46 @@ class LeapImageWithHeatmap:
             raise LeapValidationError(
                 'Number of heatmaps and labels must be equal')
 
+    def plot_visualizer(self) -> None:
+        """
+        Display the image with overlaid heatmaps contained in the LeapImageWithHeatmap object.
+
+        Returns:
+            None
+
+        Example:
+            image_data = np.random.rand(100, 100, 3).astype(np.float32)
+            heatmaps = np.random.rand(3, 100, 100).astype(np.float32)
+            labels = ["heatmap1", "heatmap2", "heatmap3"]
+            leap_image_with_heatmap = LeapImageWithHeatmap(image=image_data, heatmaps=heatmaps, labels=labels)
+            leap_image_with_heatmap.plot_visualizer()
+        """
+        image = self.image
+        heatmaps = self.heatmaps
+        labels = self.labels
+
+        # Plot the base image
+        fig, ax = plt.subplots()
+        fig.patch.set_facecolor('black')  # Set the figure background to black
+        ax.set_facecolor('black')  # Set the axis background to black
+        ax.imshow(image, cmap='gray')
+
+        # Overlay each heatmap with some transparency
+        for i in range(len(labels)):
+            heatmap = heatmaps[i]
+            ax.imshow(heatmap, cmap='jet', alpha=0.5)  # Adjust alpha for transparency
+            ax.set_title(f'Heatmap: {labels[i]}', color='white')
+
+            # Display a colorbar for the heatmap
+            cbar = plt.colorbar(ax.imshow(heatmap, cmap='jet', alpha=0.5))
+            cbar.set_label(labels[i], color='white')
+            cbar.ax.yaxis.set_tick_params(color='white')  # Set color for the colorbar ticks
+            plt.setp(plt.getp(cbar.ax.axes, 'yticklabels'), color='white')  # Set color for the colorbar labels
+
+        plt.axis('off')
+        plt.title('Leap Image With Heatmaps Visualization', color='white')
+        plt.show()
+
 
 map_leap_data_type_to_visualizer_class = {
     LeapDataType.Image.value: LeapImage,

code_loader/inner_leap_binder/leapbinder.py

@@ -9,9 +9,8 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     PreprocessHandler, VisualizerCallableInterface, CustomLossHandler, CustomCallableInterface, PredictionTypeHandler, \
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, VisualizerCallableReturnType, \
-    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
-    RawInputsForHeatmap, InstanceCallableInterface, ElementInstanceHandler
-from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, InstanceAnalysisType
+    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, RawInputsForHeatmap
+from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
 from code_loader.utils import to_numpy_return_wrapper, get_shape
@@ -203,10 +202,6 @@ class LeapBinder:
 
         self._encoder_names.append(name)
 
-    def set_instance_element(self, input_name: str, instance_function: Optional[InstanceCallableInterface] = None,
-                             analysis_type: InstanceAnalysisType = InstanceAnalysisType.MaskInput) -> None:
-        self.setup_container.element_instances.append(ElementInstanceHandler(input_name, instance_function, analysis_type))
-
     def add_custom_loss(self, function: CustomCallableInterface, name: str) -> None:
         """
         Add a custom loss function to the setup.
@@ -366,8 +361,7 @@ class LeapBinder:
             custom_layer.kernel_index = kernel_index
 
         if use_custom_latent_space and not hasattr(custom_layer, custom_latent_space_attribute):
-            raise Exception(
-                f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
+            raise Exception(f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
 
         init_args = inspect.getfullargspec(custom_layer.__init__)[0][1:]
         call_args = inspect.getfullargspec(custom_layer.call)[0][1:]
@@ -458,3 +452,5 @@ class LeapBinder:
         self.check_preprocess(preprocess_result)
         self.check_handlers(preprocess_result)
         print("Successful!")
+
+

code_loader/leaploader.py

@@ -5,16 +5,15 @@ import sys
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, Iterable, Union, Any, Optional, Tuple
+from typing import Dict, List, Iterable, Union, Any
 
 import numpy as np
 import numpy.typing as npt
 
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
     PreprocessResponse, VisualizerHandler, VisualizerCallableReturnType, CustomLossHandler, \
-    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, ElementInstance
-from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType, \
-    InstanceAnalysisType
+    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler
+from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
@@ -299,17 +298,6 @@ class LeapLoader:
     def _get_inputs(self, state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:
         return self._get_dataset_handlers(global_leap_binder.setup_container.inputs, state, idx)
 
-    def get_instance_elements(self, state: DataStateEnum, idx: int, input_name: str) \
-            -> Tuple[Optional[List[ElementInstance]], Optional[InstanceAnalysisType]]:
-        preprocess_result = self._preprocess_result()
-        preprocess_state = preprocess_result[state]
-        for element_instance in global_leap_binder.setup_container.element_instances:
-            if element_instance.input_name == input_name:
-                return element_instance.instance_function(idx, preprocess_state), element_instance.analysis_type
-
-        return None, None
-
-
     def _get_gt(self, state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:
         return self._get_dataset_handlers(global_leap_binder.setup_container.ground_truths, state, idx)
 

code_loader/rt_api/__init__.py

@@ -0,0 +1,12 @@
+
+from code_loader.rt_api.experiment_context import ExperimentContext
+from code_loader.rt_api.epoch import Epoch
+from code_loader.rt_api.experiment import Experiment
+from code_loader.rt_api.api_client import ApiClient
+
+__all__ = [
+    'ExperimentContext',
+    'Epoch',
+    'Experiment',
+    'ApiClient'
+]

code_loader/rt_api/api_client.py

@@ -0,0 +1,107 @@
+
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+from code_loader.rt_api.types import ApiMetrics
+from code_loader.rt_api.utils import join_url, to_dict_no_none
+from .cli_config_utils import get_auth_config
+import requests
+
+
+@dataclass
+class StartExperimentRequest:
+    projectId: str
+    experimentName: str
+    description: str
+    removeUntaggedUploadedModels: bool = True
+    codeIntegrationVersionId: Optional[str] = None
+
+@dataclass
+class StartExperimentResponse:
+    projectId: str
+    versionId: str
+    experimentId: str
+
+@dataclass
+class GetUploadModelSignedUrlRequest:
+    epoch: int
+    experimentId: str
+    versionId: str
+    projectId: str
+    fileType: str
+    origin: Optional[str] = None
+
+@dataclass
+class GetUploadModelSignedUrlResponse:
+    url: str
+    fileName: str
+
+@dataclass
+class AddExternalEpochDataRequest:
+  projectId: str
+  experimentId: str
+  epoch: int
+  metrics: ApiMetrics
+  force: bool = False
+
+@dataclass
+class TagModelRequest:
+  projectId: str
+  experimentId: str
+  epoch: int
+  tags: List[str]
+
+@dataclass
+class SetExperimentNotesRequest:
+    projectId: str
+    experimentId: str
+    notes: Dict[str, any] # type: ignore[valid-type]
+
+class ApiClient:
+    def __init__(self, url: Optional[str] = None, token: Optional[str] = None):
+        if url is None or token is None:
+            configAuth = get_auth_config()
+            if configAuth is None:
+                raise Exception("No auth config found, either provide url and token or use `leap auth [url] [token]` to setup a config file")
+            url = configAuth.api_url
+            token = configAuth.api_key
+        
+        self.url = url
+        self.token = token
+
+    def __add_auth(self, headers: Dict[str, str]) -> Dict[str, str]:
+        headers['Authorization'] = f'Bearer {self.token}'
+        return headers
+    
+    def __post(self, post_path: str, data: any, headers: Dict[str, str] = {})-> requests.Response: # type: ignore[valid-type]
+        headers = self.__add_auth(headers)
+        if 'Content-Type' not in headers:
+            headers['Content-Type'] = 'application/json'
+        url = join_url(self.url, post_path)
+        json_data = to_dict_no_none(data)
+        return requests.post(url, json=json_data, headers=headers)
+    
+    def __check_response(self, response: requests.Response)-> None:
+        if response.status_code >= 400:
+            raise Exception(f"Error: {response.status_code} {response.text}")
+    
+    def start_experiment(self, data: StartExperimentRequest) -> StartExperimentResponse:
+        response = self.__post('/versions/startExperiment', data)
+        self.__check_response(response)
+        return StartExperimentResponse(**response.json())
+    
+    def get_attach_model_upload_signed_url(self, data: GetUploadModelSignedUrlRequest)-> GetUploadModelSignedUrlResponse:
+        response = self.__post('/versions/getUploadModelSignedUrl', data)
+        self.__check_response(response)
+        return GetUploadModelSignedUrlResponse(**response.json())
+    
+    def add_external_epoch_data(self, data: AddExternalEpochDataRequest)-> None:
+        response = self.__post('/externalepochdata/addExternalEpochData', data)
+        self.__check_response(response)
+    
+    def tag_model(self, data: TagModelRequest)-> None:
+        response = self.__post('/versions/tagModel', data)
+        self.__check_response(response)
+
+    def set_experiment_notes(self, data: SetExperimentNotesRequest)-> None:
+        response = self.__post('/versions/setExperimentNotes', data)
+        self.__check_response(response)

code_loader/rt_api/cli_config_utils.py

@@ -0,0 +1,41 @@
+from dataclasses import dataclass
+import os
+from typing import Dict, Optional
+import yaml
+
+@dataclass
+class AuthConfig:
+    api_url: str
+    api_key: str
+
+@dataclass
+class Config:
+    current_env: str
+    envs: Dict[str, AuthConfig]
+
+def get_cli_conf_path()-> str:
+    cli_conf_path = os.getenv("TL_CLI_CONFIG_FILE") or os.path.join(os.path.expanduser("~"), ".config/tensorleap/config.yaml")
+    return cli_conf_path
+
+def get_cli_conf_file() -> Optional[Config]:
+    cli_conf_path = get_cli_conf_path()
+    if not os.path.exists(cli_conf_path):
+        return None
+    with open(cli_conf_path) as f:
+        y = yaml.safe_load(f)
+        envs_dict = y.get("envs")
+        if envs_dict is None:
+            return None
+        envs = dict()
+        for k, v in envs_dict.items():
+            envs[k] = AuthConfig(**v)
+        return Config(envs=envs, current_env=y["current_env"])
+    
+def get_auth_config() -> Optional[AuthConfig]:
+    cli_conf = get_cli_conf_file()
+    if cli_conf is None or cli_conf.current_env not in cli_conf.envs:
+        return None
+    return cli_conf.envs[cli_conf.current_env]
+
+
+    

code_loader/rt_api/epoch.py

@@ -0,0 +1,63 @@
+
+from typing import List, Optional
+from code_loader.rt_api import ExperimentContext
+from code_loader.rt_api.types import Metrics
+from code_loader.rt_api.utils import to_api_metric_value, upload_file
+from .api_client import AddExternalEpochDataRequest, GetUploadModelSignedUrlRequest, TagModelRequest
+
+
+class Epoch:
+    def __init__(self, ctx: ExperimentContext, epoch: int):
+        self.experiment = ExperimentContext
+        self.epoch = epoch
+        self.metrics: Metrics = {}
+        self.ctx = ctx
+
+    def add_metric(self, name: str, value: float)-> None:
+        self.metrics[name] = value
+
+    def set_metrics(self, metrics: Metrics)-> None:
+        self.metrics = metrics
+
+    def upload_model(self, modelFilePath: str)-> None:
+        allowed_extensions = ["h5", "onnx"]
+        modelExtension = modelFilePath.split(".")[-1]
+        if modelExtension not in allowed_extensions:
+            raise Exception(f"Model file extension not allowed. Allowed extensions are {allowed_extensions}")
+        url = self.ctx.client.get_attach_model_upload_signed_url(GetUploadModelSignedUrlRequest(
+            epoch=self.epoch,
+            experimentId=self.ctx.experiment_id,
+            versionId=self.ctx.version_id,
+            projectId=self.ctx.project_id,
+            fileType=modelExtension
+        ))
+        print(f"Uploading epoch({self.epoch}) model file")
+        upload_file(url.url, modelFilePath)
+        print("Model file uploaded")
+    
+    def tag_model(self, tags: List[str])-> None:
+        print(f"Tagging epoch({self.epoch}) model")
+        self.ctx.client.tag_model(TagModelRequest(
+            experimentId=self.ctx.experiment_id,
+            projectId=self.ctx.project_id,
+            epoch=self.epoch,
+            tags=tags
+        ))
+
+    def save(self, modelFilePath: Optional[str] = None, tags: List[str] = ['latest'])-> None:
+        if modelFilePath is not None:
+            self.upload_model(modelFilePath)
+
+        print(f"Add metrics for epoch({self.epoch}) model")
+        api_metrics ={
+            key: to_api_metric_value(value) for key, value in self.metrics.items()
+        }
+        self.ctx.client.add_external_epoch_data(AddExternalEpochDataRequest(
+            experimentId=self.ctx.experiment_id,
+            projectId=self.ctx.project_id,
+            epoch=self.epoch,
+            metrics=api_metrics
+        ))
+        if modelFilePath is not None and len(tags) > 0:
+            self.tag_model(tags)
+   

code_loader/rt_api/experiment.py

@@ -0,0 +1,47 @@
+
+from typing import Dict, List, Optional
+from code_loader.rt_api import Epoch, ExperimentContext
+from code_loader.rt_api.types import Metrics
+from code_loader.rt_api.workingspace_config_utils import load_workspace_config
+from .api_client import  SetExperimentNotesRequest, StartExperimentRequest, ApiClient
+
+        
+class Experiment:
+    def __init__(self, ctx: ExperimentContext):
+        self.ctx = ctx
+
+    def start_epoch(self, epoch: int) -> Epoch:
+        return Epoch(self.ctx, epoch)
+    
+    def add_epoch(self, epoch: int, metrics: Optional[Metrics] = None, model_path: Optional[str] = None, tags: List[str] = ['latest'])-> None:
+        epoch_o = self.start_epoch(epoch)
+        if metrics is not None:
+            epoch_o.set_metrics(metrics)
+        epoch_o.save(model_path, tags)
+
+    def set_notes(self, notes: Dict[str, any])-> None: # type: ignore[valid-type]
+        print(f"Setting experiment({self.ctx.experiment_id}) notes")
+        self.ctx.client.set_experiment_notes(SetExperimentNotesRequest(
+            experimentId=self.ctx.experiment_id,
+            projectId=self.ctx.project_id,
+            notes=notes
+        ))
+    
+    @staticmethod
+    def Start(experimentName: str, description: str, working_dir: Optional[str] = None, client: Optional[ApiClient] = None) -> 'Experiment':
+        
+        if client is None:
+            client = ApiClient()
+        
+        workspace_config = load_workspace_config(working_dir)
+        if workspace_config is None or workspace_config.projectId is None:
+            raise Exception("No leap workspace config found or projectId is missing, make sure you are in a leap workspace directory or provide a working_dir")
+
+        result = client.start_experiment(StartExperimentRequest(
+            projectId=workspace_config.projectId,
+            experimentName=experimentName,
+            description=description,
+            codeIntegrationVersionId=workspace_config.codeIntegrationId
+        ))
+        ctx = ExperimentContext(client, result.projectId, result.versionId, result.experimentId)
+        return Experiment(ctx)

code_loader/rt_api/experiment_context.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from code_loader.rt_api.api_client import ApiClient
+
+
+@dataclass
+class ExperimentContext:
+    client: ApiClient
+    project_id: str
+    version_id: str
+    experiment_id: str

code_loader/rt_api/types.py

@@ -0,0 +1,25 @@
+
+from dataclasses import dataclass
+from typing import Dict, Union
+
+
+MetricValue = Union[str, float]
+Metrics = Dict[str, MetricValue]
+
+@dataclass
+class NumericMetricValue:
+    value: float
+    type: str = "number"
+
+@dataclass
+class StringMetricValue:
+    value: str
+    type: str = "string"
+
+@dataclass
+class ImageMetricValue:
+    value: str
+    type: str = "image"
+
+ApiMetricValue = Union[NumericMetricValue, StringMetricValue, ImageMetricValue]
+ApiMetrics = Dict[str, ApiMetricValue]

code_loader/rt_api/utils.py

@@ -0,0 +1,34 @@
+from dataclasses import asdict, is_dataclass
+from typing import Dict
+from urllib.parse import urljoin
+from code_loader.rt_api.types import ApiMetricValue, MetricValue, NumericMetricValue, StringMetricValue
+import requests
+
+def upload_file(url: str, file_path: str)-> None:
+    with open(file_path, "rb") as f:
+        requests.put(url, data=f, timeout=12_000)
+
+def to_dict_no_none(data: any)-> Dict[str, any]: # type: ignore[valid-type]
+    if is_dataclass(data):
+        data = asdict(data)
+    if isinstance(data, dict):
+        return {k: to_dict_no_none(v) for k, v in data.items() if v is not None}
+    elif isinstance(data, list):
+        return [to_dict_no_none(item) for item in data if item is not None]
+    else:
+        return data
+
+def join_url(base_url: str, post_path: str)-> str:
+    if not base_url.endswith('/'):
+        base_url += '/'
+    if post_path.startswith('/'):
+        post_path = post_path[1:]
+    return urljoin(base_url, post_path)
+
+def to_api_metric_value(value: MetricValue) -> ApiMetricValue:
+    if isinstance(value, float) or isinstance(value, int):
+        return NumericMetricValue(value=value)
+    elif isinstance(value, str):
+        return StringMetricValue(value=value)
+    else:
+        raise Exception(f"Unsupported metric value type: {type(value)}")

code_loader/rt_api/workingspace_config_utils.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass
+from os import path
+import os
+from typing import List, Optional
+import yaml
+
+
+@dataclass
+class LocalProjectConfig:
+    codeIntegrationId: Optional[str] = None
+    projectId: Optional[str] = None
+    secretId: Optional[str] = None
+    secretManagerId: Optional[str] = None
+    entryFile: Optional[str] = None
+    includePatterns: Optional[List[str]] = None
+
+# Loading workspace configuration from leap.yaml
+def load_workspace_config(workspace_dir: Optional[str] = None) -> Optional[LocalProjectConfig]:
+    if workspace_dir is None:
+        workspace_dir = os.getcwd()
+    elif not path.isabs(workspace_dir):
+        workspace_dir = path.join(os.getcwd(), workspace_dir)
+
+    file_path = path.join(workspace_dir, "leap.yaml")
+    with open(file_path) as f:
+        y = yaml.safe_load(f)
+        return LocalProjectConfig(**y)

code_loader/visualizers/default_visualizers.py

@@ -1,5 +1,4 @@
 from enum import Enum
-from typing import List
 
 import numpy as np
 import numpy.typing as npt

{code_loader-1.0.42a0.dist-info → code_loader-1.0.43a0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.42a0
+Version: 1.0.43a0
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT
@@ -13,8 +13,11 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: matplotlib (>=3.3,<3.4)
 Requires-Dist: numpy (>=1.22.3,<2.0.0)
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
+Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
 Project-URL: Repository, https://github.com/tensorleap/code-loader
 Description-Content-Type: text/markdown
 

code_loader-1.0.43a0.dist-info/RECORD

@@ -0,0 +1,27 @@
+LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader/__init__.py,sha256=RKnZc9y45v19aegTyCW2F_88zRl561WGS4zVJezF3dM,171
+code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+code_loader/contract/datasetclasses.py,sha256=81TmCcVol7768lzKUp70MatLLipR3ftcR9jgE1r8Yqo,5698
+code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
+code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
+code_loader/contract/responsedataclasses.py,sha256=w7xVOv2S8Hyb5lqyomMGiKAWXDTSOG-FX1YW39bXD3A,3969
+code_loader/contract/visualizer_classes.py,sha256=Ka8fJSVKrOeZ12Eg8-dOBGMW0UswYCIkuhnNMd-7z9s,22948
+code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
+code_loader/inner_leap_binder/leapbinder.py,sha256=fHND8ayIXDlKHufHHq36u5rKNnZq64MmMuI42GctHvQ,24022
+code_loader/leaploader.py,sha256=pUySweZetJ6SsubCcZlDCJpvWmUrm5YlPlkZWQxY1hQ,17289
+code_loader/rt_api/__init__.py,sha256=VrRCxheS0dG4XDy_mGEwk6S_4Iivpfdp4QsinZokG8A,303
+code_loader/rt_api/api_client.py,sha256=ffWWqrfk8cae1-esjcSoxrWr431eXOORPDc8lEnXMPk,3573
+code_loader/rt_api/cli_config_utils.py,sha256=At09VmMuUbHvx0yq7D9hPJRlfjI6j5gRLZlRWxUplLE,1110
+code_loader/rt_api/epoch.py,sha256=xMREVKcdU4uqzy5oAyLYMMkG_-aKrRBiaRkEbS9DI-E,2449
+code_loader/rt_api/experiment.py,sha256=uRlIoC2kiuCWCxF0Ju_OVAEnbFkMLIMSEXS2PQ6NTU4,2092
+code_loader/rt_api/experiment_context.py,sha256=Lxd6YoI_Q95iBZniCAVX16ZcUs3r1QJxr7rrKPA-_N8,208
+code_loader/rt_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1oEB_NPg,485
+code_loader/rt_api/utils.py,sha256=lJKm2zNm8TW30cttnSFUBnV3eRJXhu7RFQ375h29ho0,1293
+code_loader/rt_api/workingspace_config_utils.py,sha256=fNz-xMmkKNTVjePi8r9_baYBFiiH5KO6eUOneCtIvC4,858
+code_loader/utils.py,sha256=61I4PgSl-ZBIe4DifLxMNlBELE-HQR2pB9efVYPceIU,2230
+code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+code_loader/visualizers/default_visualizers.py,sha256=VoqO9FN84yXyMjRjHjUTOt2GdTkJRMbHbXJ1cJkREkk,2230
+code_loader-1.0.43a0.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.43a0.dist-info/METADATA,sha256=EhngXZVguqOWAh9pbkvk_zywqvlpDJHhSp7jHq7HkIU,890
+code_loader-1.0.43a0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.43a0.dist-info/RECORD,,

code_loader-1.0.42a0.dist-info/RECORD

@@ -1,18 +0,0 @@
-LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader/__init__.py,sha256=V3DEXSN6Ie6PlGeSAbzjp9ufRj0XPJLpD7pDLLYxk6M,122
-code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=eoDxCGZzvogp440dhWZBBm8oTs6XiqcHxE-JXD4txeA,6128
-code_loader/contract/enums.py,sha256=2tKXamabooQjCe3EBkeBcZMA8vBbeGjDAbACvUTGqLE,1707
-code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
-code_loader/contract/responsedataclasses.py,sha256=w7xVOv2S8Hyb5lqyomMGiKAWXDTSOG-FX1YW39bXD3A,3969
-code_loader/contract/visualizer_classes.py,sha256=t0EKxTGFJoFBnKuMmjtFpj-L32xEFF3NV-E5XutVXa0,10118
-code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=7_ffUiGW54Y27f7eKnvpJ2XLOqfGfTCW5cMNaH21iIk,24464
-code_loader/leaploader.py,sha256=gQaEpHyQx50FP-d0LS2PjQdm9aOUf3jok8ZeLbWkGZc,17917
-code_loader/utils.py,sha256=61I4PgSl-ZBIe4DifLxMNlBELE-HQR2pB9efVYPceIU,2230
-code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/visualizers/default_visualizers.py,sha256=F2qRT6rNy7SOmr4QfqVxIkLlYEa00CwDkLJuA45lfSI,2254
-code_loader-1.0.42a0.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.42a0.dist-info/METADATA,sha256=HrRgFfW68Nr0KIBzK-C9Fwnxcw9SYotWTzWfwW2PF4I,770
-code_loader-1.0.42a0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.42a0.dist-info/RECORD,,