code-loader 1.0.49.dev9__tar.gz → 1.0.50__tar.gz

{code_loader-1.0.49.dev9 → code_loader-1.0.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.49.dev9
+Version: 1.0.50
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT
@@ -13,7 +13,6 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: matplotlib (>=3.3,<3.4)
 Requires-Dist: numpy (>=1.22.3,<2.0.0)
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)

{code_loader-1.0.49.dev9 → code_loader-1.0.50}/code_loader/contract/datasetclasses.py

@@ -32,38 +32,21 @@ class PreprocessResponse:
         }
         response = PreprocessResponse(length=len(preprocessed_data), data=preprocessed_data)
     """
-    length: Optional[int] = None  # Deprecated. Please use sample_ids instead
-    data: Any = None
-    sample_ids: Optional[Union[List[str], List[int]]] = None
-    state: Optional[DataStateType] = None
-    sample_id_type: Optional[Type] = None
+    length: int
+    data: Any
 
-    def __post_init__(self):
-        if self.length is not None and self.sample_ids is None:
-            self.sample_ids = [i for i in range(self.length)]
-            self.sample_id_type = int
-        elif self.length is None and self.sample_ids is not None:
-            self.length = len(self.sample_ids)
-            if self.sample_id_type is None:
-                self.sample_id_type = str
-        else:
-            raise Exception("length is deprecated. Please use sample_ids instead.")
 
-    def __len__(self):
-        return len(self.sample_ids)
-
-
-SectionCallableInterface = Callable[[Union[int, str], PreprocessResponse], npt.NDArray[np.float32]]
+SectionCallableInterface = Callable[[int, PreprocessResponse], npt.NDArray[np.float32]]
 
 MetadataSectionCallableInterface = Union[
-    Callable[[Union[int, str], PreprocessResponse], int],
-    Callable[[Union[int, str], PreprocessResponse], Dict[str, int]],
-    Callable[[Union[int, str], PreprocessResponse], str],
-    Callable[[Union[int, str], PreprocessResponse], Dict[str, str]],
-    Callable[[Union[int, str], PreprocessResponse], bool],
-    Callable[[Union[int, str], PreprocessResponse], Dict[str, bool]],
-    Callable[[Union[int, str], PreprocessResponse], float],
-    Callable[[Union[int, str], PreprocessResponse], Dict[str, float]]
+    Callable[[int, PreprocessResponse], int],
+    Callable[[int, PreprocessResponse], Dict[str, int]],
+    Callable[[int, PreprocessResponse], str],
+    Callable[[int, PreprocessResponse], Dict[str, str]],
+    Callable[[int, PreprocessResponse], bool],
+    Callable[[int, PreprocessResponse], Dict[str, bool]],
+    Callable[[int, PreprocessResponse], float],
+    Callable[[int, PreprocessResponse], Dict[str, float]]
 ]
 
 
@@ -90,8 +73,8 @@ VisualizerCallableInterface = Union[
     Callable[..., LeapImageWithHeatmap]
 ]
 
-VisualizerCallableReturnType = Union[LeapImage, LeapText, LeapGraph, LeapHorizontalBar,
-LeapImageMask, LeapTextMask, LeapImageWithBBox, LeapImageWithHeatmap]
+LeapData = Union[LeapImage, LeapText, LeapGraph, LeapHorizontalBar, LeapImageMask, LeapTextMask, LeapImageWithBBox,
+LeapImageWithHeatmap]
 
 CustomCallableInterface = Callable[..., Any]
 
@@ -198,5 +181,5 @@ class DatasetSample:
     inputs: Dict[str, npt.NDArray[np.float32]]
     gt: Optional[Dict[str, npt.NDArray[np.float32]]]
     metadata: Dict[str, Union[str, int, bool, float]]
-    index: Union[int, str]
+    index: int
     state: DataStateEnum

{code_loader-1.0.49.dev9 → code_loader-1.0.50}/code_loader/contract/visualizer_classes.py

@@ -4,8 +4,6 @@ import numpy as np
 import numpy.typing as npt
 from dataclasses import dataclass
 
-import matplotlib.pyplot as plt     # type: ignore
-
 from code_loader.contract.enums import LeapDataType
 from code_loader.contract.responsedataclasses import BoundingBox
 
@@ -51,34 +49,6 @@ class LeapImage:
         validate_type(len(self.data.shape), 3, 'Image must be of shape 3')
         validate_type(self.data.shape[2], [1, 3], 'Image channel must be either 3(rgb) or 1(gray)')
 
-    def plot_visualizer(self) -> None:
-        """
-        Display the image contained in the LeapImage object.
-
-        Returns:
-            None
-
-        Example:
-            image_data = np.random.rand(100, 100, 3).astype(np.float32)
-            leap_image = LeapImage(data=image_data)
-            leap_image.plot_visualizer()
-        """
-        image_data = self.data
-
-        # If the image has one channel, convert it to a 3-channel image for display
-        if image_data.shape[2] == 1:
-            image_data = np.repeat(image_data, 3, axis=2)
-
-        fig, ax = plt.subplots()
-        fig.patch.set_facecolor('black')
-        ax.set_facecolor('black')
-
-        ax.imshow(image_data)
-
-        plt.axis('off')
-        plt.title('Leap Image Visualization', color='white')
-        plt.show()
-
 
 @dataclass
 class LeapImageWithBBox:
@@ -106,60 +76,6 @@ class LeapImageWithBBox:
         validate_type(len(self.data.shape), 3, 'Image must be of shape 3')
         validate_type(self.data.shape[2], [1, 3], 'Image channel must be either 3(rgb) or 1(gray)')
 
-    def plot_visualizer(self) -> None:
-        """
-        Plot an image with overlaid bounding boxes.
-
-        Returns:
-            None
-
-        Example:
-            image_data = np.random.rand(100, 100, 3).astype(np.float32)
-            bbox = BoundingBox(x=0.5, y=0.5, width=0.2, height=0.2, confidence=0.9, label="object")
-            leap_image_with_bbox = LeapImageWithBBox(data=image_data, bounding_boxes=[bbox])
-            leap_image_with_bbox.plot_visualizer()
-        """
-
-        image = self.data
-        bounding_boxes = self.bounding_boxes
-
-        # Create figure and axes
-        fig, ax = plt.subplots(1)
-        fig.patch.set_facecolor('black')
-        ax.set_facecolor('black')
-
-        # Display the image
-        ax.imshow(image)
-        ax.set_title('Leap Image With BBox Visualization', color='white')
-
-        # Draw bounding boxes on the image
-        for bbox in bounding_boxes:
-            x, y, width, height = bbox.x, bbox.y, bbox.width, bbox.height
-            confidence, label = bbox.confidence, bbox.label
-
-            # Convert relative coordinates to absolute coordinates
-            abs_x = x * image.shape[1]
-            abs_y = y * image.shape[0]
-            abs_width = width * image.shape[1]
-            abs_height = height * image.shape[0]
-
-            # Create a rectangle patch
-            rect = plt.Rectangle(
-                (abs_x - abs_width / 2, abs_y - abs_height / 2),
-                abs_width, abs_height,
-                linewidth=3, edgecolor='r', facecolor='none'
-            )
-
-            # Add the rectangle to the axes
-            ax.add_patch(rect)
-
-            # Display label and confidence
-            ax.text(abs_x - abs_width / 2, abs_y - abs_height / 2 - 5,
-                    f"{label} {confidence:.2f}", color='r', fontsize=8,
-                    bbox=dict(facecolor='white', alpha=0.7, edgecolor='none', boxstyle='round,pad=0.3'))
-
-        # Show the image with bounding boxes
-        plt.show()
 
 @dataclass
 class LeapGraph:
@@ -183,40 +99,6 @@ class LeapGraph:
         validate_type(self.data.dtype, np.float32)
         validate_type(len(self.data.shape), 2, 'Graph must be of shape 2')
 
-    def plot_visualizer(self) -> None:
-        """
-        Display the line chart contained in the LeapGraph object.
-
-        Returns:
-            None
-
-        Example:
-            graph_data = np.random.rand(100, 3).astype(np.float32)
-            leap_graph = LeapGraph(data=graph_data)
-            leap_graph.plot_visualizer()
-        """
-        graph_data = self.data
-        num_variables = graph_data.shape[1]
-
-        fig, ax = plt.subplots(figsize=(10, 6))
-
-        # Set the background color to black
-        fig.patch.set_facecolor('black')
-        ax.set_facecolor('black')
-
-        for i in range(num_variables):
-            plt.plot(graph_data[:, i], label=f'Variable {i + 1}')
-
-        ax.set_xlabel('Data Points', color='white')
-        ax.set_ylabel('Values', color='white')
-        ax.set_title('Leap Graph Visualization', color='white')
-        ax.legend()
-        ax.grid(True, color='white')
-
-        # Change the color of the tick labels to white
-        ax.tick_params(colors='white')
-
-        plt.show()
 
 @dataclass
 class LeapText:
@@ -242,41 +124,6 @@ class LeapText:
         for value in self.data:
             validate_type(type(value), str)
 
-    def plot_visualizer(self) -> None:
-        """
-        Display the text contained in the LeapText object.
-
-        Returns:
-            None
-
-        Example:
-            text_data = ['I', 'ate', 'a', 'banana', '', '', '']
-            leap_text = LeapText(data=text_data)
-            leap_text.plot_visualizer()
-        """
-        text_data = self.data
-        # Join the text tokens into a single string, ignoring empty strings
-        display_text = ' '.join([token for token in text_data if token])
-
-        # Create a black image using Matplotlib
-        fig, ax = plt.subplots(figsize=(10, 5))
-        fig.patch.set_facecolor('black')
-        ax.set_facecolor('black')
-
-        # Hide the axes
-        ax.axis('off')
-
-        # Set the text properties
-        font_size = 20
-        font_color = 'white'
-
-        # Add the text to the image
-        ax.text(0.5, 0.5, display_text, color=font_color, fontsize=font_size, ha='center', va='center')
-        ax.set_title('Leap Text Visualization', color='white')
-
-        # Display the image
-        plt.show()
-
 
 @dataclass
 class LeapHorizontalBar:
@@ -309,39 +156,6 @@ class LeapHorizontalBar:
         for label in self.labels:
             validate_type(type(label), str)
 
-    def plot_visualizer(self) -> None:
-        """
-        Display the horizontal bar chart contained in the LeapHorizontalBar object.
-
-        Returns:
-            None
-
-        Example:
-            body_data = np.random.rand(5).astype(np.float32)
-            labels = ['Class A', 'Class B', 'Class C', 'Class D', 'Class E']
-            leap_horizontal_bar = LeapHorizontalBar(body=body_data, labels=labels)
-            leap_horizontal_bar.plot_visualizer()
-        """
-        body_data = self.body
-        labels = self.labels
-
-        fig, ax = plt.subplots()
-
-        fig.patch.set_facecolor('black')
-        ax.set_facecolor('black')
-
-        # Plot horizontal bar chart
-        ax.barh(labels, body_data, color='green')
-
-        # Set the color of the labels and title to white
-        ax.set_xlabel('Scores', color='white')
-        ax.set_title('Leap Horizontal Bar Visualization', color='white')
-
-        # Set the color of the ticks to white
-        ax.tick_params(axis='x', colors='white')
-        ax.tick_params(axis='y', colors='white')
-
-        plt.show()
 
 @dataclass
 class LeapImageMask:
@@ -379,51 +193,6 @@ class LeapImageMask:
         for label in self.labels:
             validate_type(type(label), str)
 
-    def plot_visualizer(self) -> None:
-        """
-        Plots an image with overlaid masks given a LeapImageMask visualizer object.
-
-        Returns:
-            None
-
-
-        Example:
-            image_data = np.random.rand(100, 100, 3).astype(np.float32)
-            mask_data = np.random.randint(0, 2, (100, 100)).astype(np.uint8)
-            labels = ["background", "object"]
-            leap_image_mask = LeapImageMask(image=image_data, mask=mask_data, labels=labels)
-            leap_image_mask.plot_visualizer()
-        """
-
-        image = self.image
-        mask = self.mask
-        labels = self.labels
-
-        # Create a color map for each label
-        colors = plt.cm.jet(np.linspace(0, 1, len(labels)))
-
-        # Make a copy of the image to draw on
-        overlayed_image = image.copy()
-
-        # Iterate through the unique values in the mask (excluding 0)
-        for i, label in enumerate(labels):
-            # Extract binary mask for the current instance
-            instance_mask = (mask == (i + 1))
-
-            # fill the instance mask with a translucent color
-            overlayed_image[instance_mask] = (
-                    overlayed_image[instance_mask] * (1 - 0.5) + np.array(colors[i][:3], dtype=np.uint8) * 0.5)
-
-        # Display the result using matplotlib
-        fig, ax = plt.subplots(1)
-        fig.patch.set_facecolor('black')  # Set the figure background to black
-        ax.set_facecolor('black')  # Set the axis background to black
-
-        ax.imshow(overlayed_image)
-        ax.set_title('Leap Image With Mask Visualization', color='white')
-        plt.axis('off')  # Hide the axis
-        plt.show()
-
 
 @dataclass
 class LeapTextMask:
@@ -461,55 +230,6 @@ class LeapTextMask:
         for label in self.labels:
             validate_type(type(label), str)
 
-    def plot_visualizer(self) -> None:
-        """
-        Plots text with overlaid masks given a LeapTextMask visualizer object.
-
-        Returns:
-            None
-
-        Example:
-            text_data = ['I', 'ate', 'a', 'banana', '', '', '']
-            mask_data = np.array([0, 0, 0, 1, 0, 0, 0]).astype(np.uint8)
-            labels = ["object"]
-            leap_text_mask = LeapTextMask(text=text_data, mask=mask_data, labels=labels)
-        """
-
-        text_data = self.text
-        mask_data = self.mask
-        labels = self.labels
-
-        # Create a color map for each label
-        colors = plt.cm.jet(np.linspace(0, 1, len(labels)))
-
-        # Create a figure and axis
-        fig, ax = plt.subplots()
-
-        # Set background to black
-        fig.patch.set_facecolor('black')
-        ax.set_facecolor('black')
-        ax.set_title('Leap Text Mask Visualization', color='white')
-        ax.axis('off')
-
-        # Set initial position
-        x_pos, y_pos = 0.01, 0.5  # Adjusted initial position for better visibility
-
-        # Display the text with colors
-        for token, mask_value in zip(text_data, mask_data):
-            if mask_value > 0:
-                color = colors[mask_value % len(colors)]
-                bbox = dict(facecolor=color, edgecolor='none',
-                            boxstyle='round,pad=0.3')  # Background color for masked tokens
-            else:
-                bbox = None
-
-            ax.text(x_pos, y_pos, token, fontsize=12, color='white', ha='left', va='center', bbox=bbox)
-
-            # Update the x position for the next token
-            x_pos += len(token) * 0.03 + 0.02  # Adjust the spacing between tokens
-
-        plt.show()
-
 
 @dataclass
 class LeapImageWithHeatmap:
@@ -547,46 +267,6 @@ class LeapImageWithHeatmap:
             raise LeapValidationError(
                 'Number of heatmaps and labels must be equal')
 
-    def plot_visualizer(self) -> None:
-        """
-        Display the image with overlaid heatmaps contained in the LeapImageWithHeatmap object.
-
-        Returns:
-            None
-
-        Example:
-            image_data = np.random.rand(100, 100, 3).astype(np.float32)
-            heatmaps = np.random.rand(3, 100, 100).astype(np.float32)
-            labels = ["heatmap1", "heatmap2", "heatmap3"]
-            leap_image_with_heatmap = LeapImageWithHeatmap(image=image_data, heatmaps=heatmaps, labels=labels)
-            leap_image_with_heatmap.plot_visualizer()
-        """
-        image = self.image
-        heatmaps = self.heatmaps
-        labels = self.labels
-
-        # Plot the base image
-        fig, ax = plt.subplots()
-        fig.patch.set_facecolor('black')  # Set the figure background to black
-        ax.set_facecolor('black')  # Set the axis background to black
-        ax.imshow(image, cmap='gray')
-
-        # Overlay each heatmap with some transparency
-        for i in range(len(labels)):
-            heatmap = heatmaps[i]
-            ax.imshow(heatmap, cmap='jet', alpha=0.5)  # Adjust alpha for transparency
-            ax.set_title(f'Heatmap: {labels[i]}', color='white')
-
-            # Display a colorbar for the heatmap
-            cbar = plt.colorbar(ax.imshow(heatmap, cmap='jet', alpha=0.5))
-            cbar.set_label(labels[i], color='white')
-            cbar.ax.yaxis.set_tick_params(color='white')  # Set color for the colorbar ticks
-            plt.setp(plt.getp(cbar.ax.axes, 'yticklabels'), color='white')  # Set color for the colorbar labels
-
-        plt.axis('off')
-        plt.title('Leap Image With Heatmaps Visualization', color='white')
-        plt.show()
-
 
 map_leap_data_type_to_visualizer_class = {
     LeapDataType.Image.value: LeapImage,

{code_loader-1.0.49.dev9 → code_loader-1.0.50}/code_loader/inner_leap_binder/leapbinder.py

@@ -8,7 +8,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     GroundTruthHandler, MetadataHandler, DatasetIntegrationSetup, VisualizerHandler, PreprocessResponse, \
     PreprocessHandler, VisualizerCallableInterface, CustomLossHandler, CustomCallableInterface, PredictionTypeHandler, \
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
-    CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, VisualizerCallableReturnType, \
+    CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, RawInputsForHeatmap
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
@@ -103,7 +103,7 @@ class LeapBinder:
         if visualizer_type.value not in map_leap_data_type_to_visualizer_class:
             raise Exception(
                 f'The visualizer_type is invalid. current visualizer_type: {visualizer_type}, '  # type: ignore[attr-defined]
-                f'should be one of : {", ".join([arg.__name__ for arg in VisualizerCallableReturnType.__args__])}')
+                f'should be one of : {", ".join([arg.__name__ for arg in LeapData.__args__])}')
 
         func_annotations = function.__annotations__
         if "return" not in func_annotations:
@@ -113,10 +113,10 @@ class LeapBinder:
                   f"https://docs.python.org/3/library/typing.html")
         else:
             return_type = func_annotations["return"]
-            if return_type not in VisualizerCallableReturnType.__args__:  # type: ignore[attr-defined]
+            if return_type not in LeapData.__args__:  # type: ignore[attr-defined]
                 raise Exception(
                     f'The return type of function {function.__name__} is invalid. current return type: {return_type}, '  # type: ignore[attr-defined]
-                    f'should be one of : {", ".join([arg.__name__ for arg in VisualizerCallableReturnType.__args__])}')
+                    f'should be one of : {", ".join([arg.__name__ for arg in LeapData.__args__])}')
 
             expected_return_type = map_leap_data_type_to_visualizer_class[visualizer_type.value]
             if not issubclass(return_type, expected_return_type):
@@ -389,35 +389,16 @@ class LeapBinder:
         if preprocess is None:
             raise Exception("Please make sure you call the leap_binder.set_preprocess method")
         preprocess_results = preprocess.function()
-        preprocess_result_dict = {}
-        for i, preprocess_result in enumerate(preprocess_results):
-            if preprocess_result.state is None:
-                state_enum = DataStateEnum(i)
-                preprocess_result.state = DataStateType(state_enum.name)
-            else:
-                state_enum = DataStateEnum[preprocess_result.state.name]
-
-            if state_enum in preprocess_result_dict:
-                raise Exception(f"Duplicate state {state_enum.name} in preprocess results")
-            preprocess_result_dict[state_enum] = preprocess_result
-
-        if DataStateEnum.unlabeled not in preprocess_result_dict:
-            preprocess_unlabeled_result = self.get_preprocess_unlabeled_result()
-            if preprocess_unlabeled_result is not None:
-                preprocess_result_dict[DataStateEnum.unlabeled] = preprocess_unlabeled_result
-
-        if DataStateEnum.training not in preprocess_result_dict:
-            raise Exception("Training data is required")
-        if DataStateEnum.validation not in preprocess_result_dict:
-            raise Exception("Validation data is required")
+        preprocess_result_dict = {
+            DataStateEnum(i): preprocess_result
+            for i, preprocess_result in enumerate(preprocess_results)
+        }
 
-        return preprocess_result_dict
-
-    def get_preprocess_unlabeled_result(self) -> Optional[PreprocessResponse]:
         unlabeled_preprocess = self.setup_container.unlabeled_data_preprocess
         if unlabeled_preprocess is not None:
-            return unlabeled_preprocess.function()
-        return None
+            preprocess_result_dict[DataStateEnum.unlabeled] = unlabeled_preprocess.function()
+
+        return preprocess_result_dict
 
     def _get_all_dataset_base_handlers(self) -> List[Union[DatasetBaseHandler, MetadataHandler]]:
         all_dataset_base_handlers: List[Union[DatasetBaseHandler, MetadataHandler]] = []
@@ -430,7 +411,7 @@ class LeapBinder:
     def check_handler(
             preprocess_response: PreprocessResponse, test_result: List[DatasetTestResultPayload],
             dataset_base_handler: Union[DatasetBaseHandler, MetadataHandler]) -> List[DatasetTestResultPayload]:
-        raw_result = dataset_base_handler.function(preprocess_response.sample_ids[0], preprocess_response)
+        raw_result = dataset_base_handler.function(0, preprocess_response)
         handler_type = 'metadata' if isinstance(dataset_base_handler, MetadataHandler) else None
         if isinstance(dataset_base_handler, MetadataHandler) and isinstance(raw_result, dict):
             metadata_test_result_payloads = [

{code_loader-1.0.49.dev9 → code_loader-1.0.50}/code_loader/leaploader.py

@@ -2,17 +2,16 @@
 import importlib.util
 import io
 import sys
-import time
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, Iterable, Union, Any, Type
+from typing import Dict, List, Iterable, Union, Any
 
 import numpy as np
 import numpy.typing as npt
 
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
-    PreprocessResponse, VisualizerHandler, VisualizerCallableReturnType, CustomLossHandler, \
+    PreprocessResponse, VisualizerHandler, LeapData, CustomLossHandler, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
@@ -28,8 +27,6 @@ class LeapLoader:
         self.code_entry_name = code_entry_name
         self.code_path = code_path
 
-        self._preprocess_result_cached = None
-
     @lru_cache()
     def exec_script(self) -> None:
         try:
@@ -106,16 +103,12 @@ class LeapLoader:
             for prediction_type in setup.prediction_types
         }
 
-    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
+    def get_sample(self, state: DataStateEnum, idx: int) -> DatasetSample:
         self.exec_script()
-        preprocess_result = self._preprocess_result()
-        if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
-            self._preprocess_result(update_unlabeled_preprocess=True)
-
-        sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
-                               gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
-                               metadata=self._get_metadata(state, sample_id),
-                               index=sample_id,
+        sample = DatasetSample(inputs=self._get_inputs(state, idx),
+                               gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, idx),
+                               metadata=self._get_metadata(state, idx),
+                               index=idx,
                                state=state)
         return sample
 
@@ -155,13 +148,6 @@ class LeapLoader:
         test_result = DatasetTestResultPayload('preprocess')
         try:
             preprocess_result = self._preprocess_result()
-            if self.get_sample_id_type() is str:
-                max_allowed_item_size = np.dtype('<U256').itemsize
-                for state, preprocess_response in preprocess_result.items():
-                    sample_ids_array = np.array(preprocess_response.sample_ids)
-                    if sample_ids_array.dtype.itemsize > max_allowed_item_size:
-                        raise Exception(f"Sample id are too long. Max allowed length is 256 charecters.")
-
             global_leap_binder.check_preprocess(preprocess_result)
         except Exception as e:
             line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
@@ -201,7 +187,7 @@ class LeapLoader:
         return all_dataset_base_handlers
 
     def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]],
-                       ) -> VisualizerCallableReturnType:
+                       ) -> LeapData:
         return self.visualizer_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
 
     def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
@@ -293,42 +279,27 @@ class LeapLoader:
         ]
         return ModelSetup(custom_layer_instances)
 
-    def _preprocess_result(self, update_unlabeled_preprocess=False) -> Dict[DataStateEnum, PreprocessResponse]:
+    @lru_cache()
+    def _preprocess_result(self) -> Dict[DataStateEnum, PreprocessResponse]:
         self.exec_script()
-
-        if self._preprocess_result_cached is None:
-            self._preprocess_result_cached = global_leap_binder.get_preprocess_result()
-
-        if update_unlabeled_preprocess:
-            self._preprocess_result_cached[
-                DataStateEnum.unlabeled] = global_leap_binder.get_preprocess_unlabeled_result()
-
-        return self._preprocess_result_cached
-
-    def get_preprocess_sample_ids(self, update_unlabeled_preprocess=False) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
-        preprocess_result = self._preprocess_result(update_unlabeled_preprocess)
-        sample_ids = {}
-        for state, preprocess_response in preprocess_result.items():
-            sample_ids[state] = preprocess_response.sample_ids
-
-        return sample_ids
+        return global_leap_binder.get_preprocess_result()
 
     def _get_dataset_handlers(self, handlers: Iterable[DatasetBaseHandler],
-                              state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
+                              state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:
         result_agg = {}
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
         for handler in handlers:
-            handler_result = handler.function(sample_id, preprocess_state)
+            handler_result = handler.function(idx, preprocess_state)
             handler_name = handler.name
             result_agg[handler_name] = handler_result
         return result_agg
 
-    def _get_inputs(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
-        return self._get_dataset_handlers(global_leap_binder.setup_container.inputs, state, sample_id)
+    def _get_inputs(self, state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:
+        return self._get_dataset_handlers(global_leap_binder.setup_container.inputs, state, idx)
 
-    def _get_gt(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
-        return self._get_dataset_handlers(global_leap_binder.setup_container.ground_truths, state, sample_id)
+    def _get_gt(self, state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:
+        return self._get_dataset_handlers(global_leap_binder.setup_container.ground_truths, state, idx)
 
     @lru_cache()
     def _metadata_name_to_type(self) -> Dict[str, DatasetMetadataType]:
@@ -363,12 +334,12 @@ class LeapLoader:
 
         return converted_value
 
-    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, Union[str, int, bool, float]]:
+    def _get_metadata(self, state: DataStateEnum, idx: int) -> Dict[str, Union[str, int, bool, float]]:
         result_agg = {}
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
         for handler in global_leap_binder.setup_container.metadata:
-            handler_result = handler.function(sample_id, preprocess_state)
+            handler_result = handler.function(idx, preprocess_state)
             if isinstance(handler_result, dict):
                 for single_metadata_name, single_metadata_result in handler_result.items():
                     handler_name = f'{handler.name}_{single_metadata_name}'
@@ -378,14 +349,3 @@ class LeapLoader:
                 result_agg[handler_name] = self._convert_metadata_to_correct_type(handler_name, handler_result)
 
         return result_agg
-
-    @lru_cache()
-    def get_sample_id_type(self) -> Type:
-        preprocess_results = list(self._preprocess_result().values())
-        id_type = preprocess_results[0].sample_id_type
-        for preprocess_result in preprocess_results:
-            if preprocess_result.sample_id_type != id_type:
-                raise Exception("Different id types in preprocess results")
-
-        return id_type
-

{code_loader-1.0.49.dev9 → code_loader-1.0.50}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.49.dev9"
+version = "1.0.50"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"
@@ -15,7 +15,6 @@ include = [
 python = ">=3.8,<3.12"
 numpy = "^1.22.3"
 psutil = "^5.9.5"
-matplotlib = ">=3.3,<3.4"
 requests = "^2.32.3"
 pyyaml = "^6.0.2"
 

code_loader-1.0.49.dev9/code_loader/code_inegration_processes_manager.py

@@ -1,83 +0,0 @@
-# mypy: ignore-errors
-import traceback
-from dataclasses import dataclass
-
-from typing import List, Tuple, Optional
-
-from multiprocessing import Process, Queue
-
-from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
-from code_loader.leaploader import LeapLoader
-from code_loader.contract.enums import DataStateEnum
-from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
-from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
-
-
-@dataclass
-class SampleSerializableError:
-    state: DataStateEnum
-    index: int
-    leap_script_trace: str
-    exception_as_str: str
-
-
-class CodeIntegrationProcessesManager:
-    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
-                 max_samples_in_queue: int = 128) -> None:
-        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
-        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
-
-    def _create_and_start_process(self) -> Process:
-        process = self.multiprocessing_context.Process(
-            target=CodeIntegrationProcessesManager._process_func,
-            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
-                  self._ready_processed_results))
-        process.daemon = True
-        process.start()
-        return process
-
-    def _run_and_warm_first_process(self):
-        process = self._create_and_start_process()
-        self.processes = [process]
-
-        # needed in order to make sure the preprocess func runs once in nonparallel
-        self._start_process_inputs([(DataStateEnum.training, 0)])
-        self._get_next_ready_processed_result()
-
-    def _operation_decider(self):
-        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
-            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
-            return 'metric'
-
-        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
-            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
-            return 'dataset'
-
-
-
-
-    @staticmethod
-    def _process_func(code_path: str, code_entry_name: str,
-                      samples_to_process: Queue, ready_samples: Queue,
-                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
-        import os
-        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
-
-        leap_loader = LeapLoader(code_path, code_entry_name)
-        while True:
-
-            # decide on sample or metric to process
-            state, idx = samples_to_process.get(block=True)
-            leap_loader._preprocess_result()
-            try:
-                sample = leap_loader.get_sample(state, idx)
-            except Exception as e:
-                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
-                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
-                continue
-
-            ready_samples.put(sample)
-
-    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
-        return self.start_process_inputs(sample_identities)
-