code-loader 1.0.24__tar.gz → 1.0.26__tar.gz

{code_loader-1.0.24 → code_loader-1.0.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.24
+Version: 1.0.26
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

code_loader-1.0.26/code_loader/code_inegration_processes_manager.py

@@ -0,0 +1,83 @@
+# mypy: ignore-errors
+import traceback
+from dataclasses import dataclass
+
+from typing import List, Tuple, Optional
+
+from multiprocessing import Process, Queue
+
+from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
+from code_loader.leaploader import LeapLoader
+from code_loader.contract.enums import DataStateEnum
+from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
+from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
+
+
+@dataclass
+class SampleSerializableError:
+    state: DataStateEnum
+    index: int
+    leap_script_trace: str
+    exception_as_str: str
+
+
+class CodeIntegrationProcessesManager:
+    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
+                 max_samples_in_queue: int = 128) -> None:
+        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
+        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
+
+    def _create_and_start_process(self) -> Process:
+        process = self.multiprocessing_context.Process(
+            target=CodeIntegrationProcessesManager._process_func,
+            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
+                  self._ready_processed_results))
+        process.daemon = True
+        process.start()
+        return process
+
+    def _run_and_warm_first_process(self):
+        process = self._create_and_start_process()
+        self.processes = [process]
+
+        # needed in order to make sure the preprocess func runs once in nonparallel
+        self._start_process_inputs([(DataStateEnum.training, 0)])
+        self._get_next_ready_processed_result()
+
+    def _operation_decider(self):
+        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
+            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'metric'
+
+        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
+            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'dataset'
+
+
+
+
+    @staticmethod
+    def _process_func(code_path: str, code_entry_name: str,
+                      samples_to_process: Queue, ready_samples: Queue,
+                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+        leap_loader = LeapLoader(code_path, code_entry_name)
+        while True:
+
+            # decide on sample or metric to process
+            state, idx = samples_to_process.get(block=True)
+            leap_loader._preprocess_result()
+            try:
+                sample = leap_loader.get_sample(state, idx)
+            except Exception as e:
+                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
+                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
+                continue
+
+            ready_samples.put(sample)
+
+    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
+        return self.start_process_inputs(sample_identities)
+

{code_loader-1.0.24 → code_loader-1.0.26}/code_loader/leaploader.py

@@ -5,7 +5,7 @@ import sys
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, Iterable, Union
+from typing import Dict, List, Iterable, Union, Any
 
 import numpy as np
 import numpy.typing as npt
@@ -19,7 +19,7 @@ from code_loader.contract.responsedataclasses import DatasetIntegParseResult, Da
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance
 from code_loader.inner_leap_binder import global_leap_binder
-from code_loader.utils import get_root_exception_line_number
+from code_loader.utils import get_root_exception_file_and_line_number
 
 
 class LeapLoader:
@@ -127,12 +127,12 @@ class LeapLoader:
                 is_valid = all([payload.is_passed for payload in test_payloads])
                 setup_response = self.get_dataset_setup_response(handlers_test_payloads)
             except DatasetScriptException as e:
-                line_number = get_root_exception_line_number()
-                general_error = f"Something went wrong, {repr(e.__cause__)} line number: {line_number}"
+                line_number, file_name = get_root_exception_file_and_line_number()
+                general_error = f"Something went wrong. {repr(e.__cause__)} in file {file_name}, line_number:  {line_number}"
                 is_valid = False
             except Exception as e:
-                line_number = get_root_exception_line_number()
-                general_error = f"Something went wrong, {repr(e)} line number: {line_number}"
+                line_number, file_name = get_root_exception_file_and_line_number()
+                general_error = f"Something went wrong. {repr(e.__cause__)} in file {file_name}, line_number:  {line_number}"
                 is_valid = False
 
         print_log = stdout_steam.getvalue()
@@ -150,8 +150,8 @@ class LeapLoader:
             preprocess_result = self._preprocess_result()
             global_leap_binder.check_preprocess(preprocess_result)
         except Exception as e:
-            line_number = get_root_exception_line_number()
-            error_string = f"{repr(e)} line number: {line_number}"
+            line_number, file_name = get_root_exception_file_and_line_number()
+            error_string = f"{repr(e)} in file {file_name}, line_number:  {line_number}"
             test_result.display[TestingSectionEnum.Errors.name] = error_string
             test_result.is_passed = False
         return test_result
@@ -170,8 +170,8 @@ class LeapLoader:
                     test_result = global_leap_binder.check_handler(
                         preprocess_response, test_result, dataset_base_handler)
                 except Exception as e:
-                    line_number = get_root_exception_line_number()
-                    test_result[0].display[state_name] = f"{repr(e)} line number: {line_number}"
+                    line_number, file_name = get_root_exception_file_and_line_number()
+                    test_result[0].display[state_name] = f"{repr(e)} in file {file_name}, line_number:  {line_number}"
                     test_result[0].is_passed = False
 
             result_payloads.extend(test_result)
@@ -301,7 +301,40 @@ class LeapLoader:
     def _get_gt(self, state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:
         return self._get_dataset_handlers(global_leap_binder.setup_container.ground_truths, state, idx)
 
+    @lru_cache()
+    def _metadata_name_to_type(self) -> Dict[str, DatasetMetadataType]:
+        global_leap_binder.check_preprocess(self._preprocess_result())
+        handlers_test_payloads = self._check_handlers()
+        metadata_setup = self.get_dataset_setup_response(handlers_test_payloads).metadata
+        metadata_name_to_type = {
+            metadata_instance.name: metadata_instance.type
+            for metadata_instance in metadata_setup
+        }
+        return metadata_name_to_type
+
     def _get_metadata(self, state: DataStateEnum, idx: int) -> Dict[str, Union[str, int, bool, float]]:
+        def _convert_metadata_to_correct_type(metadata_name: str, value: Any) -> Any:
+            metadata_name_to_type = self._metadata_name_to_type()
+            metadata_type_to_python_type = {
+                DatasetMetadataType.float: float,
+                DatasetMetadataType.string: str,
+                DatasetMetadataType.boolean: bool,
+                DatasetMetadataType.int: int
+            }
+            metadata_type_to_default_value = {
+                DatasetMetadataType.float: -1,
+                DatasetMetadataType.string: "",
+                DatasetMetadataType.boolean: False,
+                DatasetMetadataType.int: -1
+            }
+
+            try:
+                converted_value = metadata_type_to_python_type[metadata_name_to_type[metadata_name]](value)
+            except ValueError:
+                converted_value = metadata_type_to_default_value[metadata_name_to_type[metadata_name]]
+
+            return converted_value
+
         result_agg = {}
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
@@ -310,9 +343,9 @@ class LeapLoader:
             if isinstance(handler_result, dict):
                 for single_metadata_name, single_metadata_result in handler_result.items():
                     handler_name = f'{handler.name}_{single_metadata_name}'
-                    result_agg[handler_name] = single_metadata_result
+                    result_agg[handler_name] = _convert_metadata_to_correct_type(handler_name, single_metadata_result)
             else:
                 handler_name = handler.name
-                result_agg[handler_name] = handler_result
+                result_agg[handler_name] = _convert_metadata_to_correct_type(handler_name, handler_result)
 
         return result_agg

{code_loader-1.0.24 → code_loader-1.0.26}/code_loader/utils.py

@@ -1,6 +1,7 @@
 import sys
+from pathlib import Path
 from types import TracebackType
-from typing import List, Union
+from typing import List, Union, Tuple
 
 import numpy as np
 import numpy.typing as npt
@@ -24,7 +25,7 @@ def get_root_traceback(exc_tb: TracebackType) -> TracebackType:
     return return_traceback
 
 
-def get_root_exception_line_number() -> int:
+def get_root_exception_file_and_line_number() -> Tuple[int, str]:
     root_exception = sys.exc_info()[1]
     assert root_exception is not None
     if root_exception.__context__ is not None:
@@ -32,10 +33,12 @@ def get_root_exception_line_number() -> int:
     traceback = root_exception.__traceback__
 
     root_exception_line_number = -1
+    root_exception_file_name = ''
     if traceback is not None:
         root_traceback = get_root_traceback(traceback)
         root_exception_line_number = root_traceback.tb_lineno
-    return root_exception_line_number
+        root_exception_file_name = Path(root_traceback.tb_frame.f_code.co_filename).name
+    return root_exception_line_number, root_exception_file_name
 
 
 def get_shape(result: Union[npt.NDArray[np.float32], str, float, int, bool]) -> List[int]:

{code_loader-1.0.24 → code_loader-1.0.26}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.24"
+version = "1.0.26"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"