code-loader 1.0.48__tar.gz → 1.0.49.dev2__tar.gz

{code_loader-1.0.48 → code_loader-1.0.49.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.48
+Version: 1.0.49.dev2
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

code_loader-1.0.49.dev2/code_loader/code_inegration_processes_manager.py

@@ -0,0 +1,83 @@
+# mypy: ignore-errors
+import traceback
+from dataclasses import dataclass
+
+from typing import List, Tuple, Optional
+
+from multiprocessing import Process, Queue
+
+from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
+from code_loader.leaploader import LeapLoader
+from code_loader.contract.enums import DataStateEnum
+from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
+from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
+
+
+@dataclass
+class SampleSerializableError:
+    state: DataStateEnum
+    index: int
+    leap_script_trace: str
+    exception_as_str: str
+
+
+class CodeIntegrationProcessesManager:
+    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
+                 max_samples_in_queue: int = 128) -> None:
+        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
+        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
+
+    def _create_and_start_process(self) -> Process:
+        process = self.multiprocessing_context.Process(
+            target=CodeIntegrationProcessesManager._process_func,
+            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
+                  self._ready_processed_results))
+        process.daemon = True
+        process.start()
+        return process
+
+    def _run_and_warm_first_process(self):
+        process = self._create_and_start_process()
+        self.processes = [process]
+
+        # needed in order to make sure the preprocess func runs once in nonparallel
+        self._start_process_inputs([(DataStateEnum.training, 0)])
+        self._get_next_ready_processed_result()
+
+    def _operation_decider(self):
+        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
+            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'metric'
+
+        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
+            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'dataset'
+
+
+
+
+    @staticmethod
+    def _process_func(code_path: str, code_entry_name: str,
+                      samples_to_process: Queue, ready_samples: Queue,
+                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+        leap_loader = LeapLoader(code_path, code_entry_name)
+        while True:
+
+            # decide on sample or metric to process
+            state, idx = samples_to_process.get(block=True)
+            leap_loader._preprocess_result()
+            try:
+                sample = leap_loader.get_sample(state, idx)
+            except Exception as e:
+                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
+                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
+                continue
+
+            ready_samples.put(sample)
+
+    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
+        return self.start_process_inputs(sample_identities)
+

{code_loader-1.0.48 → code_loader-1.0.49.dev2}/code_loader/contract/datasetclasses.py

@@ -32,8 +32,18 @@ class PreprocessResponse:
         }
         response = PreprocessResponse(length=len(preprocessed_data), data=preprocessed_data)
     """
-    length: int
-    data: Any
+    length: Optional[int] = None  # Deprecated. Please use sample_ids instead
+    data: Any = None
+    sample_ids: Optional[Union[List[str], List[int]]] = None
+    state: Optional[DataStateType] = None
+
+    def __post_init__(self):
+        if self.length is not None and self.sample_ids is None:
+            self.sample_ids = [i for i in range(self.length)]
+        elif self.length is None and self.sample_ids is not None:
+            self.length = len(self.sample_ids)
+        else:
+            raise Exception("length is deprecated. Please use sample_ids instead.")
 
 
 SectionCallableInterface = Callable[[int, PreprocessResponse], npt.NDArray[np.float32]]

{code_loader-1.0.48 → code_loader-1.0.49.dev2}/code_loader/inner_leap_binder/leapbinder.py

@@ -389,17 +389,36 @@ class LeapBinder:
         if preprocess is None:
             raise Exception("Please make sure you call the leap_binder.set_preprocess method")
         preprocess_results = preprocess.function()
-        preprocess_result_dict = {
-            DataStateEnum(i): preprocess_result
-            for i, preprocess_result in enumerate(preprocess_results)
-        }
+        preprocess_result_dict = {}
+        for i, preprocess_result in enumerate(preprocess_results):
+            if preprocess_result.state is None:
+                state_enum = DataStateEnum(i)
+                preprocess_result.state = DataStateType(state_enum.name)
+            else:
+                state_enum = DataStateEnum[preprocess_result.state.name]
 
-        unlabeled_preprocess = self.setup_container.unlabeled_data_preprocess
-        if unlabeled_preprocess is not None:
-            preprocess_result_dict[DataStateEnum.unlabeled] = unlabeled_preprocess.function()
+            if state_enum in preprocess_result_dict:
+                raise Exception(f"Duplicate state {state_enum.name} in preprocess results")
+            preprocess_result_dict[state_enum] = preprocess_result
+
+        if DataStateEnum.unlabeled not in preprocess_result_dict:
+            preprocess_unlabeled_result = self.get_preprocess_unlabeled_result()
+            if preprocess_unlabeled_result is not None:
+                preprocess_result_dict[DataStateEnum.unlabeled] = preprocess_unlabeled_result
+
+        if DataStateEnum.training not in preprocess_result_dict:
+            raise Exception("Training data is required")
+        if DataStateEnum.validation not in preprocess_result_dict:
+            raise Exception("Validation data is required")
 
         return preprocess_result_dict
 
+    def get_preprocess_unlabeled_result(self) -> Optional[PreprocessResponse]:
+        unlabeled_preprocess = self.setup_container.unlabeled_data_preprocess
+        if unlabeled_preprocess is not None:
+            return unlabeled_preprocess.function()
+        return None
+
     def _get_all_dataset_base_handlers(self) -> List[Union[DatasetBaseHandler, MetadataHandler]]:
         all_dataset_base_handlers: List[Union[DatasetBaseHandler, MetadataHandler]] = []
         all_dataset_base_handlers.extend(self.setup_container.inputs)

{code_loader-1.0.48 → code_loader-1.0.49.dev2}/code_loader/leaploader.py

@@ -2,6 +2,7 @@
 import importlib.util
 import io
 import sys
+import time
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
@@ -27,6 +28,8 @@ class LeapLoader:
         self.code_entry_name = code_entry_name
         self.code_path = code_path
 
+        self._preprocess_result_cached = None
+
     @lru_cache()
     def exec_script(self) -> None:
         try:
@@ -105,6 +108,10 @@ class LeapLoader:
 
     def get_sample(self, state: DataStateEnum, idx: int) -> DatasetSample:
         self.exec_script()
+        preprocess_result = self._preprocess_result()
+        if state == DataStateEnum.unlabeled and idx not in preprocess_result[state].sample_ids:
+            self._preprocess_result(update_unlabeled_preprocess=True)
+
         sample = DatasetSample(inputs=self._get_inputs(state, idx),
                                gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, idx),
                                metadata=self._get_metadata(state, idx),
@@ -279,10 +286,26 @@ class LeapLoader:
         ]
         return ModelSetup(custom_layer_instances)
 
-    @lru_cache()
-    def _preprocess_result(self) -> Dict[DataStateEnum, PreprocessResponse]:
+    def _preprocess_result(self, update_unlabeled_preprocess=False) -> Dict[DataStateEnum, PreprocessResponse]:
         self.exec_script()
-        return global_leap_binder.get_preprocess_result()
+
+        if self._preprocess_result_cached is None:
+            self._preprocess_result_cached = global_leap_binder.get_preprocess_result()
+
+        if update_unlabeled_preprocess:
+            self._preprocess_result_cached[
+                DataStateEnum.unlabeled] = global_leap_binder.get_preprocess_unlabeled_result()
+
+        return self._preprocess_result_cached
+
+    def get_preprocess_sample_ids(self) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
+        preprocess_result = self._preprocess_result()
+        sample_ids = {}
+        for state, preprocess_response in preprocess_result.items():
+            sample_ids[state] = preprocess_response.sample_ids
+
+        return sample_ids
+
 
     def _get_dataset_handlers(self, handlers: Iterable[DatasetBaseHandler],
                               state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:

{code_loader-1.0.48 → code_loader-1.0.49.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.48"
+version = "1.0.49.dev2"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"