code-loader 1.0.48__py3-none-any.whl → 1.0.49.dev1__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -32,8 +32,18 @@ class PreprocessResponse:
         }
         response = PreprocessResponse(length=len(preprocessed_data), data=preprocessed_data)
     """
-    length: int
-    data: Any
+    length: Optional[int] = None  # Deprecated. Please use sample_ids instead
+    data: Any = None
+    sample_ids: Optional[Union[List[str], List[int]]] = None
+    state: Optional[DataStateType] = None
+
+    def __post_init__(self):
+        if self.length is not None and self.sample_ids is None:
+            self.sample_ids = [i for i in range(self.length)]
+        elif self.length is None and self.sample_ids is not None:
+            self.length = len(self.sample_ids)
+        else:
+            raise Exception("length is deprecated. Please use sample_ids instead.")
 
 
 SectionCallableInterface = Callable[[int, PreprocessResponse], npt.NDArray[np.float32]]

code_loader/inner_leap_binder/leapbinder.py

@@ -389,17 +389,36 @@ class LeapBinder:
         if preprocess is None:
             raise Exception("Please make sure you call the leap_binder.set_preprocess method")
         preprocess_results = preprocess.function()
-        preprocess_result_dict = {
-            DataStateEnum(i): preprocess_result
-            for i, preprocess_result in enumerate(preprocess_results)
-        }
+        preprocess_result_dict = {}
+        for i, preprocess_result in enumerate(preprocess_results):
+            if preprocess_result.state is None:
+                state_enum = DataStateEnum(i)
+                preprocess_result.state = DataStateType(state_enum.name)
+            else:
+                state_enum = DataStateEnum[preprocess_result.state.name]
 
-        unlabeled_preprocess = self.setup_container.unlabeled_data_preprocess
-        if unlabeled_preprocess is not None:
-            preprocess_result_dict[DataStateEnum.unlabeled] = unlabeled_preprocess.function()
+            if state_enum in preprocess_result_dict:
+                raise Exception(f"Duplicate state {state_enum.name} in preprocess results")
+            preprocess_result_dict[state_enum] = preprocess_result
+
+        if DataStateEnum.unlabeled not in preprocess_result_dict:
+            preprocess_unlabeled_result = self.get_preprocess_unlabeled_result()
+            if preprocess_unlabeled_result is not None:
+                preprocess_result_dict[DataStateEnum.unlabeled] = preprocess_unlabeled_result
+
+        if DataStateEnum.training not in preprocess_result_dict:
+            raise Exception("Training data is required")
+        if DataStateEnum.validation not in preprocess_result_dict:
+            raise Exception("Validation data is required")
 
         return preprocess_result_dict
 
+    def get_preprocess_unlabeled_result(self) -> Optional[PreprocessResponse]:
+        unlabeled_preprocess = self.setup_container.unlabeled_data_preprocess
+        if unlabeled_preprocess is not None:
+            return unlabeled_preprocess.function()
+        return None
+
     def _get_all_dataset_base_handlers(self) -> List[Union[DatasetBaseHandler, MetadataHandler]]:
         all_dataset_base_handlers: List[Union[DatasetBaseHandler, MetadataHandler]] = []
         all_dataset_base_handlers.extend(self.setup_container.inputs)

code_loader/leaploader.py

@@ -2,6 +2,7 @@
 import importlib.util
 import io
 import sys
+import time
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
@@ -23,10 +24,14 @@ from code_loader.utils import get_root_exception_file_and_line_number
 
 
 class LeapLoader:
-    def __init__(self, code_path: str, code_entry_name: str):
+    def __init__(self, code_path: str, code_entry_name: str, streaming_mode: bool = False):
         self.code_entry_name = code_entry_name
         self.code_path = code_path
 
+        self.streaming_mode = streaming_mode
+        self._preprocess_result_cached = None
+        self._last_updated_unlabeled_preprocess = None
+
     @lru_cache()
     def exec_script(self) -> None:
         try:
@@ -279,10 +284,34 @@ class LeapLoader:
         ]
         return ModelSetup(custom_layer_instances)
 
-    @lru_cache()
     def _preprocess_result(self) -> Dict[DataStateEnum, PreprocessResponse]:
         self.exec_script()
-        return global_leap_binder.get_preprocess_result()
+
+        if self._preprocess_result_cached is None:
+            self._preprocess_result_cached = global_leap_binder.get_preprocess_result()
+            self._last_updated_unlabeled_preprocess = time.time()
+
+        if self.streaming_mode and global_leap_binder.setup_container.unlabeled_data_preprocess is None:
+            raise Exception("unlabeled_data_preprocess is not defined in the dataset script and it "
+                            "most be defined in production monitoring.")
+
+        update_unlabeled_preprocess_interval = 120
+        if (self.streaming_mode and
+                time.time() - self._last_updated_unlabeled__preprocess > update_unlabeled_preprocess_interval):
+            self._preprocess_result_cached[
+                DataStateEnum.unlabeled] = global_leap_binder.get_preprocess_unlabeled_result()
+            self._last_updated_unlabeled_preprocess = time.time()
+
+        return self._preprocess_result_cached
+
+    def get_preprocess_sample_ids(self) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
+        preprocess_result = self._preprocess_result()
+        sample_ids = {}
+        for state, preprocess_response in preprocess_result.items():
+            sample_ids[state] = preprocess_response.sample_ids
+
+        return sample_ids
+
 
     def _get_dataset_handlers(self, handlers: Iterable[DatasetBaseHandler],
                               state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:

{code_loader-1.0.48.dist-info → code_loader-1.0.49.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.48
+Version: 1.0.49.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.48.dist-info → code_loader-1.0.49.dev1.dist-info}/RECORD

@@ -1,7 +1,7 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=81TmCcVol7768lzKUp70MatLLipR3ftcR9jgE1r8Yqo,5698
+code_loader/contract/datasetclasses.py,sha256=ooGKDBix0ZEmTqjgpYLedbu09NhiHOdhr3oJb-KRZG4,6237
 code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/responsedataclasses.py,sha256=w7xVOv2S8Hyb5lqyomMGiKAWXDTSOG-FX1YW39bXD3A,3969
@@ -17,12 +17,12 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=CDiz8AWgNW-TgSPspllUUxy2Q-9V2JcavPJ0CZOAUnU,23888
-code_loader/leaploader.py,sha256=E1DHtRDzz0hyb3kAMl6S_JDbjGxJPR_bZHU77Af4rR4,17521
+code_loader/inner_leap_binder/leapbinder.py,sha256=m-9eNKYqK71irfEd4T1bNyM9t8mh_utdUDPYvbK34UY,24902
+code_loader/leaploader.py,sha256=tE1XVLc2odt30KSUwDSrn86Dq53F5bQgMWv-p8w0Kd0,18906
 code_loader/utils.py,sha256=TZAoUbA2pE8eK3Le3s5Xr4eRaYdeDMQtxotx6rh-5oE,2185
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=VoqO9FN84yXyMjRjHjUTOt2GdTkJRMbHbXJ1cJkREkk,2230
-code_loader-1.0.48.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.48.dist-info/METADATA,sha256=s65JNpeKwOhpvCS8TY1RFkfSEUTDxGCfiBBjy-b7sh8,888
-code_loader-1.0.48.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.48.dist-info/RECORD,,
+code_loader-1.0.49.dev1.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.49.dev1.dist-info/METADATA,sha256=l0my0SEFGL5Nzwh0SW0MJhkCaX9Xiz4eqzSrQ1xnC74,893
+code_loader-1.0.49.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.49.dev1.dist-info/RECORD,,