code-loader 1.0.47__py3-none-any.whl → 1.0.49.dev1__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -32,8 +32,18 @@ class PreprocessResponse:
         }
         response = PreprocessResponse(length=len(preprocessed_data), data=preprocessed_data)
     """
-    length: int
-    data: Any
+    length: Optional[int] = None  # Deprecated. Please use sample_ids instead
+    data: Any = None
+    sample_ids: Optional[Union[List[str], List[int]]] = None
+    state: Optional[DataStateType] = None
+
+    def __post_init__(self):
+        if self.length is not None and self.sample_ids is None:
+            self.sample_ids = [i for i in range(self.length)]
+        elif self.length is None and self.sample_ids is not None:
+            self.length = len(self.sample_ids)
+        else:
+            raise Exception("length is deprecated. Please use sample_ids instead.")
 
 
 SectionCallableInterface = Callable[[int, PreprocessResponse], npt.NDArray[np.float32]]

code_loader/experiment_api/api.py

@@ -42,7 +42,7 @@ class LogExternalEpochDataRequest:
   experimentId: str
   epoch: int
   metrics: ApiMetrics
-  force: bool = False
+  override: bool = False
 
 @dataclass
 class TagModelRequest:

code_loader/experiment_api/epoch.py

@@ -91,13 +91,14 @@ class Epoch:
             tags=tags
         ))
 
-    def log(self, modelFilePath: Optional[str] = None, tags: Optional[List[str]] = None) -> None:
+    def log(self, modelFilePath: Optional[str] = None, tags: Optional[List[str]] = None, override: bool = False) -> None:
         """
         Logs the epoch with optional model file and tags.
 
         Args:
             modelFilePath (Optional[str]): The path to the model file. Defaults to None.
             tags (Optional[List[str]]): A list of tags to associate with the epoch model. Will always include the default tag. Unless the default tag is set to '', all previous epoch model with the same tag will be removed
+            override (bool): Whether to override the existing epoch model. Defaults to False.
         """
         if tags is None:
             tags = []
@@ -119,7 +120,8 @@ class Epoch:
             experimentId=self.ctx.experiment_id,
             projectId=self.ctx.project_id,
             epoch=self.epoch,
-            metrics=api_metrics
+            metrics=api_metrics,
+            override=override
         ))
         if modelFilePath is not None and len(tags) > 0:
             self._tag_model(tags)

code_loader/experiment_api/experiment.py

@@ -41,7 +41,7 @@ class Experiment:
         """
         return Epoch(self.ctx, epoch, self.default_epoch_tag)
     
-    def log_epoch(self, epoch: int, metrics: Optional[Metrics] = None, model_path: Optional[str] = None, tags: Optional[List[str]] = None) -> None:
+    def log_epoch(self, epoch: int, metrics: Optional[Metrics] = None, model_path: Optional[str] = None, tags: Optional[List[str]] = None, override: bool = False) -> None:
         """
         Logs an epoch with optional metrics, model path, and tags.
 
@@ -50,11 +50,12 @@ class Experiment:
             metrics (Optional[Metrics]): The metrics to log for the epoch. Defaults to None.
             model_path (Optional[str]): The path to the model file. Defaults to None.
             tags (Optional[List[str]]): A list of tags to associate with the epoch model. Will always include the default tag. Unless the default tag is set to '', all previous epoch model with the same tag will be removed.
+            override (bool): Whether to override the epoch if it already exists. Defaults to False.
         """
         epoch_o = self.init_epoch(epoch)
         if metrics is not None:
             epoch_o.set_metrics(metrics)
-        epoch_o.log(model_path, tags)
+        epoch_o.log(model_path, tags, override)
 
     def set_properties(self, properties: Dict[str, Any]) -> None:
         """

code_loader/inner_leap_binder/leapbinder.py

@@ -389,17 +389,36 @@ class LeapBinder:
         if preprocess is None:
             raise Exception("Please make sure you call the leap_binder.set_preprocess method")
         preprocess_results = preprocess.function()
-        preprocess_result_dict = {
-            DataStateEnum(i): preprocess_result
-            for i, preprocess_result in enumerate(preprocess_results)
-        }
+        preprocess_result_dict = {}
+        for i, preprocess_result in enumerate(preprocess_results):
+            if preprocess_result.state is None:
+                state_enum = DataStateEnum(i)
+                preprocess_result.state = DataStateType(state_enum.name)
+            else:
+                state_enum = DataStateEnum[preprocess_result.state.name]
 
-        unlabeled_preprocess = self.setup_container.unlabeled_data_preprocess
-        if unlabeled_preprocess is not None:
-            preprocess_result_dict[DataStateEnum.unlabeled] = unlabeled_preprocess.function()
+            if state_enum in preprocess_result_dict:
+                raise Exception(f"Duplicate state {state_enum.name} in preprocess results")
+            preprocess_result_dict[state_enum] = preprocess_result
+
+        if DataStateEnum.unlabeled not in preprocess_result_dict:
+            preprocess_unlabeled_result = self.get_preprocess_unlabeled_result()
+            if preprocess_unlabeled_result is not None:
+                preprocess_result_dict[DataStateEnum.unlabeled] = preprocess_unlabeled_result
+
+        if DataStateEnum.training not in preprocess_result_dict:
+            raise Exception("Training data is required")
+        if DataStateEnum.validation not in preprocess_result_dict:
+            raise Exception("Validation data is required")
 
         return preprocess_result_dict
 
+    def get_preprocess_unlabeled_result(self) -> Optional[PreprocessResponse]:
+        unlabeled_preprocess = self.setup_container.unlabeled_data_preprocess
+        if unlabeled_preprocess is not None:
+            return unlabeled_preprocess.function()
+        return None
+
     def _get_all_dataset_base_handlers(self) -> List[Union[DatasetBaseHandler, MetadataHandler]]:
         all_dataset_base_handlers: List[Union[DatasetBaseHandler, MetadataHandler]] = []
         all_dataset_base_handlers.extend(self.setup_container.inputs)

code_loader/leaploader.py

@@ -2,6 +2,7 @@
 import importlib.util
 import io
 import sys
+import time
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
@@ -23,10 +24,14 @@ from code_loader.utils import get_root_exception_file_and_line_number
 
 
 class LeapLoader:
-    def __init__(self, code_path: str, code_entry_name: str):
+    def __init__(self, code_path: str, code_entry_name: str, streaming_mode: bool = False):
         self.code_entry_name = code_entry_name
         self.code_path = code_path
 
+        self.streaming_mode = streaming_mode
+        self._preprocess_result_cached = None
+        self._last_updated_unlabeled_preprocess = None
+
     @lru_cache()
     def exec_script(self) -> None:
         try:
@@ -279,10 +284,34 @@ class LeapLoader:
         ]
         return ModelSetup(custom_layer_instances)
 
-    @lru_cache()
     def _preprocess_result(self) -> Dict[DataStateEnum, PreprocessResponse]:
         self.exec_script()
-        return global_leap_binder.get_preprocess_result()
+
+        if self._preprocess_result_cached is None:
+            self._preprocess_result_cached = global_leap_binder.get_preprocess_result()
+            self._last_updated_unlabeled_preprocess = time.time()
+
+        if self.streaming_mode and global_leap_binder.setup_container.unlabeled_data_preprocess is None:
+            raise Exception("unlabeled_data_preprocess is not defined in the dataset script and it "
+                            "most be defined in production monitoring.")
+
+        update_unlabeled_preprocess_interval = 120
+        if (self.streaming_mode and
+                time.time() - self._last_updated_unlabeled__preprocess > update_unlabeled_preprocess_interval):
+            self._preprocess_result_cached[
+                DataStateEnum.unlabeled] = global_leap_binder.get_preprocess_unlabeled_result()
+            self._last_updated_unlabeled_preprocess = time.time()
+
+        return self._preprocess_result_cached
+
+    def get_preprocess_sample_ids(self) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
+        preprocess_result = self._preprocess_result()
+        sample_ids = {}
+        for state, preprocess_response in preprocess_result.items():
+            sample_ids[state] = preprocess_response.sample_ids
+
+        return sample_ids
+
 
     def _get_dataset_handlers(self, handlers: Iterable[DatasetBaseHandler],
                               state: DataStateEnum, idx: int) -> Dict[str, npt.NDArray[np.float32]]:

{code_loader-1.0.47.dist-info → code_loader-1.0.49.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.47
+Version: 1.0.49.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.47.dist-info → code_loader-1.0.49.dev1.dist-info}/RECORD

@@ -1,28 +1,28 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=81TmCcVol7768lzKUp70MatLLipR3ftcR9jgE1r8Yqo,5698
+code_loader/contract/datasetclasses.py,sha256=ooGKDBix0ZEmTqjgpYLedbu09NhiHOdhr3oJb-KRZG4,6237
 code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/responsedataclasses.py,sha256=w7xVOv2S8Hyb5lqyomMGiKAWXDTSOG-FX1YW39bXD3A,3969
 code_loader/contract/visualizer_classes.py,sha256=Ka8fJSVKrOeZ12Eg8-dOBGMW0UswYCIkuhnNMd-7z9s,22948
 code_loader/experiment_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/experiment_api/api.py,sha256=p_III5t9WCpvd1REq9L-pF3r1RhjDlxMCBBK-Sn0B1I,2490
+code_loader/experiment_api/api.py,sha256=a7wh6Hhe7IaVxu46eV2soSz-yxnmXG3ipU1BBtsEAaQ,2493
 code_loader/experiment_api/cli_config_utils.py,sha256=n6JMyNrquxql3KKxHhAP8jAzezlRT-PV2KWI95kKsm0,1140
 code_loader/experiment_api/client.py,sha256=8kqLWh2Zg7mNA5E9Q0v0qRuPm3OE3Rs7lfGAcXOucU4,1428
-code_loader/experiment_api/epoch.py,sha256=q46cxVfpautynKL-ju0YS83vYhpuZuscLA0nff748y0,4700
-code_loader/experiment_api/experiment.py,sha256=oVi6GAJ1e1kOygnOquTvb6SgepMhTlrpH0uVqCwviYM,6349
+code_loader/experiment_api/epoch.py,sha256=mcrdgwbUgG8hcq6Nthp7bNl5KAVZ5727_ICasJp7thQ,4849
+code_loader/experiment_api/experiment.py,sha256=vyE2zyY9JcVKuHcIA8Njr5DWlmbP2cyJHnGAsnaOots,6483
 code_loader/experiment_api/experiment_context.py,sha256=kdzUbuzXo1pMVslOC3TKeJwW8sx_qWkxDVrswjduH0A,194
 code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1oEB_NPg,485
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=CDiz8AWgNW-TgSPspllUUxy2Q-9V2JcavPJ0CZOAUnU,23888
-code_loader/leaploader.py,sha256=E1DHtRDzz0hyb3kAMl6S_JDbjGxJPR_bZHU77Af4rR4,17521
+code_loader/inner_leap_binder/leapbinder.py,sha256=m-9eNKYqK71irfEd4T1bNyM9t8mh_utdUDPYvbK34UY,24902
+code_loader/leaploader.py,sha256=tE1XVLc2odt30KSUwDSrn86Dq53F5bQgMWv-p8w0Kd0,18906
 code_loader/utils.py,sha256=TZAoUbA2pE8eK3Le3s5Xr4eRaYdeDMQtxotx6rh-5oE,2185
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=VoqO9FN84yXyMjRjHjUTOt2GdTkJRMbHbXJ1cJkREkk,2230
-code_loader-1.0.47.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.47.dist-info/METADATA,sha256=bDWLdxS53o7nfne8b-M9OOGy5Rf-AqnKshHY6XtNYTM,888
-code_loader-1.0.47.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.47.dist-info/RECORD,,
+code_loader-1.0.49.dev1.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.49.dev1.dist-info/METADATA,sha256=l0my0SEFGL5Nzwh0SW0MJhkCaX9Xiz4eqzSrQ1xnC74,893
+code_loader-1.0.49.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.49.dev1.dist-info/RECORD,,