code-loader 1.0.28__py3-none-any.whl → 1.0.31__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -8,6 +8,8 @@ from code_loader.contract.enums import DataStateType, DataStateEnum, LeapDataTyp
 from code_loader.contract.visualizer_classes import LeapImage, LeapText, LeapGraph, LeapHorizontalBar, \
     LeapTextMask, LeapImageMask, LeapImageWithBBox
 
+custom_latent_space_attribute = "custom_latent_space"
+
 
 @dataclass
 class PreprocessResponse:
@@ -52,7 +54,7 @@ VisualizerCallableInterface = Union[
 ]
 
 VisualizerCallableReturnType = Union[LeapImage, LeapText, LeapGraph, LeapHorizontalBar,
-                                     LeapImageMask, LeapTextMask, LeapImageWithBBox]
+LeapImageMask, LeapTextMask, LeapImageWithBBox]
 
 CustomCallableInterface = Callable[..., Any]
 

code_loader/inner_leap_binder/leapbinder.py

@@ -9,7 +9,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     PreprocessHandler, VisualizerCallableInterface, CustomLossHandler, CustomCallableInterface, PredictionTypeHandler, \
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, VisualizerCallableReturnType, \
-    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler
+    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
@@ -120,10 +120,13 @@ class LeapBinder:
         self.setup_container.metadata.append(MetadataHandler(name, function))
 
     def set_custom_layer(self, custom_layer: Type[Any], name: str, inspect_layer: bool = False,
-                         kernel_index: Optional[int] = None) -> None:
+                         kernel_index: Optional[int] = None, use_custom_latent_space: bool = False) -> None:
         if inspect_layer and kernel_index is not None:
             custom_layer.kernel_index = kernel_index
 
+        if use_custom_latent_space and not hasattr(custom_layer, custom_latent_space_attribute):
+            raise Exception(f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
+
         init_args = inspect.getfullargspec(custom_layer.__init__)[0][1:]
         call_args = inspect.getfullargspec(custom_layer.call)[0][1:]
         self.setup_container.custom_layers[name] = CustomLayerHandler(name, custom_layer, init_args, call_args)

code_loader/leaploader.py

@@ -128,11 +128,11 @@ class LeapLoader:
                 setup_response = self.get_dataset_setup_response(handlers_test_payloads)
             except DatasetScriptException as e:
                 line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
-                general_error = f"{repr(e.__cause__)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
+                general_error = f"Something went wrong. {repr(e.__cause__)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
                 is_valid = False
             except Exception as e:
                 line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
-                general_error = f"{repr(e.__cause__)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
+                general_error = f"Something went wrong. {repr(e.__cause__)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
                 is_valid = False
 
         print_log = stdout_steam.getvalue()

{code_loader-1.0.28.dist-info → code_loader-1.0.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.28
+Version: 1.0.31
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.28.dist-info → code_loader-1.0.31.dist-info}/RECORD

@@ -1,19 +1,18 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=V3DEXSN6Ie6PlGeSAbzjp9ufRj0XPJLpD7pDLLYxk6M,122
-code_loader/code_inegration_processes_manager.py,sha256=XslWOPeNQk4RAFJ_f3tP5Oe3EgcIR7BE7Y8r9Ty73-o,3261
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=xuPrL3cKZ-tNBMQ5FsSh7oI8U6ITA_QPI57LN2T8aF8,4454
+code_loader/contract/datasetclasses.py,sha256=LbfWSWuqtvvn-hurYnkJVltLZNzjRF76h8HAlZdPwiU,4472
 code_loader/contract/enums.py,sha256=KDIeNl79e8rCp4ybJat7U03j50McrcBTWUxko5SXrug,1481
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/responsedataclasses.py,sha256=0bo4iLbfu6-3tpgzeKakIH2i_1jtR8BiLdupa1TVT0o,2679
 code_loader/contract/visualizer_classes.py,sha256=1FjVO744J_EMuJfHWXGdvSz6vl3Vu7iS3CDfs8MzEEQ,5138
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=Nb3UXrtoYMDNLGbL6JvIIaVMEAQ_B8YpUPXxZ5DThrE,12540
-code_loader/leaploader.py,sha256=KzCHMACx9_YdLG-TJ5LNF6IgvO41tYpmofZ_a5K47hA,17199
+code_loader/inner_leap_binder/leapbinder.py,sha256=MYYOZA2vpQo06p6kdOGqYUS7-QWlIlmGO99p_ITe3JQ,12840
+code_loader/leaploader.py,sha256=S8gpDJCk0K2prvja7GzNWqLlBRx7h2jAd-PoL0RevY0,17243
 code_loader/utils.py,sha256=61I4PgSl-ZBIe4DifLxMNlBELE-HQR2pB9efVYPceIU,2230
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=HqWx2qfTrroGl2n8Fpmr_4X-rk7tE2oGapjO3gzz4WY,2226
-code_loader-1.0.28.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.28.dist-info/METADATA,sha256=2RyxStF5mGoJqPpY3dwB8EWT5vEnepZ5hJb2L9-d3kI,768
-code_loader-1.0.28.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-code_loader-1.0.28.dist-info/RECORD,,
+code_loader-1.0.31.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.31.dist-info/METADATA,sha256=UjP-_7vnZtTuKG3oVVfB-928yBG4vyvDA2T4vPgA_Oo,768
+code_loader-1.0.31.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+code_loader-1.0.31.dist-info/RECORD,,

code_loader/code_inegration_processes_manager.py

@@ -1,83 +0,0 @@
-# mypy: ignore-errors
-import traceback
-from dataclasses import dataclass
-
-from typing import List, Tuple, Optional
-
-from multiprocessing import Process, Queue
-
-from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
-from code_loader.leaploader import LeapLoader
-from code_loader.contract.enums import DataStateEnum
-from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
-from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
-
-
-@dataclass
-class SampleSerializableError:
-    state: DataStateEnum
-    index: int
-    leap_script_trace: str
-    exception_as_str: str
-
-
-class CodeIntegrationProcessesManager:
-    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
-                 max_samples_in_queue: int = 128) -> None:
-        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
-        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
-
-    def _create_and_start_process(self) -> Process:
-        process = self.multiprocessing_context.Process(
-            target=CodeIntegrationProcessesManager._process_func,
-            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
-                  self._ready_processed_results))
-        process.daemon = True
-        process.start()
-        return process
-
-    def _run_and_warm_first_process(self):
-        process = self._create_and_start_process()
-        self.processes = [process]
-
-        # needed in order to make sure the preprocess func runs once in nonparallel
-        self._start_process_inputs([(DataStateEnum.training, 0)])
-        self._get_next_ready_processed_result()
-
-    def _operation_decider(self):
-        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
-            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
-            return 'metric'
-
-        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
-            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
-            return 'dataset'
-
-
-
-
-    @staticmethod
-    def _process_func(code_path: str, code_entry_name: str,
-                      samples_to_process: Queue, ready_samples: Queue,
-                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
-        import os
-        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
-
-        leap_loader = LeapLoader(code_path, code_entry_name)
-        while True:
-
-            # decide on sample or metric to process
-            state, idx = samples_to_process.get(block=True)
-            leap_loader._preprocess_result()
-            try:
-                sample = leap_loader.get_sample(state, idx)
-            except Exception as e:
-                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
-                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
-                continue
-
-            ready_samples.put(sample)
-
-    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
-        return self.start_process_inputs(sample_identities)
-