code-loader 1.0.66.dev3__py3-none-any.whl → 1.0.68a0__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -196,6 +196,7 @@ class CustomLayerHandler:
     init_arg_names: List[str]
     call_arg_names: List[str]
     use_custom_latent_space: bool = False
+    build: bool = True
 
 
 @dataclass

code_loader/default_losses.py

@@ -1,7 +1,5 @@
 from enum import Enum
 
-from code_loader.contract.datasetclasses import ConfusionMatrixElement  # type: ignore
-from code_loader.contract.enums import ConfusionMatrixValue, MetricDirection  # type: ignore
 from code_loader.default_metrics import mean_absolute_percentage_error_dimension_reduced, \
     mean_absolute_error_dimension_reduced, mean_squared_logarithmic_error_dimension_reduced, \
     mean_squared_error_dimension_reduced, categorical_crossentropy, binary_crossentropy

code_loader/default_metrics.py

@@ -21,12 +21,18 @@ class Metric(Enum):
 
 def binary_crossentropy(ground_truth: np.array, prediction: np.array) -> np.array:
     ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
-    return -(ground_truth * np.log(prediction) + (1 - ground_truth) * np.log(1 - prediction)).mean(axis=1).astype(np.float32)
+    epsilon = 1e-07
+    prediction = np.clip(prediction, epsilon, 1.0 - epsilon)
+    return -(ground_truth * np.log(prediction) + (1 - ground_truth) *
+             np.log(1 - prediction)).sum(axis=1).astype(np.float32)
 
 
 def categorical_crossentropy(ground_truth: np.array, prediction: np.array) -> np.array:
     ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
-    return -(ground_truth * np.log(prediction)).mean(axis=1).astype(np.float32)
+    prediction = prediction / np.sum(prediction, axis=1)
+    epsilon = 1e-07
+    prediction = np.clip(prediction, epsilon, 1.0 - epsilon)
+    return -(ground_truth * np.log(prediction)).sum(axis=1).astype(np.float32)
 
 def accuracy_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
     ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)

code_loader/inner_leap_binder/leapbinder.py

@@ -33,6 +33,7 @@ class LeapBinder:
     setup_container (DatasetIntegrationSetup): Container to hold setup configurations.
     cache_container (Dict[str, Any]): Cache container to store intermediate data.
     """
+
     def __init__(self) -> None:
         self.setup_container = DatasetIntegrationSetup()
         self.cache_container: Dict[str, Any] = {"word_to_index": {}}
@@ -352,7 +353,8 @@ class LeapBinder:
         self.setup_container.metadata.append(MetadataHandler(name, function))
 
     def set_custom_layer(self, custom_layer: Type[Any], name: str, inspect_layer: bool = False,
-                         kernel_index: Optional[int] = None, use_custom_latent_space: bool = False) -> None:
+                         kernel_index: Optional[int] = None, use_custom_latent_space: bool = False,
+                         build: bool = True) -> None:
         """
         Set a custom layer for the model.
 
@@ -377,12 +379,14 @@ class LeapBinder:
             custom_layer.kernel_index = kernel_index
 
         if use_custom_latent_space and not hasattr(custom_layer, custom_latent_space_attribute):
-            raise Exception(f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
+            raise Exception(
+                f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
 
         init_args = inspect.getfullargspec(custom_layer.__init__)[0][1:]
         call_args = inspect.getfullargspec(custom_layer.call)[0][1:]
         self.setup_container.custom_layers[name] = CustomLayerHandler(name, custom_layer, init_args, call_args,
-                                                                      use_custom_latent_space=use_custom_latent_space)
+                                                                      use_custom_latent_space=use_custom_latent_space,
+                                                                      build=build)
 
     def check_preprocess(self, preprocess_result: Dict[DataStateEnum, PreprocessResponse]) -> None:
         preprocess_handler = self.setup_container.preprocess
@@ -490,7 +494,3 @@ class LeapBinder:
 
     def set_batch_size_to_validate(self, batch_size: int) -> None:
         self.batch_size_to_validate = batch_size
-
-
-
-

{code_loader-1.0.66.dev3.dist-info → code_loader-1.0.68a0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.66.dev3
+Version: 1.0.68a0
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.66.dev3.dist-info → code_loader-1.0.68a0.dist-info}/RECORD

@@ -1,14 +1,13 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
-code_loader/code_inegration_processes_manager.py,sha256=XslWOPeNQk4RAFJ_f3tP5Oe3EgcIR7BE7Y8r9Ty73-o,3261
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=L_fSdSvf-eKoez2uBJ8VjfrKedEP0szNOPvaUvsWeRQ,6973
+code_loader/contract/datasetclasses.py,sha256=3tL8v5bKoWUNiHln5CF95-CrvOP4jai3iIqc5KTKGr0,6996
 code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/responsedataclasses.py,sha256=RSx9m_R3LawhK5o1nAcO3hfp2F9oJYtxZr_bpP3bTmw,4005
 code_loader/contract/visualizer_classes.py,sha256=zD7SVgI1R_DaGby3FPJY2rFphK162P7F2hRTHp2xUuM,12428
-code_loader/default_losses.py,sha256=dyYfYkud0JvNnogsAnin2KGJ9SZ0_uSJI2cJxqH7-xs,1319
-code_loader/default_metrics.py,sha256=_evlQ3m5vaOU-5VeLHzlO38A-00_XKarkcRXFznhjr0,4780
+code_loader/default_losses.py,sha256=NoOQym1106bDN5dcIk56Elr7ZG5quUHArqfP5-Nyxyo,1139
+code_loader/default_metrics.py,sha256=v16Mrt2Ze1tXPgfKywGVdRSrkaK4CKLNQztN1UdVqIY,5010
 code_loader/experiment_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/experiment_api/api.py,sha256=a7wh6Hhe7IaVxu46eV2soSz-yxnmXG3ipU1BBtsEAaQ,2493
 code_loader/experiment_api/cli_config_utils.py,sha256=n6JMyNrquxql3KKxHhAP8jAzezlRT-PV2KWI95kKsm0,1140
@@ -20,14 +19,14 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=o57Pj-iY61-OBuTjK-jYUKCJ0g2pPWbbqitv_e75Bps,25959
+code_loader/inner_leap_binder/leapbinder.py,sha256=uU1jU6rsw8kFd_iUBE_m6SzOirIdiiccqgzLRAS8zEY,26101
 code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=P42U-sqsPKZgsaAzmX8y13F2N3CogeTzU1Iozj4S0GY,20778
 code_loader/leaploader.py,sha256=K__WKfqtKwEch40au177Po10EUX7gm0PJzcV6kpUMlo,22212
 code_loader/leaploaderbase.py,sha256=aHlqWDZRacIdBefeB9goYVnpApaNN2FT24uPIWKkCeQ,3090
 code_loader/utils.py,sha256=aw2i_fqW_ADjLB66FWZd9DfpCQ7mPdMyauROC5Nd51I,2197
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=Ffx5VHVOe5ujBOsjBSxN_aIEVwFSQ6gbhTMG5aUS-po,2305
-code_loader-1.0.66.dev3.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.66.dev3.dist-info/METADATA,sha256=ddrUc-_y21jCdkK0nIhs6qNmguXO02A1sKC4LRW-lEQ,854
-code_loader-1.0.66.dev3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.66.dev3.dist-info/RECORD,,
+code_loader-1.0.68a0.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.68a0.dist-info/METADATA,sha256=AfcuMlL6hB8YgvZ9wFa1dSs2PAIJUKlcjuf0D30T2DQ,851
+code_loader-1.0.68a0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.68a0.dist-info/RECORD,,

code_loader/code_inegration_processes_manager.py

@@ -1,83 +0,0 @@
-# mypy: ignore-errors
-import traceback
-from dataclasses import dataclass
-
-from typing import List, Tuple, Optional
-
-from multiprocessing import Process, Queue
-
-from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
-from code_loader.leaploader import LeapLoader
-from code_loader.contract.enums import DataStateEnum
-from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
-from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
-
-
-@dataclass
-class SampleSerializableError:
-    state: DataStateEnum
-    index: int
-    leap_script_trace: str
-    exception_as_str: str
-
-
-class CodeIntegrationProcessesManager:
-    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
-                 max_samples_in_queue: int = 128) -> None:
-        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
-        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
-
-    def _create_and_start_process(self) -> Process:
-        process = self.multiprocessing_context.Process(
-            target=CodeIntegrationProcessesManager._process_func,
-            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
-                  self._ready_processed_results))
-        process.daemon = True
-        process.start()
-        return process
-
-    def _run_and_warm_first_process(self):
-        process = self._create_and_start_process()
-        self.processes = [process]
-
-        # needed in order to make sure the preprocess func runs once in nonparallel
-        self._start_process_inputs([(DataStateEnum.training, 0)])
-        self._get_next_ready_processed_result()
-
-    def _operation_decider(self):
-        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
-            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
-            return 'metric'
-
-        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
-            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
-            return 'dataset'
-
-
-
-
-    @staticmethod
-    def _process_func(code_path: str, code_entry_name: str,
-                      samples_to_process: Queue, ready_samples: Queue,
-                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
-        import os
-        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
-
-        leap_loader = LeapLoader(code_path, code_entry_name)
-        while True:
-
-            # decide on sample or metric to process
-            state, idx = samples_to_process.get(block=True)
-            leap_loader._preprocess_result()
-            try:
-                sample = leap_loader.get_sample(state, idx)
-            except Exception as e:
-                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
-                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
-                continue
-
-            ready_samples.put(sample)
-
-    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
-        return self.start_process_inputs(sample_identities)
-