code-loader 1.0.68a0__tar.gz → 1.0.69__tar.gz

{code_loader-1.0.68a0 → code_loader-1.0.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.68a0
+Version: 1.0.69
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

code_loader-1.0.69/code_loader/code_inegration_processes_manager.py

@@ -0,0 +1,83 @@
+# mypy: ignore-errors
+import traceback
+from dataclasses import dataclass
+
+from typing import List, Tuple, Optional
+
+from multiprocessing import Process, Queue
+
+from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
+from code_loader.leaploader import LeapLoader
+from code_loader.contract.enums import DataStateEnum
+from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
+from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
+
+
+@dataclass
+class SampleSerializableError:
+    state: DataStateEnum
+    index: int
+    leap_script_trace: str
+    exception_as_str: str
+
+
+class CodeIntegrationProcessesManager:
+    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
+                 max_samples_in_queue: int = 128) -> None:
+        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
+        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
+
+    def _create_and_start_process(self) -> Process:
+        process = self.multiprocessing_context.Process(
+            target=CodeIntegrationProcessesManager._process_func,
+            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
+                  self._ready_processed_results))
+        process.daemon = True
+        process.start()
+        return process
+
+    def _run_and_warm_first_process(self):
+        process = self._create_and_start_process()
+        self.processes = [process]
+
+        # needed in order to make sure the preprocess func runs once in nonparallel
+        self._start_process_inputs([(DataStateEnum.training, 0)])
+        self._get_next_ready_processed_result()
+
+    def _operation_decider(self):
+        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
+            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'metric'
+
+        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
+            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'dataset'
+
+
+
+
+    @staticmethod
+    def _process_func(code_path: str, code_entry_name: str,
+                      samples_to_process: Queue, ready_samples: Queue,
+                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+        leap_loader = LeapLoader(code_path, code_entry_name)
+        while True:
+
+            # decide on sample or metric to process
+            state, idx = samples_to_process.get(block=True)
+            leap_loader._preprocess_result()
+            try:
+                sample = leap_loader.get_sample(state, idx)
+            except Exception as e:
+                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
+                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
+                continue
+
+            ready_samples.put(sample)
+
+    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
+        return self.start_process_inputs(sample_identities)
+

{code_loader-1.0.68a0 → code_loader-1.0.69}/code_loader/contract/datasetclasses.py

@@ -196,7 +196,6 @@ class CustomLayerHandler:
     init_arg_names: List[str]
     call_arg_names: List[str]
     use_custom_latent_space: bool = False
-    build: bool = True
 
 
 @dataclass

{code_loader-1.0.68a0 → code_loader-1.0.69}/code_loader/inner_leap_binder/leapbinder.py

@@ -33,7 +33,6 @@ class LeapBinder:
     setup_container (DatasetIntegrationSetup): Container to hold setup configurations.
     cache_container (Dict[str, Any]): Cache container to store intermediate data.
     """
-
     def __init__(self) -> None:
         self.setup_container = DatasetIntegrationSetup()
         self.cache_container: Dict[str, Any] = {"word_to_index": {}}
@@ -353,8 +352,7 @@ class LeapBinder:
         self.setup_container.metadata.append(MetadataHandler(name, function))
 
     def set_custom_layer(self, custom_layer: Type[Any], name: str, inspect_layer: bool = False,
-                         kernel_index: Optional[int] = None, use_custom_latent_space: bool = False,
-                         build: bool = True) -> None:
+                         kernel_index: Optional[int] = None, use_custom_latent_space: bool = False) -> None:
         """
         Set a custom layer for the model.
 
@@ -379,14 +377,12 @@ class LeapBinder:
             custom_layer.kernel_index = kernel_index
 
         if use_custom_latent_space and not hasattr(custom_layer, custom_latent_space_attribute):
-            raise Exception(
-                f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
+            raise Exception(f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
 
         init_args = inspect.getfullargspec(custom_layer.__init__)[0][1:]
         call_args = inspect.getfullargspec(custom_layer.call)[0][1:]
         self.setup_container.custom_layers[name] = CustomLayerHandler(name, custom_layer, init_args, call_args,
-                                                                      use_custom_latent_space=use_custom_latent_space,
-                                                                      build=build)
+                                                                      use_custom_latent_space=use_custom_latent_space)
 
     def check_preprocess(self, preprocess_result: Dict[DataStateEnum, PreprocessResponse]) -> None:
         preprocess_handler = self.setup_container.preprocess
@@ -494,3 +490,7 @@ class LeapBinder:
 
     def set_batch_size_to_validate(self, batch_size: int) -> None:
         self.batch_size_to_validate = batch_size
+
+
+
+

{code_loader-1.0.68a0 → code_loader-1.0.69}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -355,7 +355,7 @@ def tensorleap_custom_loss(name: str):
                         assert isinstance(elem, valid_types), (f'tensorleap_custom_loss validation failed: '
                                                                f'Element #{y} of list should be a numpy array. Got {type(elem)}.')
                 else:
-                    assert isinstance(arg, tf.Tensor), (f'tensorleap_custom_loss validation failed: '
+                    assert isinstance(arg, valid_types), (f'tensorleap_custom_loss validation failed: '
                                                         f'Argument #{i} should be a numpy array. Got {type(arg)}.')
             for _arg_name, arg in kwargs.items():
                 if isinstance(arg, list):

{code_loader-1.0.68a0 → code_loader-1.0.69}/code_loader/leaploader.py

@@ -270,6 +270,8 @@ class LeapLoader(LeapLoaderBase):
         unlabeled_length = None
         if global_leap_binder.setup_container.unlabeled_data_preprocess:
             unlabeled_length = global_leap_binder.setup_container.unlabeled_data_preprocess.data_length
+        else:
+            unlabeled_length = setup.preprocess.data_length.get(DataStateType.unlabeled)
         dataset_preprocess = DatasetPreprocess(
             training_length=setup.preprocess.data_length.get(DataStateType.training, 0),
             validation_length=setup.preprocess.data_length.get(DataStateType.validation, 0),

{code_loader-1.0.68a0 → code_loader-1.0.69}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.68a0"
+version = "1.0.69"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"