code-loader 0.2.89.dev1__tar.gz → 0.2.89.dev3__tar.gz

{code_loader-0.2.89.dev1 → code_loader-0.2.89.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 0.2.89.dev1
+Version: 0.2.89.dev3
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-0.2.89.dev1 → code_loader-0.2.89.dev3}/code_loader/leaploader.py

@@ -159,6 +159,7 @@ class LeapLoader:
                                        print_log=print_log)
 
     def _check_preprocess(self) -> DatasetTestResultPayload:
+        self.exec_script()
         preprocess_handler = global_leap_binder.setup_container.preprocess
         unlabeled_preprocess_handler = global_leap_binder.setup_container.unlabeled_data_preprocess
         test_result = DatasetTestResultPayload('preprocess')

{code_loader-0.2.89.dev1 → code_loader-0.2.89.dev3}/code_loader/visualizer_calculator_parallelized.py

@@ -1,6 +1,9 @@
 # mypy: ignore-errors
 from typing import Optional, List, Tuple, Dict
 from multiprocessing import Process, Queue
+
+import numpy as np
+
 from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
 from dataclasses import dataclass
 import tensorflow as tf
@@ -11,6 +14,7 @@ from code_loader.leaploader import LeapLoader
 class VisualizerSerializableError:
     visualizer_id: str
     visualizer_name: str
+    index_in_batch: int
     exception_as_str: str
 
 
@@ -26,6 +30,12 @@ class VisualizerCalculatorParallelized(LeapLoaderParallelizedBase):
         os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
 
         leap_loader = LeapLoader(code_path, code_entry_name)
+
+        # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
+        # global param that visualizer is using)
+        leap_loader._preprocess_result()
+        leap_loader._preprocess_result.cache_clear()
+
         while True:
             index_in_batch, visualizer_id, visualizer_name, input_arg_name_to_tensor = \
                 visualizers_to_process.get(block=True)
@@ -34,7 +44,8 @@ class VisualizerCalculatorParallelized(LeapLoaderParallelizedBase):
                     visualizer_result = \
                         leap_loader.visualizer_by_name()[visualizer_name].function(**input_arg_name_to_tensor)
             except Exception as e:
-                ready_visualizations.put(VisualizerSerializableError(visualizer_id, visualizer_name, str(e)))
+                ready_visualizations.put(VisualizerSerializableError(
+                    visualizer_id, visualizer_name, index_in_batch, str(e)))
                 continue
 
             ready_visualizations.put((index_in_batch, visualizer_id, visualizer_result))
@@ -48,5 +59,5 @@ class VisualizerCalculatorParallelized(LeapLoaderParallelizedBase):
         process.start()
         return process
 
-    def calculate_visualizers(self, input_arg_name_to_tensor_list: List[Tuple[int, str, str, Dict[str, tf.Tensor]]]):
+    def calculate_visualizers(self, input_arg_name_to_tensor_list: List[Tuple[int, str, str, Dict[str, np.array]]]):
         return self.start_process_inputs(input_arg_name_to_tensor_list)

{code_loader-0.2.89.dev1 → code_loader-0.2.89.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "0.2.89.dev1"
+version = "0.2.89.dev3"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"