code-loader 1.0.58__tar.gz → 1.0.60.dev1__tar.gz

{code_loader-1.0.58 → code_loader-1.0.60.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.58
+Version: 1.0.60.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.58 → code_loader-1.0.60.dev1}/code_loader/contract/datasetclasses.py

@@ -154,7 +154,7 @@ class DatasetBaseHandler:
 @dataclass
 class InputHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None
-
+    channel_dim: Optional[int] = -1
 
 @dataclass
 class GroundTruthHandler(DatasetBaseHandler):

{code_loader-1.0.58 → code_loader-1.0.60.dev1}/code_loader/contract/responsedataclasses.py

@@ -20,6 +20,7 @@ class DatasetBaseSectionInstance:
 @dataclass
 class DatasetInputInstance(DatasetBaseSectionInstance):
     shape: List[int]
+    channel_dim: Optional[bool] = -1
 
 
 @dataclass

{code_loader-1.0.58 → code_loader-1.0.60.dev1}/code_loader/inner_leap_binder/leapbinder.py

@@ -181,13 +181,14 @@ class LeapBinder:
         """
         self.setup_container.unlabeled_data_preprocess = UnlabeledDataPreprocessHandler(function)
 
-    def set_input(self, function: SectionCallableInterface, name: str) -> None:
+    def set_input(self, function: SectionCallableInterface, name: str, channel_dim: int = -1) -> None:
         """
         Set the input handler function.
 
         Args:
         function (SectionCallableInterface): The input handler function.
         name (str): The name of the input section.
+        channel_dim (int): The dimension of the channels axis
 
         Example:
             def input_encoder(subset: PreprocessResponse, index: int) -> np.ndarray:
@@ -197,10 +198,10 @@ class LeapBinder:
                 img = normalize(img)
                 return img
 
-            leap_binder.set_input(input_encoder, name='input_encoder')
+            leap_binder.set_input(input_encoder, name='input_encoder', channel_dim=-1)
         """
         function = to_numpy_return_wrapper(function)
-        self.setup_container.inputs.append(InputHandler(name, function))
+        self.setup_container.inputs.append(InputHandler(name, function, channel_dim=channel_dim))
 
         self._encoder_names.append(name)
 

{code_loader-1.0.58 → code_loader-1.0.60.dev1}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -245,14 +245,16 @@ def tensorleap_unlabeled_preprocess():
     return decorating_function
 
 
-def tensorleap_input_encoder(name: str):
+def tensorleap_input_encoder(name: str, channel_dim=-1):
     def decorating_function(user_function: SectionCallableInterface):
         for input_handler in leap_binder.setup_container.inputs:
             if input_handler.name == name:
                 raise Exception(f'Input with name {name} already exists. '
                                 f'Please choose another')
+        if channel_dim < 0 and channel_dim != -1:
+            raise Exception(f"Channel dim for input {name} is expected to be either -1 or positive")
 
-        leap_binder.set_input(user_function, name)
+        leap_binder.set_input(user_function, name, channel_dim=channel_dim)
 
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
             assert isinstance(sample_id, (int, str)), \
@@ -273,6 +275,8 @@ def tensorleap_input_encoder(name: str):
             assert result.dtype == np.float32, \
                 (f'tensorleap_input_encoder validation failed: '
                  f'The return type should be a numpy array of type float32. Got {result.dtype}.')
+            assert channel_dim - 1 <= len(result.shape), (f'tensorleap_input_encoder validation failed: '
+                 f'The channel_dim ({channel_dim}) should be <= to the rank of the resulting input rank ({len(result.shape)}).')
 
         def inner(sample_id, preprocess_response):
             _validate_input_args(sample_id, preprocess_response)
@@ -357,9 +361,9 @@ def tensorleap_custom_loss(name: str):
                 (f'tensorleap_custom_loss validation failed: '
                  f'The return type should be a numpy array or a tensorflow tensor. Got {type(result)}.')
 
-        def inner(sample_id, preprocess_response):
-            _validate_input_args(sample_id, preprocess_response)
-            result = user_function(sample_id, preprocess_response)
+        def inner(*args, **kwargs):
+            _validate_input_args(*args, **kwargs)
+            result = user_function(*args, **kwargs)
             _validate_result(result)
             return result
 

{code_loader-1.0.58 → code_loader-1.0.60.dev1}/code_loader/leaploader.py

@@ -19,7 +19,6 @@ from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance
-from code_loader.dualstream import DualStream
 from code_loader.inner_leap_binder import global_leap_binder
 from code_loader.utils import get_root_exception_file_and_line_number
 
@@ -124,9 +123,8 @@ class LeapLoader:
         test_payloads: List[DatasetTestResultPayload] = []
         setup_response = None
         general_error = None
-        stdout_stream = io.StringIO()
-        dual_stream = DualStream(sys.stdout, stdout_stream)
-        with redirect_stdout(dual_stream):
+        stdout_steam = io.StringIO()
+        with redirect_stdout(stdout_steam):
             try:
                 self.exec_script()
                 preprocess_test_payload = self._check_preprocess()
@@ -144,7 +142,7 @@ class LeapLoader:
                 general_error = f"Something went wrong. {repr(e.__cause__)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
                 is_valid = False
 
-        print_log = dual_stream.stream2.getvalue()
+        print_log = stdout_steam.getvalue()
         is_valid_for_model = bool(global_leap_binder.setup_container.custom_layers)
         model_setup = self.get_model_setup_response()
 
@@ -223,8 +221,8 @@ class LeapLoader:
         if global_leap_binder.setup_container.unlabeled_data_preprocess:
             unlabeled_length = global_leap_binder.setup_container.unlabeled_data_preprocess.data_length
         dataset_preprocess = DatasetPreprocess(
-            training_length=setup.preprocess.data_length[DataStateType.training],
-            validation_length=setup.preprocess.data_length[DataStateType.validation],
+            training_length=setup.preprocess.data_length.get(DataStateType.training, 0),
+            validation_length=setup.preprocess.data_length.get(DataStateType.validation, 0),
             test_length=setup.preprocess.data_length.get(DataStateType.test),
             unlabeled_length=unlabeled_length
         )
@@ -233,7 +231,7 @@ class LeapLoader:
         for inp in setup.inputs:
             if inp.shape is None:
                 raise Exception(f"cant calculate shape for input, input name:{inp.name}")
-            inputs.append(DatasetInputInstance(name=inp.name, shape=inp.shape))
+            inputs.append(DatasetInputInstance(name=inp.name, shape=inp.shape, channel_dim=inp.channel_dim))
 
         ground_truths = []
         for gt in setup.ground_truths:

{code_loader-1.0.58 → code_loader-1.0.60.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.58"
+version = "1.0.60.dev1"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"

code_loader-1.0.58/code_loader/dualstream.py

@@ -1,32 +0,0 @@
-from io import StringIO
-from typing import IO
-
-
-class DualStream(StringIO):
-    def __init__(self, stream1: IO[str], stream2: StringIO):
-        super().__init__()
-        self.stream1 = stream1  # Usually sys.stdout
-        self.stream2 = stream2  # The StringIO stream
-
-    def write(self, s: str) -> int:
-        # Write to both streams and return the length of the written string
-        self.stream1.write(s)
-        self.stream2.write(s)
-        return len(s)
-
-    def flush(self) -> None:
-        self.stream1.flush()
-        self.stream2.flush()
-
-    def close(self) -> None:
-        # Do not close sys.stdout
-        self.stream2.close()
-
-    def readable(self) -> bool:
-        return False
-
-    def writable(self) -> bool:
-        return True
-
-    def seekable(self) -> bool:
-        return False