code-loader 1.0.97.dev0__py3-none-any.whl → 1.0.98__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of code-loader might be problematic. Click here for more details.
- code_loader/contract/datasetclasses.py +24 -0
- code_loader/inner_leap_binder/leapbinder.py +19 -2
- code_loader/inner_leap_binder/leapbinder_decorators.py +91 -1
- code_loader/leaploader.py +54 -8
- code_loader/leaploaderbase.py +9 -1
- code_loader/utils.py +12 -27
- {code_loader-1.0.97.dev0.dist-info → code_loader-1.0.98.dist-info}/METADATA +1 -1
- {code_loader-1.0.97.dev0.dist-info → code_loader-1.0.98.dist-info}/RECORD +10 -10
- {code_loader-1.0.97.dev0.dist-info → code_loader-1.0.98.dist-info}/LICENSE +0 -0
- {code_loader-1.0.97.dev0.dist-info → code_loader-1.0.98.dist-info}/WHEEL +0 -0
|
@@ -38,8 +38,18 @@ class PreprocessResponse:
|
|
|
38
38
|
sample_ids: Optional[Union[List[str], List[int]]] = None
|
|
39
39
|
state: Optional[DataStateType] = None
|
|
40
40
|
sample_id_type: Optional[Union[Type[str], Type[int]]] = None
|
|
41
|
+
sample_ids_to_instance_mappings: Optional[Dict[str, List[str]]] = None # in use only for element instance
|
|
42
|
+
instance_to_sample_ids_mappings: Optional[Dict[str, str]] = None # in use only for element instance
|
|
43
|
+
instance_ids_to_names: Optional[Dict[str, str]] = None # in use only for element instance
|
|
41
44
|
|
|
42
45
|
def __post_init__(self) -> None:
|
|
46
|
+
def is_valid_string(s: str) -> bool:
|
|
47
|
+
return bool(re.match(r'^[A-Za-z0-9_]+$', s))
|
|
48
|
+
|
|
49
|
+
assert self.sample_ids_to_instance_mappings is None, f"Keep sample_ids_to_instance_mappings None when initializing PreprocessResponse"
|
|
50
|
+
assert self.instance_to_sample_ids_mappings is None, f"Keep instance_to_sample_ids_mappings None when initializing PreprocessResponse"
|
|
51
|
+
assert self.instance_ids_to_names is None, f"Keep instance_ids_to_names None when initializing PreprocessResponse"
|
|
52
|
+
|
|
43
53
|
if self.length is not None and self.sample_ids is None:
|
|
44
54
|
self.sample_ids = [i for i in range(self.length)]
|
|
45
55
|
self.sample_id_type = int
|
|
@@ -50,6 +60,8 @@ class PreprocessResponse:
|
|
|
50
60
|
if self.sample_id_type == str:
|
|
51
61
|
for sample_id in self.sample_ids:
|
|
52
62
|
assert isinstance(sample_id, str), f"Sample id should be of type str. Got: {type(sample_id)}"
|
|
63
|
+
if not is_valid_string(sample_id):
|
|
64
|
+
raise Exception(f"Sample id should contain only letters (A-Z, a-z), numbers or '_'. Got: {sample_id}")
|
|
53
65
|
else:
|
|
54
66
|
raise Exception("length is deprecated.")
|
|
55
67
|
|
|
@@ -60,8 +72,13 @@ class PreprocessResponse:
|
|
|
60
72
|
assert self.sample_ids is not None
|
|
61
73
|
return len(self.sample_ids)
|
|
62
74
|
|
|
75
|
+
@dataclass
|
|
76
|
+
class ElementInstance:
|
|
77
|
+
name: str
|
|
78
|
+
mask: npt.NDArray[np.float32]
|
|
63
79
|
|
|
64
80
|
SectionCallableInterface = Callable[[Union[int, str], PreprocessResponse], npt.NDArray[np.float32]]
|
|
81
|
+
InstanceCallableInterface = Callable[[Union[int, str], PreprocessResponse], List[ElementInstance]]
|
|
65
82
|
|
|
66
83
|
MetadataSectionCallableInterface = Union[
|
|
67
84
|
Callable[[Union[int, str], PreprocessResponse], int],
|
|
@@ -183,6 +200,10 @@ class InputHandler(DatasetBaseHandler):
|
|
|
183
200
|
shape: Optional[List[int]] = None
|
|
184
201
|
channel_dim: Optional[int] = -1
|
|
185
202
|
|
|
203
|
+
@dataclass
|
|
204
|
+
class ElementInstanceMasksHandler:
|
|
205
|
+
name: str
|
|
206
|
+
function: InstanceCallableInterface
|
|
186
207
|
|
|
187
208
|
@dataclass
|
|
188
209
|
class GroundTruthHandler(DatasetBaseHandler):
|
|
@@ -218,6 +239,7 @@ class DatasetIntegrationSetup:
|
|
|
218
239
|
unlabeled_data_preprocess: Optional[UnlabeledDataPreprocessHandler] = None
|
|
219
240
|
visualizers: List[VisualizerHandler] = field(default_factory=list)
|
|
220
241
|
inputs: List[InputHandler] = field(default_factory=list)
|
|
242
|
+
instance_masks: List[ElementInstanceMasksHandler] = field(default_factory=list)
|
|
221
243
|
ground_truths: List[GroundTruthHandler] = field(default_factory=list)
|
|
222
244
|
metadata: List[MetadataHandler] = field(default_factory=list)
|
|
223
245
|
prediction_types: List[PredictionTypeHandler] = field(default_factory=list)
|
|
@@ -234,3 +256,5 @@ class DatasetSample:
|
|
|
234
256
|
metadata_is_none: Dict[str, bool]
|
|
235
257
|
index: Union[int, str]
|
|
236
258
|
state: DataStateEnum
|
|
259
|
+
instance_masks: Optional[Dict[str, List[ElementInstance]]] = None
|
|
260
|
+
|
|
@@ -10,14 +10,15 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
|
|
|
10
10
|
MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
|
|
11
11
|
CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
|
|
12
12
|
CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
|
|
13
|
-
RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse
|
|
13
|
+
RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
|
|
14
|
+
ElementInstanceMasksHandler, InstanceCallableInterface
|
|
14
15
|
from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
|
|
15
16
|
from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
|
|
16
17
|
from code_loader.contract.responsedataclasses import DatasetTestResultPayload
|
|
17
18
|
from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
|
|
18
19
|
from code_loader.default_losses import loss_name_to_function
|
|
19
20
|
from code_loader.default_metrics import metrics_names_to_functions_and_direction
|
|
20
|
-
from code_loader.utils import to_numpy_return_wrapper, get_shape
|
|
21
|
+
from code_loader.utils import to_numpy_return_wrapper, get_shape, to_numpy_return_masks_wrapper
|
|
21
22
|
from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
|
|
22
23
|
default_graph_visualizer, \
|
|
23
24
|
default_image_visualizer, default_horizontal_bar_visualizer, default_word_visualizer, \
|
|
@@ -234,6 +235,22 @@ class LeapBinder:
|
|
|
234
235
|
|
|
235
236
|
self._encoder_names.append(name)
|
|
236
237
|
|
|
238
|
+
|
|
239
|
+
def set_instance_masks(self, function: InstanceCallableInterface, name: str) -> None:
|
|
240
|
+
"""
|
|
241
|
+
Set the instance mask handler function.
|
|
242
|
+
|
|
243
|
+
Args:
|
|
244
|
+
function (SectionCallableInterface): The input handler function.
|
|
245
|
+
name (str): The name of the instance mask section.
|
|
246
|
+
|
|
247
|
+
Example:
|
|
248
|
+
leap_binder.set_input(input_encoder, name='input_encoder')
|
|
249
|
+
"""
|
|
250
|
+
function = to_numpy_return_masks_wrapper(function)
|
|
251
|
+
self.setup_container.instance_masks.append(ElementInstanceMasksHandler(name, function))
|
|
252
|
+
|
|
253
|
+
|
|
237
254
|
def add_custom_loss(self, function: CustomCallableInterface, name: str) -> None:
|
|
238
255
|
"""
|
|
239
256
|
Add a custom loss function to the setup.
|
|
@@ -8,7 +8,7 @@ import numpy.typing as npt
|
|
|
8
8
|
from code_loader.contract.datasetclasses import CustomCallableInterfaceMultiArgs, \
|
|
9
9
|
CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, CustomCallableInterface, \
|
|
10
10
|
VisualizerCallableInterface, MetadataSectionCallableInterface, PreprocessResponse, SectionCallableInterface, \
|
|
11
|
-
ConfusionMatrixElement, SamplePreprocessResponse
|
|
11
|
+
ConfusionMatrixElement, SamplePreprocessResponse, InstanceCallableInterface, ElementInstance
|
|
12
12
|
from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType
|
|
13
13
|
from code_loader import leap_binder
|
|
14
14
|
from code_loader.contract.mapping import NodeMapping, NodeMappingType, NodeConnection
|
|
@@ -270,6 +270,64 @@ def tensorleap_preprocess():
|
|
|
270
270
|
|
|
271
271
|
return decorating_function
|
|
272
272
|
|
|
273
|
+
def tensorleap_element_instance_preprocess(instance_mask_encoder: Callable[[str, PreprocessResponse], List[ElementInstance]]):
|
|
274
|
+
def decorating_function(user_function: Callable[[], List[PreprocessResponse]]):
|
|
275
|
+
def user_function_instance() -> List[PreprocessResponse]:
|
|
276
|
+
result = user_function()
|
|
277
|
+
for preprocess_response in result:
|
|
278
|
+
sample_ids_to_instance_mappings = {}
|
|
279
|
+
instance_to_sample_ids_mappings = {}
|
|
280
|
+
instance_ids_to_names = {}
|
|
281
|
+
all_sample_ids = preprocess_response.sample_ids.copy()
|
|
282
|
+
for sample_id in preprocess_response.sample_ids:
|
|
283
|
+
instances_masks = instance_mask_encoder(sample_id, preprocess_response)
|
|
284
|
+
instances_ids = [f'{sample_id}_{instance_id}' for instance_id in range(len(instances_masks))]
|
|
285
|
+
sample_ids_to_instance_mappings[sample_id] = instances_ids
|
|
286
|
+
instance_to_sample_ids_mappings[sample_id] = sample_id
|
|
287
|
+
instance_names = [instance.name for instance in instances_masks]
|
|
288
|
+
instance_ids_to_names[sample_id] = 'none'
|
|
289
|
+
for instance_id, instance_name in zip(instances_ids, instance_names):
|
|
290
|
+
instance_to_sample_ids_mappings[instance_id] = sample_id
|
|
291
|
+
instance_ids_to_names[instance_id] = instance_name
|
|
292
|
+
all_sample_ids.extend(instances_ids)
|
|
293
|
+
preprocess_response.sample_ids_to_instance_mappings = sample_ids_to_instance_mappings
|
|
294
|
+
preprocess_response.instance_to_sample_ids_mappings = instance_to_sample_ids_mappings
|
|
295
|
+
preprocess_response.instance_ids_to_names = instance_ids_to_names
|
|
296
|
+
preprocess_response.sample_ids = all_sample_ids
|
|
297
|
+
return result
|
|
298
|
+
|
|
299
|
+
def builtin_instance_metadata(idx: str, preprocess: PreprocessResponse) -> Dict[str, str]:
|
|
300
|
+
return {'is_instance': '0', 'original_sample_id': idx, 'instance_name': 'none'}
|
|
301
|
+
leap_binder.set_preprocess(user_function_instance)
|
|
302
|
+
leap_binder.set_metadata(builtin_instance_metadata, "builtin_instance_metadata")
|
|
303
|
+
|
|
304
|
+
def _validate_input_args(*args, **kwargs):
|
|
305
|
+
assert len(args) == 0 and len(kwargs) == 0, \
|
|
306
|
+
(f'tensorleap_element_instance_preprocess validation failed: '
|
|
307
|
+
f'The function should not take any arguments. Got {args} and {kwargs}.')
|
|
308
|
+
|
|
309
|
+
def _validate_result(result):
|
|
310
|
+
assert isinstance(result, list), \
|
|
311
|
+
(f'tensorleap_element_instance_preprocess validation failed: '
|
|
312
|
+
f'The return type should be a list. Got {type(result)}.')
|
|
313
|
+
for i, response in enumerate(result):
|
|
314
|
+
assert isinstance(response, PreprocessResponse), \
|
|
315
|
+
(f'tensorleap_element_instance_preprocess validation failed: '
|
|
316
|
+
f'Element #{i} in the return list should be a PreprocessResponse. Got {type(response)}.')
|
|
317
|
+
assert len(set(result)) == len(result), \
|
|
318
|
+
(f'tensorleap_element_instance_preprocess validation failed: '
|
|
319
|
+
f'The return list should not contain duplicate PreprocessResponse objects.')
|
|
320
|
+
|
|
321
|
+
def inner(*args, **kwargs):
|
|
322
|
+
_validate_input_args(*args, **kwargs)
|
|
323
|
+
result = user_function_instance()
|
|
324
|
+
_validate_result(result)
|
|
325
|
+
return result
|
|
326
|
+
|
|
327
|
+
return inner
|
|
328
|
+
|
|
329
|
+
return decorating_function
|
|
330
|
+
|
|
273
331
|
|
|
274
332
|
def tensorleap_unlabeled_preprocess():
|
|
275
333
|
def decorating_function(user_function: Callable[[], PreprocessResponse]):
|
|
@@ -296,6 +354,38 @@ def tensorleap_unlabeled_preprocess():
|
|
|
296
354
|
return decorating_function
|
|
297
355
|
|
|
298
356
|
|
|
357
|
+
def tensorleap_instances_masks_encoder(name: str):
|
|
358
|
+
def decorating_function(user_function: InstanceCallableInterface):
|
|
359
|
+
leap_binder.set_instance_masks(user_function, name)
|
|
360
|
+
|
|
361
|
+
def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse):
|
|
362
|
+
assert isinstance(sample_id, str), \
|
|
363
|
+
(f'tensorleap_instances_masks_encoder validation failed: '
|
|
364
|
+
f'Argument sample_id should be str. Got {type(sample_id)}.')
|
|
365
|
+
assert isinstance(preprocess_response, PreprocessResponse), \
|
|
366
|
+
(f'tensorleap_instances_masks_encoder validation failed: '
|
|
367
|
+
f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
|
|
368
|
+
assert type(sample_id) == preprocess_response.sample_id_type, \
|
|
369
|
+
(f'tensorleap_instances_masks_encoder validation failed: '
|
|
370
|
+
f'Argument sample_id should be as the same type as defined in the preprocess response '
|
|
371
|
+
f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
|
|
372
|
+
|
|
373
|
+
def _validate_result(result):
|
|
374
|
+
assert isinstance(result, list), \
|
|
375
|
+
(f'tensorleap_instances_masks_encoder validation failed: '
|
|
376
|
+
f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
|
|
377
|
+
|
|
378
|
+
def inner(sample_id, preprocess_response):
|
|
379
|
+
_validate_input_args(sample_id, preprocess_response)
|
|
380
|
+
result = user_function(sample_id, preprocess_response)
|
|
381
|
+
_validate_result(result)
|
|
382
|
+
return result
|
|
383
|
+
|
|
384
|
+
return inner
|
|
385
|
+
|
|
386
|
+
return decorating_function
|
|
387
|
+
|
|
388
|
+
|
|
299
389
|
def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
|
|
300
390
|
def decorating_function(user_function: SectionCallableInterface):
|
|
301
391
|
for input_handler in leap_binder.setup_container.inputs:
|
code_loader/leaploader.py
CHANGED
|
@@ -14,7 +14,8 @@ import numpy.typing as npt
|
|
|
14
14
|
from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
|
|
15
15
|
PreprocessResponse, VisualizerHandler, LeapData, \
|
|
16
16
|
PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
|
|
17
|
-
MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse
|
|
17
|
+
MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse, \
|
|
18
|
+
ElementInstance
|
|
18
19
|
from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
|
|
19
20
|
from code_loader.contract.exceptions import DatasetScriptException
|
|
20
21
|
from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
|
|
@@ -23,7 +24,7 @@ from code_loader.contract.responsedataclasses import DatasetIntegParseResult, Da
|
|
|
23
24
|
EngineFileContract
|
|
24
25
|
from code_loader.inner_leap_binder import global_leap_binder
|
|
25
26
|
from code_loader.leaploaderbase import LeapLoaderBase
|
|
26
|
-
from code_loader.utils import get_root_exception_file_and_line_number
|
|
27
|
+
from code_loader.utils import get_root_exception_file_and_line_number
|
|
27
28
|
|
|
28
29
|
|
|
29
30
|
class LeapLoader(LeapLoaderBase):
|
|
@@ -150,6 +151,22 @@ class LeapLoader(LeapLoaderBase):
|
|
|
150
151
|
state=state)
|
|
151
152
|
return sample
|
|
152
153
|
|
|
154
|
+
def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
|
|
155
|
+
self.exec_script()
|
|
156
|
+
preprocess_result = self._preprocess_result()
|
|
157
|
+
if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
|
|
158
|
+
self._preprocess_result(update_unlabeled_preprocess=True)
|
|
159
|
+
|
|
160
|
+
metadata, metadata_is_none = self._get_metadata(state, sample_id)
|
|
161
|
+
sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
|
|
162
|
+
gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
|
|
163
|
+
metadata=metadata,
|
|
164
|
+
metadata_is_none=metadata_is_none,
|
|
165
|
+
index=sample_id,
|
|
166
|
+
state=state,
|
|
167
|
+
instance_masks=self._get_instances_masks(state, sample_id))
|
|
168
|
+
return sample
|
|
169
|
+
|
|
153
170
|
def check_dataset(self) -> DatasetIntegParseResult:
|
|
154
171
|
test_payloads: List[DatasetTestResultPayload] = []
|
|
155
172
|
setup_response = None
|
|
@@ -437,6 +454,16 @@ class LeapLoader(LeapLoaderBase):
|
|
|
437
454
|
def _get_inputs(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
|
|
438
455
|
return self._get_dataset_handlers(global_leap_binder.setup_container.inputs, state, sample_id)
|
|
439
456
|
|
|
457
|
+
def _get_instances_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, List[ElementInstance]]:
|
|
458
|
+
preprocess_result = self._preprocess_result()
|
|
459
|
+
preprocess_state = preprocess_result[state]
|
|
460
|
+
result_agg = {}
|
|
461
|
+
for handler in global_leap_binder.setup_container.instance_masks:
|
|
462
|
+
handler_result = handler.function(sample_id, preprocess_state)
|
|
463
|
+
handler_name = handler.name
|
|
464
|
+
result_agg[handler_name] = handler_result
|
|
465
|
+
return result_agg
|
|
466
|
+
|
|
440
467
|
def _get_gt(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
|
|
441
468
|
return self._get_dataset_handlers(global_leap_binder.setup_container.ground_truths, state, sample_id)
|
|
442
469
|
|
|
@@ -477,18 +504,22 @@ class LeapLoader(LeapLoaderBase):
|
|
|
477
504
|
|
|
478
505
|
return converted_value, is_none
|
|
479
506
|
|
|
480
|
-
def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[
|
|
481
|
-
Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
|
|
507
|
+
def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
|
|
482
508
|
result_agg = {}
|
|
483
509
|
is_none = {}
|
|
484
510
|
preprocess_result = self._preprocess_result()
|
|
485
511
|
preprocess_state = preprocess_result[state]
|
|
486
512
|
for handler in global_leap_binder.setup_container.metadata:
|
|
487
513
|
handler_result = handler.function(sample_id, preprocess_state)
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
514
|
+
if isinstance(handler_result, dict):
|
|
515
|
+
for single_metadata_name, single_metadata_result in handler_result.items():
|
|
516
|
+
handler_name = f'{handler.name}_{single_metadata_name}'
|
|
517
|
+
result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
|
|
518
|
+
handler_name, single_metadata_result)
|
|
519
|
+
else:
|
|
520
|
+
handler_name = handler.name
|
|
521
|
+
result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
|
|
522
|
+
handler_name, handler_result)
|
|
492
523
|
|
|
493
524
|
return result_agg, is_none
|
|
494
525
|
|
|
@@ -501,3 +532,18 @@ class LeapLoader(LeapLoaderBase):
|
|
|
501
532
|
raise Exception("Different id types in preprocess results")
|
|
502
533
|
|
|
503
534
|
return id_type
|
|
535
|
+
|
|
536
|
+
def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str], Dict[str, str]]:
|
|
537
|
+
"""
|
|
538
|
+
This Method get the data state and returns two dictionaries that holds the mapping of the sample ids to their
|
|
539
|
+
instances and the other way around and the sample ids array.
|
|
540
|
+
Args:
|
|
541
|
+
state: DataStateEnum state
|
|
542
|
+
Returns:
|
|
543
|
+
sample_ids_to_instance_mappings: sample id to instance mappings
|
|
544
|
+
instance_to_sample_ids_mappings: instance to sample ids mappings
|
|
545
|
+
sample_ids: sample ids array
|
|
546
|
+
"""
|
|
547
|
+
preprocess_result = self._preprocess_result()
|
|
548
|
+
preprocess_state = preprocess_result[state]
|
|
549
|
+
return preprocess_state.sample_ids_to_instance_mappings, preprocess_state.instance_to_sample_ids_mappings, preprocess_state.instance_ids_to_names
|
code_loader/leaploaderbase.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from abc import abstractmethod
|
|
4
4
|
|
|
5
|
-
from typing import Dict, List, Union, Type, Optional
|
|
5
|
+
from typing import Dict, List, Union, Type, Optional, Tuple
|
|
6
6
|
|
|
7
7
|
import numpy as np
|
|
8
8
|
import numpy.typing as npt
|
|
@@ -64,6 +64,14 @@ class LeapLoaderBase:
|
|
|
64
64
|
def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
|
|
65
65
|
pass
|
|
66
66
|
|
|
67
|
+
@abstractmethod
|
|
68
|
+
def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
|
|
69
|
+
pass
|
|
70
|
+
|
|
71
|
+
@abstractmethod
|
|
72
|
+
def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str], Dict[str, str]]:
|
|
73
|
+
pass
|
|
74
|
+
|
|
67
75
|
@abstractmethod
|
|
68
76
|
def check_dataset(self) -> DatasetIntegParseResult:
|
|
69
77
|
pass
|
code_loader/utils.py
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import sys
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from types import TracebackType
|
|
4
|
-
from typing import List, Union, Tuple, Any,
|
|
4
|
+
from typing import List, Union, Tuple, Any, Callable
|
|
5
5
|
import traceback
|
|
6
6
|
import numpy as np
|
|
7
7
|
import numpy.typing as npt
|
|
8
8
|
|
|
9
|
-
from code_loader.contract.datasetclasses import SectionCallableInterface, PreprocessResponse
|
|
9
|
+
from code_loader.contract.datasetclasses import SectionCallableInterface, PreprocessResponse, \
|
|
10
|
+
InstanceCallableInterface, ElementInstance
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> SectionCallableInterface:
|
|
@@ -17,6 +18,15 @@ def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> Secti
|
|
|
17
18
|
|
|
18
19
|
return numpy_encoder_function
|
|
19
20
|
|
|
21
|
+
def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -> Callable[
|
|
22
|
+
[Union[int, str], PreprocessResponse], List[ElementInstance]]:
|
|
23
|
+
def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse) -> List[ElementInstance]:
|
|
24
|
+
result = encoder_function(idx, samples)
|
|
25
|
+
for res in result:
|
|
26
|
+
res.mask = np.array(res.mask)
|
|
27
|
+
return result
|
|
28
|
+
return numpy_encoder_function
|
|
29
|
+
|
|
20
30
|
|
|
21
31
|
def get_root_traceback(exc_tb: TracebackType) -> TracebackType:
|
|
22
32
|
return_traceback = exc_tb
|
|
@@ -66,28 +76,3 @@ def rescale_min_max(image: npt.NDArray[np.float32]) -> npt.NDArray[np.float32]:
|
|
|
66
76
|
return image
|
|
67
77
|
|
|
68
78
|
|
|
69
|
-
def flatten(
|
|
70
|
-
value: Any,
|
|
71
|
-
*,
|
|
72
|
-
prefix: str = "",
|
|
73
|
-
list_token: str = "e",
|
|
74
|
-
) -> Iterator[Tuple[str, Any]]:
|
|
75
|
-
"""
|
|
76
|
-
Recursively walk `value` and yield (flat_key, leaf_value) pairs.
|
|
77
|
-
|
|
78
|
-
• Dicts → descend with new_prefix = f"{prefix}_{key}" (or just key if top level)
|
|
79
|
-
• Sequences → descend with new_prefix = f"{prefix}_{list_token}{idx}"
|
|
80
|
-
• Leaf scalars → yield the accumulated flat key and the scalar itself
|
|
81
|
-
"""
|
|
82
|
-
if isinstance(value, dict):
|
|
83
|
-
for k, v in value.items():
|
|
84
|
-
new_prefix = f"{prefix}_{k}" if prefix else k
|
|
85
|
-
yield from flatten(v, prefix=new_prefix, list_token=list_token)
|
|
86
|
-
|
|
87
|
-
elif isinstance(value, (list, tuple)):
|
|
88
|
-
for idx, v in enumerate(value):
|
|
89
|
-
new_prefix = f"{prefix}_{list_token}{idx}"
|
|
90
|
-
yield from flatten(v, prefix=new_prefix, list_token=list_token)
|
|
91
|
-
|
|
92
|
-
else: # primitive leaf (str, int, float, bool, None…)
|
|
93
|
-
yield prefix, value
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
|
|
2
2
|
code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
|
|
3
3
|
code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
-
code_loader/contract/datasetclasses.py,sha256=
|
|
4
|
+
code_loader/contract/datasetclasses.py,sha256=eioYa_gXpiFsRoGhkvQ7RMBfhFL6t9WsqKBddUlmnYA,9118
|
|
5
5
|
code_loader/contract/enums.py,sha256=GEFkvUMXnCNt-GOoz7NJ9ecQZ2PPDettJNOsxsiM0wk,1622
|
|
6
6
|
code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
|
|
7
7
|
code_loader/contract/mapping.py,sha256=e11h_sprwOyE32PcqgRq9JvyahQrPzwqgkhmbQLKLQY,1165
|
|
@@ -20,14 +20,14 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
|
|
|
20
20
|
code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
|
|
21
21
|
code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
|
|
22
22
|
code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
|
|
23
|
-
code_loader/inner_leap_binder/leapbinder.py,sha256=
|
|
24
|
-
code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=
|
|
25
|
-
code_loader/leaploader.py,sha256=
|
|
26
|
-
code_loader/leaploaderbase.py,sha256=
|
|
27
|
-
code_loader/utils.py,sha256=
|
|
23
|
+
code_loader/inner_leap_binder/leapbinder.py,sha256=mOEo0kkfQA1OYLWvoOOsnDdP9_itLPCPAvsfE2Xx0yQ,32443
|
|
24
|
+
code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=n0S0L6YGMxgxTliQeQAiWIK9SMrj5ZjOl9wXqD_Tvjw,29521
|
|
25
|
+
code_loader/leaploader.py,sha256=FiENUvp_TvnPXyX9n5QDQCdUQ_stWv_LazaKnCesCQ4,28822
|
|
26
|
+
code_loader/leaploaderbase.py,sha256=lKdw2pd6H9hFsxVmc7jJMoZd_vlG5He1ooqT-cR_yq8,4496
|
|
27
|
+
code_loader/utils.py,sha256=_j8b60pimoNAvWMRj7hEkkT6C76qES6cZoBFHpXHMxA,2698
|
|
28
28
|
code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
29
29
|
code_loader/visualizers/default_visualizers.py,sha256=669lBpLISLO6my5Qcgn1FLDDeZgHumPf252m4KHY4YM,2555
|
|
30
|
-
code_loader-1.0.
|
|
31
|
-
code_loader-1.0.
|
|
32
|
-
code_loader-1.0.
|
|
33
|
-
code_loader-1.0.
|
|
30
|
+
code_loader-1.0.98.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
|
|
31
|
+
code_loader-1.0.98.dist-info/METADATA,sha256=D6yEtbhJsxugq22Wk9bAEv890unZJLXus6cTBOL0eP0,849
|
|
32
|
+
code_loader-1.0.98.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
33
|
+
code_loader-1.0.98.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|