code-loader 1.0.94.dev5__py3-none-any.whl → 1.0.97.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of code-loader might be problematic. Click here for more details.
- code_loader/contract/datasetclasses.py +0 -24
- code_loader/inner_leap_binder/leapbinder.py +5 -21
- code_loader/inner_leap_binder/leapbinder_decorators.py +1 -97
- code_loader/leaploader.py +8 -54
- code_loader/leaploaderbase.py +1 -9
- code_loader/utils.py +27 -12
- {code_loader-1.0.94.dev5.dist-info → code_loader-1.0.97.dev0.dist-info}/METADATA +1 -1
- {code_loader-1.0.94.dev5.dist-info → code_loader-1.0.97.dev0.dist-info}/RECORD +10 -10
- {code_loader-1.0.94.dev5.dist-info → code_loader-1.0.97.dev0.dist-info}/LICENSE +0 -0
- {code_loader-1.0.94.dev5.dist-info → code_loader-1.0.97.dev0.dist-info}/WHEEL +0 -0
|
@@ -38,18 +38,8 @@ class PreprocessResponse:
|
|
|
38
38
|
sample_ids: Optional[Union[List[str], List[int]]] = None
|
|
39
39
|
state: Optional[DataStateType] = None
|
|
40
40
|
sample_id_type: Optional[Union[Type[str], Type[int]]] = None
|
|
41
|
-
sample_ids_to_instance_mappings: Optional[Dict[str, List[str]]] = None # in use only for element instance
|
|
42
|
-
instance_to_sample_ids_mappings: Optional[Dict[str, str]] = None # in use only for element instance
|
|
43
|
-
instance_id_to_instance_name: Optional[Dict[str, str]] = None # in use only for element instance
|
|
44
41
|
|
|
45
42
|
def __post_init__(self) -> None:
|
|
46
|
-
def is_valid_string(s: str) -> bool:
|
|
47
|
-
return bool(re.match(r'^[A-Za-z0-9_]+$', s))
|
|
48
|
-
|
|
49
|
-
assert self.sample_ids_to_instance_mappings is None, f"Keep sample_ids_to_instance_mappings None when initializing PreprocessResponse"
|
|
50
|
-
assert self.instance_to_sample_ids_mappings is None, f"Keep instance_to_sample_ids_mappings None when initializing PreprocessResponse"
|
|
51
|
-
assert self.instance_id_to_instance_name is None, f"Keep instance_id_to_instance_name None when initializing PreprocessResponse"
|
|
52
|
-
|
|
53
43
|
if self.length is not None and self.sample_ids is None:
|
|
54
44
|
self.sample_ids = [i for i in range(self.length)]
|
|
55
45
|
self.sample_id_type = int
|
|
@@ -60,8 +50,6 @@ class PreprocessResponse:
|
|
|
60
50
|
if self.sample_id_type == str:
|
|
61
51
|
for sample_id in self.sample_ids:
|
|
62
52
|
assert isinstance(sample_id, str), f"Sample id should be of type str. Got: {type(sample_id)}"
|
|
63
|
-
if not is_valid_string(sample_id):
|
|
64
|
-
raise Exception(f"Sample id should contain only letters (A-Z, a-z), numbers or '_'. Got: {sample_id}")
|
|
65
53
|
else:
|
|
66
54
|
raise Exception("length is deprecated.")
|
|
67
55
|
|
|
@@ -72,13 +60,8 @@ class PreprocessResponse:
|
|
|
72
60
|
assert self.sample_ids is not None
|
|
73
61
|
return len(self.sample_ids)
|
|
74
62
|
|
|
75
|
-
@dataclass
|
|
76
|
-
class ElementInstance:
|
|
77
|
-
name: str
|
|
78
|
-
mask: npt.NDArray[np.float32]
|
|
79
63
|
|
|
80
64
|
SectionCallableInterface = Callable[[Union[int, str], PreprocessResponse], npt.NDArray[np.float32]]
|
|
81
|
-
InstanceCallableInterface = Callable[[Union[int, str], PreprocessResponse], List[ElementInstance]]
|
|
82
65
|
|
|
83
66
|
MetadataSectionCallableInterface = Union[
|
|
84
67
|
Callable[[Union[int, str], PreprocessResponse], int],
|
|
@@ -200,10 +183,6 @@ class InputHandler(DatasetBaseHandler):
|
|
|
200
183
|
shape: Optional[List[int]] = None
|
|
201
184
|
channel_dim: Optional[int] = -1
|
|
202
185
|
|
|
203
|
-
@dataclass
|
|
204
|
-
class ElementInstanceMasksHandler:
|
|
205
|
-
name: str
|
|
206
|
-
function: InstanceCallableInterface
|
|
207
186
|
|
|
208
187
|
@dataclass
|
|
209
188
|
class GroundTruthHandler(DatasetBaseHandler):
|
|
@@ -239,7 +218,6 @@ class DatasetIntegrationSetup:
|
|
|
239
218
|
unlabeled_data_preprocess: Optional[UnlabeledDataPreprocessHandler] = None
|
|
240
219
|
visualizers: List[VisualizerHandler] = field(default_factory=list)
|
|
241
220
|
inputs: List[InputHandler] = field(default_factory=list)
|
|
242
|
-
instance_masks: List[ElementInstanceMasksHandler] = field(default_factory=list)
|
|
243
221
|
ground_truths: List[GroundTruthHandler] = field(default_factory=list)
|
|
244
222
|
metadata: List[MetadataHandler] = field(default_factory=list)
|
|
245
223
|
prediction_types: List[PredictionTypeHandler] = field(default_factory=list)
|
|
@@ -256,5 +234,3 @@ class DatasetSample:
|
|
|
256
234
|
metadata_is_none: Dict[str, bool]
|
|
257
235
|
index: Union[int, str]
|
|
258
236
|
state: DataStateEnum
|
|
259
|
-
instance_masks: Optional[Dict[str, List[ElementInstance]]] = None
|
|
260
|
-
|
|
@@ -10,15 +10,14 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
|
|
|
10
10
|
MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
|
|
11
11
|
CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
|
|
12
12
|
CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
|
|
13
|
-
RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse
|
|
14
|
-
ElementInstanceMasksHandler, InstanceCallableInterface
|
|
13
|
+
RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse
|
|
15
14
|
from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
|
|
16
15
|
from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
|
|
17
16
|
from code_loader.contract.responsedataclasses import DatasetTestResultPayload
|
|
18
17
|
from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
|
|
19
18
|
from code_loader.default_losses import loss_name_to_function
|
|
20
19
|
from code_loader.default_metrics import metrics_names_to_functions_and_direction
|
|
21
|
-
from code_loader.utils import to_numpy_return_wrapper, get_shape
|
|
20
|
+
from code_loader.utils import to_numpy_return_wrapper, get_shape
|
|
22
21
|
from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
|
|
23
22
|
default_graph_visualizer, \
|
|
24
23
|
default_image_visualizer, default_horizontal_bar_visualizer, default_word_visualizer, \
|
|
@@ -235,22 +234,6 @@ class LeapBinder:
|
|
|
235
234
|
|
|
236
235
|
self._encoder_names.append(name)
|
|
237
236
|
|
|
238
|
-
|
|
239
|
-
def set_instance_masks(self, function: InstanceCallableInterface, name: str) -> None:
|
|
240
|
-
"""
|
|
241
|
-
Set the instance mask handler function.
|
|
242
|
-
|
|
243
|
-
Args:
|
|
244
|
-
function (SectionCallableInterface): The input handler function.
|
|
245
|
-
name (str): The name of the instance mask section.
|
|
246
|
-
|
|
247
|
-
Example:
|
|
248
|
-
leap_binder.set_input(input_encoder, name='input_encoder')
|
|
249
|
-
"""
|
|
250
|
-
function = to_numpy_return_masks_wrapper(function)
|
|
251
|
-
self.setup_container.instance_masks.append(ElementInstanceMasksHandler(name, function))
|
|
252
|
-
|
|
253
|
-
|
|
254
237
|
def add_custom_loss(self, function: CustomCallableInterface, name: str) -> None:
|
|
255
238
|
"""
|
|
256
239
|
Add a custom loss function to the setup.
|
|
@@ -530,9 +513,10 @@ class LeapBinder:
|
|
|
530
513
|
for i, (single_metadata_name, single_metadata_result) in enumerate(raw_result.items()):
|
|
531
514
|
metadata_test_result = metadata_test_result_payloads[i]
|
|
532
515
|
|
|
533
|
-
if not isinstance(single_metadata_result, (int, str, bool, float, np.unsignedinteger,
|
|
516
|
+
if not isinstance(single_metadata_result, (int, str, bool, float, np.unsignedinteger,
|
|
517
|
+
np.signedinteger, np.bool_, np.floating, type(None))):
|
|
534
518
|
raise Exception(f"Unsupported return type of metadata {single_metadata_name}."
|
|
535
|
-
f"The return type should be one of [int, float, str, bool]. Got {type(single_metadata_result)}")
|
|
519
|
+
f"The return type should be one of [int, float, str, bool, None]. Got {type(single_metadata_result)}")
|
|
536
520
|
|
|
537
521
|
metadata_type = None
|
|
538
522
|
if single_metadata_result is None:
|
|
@@ -8,7 +8,7 @@ import numpy.typing as npt
|
|
|
8
8
|
from code_loader.contract.datasetclasses import CustomCallableInterfaceMultiArgs, \
|
|
9
9
|
CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, CustomCallableInterface, \
|
|
10
10
|
VisualizerCallableInterface, MetadataSectionCallableInterface, PreprocessResponse, SectionCallableInterface, \
|
|
11
|
-
ConfusionMatrixElement, SamplePreprocessResponse
|
|
11
|
+
ConfusionMatrixElement, SamplePreprocessResponse
|
|
12
12
|
from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType
|
|
13
13
|
from code_loader import leap_binder
|
|
14
14
|
from code_loader.contract.mapping import NodeMapping, NodeMappingType, NodeConnection
|
|
@@ -270,70 +270,6 @@ def tensorleap_preprocess():
|
|
|
270
270
|
|
|
271
271
|
return decorating_function
|
|
272
272
|
|
|
273
|
-
def tensorleap_element_instance_preprocess(instance_mask_encoder: Callable[[int, PreprocessResponse], List[ElementInstance]]):
|
|
274
|
-
def decorating_function(user_function: Callable[[], List[PreprocessResponse]]):
|
|
275
|
-
def user_function_instance() -> List[PreprocessResponse]:
|
|
276
|
-
result = user_function()
|
|
277
|
-
for preprocess_response in result:
|
|
278
|
-
sample_ids_to_instance_mappings = {}
|
|
279
|
-
instance_to_sample_ids_mappings = {}
|
|
280
|
-
instance_id_to_instance_name = {}
|
|
281
|
-
all_sample_ids = preprocess_response.sample_ids.copy()
|
|
282
|
-
for sample_id in preprocess_response.sample_ids:
|
|
283
|
-
instances_masks = instance_mask_encoder(sample_id, preprocess_response)
|
|
284
|
-
instance_names = [instance.name for instance in instances_masks]
|
|
285
|
-
instances_ids = [f'{sample_id}_{instance_id}' for instance_id in range(len(instances_masks))]
|
|
286
|
-
sample_ids_to_instance_mappings[sample_id] = instances_ids
|
|
287
|
-
instance_to_sample_ids_mappings[sample_id] = sample_id
|
|
288
|
-
instance_id_to_instance_name[sample_id] = None
|
|
289
|
-
for instance_id, instance_name in zip(instances_ids, instance_names):
|
|
290
|
-
instance_to_sample_ids_mappings[instance_id] = sample_id
|
|
291
|
-
instance_id_to_instance_name[instance_id] = instance_name
|
|
292
|
-
all_sample_ids.extend(instances_ids)
|
|
293
|
-
preprocess_response.sample_ids_to_instance_mappings = sample_ids_to_instance_mappings
|
|
294
|
-
preprocess_response.instance_to_sample_ids_mappings = instance_to_sample_ids_mappings
|
|
295
|
-
preprocess_response.instance_id_to_instance_name = instance_id_to_instance_name
|
|
296
|
-
preprocess_response.sample_ids = all_sample_ids
|
|
297
|
-
return result
|
|
298
|
-
|
|
299
|
-
def metadata_is_instance(idx: str, preprocess: PreprocessResponse) -> Dict[str, str]:
|
|
300
|
-
# return {'is_instance': '0',
|
|
301
|
-
# 'orig_sample_id': preprocess.instance_to_sample_ids_mappings[idx],
|
|
302
|
-
# 'instance_name': preprocess.instance_id_to_instance_name[idx]}
|
|
303
|
-
# return '0'
|
|
304
|
-
return {'is_instance': '0',
|
|
305
|
-
'orig_sample_id': '0',
|
|
306
|
-
'instance_name': '0'}
|
|
307
|
-
leap_binder.set_preprocess(user_function_instance)
|
|
308
|
-
leap_binder.set_metadata(metadata_is_instance, "metadata_is_instance")
|
|
309
|
-
|
|
310
|
-
def _validate_input_args(*args, **kwargs):
|
|
311
|
-
assert len(args) == 0 and len(kwargs) == 0, \
|
|
312
|
-
(f'tensorleap_element_instance_preprocess validation failed: '
|
|
313
|
-
f'The function should not take any arguments. Got {args} and {kwargs}.')
|
|
314
|
-
|
|
315
|
-
def _validate_result(result):
|
|
316
|
-
assert isinstance(result, list), \
|
|
317
|
-
(f'tensorleap_element_instance_preprocess validation failed: '
|
|
318
|
-
f'The return type should be a list. Got {type(result)}.')
|
|
319
|
-
for i, response in enumerate(result):
|
|
320
|
-
assert isinstance(response, PreprocessResponse), \
|
|
321
|
-
(f'tensorleap_element_instance_preprocess validation failed: '
|
|
322
|
-
f'Element #{i} in the return list should be a PreprocessResponse. Got {type(response)}.')
|
|
323
|
-
assert len(set(result)) == len(result), \
|
|
324
|
-
(f'tensorleap_element_instance_preprocess validation failed: '
|
|
325
|
-
f'The return list should not contain duplicate PreprocessResponse objects.')
|
|
326
|
-
|
|
327
|
-
def inner(*args, **kwargs):
|
|
328
|
-
_validate_input_args(*args, **kwargs)
|
|
329
|
-
result = user_function_instance()
|
|
330
|
-
_validate_result(result)
|
|
331
|
-
return result
|
|
332
|
-
|
|
333
|
-
return inner
|
|
334
|
-
|
|
335
|
-
return decorating_function
|
|
336
|
-
|
|
337
273
|
|
|
338
274
|
def tensorleap_unlabeled_preprocess():
|
|
339
275
|
def decorating_function(user_function: Callable[[], PreprocessResponse]):
|
|
@@ -360,38 +296,6 @@ def tensorleap_unlabeled_preprocess():
|
|
|
360
296
|
return decorating_function
|
|
361
297
|
|
|
362
298
|
|
|
363
|
-
def tensorleap_instances_masks_encoder(name: str):
|
|
364
|
-
def decorating_function(user_function: InstanceCallableInterface):
|
|
365
|
-
leap_binder.set_instance_masks(user_function, name)
|
|
366
|
-
|
|
367
|
-
def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse):
|
|
368
|
-
assert isinstance(sample_id, str), \
|
|
369
|
-
(f'tensorleap_instances_masks_encoder validation failed: '
|
|
370
|
-
f'Argument sample_id should be str. Got {type(sample_id)}.')
|
|
371
|
-
assert isinstance(preprocess_response, PreprocessResponse), \
|
|
372
|
-
(f'tensorleap_instances_masks_encoder validation failed: '
|
|
373
|
-
f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
|
|
374
|
-
assert type(sample_id) == preprocess_response.sample_id_type, \
|
|
375
|
-
(f'tensorleap_instances_masks_encoder validation failed: '
|
|
376
|
-
f'Argument sample_id should be as the same type as defined in the preprocess response '
|
|
377
|
-
f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
|
|
378
|
-
|
|
379
|
-
def _validate_result(result):
|
|
380
|
-
assert isinstance(result, list), \
|
|
381
|
-
(f'tensorleap_instances_masks_encoder validation failed: '
|
|
382
|
-
f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
|
|
383
|
-
|
|
384
|
-
def inner(sample_id, preprocess_response):
|
|
385
|
-
_validate_input_args(sample_id, preprocess_response)
|
|
386
|
-
result = user_function(sample_id, preprocess_response)
|
|
387
|
-
_validate_result(result)
|
|
388
|
-
return result
|
|
389
|
-
|
|
390
|
-
return inner
|
|
391
|
-
|
|
392
|
-
return decorating_function
|
|
393
|
-
|
|
394
|
-
|
|
395
299
|
def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
|
|
396
300
|
def decorating_function(user_function: SectionCallableInterface):
|
|
397
301
|
for input_handler in leap_binder.setup_container.inputs:
|
code_loader/leaploader.py
CHANGED
|
@@ -14,8 +14,7 @@ import numpy.typing as npt
|
|
|
14
14
|
from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
|
|
15
15
|
PreprocessResponse, VisualizerHandler, LeapData, \
|
|
16
16
|
PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
|
|
17
|
-
MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse
|
|
18
|
-
ElementInstance
|
|
17
|
+
MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse
|
|
19
18
|
from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
|
|
20
19
|
from code_loader.contract.exceptions import DatasetScriptException
|
|
21
20
|
from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
|
|
@@ -24,7 +23,7 @@ from code_loader.contract.responsedataclasses import DatasetIntegParseResult, Da
|
|
|
24
23
|
EngineFileContract
|
|
25
24
|
from code_loader.inner_leap_binder import global_leap_binder
|
|
26
25
|
from code_loader.leaploaderbase import LeapLoaderBase
|
|
27
|
-
from code_loader.utils import get_root_exception_file_and_line_number
|
|
26
|
+
from code_loader.utils import get_root_exception_file_and_line_number, flatten
|
|
28
27
|
|
|
29
28
|
|
|
30
29
|
class LeapLoader(LeapLoaderBase):
|
|
@@ -151,22 +150,6 @@ class LeapLoader(LeapLoaderBase):
|
|
|
151
150
|
state=state)
|
|
152
151
|
return sample
|
|
153
152
|
|
|
154
|
-
def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
|
|
155
|
-
self.exec_script()
|
|
156
|
-
preprocess_result = self._preprocess_result()
|
|
157
|
-
if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
|
|
158
|
-
self._preprocess_result(update_unlabeled_preprocess=True)
|
|
159
|
-
|
|
160
|
-
metadata, metadata_is_none = self._get_metadata(state, sample_id)
|
|
161
|
-
sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
|
|
162
|
-
gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
|
|
163
|
-
metadata=metadata,
|
|
164
|
-
metadata_is_none=metadata_is_none,
|
|
165
|
-
index=sample_id,
|
|
166
|
-
state=state,
|
|
167
|
-
instance_masks=self._get_instances_masks(state, sample_id))
|
|
168
|
-
return sample
|
|
169
|
-
|
|
170
153
|
def check_dataset(self) -> DatasetIntegParseResult:
|
|
171
154
|
test_payloads: List[DatasetTestResultPayload] = []
|
|
172
155
|
setup_response = None
|
|
@@ -454,16 +437,6 @@ class LeapLoader(LeapLoaderBase):
|
|
|
454
437
|
def _get_inputs(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
|
|
455
438
|
return self._get_dataset_handlers(global_leap_binder.setup_container.inputs, state, sample_id)
|
|
456
439
|
|
|
457
|
-
def _get_instances_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, List[ElementInstance]]:
|
|
458
|
-
preprocess_result = self._preprocess_result()
|
|
459
|
-
preprocess_state = preprocess_result[state]
|
|
460
|
-
result_agg = {}
|
|
461
|
-
for handler in global_leap_binder.setup_container.instance_masks:
|
|
462
|
-
handler_result = handler.function(sample_id, preprocess_state)
|
|
463
|
-
handler_name = handler.name
|
|
464
|
-
result_agg[handler_name] = handler_result
|
|
465
|
-
return result_agg
|
|
466
|
-
|
|
467
440
|
def _get_gt(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
|
|
468
441
|
return self._get_dataset_handlers(global_leap_binder.setup_container.ground_truths, state, sample_id)
|
|
469
442
|
|
|
@@ -504,22 +477,18 @@ class LeapLoader(LeapLoaderBase):
|
|
|
504
477
|
|
|
505
478
|
return converted_value, is_none
|
|
506
479
|
|
|
507
|
-
def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[
|
|
480
|
+
def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[
|
|
481
|
+
Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
|
|
508
482
|
result_agg = {}
|
|
509
483
|
is_none = {}
|
|
510
484
|
preprocess_result = self._preprocess_result()
|
|
511
485
|
preprocess_state = preprocess_result[state]
|
|
512
486
|
for handler in global_leap_binder.setup_container.metadata:
|
|
513
487
|
handler_result = handler.function(sample_id, preprocess_state)
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
handler_name, single_metadata_result)
|
|
519
|
-
else:
|
|
520
|
-
handler_name = handler.name
|
|
521
|
-
result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
|
|
522
|
-
handler_name, handler_result)
|
|
488
|
+
|
|
489
|
+
for flat_name, flat_result in flatten(handler_result, prefix=handler.name):
|
|
490
|
+
result_agg[flat_name], is_none[flat_name] = self._convert_metadata_to_correct_type(
|
|
491
|
+
flat_name, flat_result)
|
|
523
492
|
|
|
524
493
|
return result_agg, is_none
|
|
525
494
|
|
|
@@ -532,18 +501,3 @@ class LeapLoader(LeapLoaderBase):
|
|
|
532
501
|
raise Exception("Different id types in preprocess results")
|
|
533
502
|
|
|
534
503
|
return id_type
|
|
535
|
-
|
|
536
|
-
def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[Union[int, str], List[Union[int, str]]], Dict[Union[int, str], Union[int, str]], List[Union[int, str]]]:
|
|
537
|
-
"""
|
|
538
|
-
This Method get the data state and returns two dictionaries that holds the mapping of the sample ids to their
|
|
539
|
-
instances and the other way around and the sample ids array.
|
|
540
|
-
Args:
|
|
541
|
-
state: DataStateEnum state
|
|
542
|
-
Returns:
|
|
543
|
-
sample_ids_to_instance_mappings: sample id to instance mappings
|
|
544
|
-
instance_to_sample_ids_mappings: instance to sample ids mappings
|
|
545
|
-
sample_ids: sample ids array
|
|
546
|
-
"""
|
|
547
|
-
preprocess_result = self._preprocess_result()
|
|
548
|
-
preprocess_state = preprocess_result[state]
|
|
549
|
-
return preprocess_state.sample_ids_to_instance_mappings, preprocess_state.instance_to_sample_ids_mappings, preprocess_state.sample_ids
|
code_loader/leaploaderbase.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from abc import abstractmethod
|
|
4
4
|
|
|
5
|
-
from typing import Dict, List, Union, Type, Optional
|
|
5
|
+
from typing import Dict, List, Union, Type, Optional
|
|
6
6
|
|
|
7
7
|
import numpy as np
|
|
8
8
|
import numpy.typing as npt
|
|
@@ -64,14 +64,6 @@ class LeapLoaderBase:
|
|
|
64
64
|
def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
|
|
65
65
|
pass
|
|
66
66
|
|
|
67
|
-
@abstractmethod
|
|
68
|
-
def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
|
|
69
|
-
pass
|
|
70
|
-
|
|
71
|
-
@abstractmethod
|
|
72
|
-
def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[Union[int, str], List[Union[int, str]]], Dict[Union[int, str], Union[int, str]], List[Union[int, str]]]:
|
|
73
|
-
pass
|
|
74
|
-
|
|
75
67
|
@abstractmethod
|
|
76
68
|
def check_dataset(self) -> DatasetIntegParseResult:
|
|
77
69
|
pass
|
code_loader/utils.py
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import sys
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from types import TracebackType
|
|
4
|
-
from typing import List, Union, Tuple, Any,
|
|
4
|
+
from typing import List, Union, Tuple, Any, Iterator
|
|
5
5
|
import traceback
|
|
6
6
|
import numpy as np
|
|
7
7
|
import numpy.typing as npt
|
|
8
8
|
|
|
9
|
-
from code_loader.contract.datasetclasses import SectionCallableInterface, PreprocessResponse
|
|
10
|
-
InstanceCallableInterface, ElementInstance
|
|
9
|
+
from code_loader.contract.datasetclasses import SectionCallableInterface, PreprocessResponse
|
|
11
10
|
|
|
12
11
|
|
|
13
12
|
def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> SectionCallableInterface:
|
|
@@ -18,15 +17,6 @@ def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> Secti
|
|
|
18
17
|
|
|
19
18
|
return numpy_encoder_function
|
|
20
19
|
|
|
21
|
-
def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -> Callable[
|
|
22
|
-
[Union[int, str], PreprocessResponse], List[ElementInstance]]:
|
|
23
|
-
def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse) -> List[ElementInstance]:
|
|
24
|
-
result = encoder_function(idx, samples)
|
|
25
|
-
for res in result:
|
|
26
|
-
res.mask = np.array(res.mask)
|
|
27
|
-
return result
|
|
28
|
-
return numpy_encoder_function
|
|
29
|
-
|
|
30
20
|
|
|
31
21
|
def get_root_traceback(exc_tb: TracebackType) -> TracebackType:
|
|
32
22
|
return_traceback = exc_tb
|
|
@@ -76,3 +66,28 @@ def rescale_min_max(image: npt.NDArray[np.float32]) -> npt.NDArray[np.float32]:
|
|
|
76
66
|
return image
|
|
77
67
|
|
|
78
68
|
|
|
69
|
+
def flatten(
|
|
70
|
+
value: Any,
|
|
71
|
+
*,
|
|
72
|
+
prefix: str = "",
|
|
73
|
+
list_token: str = "e",
|
|
74
|
+
) -> Iterator[Tuple[str, Any]]:
|
|
75
|
+
"""
|
|
76
|
+
Recursively walk `value` and yield (flat_key, leaf_value) pairs.
|
|
77
|
+
|
|
78
|
+
• Dicts → descend with new_prefix = f"{prefix}_{key}" (or just key if top level)
|
|
79
|
+
• Sequences → descend with new_prefix = f"{prefix}_{list_token}{idx}"
|
|
80
|
+
• Leaf scalars → yield the accumulated flat key and the scalar itself
|
|
81
|
+
"""
|
|
82
|
+
if isinstance(value, dict):
|
|
83
|
+
for k, v in value.items():
|
|
84
|
+
new_prefix = f"{prefix}_{k}" if prefix else k
|
|
85
|
+
yield from flatten(v, prefix=new_prefix, list_token=list_token)
|
|
86
|
+
|
|
87
|
+
elif isinstance(value, (list, tuple)):
|
|
88
|
+
for idx, v in enumerate(value):
|
|
89
|
+
new_prefix = f"{prefix}_{list_token}{idx}"
|
|
90
|
+
yield from flatten(v, prefix=new_prefix, list_token=list_token)
|
|
91
|
+
|
|
92
|
+
else: # primitive leaf (str, int, float, bool, None…)
|
|
93
|
+
yield prefix, value
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
|
|
2
2
|
code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
|
|
3
3
|
code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
-
code_loader/contract/datasetclasses.py,sha256=
|
|
4
|
+
code_loader/contract/datasetclasses.py,sha256=3BWSCHaKtNWlKucMkPKSMiuvZosnnQgXFq2R-GbVOgg,7679
|
|
5
5
|
code_loader/contract/enums.py,sha256=GEFkvUMXnCNt-GOoz7NJ9ecQZ2PPDettJNOsxsiM0wk,1622
|
|
6
6
|
code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
|
|
7
7
|
code_loader/contract/mapping.py,sha256=e11h_sprwOyE32PcqgRq9JvyahQrPzwqgkhmbQLKLQY,1165
|
|
@@ -20,14 +20,14 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
|
|
|
20
20
|
code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
|
|
21
21
|
code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
|
|
22
22
|
code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
|
|
23
|
-
code_loader/inner_leap_binder/leapbinder.py,sha256=
|
|
24
|
-
code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=
|
|
25
|
-
code_loader/leaploader.py,sha256=
|
|
26
|
-
code_loader/leaploaderbase.py,sha256=
|
|
27
|
-
code_loader/utils.py,sha256=
|
|
23
|
+
code_loader/inner_leap_binder/leapbinder.py,sha256=Acg5C8pMlQHSCTNmTXlMiLdS7P_k6sBUahD5ffr6mN4,31794
|
|
24
|
+
code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=y5TuhJe-J3xEE0Oj7bxIfhyN1lXiLJgfgy3ZiAs-yic,24364
|
|
25
|
+
code_loader/leaploader.py,sha256=qUV-MmxZzESxi8zciKiitGObgks_5d58tgrZyJwRpuU,26125
|
|
26
|
+
code_loader/leaploaderbase.py,sha256=VH0vddRmkqLtcDlYPCO7hfz1_VbKo43lUdHDAbd4iJc,4198
|
|
27
|
+
code_loader/utils.py,sha256=4tXLum2AT3Z1ldD6BeYScNg0ATyE4oM8cuIGQxrXyjM,3163
|
|
28
28
|
code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
29
29
|
code_loader/visualizers/default_visualizers.py,sha256=669lBpLISLO6my5Qcgn1FLDDeZgHumPf252m4KHY4YM,2555
|
|
30
|
-
code_loader-1.0.
|
|
31
|
-
code_loader-1.0.
|
|
32
|
-
code_loader-1.0.
|
|
33
|
-
code_loader-1.0.
|
|
30
|
+
code_loader-1.0.97.dev0.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
|
|
31
|
+
code_loader-1.0.97.dev0.dist-info/METADATA,sha256=VjlJJJzJiQXZgcE2dzItxMz4B2mrry2W3GSEK_YkFyk,854
|
|
32
|
+
code_loader-1.0.97.dev0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
33
|
+
code_loader-1.0.97.dev0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|