code-loader 1.0.147__tar.gz → 1.0.148__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {code_loader-1.0.147 → code_loader-1.0.148}/PKG-INFO +1 -1
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/contract/datasetclasses.py +9 -1
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/inner_leap_binder/leapbinder_decorators.py +508 -121
- {code_loader-1.0.147 → code_loader-1.0.148}/pyproject.toml +1 -1
- {code_loader-1.0.147 → code_loader-1.0.148}/LICENSE +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/README.md +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/__init__.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/contract/__init__.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/contract/enums.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/contract/exceptions.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/contract/mapping.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/contract/responsedataclasses.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/contract/visualizer_classes.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/default_losses.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/default_metrics.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/__init__.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/api.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/cli_config_utils.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/client.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/epoch.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/experiment.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/experiment_context.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/types.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/utils.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/workingspace_config_utils.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/inner_leap_binder/__init__.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/inner_leap_binder/leapbinder.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/leaploader.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/leaploaderbase.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/mixpanel_tracker.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/plot_functions/__init__.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/plot_functions/plot_functions.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/plot_functions/visualize.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/utils.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/visualizers/__init__.py +0 -0
- {code_loader-1.0.147 → code_loader-1.0.148}/code_loader/visualizers/default_visualizers.py +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import warnings
|
|
1
2
|
from dataclasses import dataclass, field
|
|
2
3
|
from typing import Any, Callable, List, Optional, Dict, Union, Type
|
|
3
4
|
import re
|
|
@@ -56,7 +57,14 @@ class PreprocessResponse:
|
|
|
56
57
|
for sample_id in self.sample_ids:
|
|
57
58
|
assert isinstance(sample_id, str), f"Sample id should be of type str. Got: {type(sample_id)}"
|
|
58
59
|
else:
|
|
59
|
-
raise Exception("length is deprecated.")
|
|
60
|
+
raise Exception("length is deprecated, please use sample_ids instead.")
|
|
61
|
+
|
|
62
|
+
if self.state is None:
|
|
63
|
+
warnings.warn(
|
|
64
|
+
"PreprocessResponse.state is not set. For best practice, assign a unique `state` value to each PreprocessResponse instance."
|
|
65
|
+
)
|
|
66
|
+
else:
|
|
67
|
+
assert isinstance(self.state, DataStateType), f"PreprocessResponse.state must be of type {DataStateType.__name__} but got {type(self.state)}"
|
|
60
68
|
|
|
61
69
|
def __hash__(self) -> int:
|
|
62
70
|
return id(self)
|
{code_loader-1.0.147 → code_loader-1.0.148}/code_loader/inner_leap_binder/leapbinder_decorators.py
RENAMED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
# mypy: ignore-errors
|
|
2
2
|
import os
|
|
3
|
+
import warnings
|
|
3
4
|
import logging
|
|
4
5
|
from collections import defaultdict
|
|
5
6
|
from functools import lru_cache
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
from typing import Optional, Union, Callable, List, Dict, Set, Any
|
|
9
|
+
from typing import Optional, Union, Callable, List, Dict, get_args, get_origin
|
|
8
10
|
|
|
9
11
|
import numpy as np
|
|
10
12
|
import numpy.typing as npt
|
|
@@ -29,6 +31,115 @@ import functools
|
|
|
29
31
|
|
|
30
32
|
_called_from_inside_tl_decorator = 0
|
|
31
33
|
_called_from_inside_tl_integration_test_decorator = False
|
|
34
|
+
_call_from_tl_platform = os.environ.get('IS_TENSORLEAP_PLATFORM') == 'true'
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def validate_args_structure(*args, types_order, func_name, expected_names, **kwargs):
|
|
38
|
+
def _type_to_str(t):
|
|
39
|
+
origin = get_origin(t)
|
|
40
|
+
if origin is Union:
|
|
41
|
+
return " | ".join(tt.__name__ for tt in get_args(t))
|
|
42
|
+
elif hasattr(t, "__name__"):
|
|
43
|
+
return t.__name__
|
|
44
|
+
else:
|
|
45
|
+
return str(t)
|
|
46
|
+
|
|
47
|
+
def _format_types(types, names=None):
|
|
48
|
+
return ", ".join(
|
|
49
|
+
f"{(names[i] + ': ') if names else f'arg{i}: '}{_type_to_str(ty)}"
|
|
50
|
+
for i, ty in enumerate(types)
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
if expected_names:
|
|
54
|
+
normalized_args = []
|
|
55
|
+
for i, name in enumerate(expected_names):
|
|
56
|
+
if i < len(args):
|
|
57
|
+
normalized_args.append(args[i])
|
|
58
|
+
elif name in kwargs:
|
|
59
|
+
normalized_args.append(kwargs[name])
|
|
60
|
+
else:
|
|
61
|
+
raise AssertionError(
|
|
62
|
+
f"{func_name} validation failed: "
|
|
63
|
+
f"Missing required argument '{name}'. "
|
|
64
|
+
f"Expected arguments: {expected_names}."
|
|
65
|
+
)
|
|
66
|
+
else:
|
|
67
|
+
normalized_args = list(args)
|
|
68
|
+
if len(normalized_args) != len(types_order):
|
|
69
|
+
expected = _format_types(types_order, expected_names)
|
|
70
|
+
got_types = ", ".join(type(arg).__name__ for arg in normalized_args)
|
|
71
|
+
raise AssertionError(
|
|
72
|
+
f"{func_name} validation failed: "
|
|
73
|
+
f"Expected exactly {len(types_order)} arguments ({expected}), "
|
|
74
|
+
f"but got {len(normalized_args)} argument(s) of type(s): ({got_types}). "
|
|
75
|
+
f"Correct usage example: {func_name}({expected})"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
for i, (arg, expected_type) in enumerate(zip(normalized_args, types_order)):
|
|
79
|
+
origin = get_origin(expected_type)
|
|
80
|
+
if origin is Union:
|
|
81
|
+
allowed_types = get_args(expected_type)
|
|
82
|
+
else:
|
|
83
|
+
allowed_types = (expected_type,)
|
|
84
|
+
|
|
85
|
+
if not isinstance(arg, allowed_types):
|
|
86
|
+
allowed_str = " | ".join(t.__name__ for t in allowed_types)
|
|
87
|
+
raise AssertionError(
|
|
88
|
+
f"{func_name} validation failed: "
|
|
89
|
+
f"Argument '{expected_names[i] if expected_names else f'arg{i}'}' "
|
|
90
|
+
f"expected type {allowed_str}, but got {type(arg).__name__}. "
|
|
91
|
+
f"Correct usage example: {func_name}({_format_types(types_order, expected_names)})"
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def validate_output_structure(result, func_name: str, expected_type_name="np.ndarray", gt_flag=False):
|
|
96
|
+
if result is None or (isinstance(result, float) and np.isnan(result)):
|
|
97
|
+
if gt_flag:
|
|
98
|
+
raise AssertionError(
|
|
99
|
+
f"{func_name} validation failed: "
|
|
100
|
+
f"The function returned {result!r}. "
|
|
101
|
+
f"If you are working with an unlabeled dataset and no ground truth is available, "
|
|
102
|
+
f"use 'return np.array([], dtype=np.float32)' instead. "
|
|
103
|
+
f"Otherwise, {func_name} expected a single {expected_type_name} object. "
|
|
104
|
+
f"Make sure the function ends with 'return <{expected_type_name}>'."
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
raise AssertionError(
|
|
108
|
+
f"{func_name} validation failed: "
|
|
109
|
+
f"The function returned None. "
|
|
110
|
+
f"Expected a single {expected_type_name} object. "
|
|
111
|
+
f"Make sure the function ends with 'return <{expected_type_name}>'."
|
|
112
|
+
)
|
|
113
|
+
if isinstance(result, tuple):
|
|
114
|
+
element_descriptions = [
|
|
115
|
+
f"[{i}] type: {type(r).__name__}"
|
|
116
|
+
for i, r in enumerate(result)
|
|
117
|
+
]
|
|
118
|
+
element_summary = "\n ".join(element_descriptions)
|
|
119
|
+
|
|
120
|
+
raise AssertionError(
|
|
121
|
+
f"{func_name} validation failed: "
|
|
122
|
+
f"The function returned multiple outputs ({len(result)} values), "
|
|
123
|
+
f"but only a single {expected_type_name} is allowed.\n\n"
|
|
124
|
+
f"Returned elements:\n"
|
|
125
|
+
f" {element_summary}\n\n"
|
|
126
|
+
f"Correct usage example:\n"
|
|
127
|
+
f" def {func_name}(...):\n"
|
|
128
|
+
f" return <{expected_type_name}>\n\n"
|
|
129
|
+
f"If you intended to return multiple values, combine them into a single "
|
|
130
|
+
f"{expected_type_name} (e.g., by concatenation or stacking)."
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def batch_warning(result, func_name):
|
|
135
|
+
if result.shape[0] == 1:
|
|
136
|
+
warnings.warn(
|
|
137
|
+
f"{func_name} warning: Tensorleap will add a batch dimension at axis 0 to the output of {func_name}, "
|
|
138
|
+
f"although the detected size of axis 0 is already 1. "
|
|
139
|
+
f"This may lead to an extra batch dimension (e.g., shape (1, 1, ...)). "
|
|
140
|
+
f"Please ensure that the output of '{func_name}' is not already batched "
|
|
141
|
+
f"to avoid computation errors."
|
|
142
|
+
)
|
|
32
143
|
|
|
33
144
|
|
|
34
145
|
def _add_mapping_connection(user_unique_name, connection_destinations, arg_names, name, node_mapping_type):
|
|
@@ -53,7 +164,19 @@ def tensorleap_integration_test():
|
|
|
53
164
|
def decorating_function(integration_test_function: Callable):
|
|
54
165
|
leap_binder.integration_test_func = integration_test_function
|
|
55
166
|
|
|
167
|
+
def _validate_input_args(*args, **kwargs):
|
|
168
|
+
sample_id, preprocess_response = args
|
|
169
|
+
assert type(sample_id) == preprocess_response.sample_id_type, (
|
|
170
|
+
f"tensorleap_integration_test validation failed: "
|
|
171
|
+
f"sample_id type ({type(sample_id).__name__}) does not match the expected "
|
|
172
|
+
f"type ({preprocess_response.sample_id_type}) from the PreprocessResponse."
|
|
173
|
+
)
|
|
174
|
+
|
|
56
175
|
def inner(*args, **kwargs):
|
|
176
|
+
validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
|
|
177
|
+
func_name='integration_test', expected_names=["idx", "preprocess"], **kwargs)
|
|
178
|
+
_validate_input_args(*args, **kwargs)
|
|
179
|
+
|
|
57
180
|
global _called_from_inside_tl_integration_test_decorator
|
|
58
181
|
# Clear integration test events for new test
|
|
59
182
|
try:
|
|
@@ -62,6 +185,9 @@ def tensorleap_integration_test():
|
|
|
62
185
|
logger.debug(f"Failed to clear integration events: {e}")
|
|
63
186
|
try:
|
|
64
187
|
_called_from_inside_tl_integration_test_decorator = True
|
|
188
|
+
if not _call_from_tl_platform:
|
|
189
|
+
update_env_params_func("tensorleap_integration_test",
|
|
190
|
+
"v") # put here because otherwise it will become v only if it finishes all the script
|
|
65
191
|
ret = integration_test_function(*args, **kwargs)
|
|
66
192
|
|
|
67
193
|
try:
|
|
@@ -73,12 +199,15 @@ def tensorleap_integration_test():
|
|
|
73
199
|
file_name = Path(first_tb.filename).name
|
|
74
200
|
line_number = first_tb.lineno
|
|
75
201
|
if isinstance(e, TypeError) and 'is not subscriptable' in str(e):
|
|
76
|
-
|
|
77
|
-
|
|
202
|
+
update_env_params_func("code_mapping", "x")
|
|
203
|
+
raise (f'Invalid integration code. File {file_name}, line {line_number}: '
|
|
204
|
+
f"indexing is supported only on the model's predictions inside the integration test. Please remove this indexing operation usage from the integration test code.")
|
|
78
205
|
else:
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
206
|
+
update_env_params_func("code_mapping", "x")
|
|
207
|
+
|
|
208
|
+
raise (f'Invalid integration code. File {file_name}, line {line_number}: '
|
|
209
|
+
f'Integration test is only allowed to call Tensorleap decorators. '
|
|
210
|
+
f'Ensure any arithmetics, external library use, Python logic is placed within Tensorleap decoders')
|
|
82
211
|
finally:
|
|
83
212
|
if mapping_runtime_mode_env_var_mame in os.environ:
|
|
84
213
|
del os.environ[mapping_runtime_mode_env_var_mame]
|
|
@@ -91,6 +220,7 @@ def tensorleap_integration_test():
|
|
|
91
220
|
|
|
92
221
|
return decorating_function
|
|
93
222
|
|
|
223
|
+
|
|
94
224
|
def _safe_get_item(key):
|
|
95
225
|
try:
|
|
96
226
|
return NodeMappingType[f'Input{str(key)}']
|
|
@@ -99,34 +229,76 @@ def _safe_get_item(key):
|
|
|
99
229
|
|
|
100
230
|
|
|
101
231
|
def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]] = []):
|
|
232
|
+
assert isinstance(prediction_types, list), (
|
|
233
|
+
f"tensorleap_load_model validation failed: "
|
|
234
|
+
f" prediction_types is an optional argument of type List[PredictionTypeHandler]] but got {type(prediction_types).__name__}."
|
|
235
|
+
)
|
|
102
236
|
for i, prediction_type in enumerate(prediction_types):
|
|
237
|
+
assert isinstance(prediction_type, PredictionTypeHandler), (f"tensorleap_load_model validation failed: "
|
|
238
|
+
f" prediction_types at position {i} must be of type PredictionTypeHandler but got {type(prediction_types[i]).__name__}.")
|
|
103
239
|
leap_binder.add_prediction(prediction_type.name, prediction_type.labels, prediction_type.channel_dim, i)
|
|
104
240
|
|
|
105
|
-
def
|
|
241
|
+
def _validate_result(result) -> None:
|
|
242
|
+
valid_types = ["onnxruntime", "keras"]
|
|
243
|
+
err_message = f"tensorleap_load_model validation failed:\nSupported models are Keras and onnxruntime only and non of them was returned."
|
|
244
|
+
validate_output_structure(result, func_name="tensorleap_load_model",
|
|
245
|
+
expected_type_name=[" | ".join(t for t in valid_types)][0])
|
|
246
|
+
try:
|
|
247
|
+
import keras
|
|
248
|
+
except ImportError:
|
|
249
|
+
keras = None
|
|
250
|
+
try:
|
|
251
|
+
import tensorflow as tf
|
|
252
|
+
except ImportError:
|
|
253
|
+
tf = None
|
|
254
|
+
try:
|
|
255
|
+
import onnxruntime
|
|
256
|
+
except ImportError:
|
|
257
|
+
onnxruntime = None
|
|
258
|
+
|
|
259
|
+
if not keras and not onnxruntime:
|
|
260
|
+
raise AssertionError(err_message)
|
|
261
|
+
|
|
262
|
+
is_keras_model = (
|
|
263
|
+
bool(keras and isinstance(result, getattr(keras, "Model", tuple())))
|
|
264
|
+
or bool(tf and isinstance(result, getattr(tf.keras, "Model", tuple())))
|
|
265
|
+
)
|
|
266
|
+
is_onnx_model = bool(onnxruntime and isinstance(result, onnxruntime.InferenceSession))
|
|
267
|
+
|
|
268
|
+
if not any([is_keras_model, is_onnx_model]):
|
|
269
|
+
raise AssertionError(err_message)
|
|
270
|
+
|
|
271
|
+
def decorating_function(load_model_func, prediction_types=prediction_types):
|
|
106
272
|
class TempMapping:
|
|
107
273
|
pass
|
|
108
274
|
|
|
109
275
|
@lru_cache()
|
|
110
|
-
def inner():
|
|
276
|
+
def inner(*args, **kwargs):
|
|
277
|
+
validate_args_structure(*args, types_order=[],
|
|
278
|
+
func_name='tensorleap_load_model', expected_names=[], **kwargs)
|
|
279
|
+
|
|
111
280
|
class ModelPlaceholder:
|
|
112
|
-
def __init__(self):
|
|
113
|
-
self.model = load_model_func()
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
emit_integration_event_once(AnalyticsEvent.LOAD_MODEL_INTEGRATION_TEST, {
|
|
117
|
-
'prediction_types_count': len(prediction_types)
|
|
118
|
-
})
|
|
119
|
-
except Exception as e:
|
|
120
|
-
logger.debug(f"Failed to emit load_model integration test event: {e}")
|
|
281
|
+
def __init__(self, prediction_types):
|
|
282
|
+
self.model = load_model_func() # TODO- check why this fails on onnx model
|
|
283
|
+
self.prediction_types = prediction_types
|
|
284
|
+
_validate_result(self.model)
|
|
121
285
|
|
|
122
286
|
# keras interface
|
|
123
287
|
def __call__(self, arg):
|
|
124
288
|
ret = self.model(arg)
|
|
289
|
+
self.validate_declared_prediction_types(ret)
|
|
125
290
|
if isinstance(ret, list):
|
|
126
291
|
return [r.numpy() for r in ret]
|
|
127
|
-
|
|
128
292
|
return ret.numpy()
|
|
129
293
|
|
|
294
|
+
def validate_declared_prediction_types(self, ret):
|
|
295
|
+
if not (len(self.prediction_types) == len(ret) if isinstance(ret, list) else 1) or len(
|
|
296
|
+
self.prediction_types) == 0:
|
|
297
|
+
if not _call_from_tl_platform:
|
|
298
|
+
update_env_params_func("tensorleap_load_model", "x")
|
|
299
|
+
raise Exception(
|
|
300
|
+
f"tensorleap_load_model validation failed: number of declared prediction types({len(prediction_types)}) != number of model outputs({len(ret) if isinstance(ret, list) else 1})")
|
|
301
|
+
|
|
130
302
|
def _convert_onnx_inputs_to_correct_type(
|
|
131
303
|
self, float_arrays_inputs: Dict[str, np.ndarray]
|
|
132
304
|
) -> Dict[str, np.ndarray]:
|
|
@@ -180,12 +352,17 @@ def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]
|
|
|
180
352
|
# onnx runtime interface
|
|
181
353
|
def run(self, output_names, input_dict):
|
|
182
354
|
corrected_type_inputs = self._convert_onnx_inputs_to_correct_type(input_dict)
|
|
183
|
-
|
|
355
|
+
ret = self.model.run(output_names, corrected_type_inputs)
|
|
356
|
+
self.validate_declared_prediction_types(ret)
|
|
357
|
+
return ret
|
|
184
358
|
|
|
185
359
|
def get_inputs(self):
|
|
186
360
|
return self.model.get_inputs()
|
|
187
361
|
|
|
188
|
-
|
|
362
|
+
model_placeholder = ModelPlaceholder(prediction_types)
|
|
363
|
+
if not _call_from_tl_platform:
|
|
364
|
+
update_env_params_func("tensorleap_load_model", "v")
|
|
365
|
+
return model_placeholder
|
|
189
366
|
|
|
190
367
|
def mapping_inner():
|
|
191
368
|
class ModelOutputPlaceholder:
|
|
@@ -232,7 +409,7 @@ def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]
|
|
|
232
409
|
def get_inputs(self):
|
|
233
410
|
class FollowIndex:
|
|
234
411
|
def __init__(self, index):
|
|
235
|
-
self.name =
|
|
412
|
+
self.name = _safe_get_item(index)
|
|
236
413
|
|
|
237
414
|
class FollowInputIndex:
|
|
238
415
|
def __init__(self):
|
|
@@ -248,11 +425,11 @@ def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]
|
|
|
248
425
|
|
|
249
426
|
return ModelPlaceholder()
|
|
250
427
|
|
|
251
|
-
def final_inner():
|
|
428
|
+
def final_inner(*args, **kwargs):
|
|
252
429
|
if os.environ.get(mapping_runtime_mode_env_var_mame):
|
|
253
430
|
return mapping_inner()
|
|
254
431
|
else:
|
|
255
|
-
return inner()
|
|
432
|
+
return inner(*args, **kwargs)
|
|
256
433
|
|
|
257
434
|
return final_inner
|
|
258
435
|
|
|
@@ -268,77 +445,164 @@ def tensorleap_custom_metric(name: str,
|
|
|
268
445
|
def decorating_function(
|
|
269
446
|
user_function: Union[CustomCallableInterfaceMultiArgs, CustomMultipleReturnCallableInterfaceMultiArgs,
|
|
270
447
|
ConfusionMatrixCallableInterfaceMultiArgs]):
|
|
448
|
+
|
|
449
|
+
def _validate_decorators_signature():
|
|
450
|
+
err_message = f"{user_function.__name__} validation failed.\n"
|
|
451
|
+
if not isinstance(name, str):
|
|
452
|
+
raise TypeError(err_message + f"`name` must be a string, got type {type(name).__name__}.")
|
|
453
|
+
valid_directions = {MetricDirection.Upward, MetricDirection.Downward}
|
|
454
|
+
if isinstance(direction, MetricDirection):
|
|
455
|
+
if direction not in valid_directions:
|
|
456
|
+
raise ValueError(
|
|
457
|
+
err_message +
|
|
458
|
+
f"Invalid MetricDirection: {direction}. Must be one of {valid_directions}, "
|
|
459
|
+
f"got type {type(direction).__name__}."
|
|
460
|
+
)
|
|
461
|
+
elif isinstance(direction, dict):
|
|
462
|
+
if not all(isinstance(k, str) for k in direction.keys()):
|
|
463
|
+
invalid_keys = {k: type(k).__name__ for k in direction.keys() if not isinstance(k, str)}
|
|
464
|
+
raise TypeError(
|
|
465
|
+
err_message +
|
|
466
|
+
f"All keys in `direction` must be strings, got invalid key types: {invalid_keys}."
|
|
467
|
+
)
|
|
468
|
+
for k, v in direction.items():
|
|
469
|
+
if v not in valid_directions:
|
|
470
|
+
raise ValueError(
|
|
471
|
+
err_message +
|
|
472
|
+
f"Invalid direction for key '{k}': {v}. Must be one of {valid_directions}, "
|
|
473
|
+
f"got type {type(v).__name__}."
|
|
474
|
+
)
|
|
475
|
+
else:
|
|
476
|
+
raise TypeError(
|
|
477
|
+
err_message +
|
|
478
|
+
f"`direction` must be a MetricDirection or a Dict[str, MetricDirection], "
|
|
479
|
+
f"got type {type(direction).__name__}."
|
|
480
|
+
)
|
|
481
|
+
if compute_insights is not None:
|
|
482
|
+
if not isinstance(compute_insights, (bool, dict)):
|
|
483
|
+
raise TypeError(
|
|
484
|
+
err_message +
|
|
485
|
+
f"`compute_insights` must be a bool or a Dict[str, bool], "
|
|
486
|
+
f"got type {type(compute_insights).__name__}."
|
|
487
|
+
)
|
|
488
|
+
if isinstance(compute_insights, dict):
|
|
489
|
+
if not all(isinstance(k, str) for k in compute_insights.keys()):
|
|
490
|
+
invalid_keys = {k: type(k).__name__ for k in compute_insights.keys() if not isinstance(k, str)}
|
|
491
|
+
raise TypeError(
|
|
492
|
+
err_message +
|
|
493
|
+
f"All keys in `compute_insights` must be strings, got invalid key types: {invalid_keys}."
|
|
494
|
+
)
|
|
495
|
+
for k, v in compute_insights.items():
|
|
496
|
+
if not isinstance(v, bool):
|
|
497
|
+
raise TypeError(
|
|
498
|
+
err_message +
|
|
499
|
+
f"Invalid type for compute_insights['{k}']: expected bool, got type {type(v).__name__}."
|
|
500
|
+
)
|
|
501
|
+
if connects_to is not None:
|
|
502
|
+
valid_types = (str, list, tuple, set)
|
|
503
|
+
if not isinstance(connects_to, valid_types):
|
|
504
|
+
raise TypeError(
|
|
505
|
+
err_message +
|
|
506
|
+
f"`connects_to` must be one of {valid_types}, got type {type(connects_to).__name__}."
|
|
507
|
+
)
|
|
508
|
+
if isinstance(connects_to, (list, tuple, set)):
|
|
509
|
+
invalid_elems = [f"{type(e).__name__}" for e in connects_to if not isinstance(e, str)]
|
|
510
|
+
if invalid_elems:
|
|
511
|
+
raise TypeError(
|
|
512
|
+
err_message +
|
|
513
|
+
f"All elements in `connects_to` must be strings, "
|
|
514
|
+
f"but found element types: {invalid_elems}."
|
|
515
|
+
)
|
|
516
|
+
|
|
517
|
+
_validate_decorators_signature()
|
|
518
|
+
|
|
271
519
|
for metric_handler in leap_binder.setup_container.metrics:
|
|
272
520
|
if metric_handler.metric_handler_data.name == name:
|
|
273
521
|
raise Exception(f'Metric with name {name} already exists. '
|
|
274
522
|
f'Please choose another')
|
|
275
523
|
|
|
276
524
|
def _validate_input_args(*args, **kwargs) -> None:
|
|
525
|
+
assert len(args) + len(kwargs) > 0, (
|
|
526
|
+
f"{user_function.__name__}() validation failed: "
|
|
527
|
+
f"Expected at least one positional|key-word argument of type np.ndarray, "
|
|
528
|
+
f"but received none. "
|
|
529
|
+
f"Correct usage example: tensorleap_custom_metric(input_array: np.ndarray, ...)"
|
|
530
|
+
)
|
|
277
531
|
for i, arg in enumerate(args):
|
|
278
532
|
assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
|
|
279
|
-
f'
|
|
533
|
+
f'{user_function.__name__}() validation failed: '
|
|
280
534
|
f'Argument #{i} should be a numpy array. Got {type(arg)}.')
|
|
281
535
|
if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
|
|
282
536
|
assert arg.shape[0] == leap_binder.batch_size_to_validate, \
|
|
283
|
-
(f'
|
|
537
|
+
(f'{user_function.__name__}() validation failed: Argument #{i} '
|
|
284
538
|
f'first dim should be as the batch size. Got {arg.shape[0]} '
|
|
285
539
|
f'instead of {leap_binder.batch_size_to_validate}')
|
|
286
540
|
|
|
287
541
|
for _arg_name, arg in kwargs.items():
|
|
288
542
|
assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
|
|
289
|
-
f'
|
|
543
|
+
f'{user_function.__name__}() validation failed: '
|
|
290
544
|
f'Argument {_arg_name} should be a numpy array. Got {type(arg)}.')
|
|
291
545
|
if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
|
|
292
546
|
assert arg.shape[0] == leap_binder.batch_size_to_validate, \
|
|
293
|
-
(f'
|
|
547
|
+
(f'{user_function.__name__}() validation failed: Argument {_arg_name} '
|
|
294
548
|
f'first dim should be as the batch size. Got {arg.shape[0]} '
|
|
295
549
|
f'instead of {leap_binder.batch_size_to_validate}')
|
|
296
550
|
|
|
297
551
|
def _validate_result(result) -> None:
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
552
|
+
validate_output_structure(result, func_name=user_function.__name__,
|
|
553
|
+
expected_type_name="List[float | int | None | List[ConfusionMatrixElement] ] | NDArray[np.float32] or dictonary with one of these types as its values types")
|
|
554
|
+
supported_types_message = (f'{user_function.__name__}() validation failed: '
|
|
555
|
+
f'{user_function.__name__}() has returned unsupported type.\nSupported types are List[float|int|None], '
|
|
556
|
+
f'List[List[ConfusionMatrixElement]], NDArray[np.float32] or dictonary with one of these types as its values types. ')
|
|
301
557
|
|
|
302
|
-
def _validate_single_metric(single_metric_result):
|
|
558
|
+
def _validate_single_metric(single_metric_result, key=None):
|
|
303
559
|
if isinstance(single_metric_result, list):
|
|
304
560
|
if isinstance(single_metric_result[0], list):
|
|
305
|
-
assert isinstance(single_metric_result[0]
|
|
306
|
-
f
|
|
561
|
+
assert all(isinstance(cm, ConfusionMatrixElement) for cm in single_metric_result[0]), (
|
|
562
|
+
f"{supported_types_message} "
|
|
563
|
+
f"Got {'a dict where the value of ' + str(key) + ' is of type ' if key is not None else ''}"
|
|
564
|
+
f"List[List[{', '.join(type(cm).__name__ for cm in single_metric_result[0])}]]."
|
|
565
|
+
)
|
|
566
|
+
|
|
307
567
|
else:
|
|
308
|
-
assert isinstance(single_metric_result
|
|
309
|
-
|
|
310
|
-
|
|
568
|
+
assert all(isinstance(v, (float, int, type(None), np.float32)) for v in single_metric_result), (
|
|
569
|
+
f"{supported_types_message}\n"
|
|
570
|
+
f"Got {'a dict where the value of ' + str(key) + ' is of type ' if key is not None else ''}"
|
|
571
|
+
f"List[{', '.join(type(v).__name__ for v in single_metric_result)}]."
|
|
572
|
+
)
|
|
311
573
|
else:
|
|
312
574
|
assert isinstance(single_metric_result,
|
|
313
|
-
np.ndarray), f'{supported_types_message}
|
|
314
|
-
assert len(single_metric_result.shape) == 1, (f'
|
|
575
|
+
np.ndarray), f'{supported_types_message}\nGot {type(single_metric_result)}.'
|
|
576
|
+
assert len(single_metric_result.shape) == 1, (f'{user_function.__name__}() validation failed: '
|
|
315
577
|
f'The return shape should be 1D. Got {len(single_metric_result.shape)}D.')
|
|
316
578
|
|
|
317
579
|
if leap_binder.batch_size_to_validate:
|
|
318
580
|
assert len(single_metric_result) == leap_binder.batch_size_to_validate, \
|
|
319
|
-
f'
|
|
581
|
+
f'{user_function.__name__}() validation failed: The return len {f"of srt{key} value" if key is not None else ""} should be as the batch size.'
|
|
320
582
|
|
|
321
583
|
if isinstance(result, dict):
|
|
322
584
|
for key, value in result.items():
|
|
585
|
+
_validate_single_metric(value, key)
|
|
586
|
+
|
|
323
587
|
assert isinstance(key, str), \
|
|
324
|
-
(f'
|
|
588
|
+
(f'{user_function.__name__}() validation failed: '
|
|
325
589
|
f'Keys in the return dict should be of type str. Got {type(key)}.')
|
|
326
590
|
_validate_single_metric(value)
|
|
327
591
|
|
|
328
592
|
if isinstance(direction, dict):
|
|
329
593
|
for direction_key in direction:
|
|
330
594
|
assert direction_key in result, \
|
|
331
|
-
(f'
|
|
595
|
+
(f'{user_function.__name__}() validation failed: '
|
|
332
596
|
f'Keys in the direction mapping should be part of result keys. Got key {direction_key}.')
|
|
333
597
|
|
|
334
598
|
if compute_insights is not None:
|
|
335
599
|
assert isinstance(compute_insights, dict), \
|
|
336
|
-
(f'
|
|
600
|
+
(f'{user_function.__name__}() validation failed: '
|
|
337
601
|
f'compute_insights should be dict if using the dict results. Got {type(compute_insights)}.')
|
|
338
602
|
|
|
339
603
|
for ci_key in compute_insights:
|
|
340
604
|
assert ci_key in result, \
|
|
341
|
-
(f'
|
|
605
|
+
(f'{user_function.__name__}() validation failed: '
|
|
342
606
|
f'Keys in the compute_insights mapping should be part of result keys. Got key {ci_key}.')
|
|
343
607
|
|
|
344
608
|
else:
|
|
@@ -346,7 +610,7 @@ def tensorleap_custom_metric(name: str,
|
|
|
346
610
|
|
|
347
611
|
if compute_insights is not None:
|
|
348
612
|
assert isinstance(compute_insights, bool), \
|
|
349
|
-
(f'
|
|
613
|
+
(f'{user_function.__name__}() validation failed: '
|
|
350
614
|
f'compute_insights should be boolean. Got {type(compute_insights)}.')
|
|
351
615
|
|
|
352
616
|
@functools.wraps(user_function)
|
|
@@ -378,6 +642,8 @@ def tensorleap_custom_metric(name: str,
|
|
|
378
642
|
result = inner_without_validate(*args, **kwargs)
|
|
379
643
|
|
|
380
644
|
_validate_result(result)
|
|
645
|
+
if not _call_from_tl_platform:
|
|
646
|
+
update_env_params_func("tensorleap_custom_metric", "v")
|
|
381
647
|
return result
|
|
382
648
|
|
|
383
649
|
def mapping_inner(*args, **kwargs):
|
|
@@ -417,28 +683,38 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
|
|
|
417
683
|
name_to_unique_name = defaultdict(set)
|
|
418
684
|
|
|
419
685
|
def decorating_function(user_function: VisualizerCallableInterface):
|
|
686
|
+
assert isinstance(visualizer_type, LeapDataType), (f"{user_function.__name__} validation failed: "
|
|
687
|
+
f"visualizer_type should be of type {LeapDataType.__name__} but got {type(visualizer_type)}"
|
|
688
|
+
)
|
|
689
|
+
|
|
420
690
|
for viz_handler in leap_binder.setup_container.visualizers:
|
|
421
691
|
if viz_handler.visualizer_handler_data.name == name:
|
|
422
692
|
raise Exception(f'Visualizer with name {name} already exists. '
|
|
423
693
|
f'Please choose another')
|
|
424
694
|
|
|
425
695
|
def _validate_input_args(*args, **kwargs):
|
|
696
|
+
assert len(args) + len(kwargs) > 0, (
|
|
697
|
+
f"{user_function.__name__}() validation failed: "
|
|
698
|
+
f"Expected at least one positional|key-word argument of type np.ndarray, "
|
|
699
|
+
f"but received none. "
|
|
700
|
+
f"Correct usage example: {user_function.__name__}(input_array: np.ndarray, ...)"
|
|
701
|
+
)
|
|
426
702
|
for i, arg in enumerate(args):
|
|
427
703
|
assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
|
|
428
|
-
f'
|
|
704
|
+
f'{user_function.__name__}() validation failed: '
|
|
429
705
|
f'Argument #{i} should be a numpy array. Got {type(arg)}.')
|
|
430
706
|
if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
|
|
431
707
|
assert arg.shape[0] != leap_binder.batch_size_to_validate, \
|
|
432
|
-
(f'
|
|
708
|
+
(f'{user_function.__name__}() validation failed: '
|
|
433
709
|
f'Argument #{i} should be without batch dimension. ')
|
|
434
710
|
|
|
435
711
|
for _arg_name, arg in kwargs.items():
|
|
436
712
|
assert isinstance(arg, (np.ndarray, SamplePreprocessResponse)), (
|
|
437
|
-
f'
|
|
713
|
+
f'{user_function.__name__}() validation failed: '
|
|
438
714
|
f'Argument {_arg_name} should be a numpy array. Got {type(arg)}.')
|
|
439
715
|
if leap_binder.batch_size_to_validate and isinstance(arg, np.ndarray):
|
|
440
716
|
assert arg.shape[0] != leap_binder.batch_size_to_validate, \
|
|
441
|
-
(f'
|
|
717
|
+
(f'{user_function.__name__}() validation failed: Argument {_arg_name} '
|
|
442
718
|
f'should be without batch dimension. ')
|
|
443
719
|
|
|
444
720
|
def _validate_result(result):
|
|
@@ -452,8 +728,11 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
|
|
|
452
728
|
LeapDataType.ImageWithBBox: LeapImageWithBBox,
|
|
453
729
|
LeapDataType.ImageWithHeatmap: LeapImageWithHeatmap
|
|
454
730
|
}
|
|
731
|
+
validate_output_structure(result, func_name=user_function.__name__,
|
|
732
|
+
expected_type_name=result_type_map[visualizer_type])
|
|
733
|
+
|
|
455
734
|
assert isinstance(result, result_type_map[visualizer_type]), \
|
|
456
|
-
(f'
|
|
735
|
+
(f'{user_function.__name__}() validation failed: '
|
|
457
736
|
f'The return type should be {result_type_map[visualizer_type]}. Got {type(result)}.')
|
|
458
737
|
|
|
459
738
|
@functools.wraps(user_function)
|
|
@@ -485,6 +764,8 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
|
|
|
485
764
|
result = inner_without_validate(*args, **kwargs)
|
|
486
765
|
|
|
487
766
|
_validate_result(result)
|
|
767
|
+
if not _call_from_tl_platform:
|
|
768
|
+
update_env_params_func("tensorleap_custom_visualizer", "v")
|
|
488
769
|
return result
|
|
489
770
|
|
|
490
771
|
def mapping_inner(*args, **kwargs):
|
|
@@ -526,30 +807,26 @@ def tensorleap_metadata(
|
|
|
526
807
|
f'Please choose another')
|
|
527
808
|
|
|
528
809
|
def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
|
|
529
|
-
assert isinstance(sample_id, (int, str)), \
|
|
530
|
-
(f'tensorleap_metadata validation failed: '
|
|
531
|
-
f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
|
|
532
|
-
assert isinstance(preprocess_response, PreprocessResponse), \
|
|
533
|
-
(f'tensorleap_metadata validation failed: '
|
|
534
|
-
f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
|
|
535
810
|
assert type(sample_id) == preprocess_response.sample_id_type, \
|
|
536
|
-
(f'
|
|
811
|
+
(f'{user_function.__name__}() validation failed: '
|
|
537
812
|
f'Argument sample_id should be as the same type as defined in the preprocess response '
|
|
538
813
|
f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
|
|
539
814
|
|
|
540
815
|
def _validate_result(result):
|
|
541
816
|
supported_result_types = (type(None), int, str, bool, float, dict, np.floating,
|
|
542
817
|
np.bool_, np.unsignedinteger, np.signedinteger, np.integer)
|
|
818
|
+
validate_output_structure(result, func_name=user_function.__name__,
|
|
819
|
+
expected_type_name=supported_result_types)
|
|
543
820
|
assert isinstance(result, supported_result_types), \
|
|
544
|
-
(f'
|
|
821
|
+
(f'{user_function.__name__}() validation failed: '
|
|
545
822
|
f'Unsupported return type. Got {type(result)}. should be any of {str(supported_result_types)}')
|
|
546
823
|
if isinstance(result, dict):
|
|
547
824
|
for key, value in result.items():
|
|
548
825
|
assert isinstance(key, str), \
|
|
549
|
-
(f'
|
|
826
|
+
(f'{user_function.__name__}() validation failed: '
|
|
550
827
|
f'Keys in the return dict should be of type str. Got {type(key)}.')
|
|
551
828
|
assert isinstance(value, supported_result_types), \
|
|
552
|
-
(f'
|
|
829
|
+
(f'{user_function.__name__}() validation failed: '
|
|
553
830
|
f'Values in the return dict should be of type {str(supported_result_types)}. Got {type(value)}.')
|
|
554
831
|
|
|
555
832
|
def inner_without_validate(sample_id, preprocess_response):
|
|
@@ -566,15 +843,19 @@ def tensorleap_metadata(
|
|
|
566
843
|
|
|
567
844
|
leap_binder.set_metadata(inner_without_validate, name, metadata_type)
|
|
568
845
|
|
|
569
|
-
def inner(
|
|
846
|
+
def inner(*args, **kwargs):
|
|
570
847
|
if os.environ.get(mapping_runtime_mode_env_var_mame):
|
|
571
848
|
return None
|
|
572
|
-
|
|
849
|
+
validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
|
|
850
|
+
func_name=user_function.__name__, expected_names=["idx", "preprocess"], **kwargs)
|
|
851
|
+
sample_id, preprocess_response = args if len(args) != 0 else kwargs.values()
|
|
573
852
|
_validate_input_args(sample_id, preprocess_response)
|
|
574
853
|
|
|
575
854
|
result = inner_without_validate(sample_id, preprocess_response)
|
|
576
855
|
|
|
577
856
|
_validate_result(result)
|
|
857
|
+
if not _call_from_tl_platform:
|
|
858
|
+
update_env_params_func("tensorleap_metadata", "v")
|
|
578
859
|
return result
|
|
579
860
|
|
|
580
861
|
return inner
|
|
@@ -582,7 +863,6 @@ def tensorleap_metadata(
|
|
|
582
863
|
return decorating_function
|
|
583
864
|
|
|
584
865
|
|
|
585
|
-
|
|
586
866
|
def tensorleap_custom_latent_space():
|
|
587
867
|
def decorating_function(user_function: SectionCallableInterface):
|
|
588
868
|
def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
|
|
@@ -636,20 +916,24 @@ def tensorleap_preprocess():
|
|
|
636
916
|
leap_binder.set_preprocess(user_function)
|
|
637
917
|
|
|
638
918
|
def _validate_input_args(*args, **kwargs):
|
|
639
|
-
assert len(args)
|
|
640
|
-
(f'
|
|
919
|
+
assert len(args) + len(kwargs) == 0, \
|
|
920
|
+
(f'{user_function.__name__}() validation failed: '
|
|
641
921
|
f'The function should not take any arguments. Got {args} and {kwargs}.')
|
|
642
922
|
|
|
643
923
|
def _validate_result(result):
|
|
644
|
-
assert isinstance(result, list),
|
|
645
|
-
(
|
|
646
|
-
|
|
924
|
+
assert isinstance(result, list), (
|
|
925
|
+
f"{user_function.__name__}() validation failed: expected return type list[{PreprocessResponse.__name__}]"
|
|
926
|
+
f"(e.g., [PreprocessResponse1, PreprocessResponse2, ...]), but returned type is {type(result).__name__}."
|
|
927
|
+
if not isinstance(result, tuple)
|
|
928
|
+
else f"{user_function.__name__}() validation failed: expected to return a single list[{PreprocessResponse.__name__}] object, "
|
|
929
|
+
f"but returned {len(result)} objects instead."
|
|
930
|
+
)
|
|
647
931
|
for i, response in enumerate(result):
|
|
648
932
|
assert isinstance(response, PreprocessResponse), \
|
|
649
|
-
(f'
|
|
933
|
+
(f'{user_function.__name__}() validation failed: '
|
|
650
934
|
f'Element #{i} in the return list should be a PreprocessResponse. Got {type(response)}.')
|
|
651
935
|
assert len(set(result)) == len(result), \
|
|
652
|
-
(f'
|
|
936
|
+
(f'{user_function.__name__}() validation failed: '
|
|
653
937
|
f'The return list should not contain duplicate PreprocessResponse objects.')
|
|
654
938
|
|
|
655
939
|
def inner(*args, **kwargs):
|
|
@@ -657,7 +941,6 @@ def tensorleap_preprocess():
|
|
|
657
941
|
return [None, None, None, None]
|
|
658
942
|
|
|
659
943
|
_validate_input_args(*args, **kwargs)
|
|
660
|
-
|
|
661
944
|
result = user_function()
|
|
662
945
|
_validate_result(result)
|
|
663
946
|
|
|
@@ -668,7 +951,8 @@ def tensorleap_preprocess():
|
|
|
668
951
|
})
|
|
669
952
|
except Exception as e:
|
|
670
953
|
logger.debug(f"Failed to emit preprocess integration test event: {e}")
|
|
671
|
-
|
|
954
|
+
if not _call_from_tl_platform:
|
|
955
|
+
update_env_params_func("tensorleap_preprocess", "v")
|
|
672
956
|
return result
|
|
673
957
|
|
|
674
958
|
return inner
|
|
@@ -730,6 +1014,8 @@ def tensorleap_element_instance_preprocess(
|
|
|
730
1014
|
|
|
731
1015
|
result = user_function_instance()
|
|
732
1016
|
_validate_result(result)
|
|
1017
|
+
if not _call_from_tl_platform:
|
|
1018
|
+
update_env_params_func("tensorleap_preprocess", "v")
|
|
733
1019
|
return result
|
|
734
1020
|
|
|
735
1021
|
return inner
|
|
@@ -812,6 +1098,7 @@ def tensorleap_instances_masks_encoder(name: str):
|
|
|
812
1098
|
|
|
813
1099
|
return decorating_function
|
|
814
1100
|
|
|
1101
|
+
|
|
815
1102
|
def tensorleap_instances_length_encoder(name: str):
|
|
816
1103
|
def decorating_function(user_function: InstanceLengthCallableInterface):
|
|
817
1104
|
def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse):
|
|
@@ -857,6 +1144,7 @@ def tensorleap_instances_length_encoder(name: str):
|
|
|
857
1144
|
|
|
858
1145
|
return decorating_function
|
|
859
1146
|
|
|
1147
|
+
|
|
860
1148
|
def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
|
|
861
1149
|
def decorating_function(user_function: SectionCallableInterface):
|
|
862
1150
|
for input_handler in leap_binder.setup_container.inputs:
|
|
@@ -867,29 +1155,23 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
|
|
|
867
1155
|
raise Exception(f"Channel dim for input {name} is expected to be either -1 or positive")
|
|
868
1156
|
|
|
869
1157
|
def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
|
|
870
|
-
assert isinstance(sample_id, (int, str)), \
|
|
871
|
-
(f'tensorleap_input_encoder validation failed: '
|
|
872
|
-
f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
|
|
873
|
-
assert isinstance(preprocess_response, PreprocessResponse), \
|
|
874
|
-
(f'tensorleap_input_encoder validation failed: '
|
|
875
|
-
f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
|
|
876
1158
|
assert type(sample_id) == preprocess_response.sample_id_type, \
|
|
877
|
-
(f'
|
|
1159
|
+
(f'{user_function.__name__}() validation failed: '
|
|
878
1160
|
f'Argument sample_id should be as the same type as defined in the preprocess response '
|
|
879
1161
|
f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
|
|
880
1162
|
|
|
881
1163
|
def _validate_result(result):
|
|
1164
|
+
validate_output_structure(result, func_name=user_function.__name__, expected_type_name="np.ndarray")
|
|
882
1165
|
assert isinstance(result, np.ndarray), \
|
|
883
|
-
(f'
|
|
1166
|
+
(f'{user_function.__name__}() validation failed: '
|
|
884
1167
|
f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
|
|
885
1168
|
assert result.dtype == np.float32, \
|
|
886
|
-
(f'
|
|
1169
|
+
(f'{user_function.__name__}() validation failed: '
|
|
887
1170
|
f'The return type should be a numpy array of type float32. Got {result.dtype}.')
|
|
888
|
-
assert channel_dim - 1 <= len(result.shape), (f'
|
|
1171
|
+
assert channel_dim - 1 <= len(result.shape), (f'{user_function.__name__}() validation failed: '
|
|
889
1172
|
f'The channel_dim ({channel_dim}) should be <= to the rank of the resulting input rank ({len(result.shape)}).')
|
|
890
1173
|
|
|
891
1174
|
def inner_without_validate(sample_id, preprocess_response):
|
|
892
|
-
|
|
893
1175
|
global _called_from_inside_tl_decorator
|
|
894
1176
|
_called_from_inside_tl_decorator += 1
|
|
895
1177
|
|
|
@@ -902,8 +1184,10 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
|
|
|
902
1184
|
|
|
903
1185
|
leap_binder.set_input(inner_without_validate, name, channel_dim=channel_dim)
|
|
904
1186
|
|
|
905
|
-
|
|
906
|
-
|
|
1187
|
+
def inner(*args, **kwargs):
|
|
1188
|
+
validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
|
|
1189
|
+
func_name=user_function.__name__, expected_names=["idx", "preprocess"], **kwargs)
|
|
1190
|
+
sample_id, preprocess_response = args if len(args) != 0 else kwargs.values()
|
|
907
1191
|
_validate_input_args(sample_id, preprocess_response)
|
|
908
1192
|
|
|
909
1193
|
result = inner_without_validate(sample_id, preprocess_response)
|
|
@@ -911,6 +1195,7 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
|
|
|
911
1195
|
_validate_result(result)
|
|
912
1196
|
|
|
913
1197
|
if _called_from_inside_tl_decorator == 0 and _called_from_inside_tl_integration_test_decorator:
|
|
1198
|
+
batch_warning(result, user_function.__name__)
|
|
914
1199
|
result = np.expand_dims(result, axis=0)
|
|
915
1200
|
# Emit integration test event once per test
|
|
916
1201
|
try:
|
|
@@ -921,17 +1206,17 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
|
|
|
921
1206
|
})
|
|
922
1207
|
except Exception as e:
|
|
923
1208
|
logger.debug(f"Failed to emit input_encoder integration test event: {e}")
|
|
1209
|
+
if not _call_from_tl_platform:
|
|
1210
|
+
update_env_params_func("tensorleap_input_encoder", "v")
|
|
924
1211
|
|
|
925
1212
|
return result
|
|
926
1213
|
|
|
927
|
-
|
|
928
|
-
|
|
929
1214
|
node_mapping_type = NodeMappingType.Input
|
|
930
1215
|
if model_input_index is not None:
|
|
931
1216
|
node_mapping_type = NodeMappingType(f'Input{str(model_input_index)}')
|
|
932
1217
|
inner.node_mapping = NodeMapping(name, node_mapping_type)
|
|
933
1218
|
|
|
934
|
-
def mapping_inner(
|
|
1219
|
+
def mapping_inner(*args, **kwargs):
|
|
935
1220
|
class TempMapping:
|
|
936
1221
|
pass
|
|
937
1222
|
|
|
@@ -943,11 +1228,11 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
|
|
|
943
1228
|
|
|
944
1229
|
mapping_inner.node_mapping = NodeMapping(name, node_mapping_type)
|
|
945
1230
|
|
|
946
|
-
def final_inner(
|
|
1231
|
+
def final_inner(*args, **kwargs):
|
|
947
1232
|
if os.environ.get(mapping_runtime_mode_env_var_mame):
|
|
948
|
-
return mapping_inner(
|
|
1233
|
+
return mapping_inner(*args, **kwargs)
|
|
949
1234
|
else:
|
|
950
|
-
return inner(
|
|
1235
|
+
return inner(*args, **kwargs)
|
|
951
1236
|
|
|
952
1237
|
final_inner.node_mapping = NodeMapping(name, node_mapping_type)
|
|
953
1238
|
|
|
@@ -964,23 +1249,19 @@ def tensorleap_gt_encoder(name: str):
|
|
|
964
1249
|
f'Please choose another')
|
|
965
1250
|
|
|
966
1251
|
def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
|
|
967
|
-
assert isinstance(sample_id, (int, str)), \
|
|
968
|
-
(f'tensorleap_gt_encoder validation failed: '
|
|
969
|
-
f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
|
|
970
|
-
assert isinstance(preprocess_response, PreprocessResponse), \
|
|
971
|
-
(f'tensorleap_gt_encoder validation failed: '
|
|
972
|
-
f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
|
|
973
1252
|
assert type(sample_id) == preprocess_response.sample_id_type, \
|
|
974
|
-
(f'
|
|
1253
|
+
(f'{user_function.__name__}() validation failed: '
|
|
975
1254
|
f'Argument sample_id should be as the same type as defined in the preprocess response '
|
|
976
1255
|
f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
|
|
977
1256
|
|
|
978
1257
|
def _validate_result(result):
|
|
1258
|
+
validate_output_structure(result, func_name=user_function.__name__, expected_type_name="np.ndarray",
|
|
1259
|
+
gt_flag=True)
|
|
979
1260
|
assert isinstance(result, np.ndarray), \
|
|
980
|
-
(f'
|
|
1261
|
+
(f'{user_function.__name__}() validation failed: '
|
|
981
1262
|
f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
|
|
982
1263
|
assert result.dtype == np.float32, \
|
|
983
|
-
(f'
|
|
1264
|
+
(f'{user_function.__name__}() validation failed: '
|
|
984
1265
|
f'The return type should be a numpy array of type float32. Got {result.dtype}.')
|
|
985
1266
|
|
|
986
1267
|
def inner_without_validate(sample_id, preprocess_response):
|
|
@@ -996,8 +1277,10 @@ def tensorleap_gt_encoder(name: str):
|
|
|
996
1277
|
|
|
997
1278
|
leap_binder.set_ground_truth(inner_without_validate, name)
|
|
998
1279
|
|
|
999
|
-
|
|
1000
|
-
|
|
1280
|
+
def inner(*args, **kwargs):
|
|
1281
|
+
validate_args_structure(*args, types_order=[Union[int, str], PreprocessResponse],
|
|
1282
|
+
func_name=user_function.__name__, expected_names=["idx", "preprocess"], **kwargs)
|
|
1283
|
+
sample_id, preprocess_response = args
|
|
1001
1284
|
_validate_input_args(sample_id, preprocess_response)
|
|
1002
1285
|
|
|
1003
1286
|
result = inner_without_validate(sample_id, preprocess_response)
|
|
@@ -1005,6 +1288,7 @@ def tensorleap_gt_encoder(name: str):
|
|
|
1005
1288
|
_validate_result(result)
|
|
1006
1289
|
|
|
1007
1290
|
if _called_from_inside_tl_decorator == 0 and _called_from_inside_tl_integration_test_decorator:
|
|
1291
|
+
batch_warning(result, user_function.__name__)
|
|
1008
1292
|
result = np.expand_dims(result, axis=0)
|
|
1009
1293
|
# Emit integration test event once per test
|
|
1010
1294
|
try:
|
|
@@ -1013,12 +1297,13 @@ def tensorleap_gt_encoder(name: str):
|
|
|
1013
1297
|
})
|
|
1014
1298
|
except Exception as e:
|
|
1015
1299
|
logger.debug(f"Failed to emit gt_encoder integration test event: {e}")
|
|
1016
|
-
|
|
1300
|
+
if not _call_from_tl_platform:
|
|
1301
|
+
update_env_params_func("tensorleap_gt_encoder", "v")
|
|
1017
1302
|
return result
|
|
1018
1303
|
|
|
1019
1304
|
inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
|
|
1020
1305
|
|
|
1021
|
-
def mapping_inner(
|
|
1306
|
+
def mapping_inner(*args, **kwargs):
|
|
1022
1307
|
class TempMapping:
|
|
1023
1308
|
pass
|
|
1024
1309
|
|
|
@@ -1029,11 +1314,11 @@ def tensorleap_gt_encoder(name: str):
|
|
|
1029
1314
|
|
|
1030
1315
|
mapping_inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
|
|
1031
1316
|
|
|
1032
|
-
def final_inner(
|
|
1317
|
+
def final_inner(*args, **kwargs):
|
|
1033
1318
|
if os.environ.get(mapping_runtime_mode_env_var_mame):
|
|
1034
|
-
return mapping_inner(
|
|
1319
|
+
return mapping_inner(*args, **kwargs)
|
|
1035
1320
|
else:
|
|
1036
|
-
return inner(
|
|
1321
|
+
return inner(*args, **kwargs)
|
|
1037
1322
|
|
|
1038
1323
|
final_inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
|
|
1039
1324
|
|
|
@@ -1054,28 +1339,27 @@ def tensorleap_custom_loss(name: str, connects_to=None):
|
|
|
1054
1339
|
valid_types = (np.ndarray, SamplePreprocessResponse)
|
|
1055
1340
|
|
|
1056
1341
|
def _validate_input_args(*args, **kwargs):
|
|
1342
|
+
assert len(args) + len(kwargs) > 0, (
|
|
1343
|
+
f"{user_function.__name__}() validation failed: "
|
|
1344
|
+
f"Expected at least one positional|key-word argument of the allowed types (np.ndarray|SamplePreprocessResponse|). "
|
|
1345
|
+
f"but received none. "
|
|
1346
|
+
f"Correct usage example: {user_function.__name__}(input_array: np.ndarray, ...)"
|
|
1347
|
+
)
|
|
1057
1348
|
for i, arg in enumerate(args):
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
assert isinstance(elem, valid_types), (f'tensorleap_custom_loss validation failed: '
|
|
1061
|
-
f'Element #{y} of list should be a numpy array. Got {type(elem)}.')
|
|
1062
|
-
else:
|
|
1063
|
-
assert isinstance(arg, valid_types), (f'tensorleap_custom_loss validation failed: '
|
|
1064
|
-
f'Argument #{i} should be a numpy array. Got {type(arg)}.')
|
|
1349
|
+
assert isinstance(arg, valid_types), (f'{user_function.__name__}() validation failed: '
|
|
1350
|
+
f'Argument #{i} should be a numpy array. Got {type(arg)}.')
|
|
1065
1351
|
for _arg_name, arg in kwargs.items():
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
assert isinstance(elem, valid_types), (f'tensorleap_custom_loss validation failed: '
|
|
1069
|
-
f'Element #{y} of list should be a numpy array. Got {type(elem)}.')
|
|
1070
|
-
else:
|
|
1071
|
-
assert isinstance(arg, valid_types), (f'tensorleap_custom_loss validation failed: '
|
|
1072
|
-
f'Argument #{_arg_name} should be a numpy array. Got {type(arg)}.')
|
|
1352
|
+
assert isinstance(arg, valid_types), (f'{user_function.__name__}() validation failed: '
|
|
1353
|
+
f'Argument #{_arg_name} should be a numpy array. Got {type(arg)}.')
|
|
1073
1354
|
|
|
1074
1355
|
def _validate_result(result):
|
|
1356
|
+
validate_output_structure(result, func_name=user_function.__name__,
|
|
1357
|
+
expected_type_name="np.ndarray")
|
|
1075
1358
|
assert isinstance(result, np.ndarray), \
|
|
1076
|
-
(f'
|
|
1359
|
+
(f'{user_function.__name__} validation failed: '
|
|
1077
1360
|
f'The return type should be a numpy array. Got {type(result)}.')
|
|
1078
|
-
|
|
1361
|
+
assert result.ndim < 2, (f'{user_function.__name__} validation failed: '
|
|
1362
|
+
f'The return type should be a 1Dim numpy array but got {result.ndim}Dim.')
|
|
1079
1363
|
|
|
1080
1364
|
@functools.wraps(user_function)
|
|
1081
1365
|
def inner_without_validate(*args, **kwargs):
|
|
@@ -1106,6 +1390,9 @@ def tensorleap_custom_loss(name: str, connects_to=None):
|
|
|
1106
1390
|
result = inner_without_validate(*args, **kwargs)
|
|
1107
1391
|
|
|
1108
1392
|
_validate_result(result)
|
|
1393
|
+
if not _call_from_tl_platform:
|
|
1394
|
+
update_env_params_func("tensorleap_custom_loss", "v")
|
|
1395
|
+
|
|
1109
1396
|
return result
|
|
1110
1397
|
|
|
1111
1398
|
def mapping_inner(*args, **kwargs):
|
|
@@ -1162,3 +1449,103 @@ def tensorleap_custom_layer(name: str):
|
|
|
1162
1449
|
return custom_layer
|
|
1163
1450
|
|
|
1164
1451
|
return decorating_function
|
|
1452
|
+
|
|
1453
|
+
|
|
1454
|
+
def tensorleap_status_table():
|
|
1455
|
+
'''
|
|
1456
|
+
Usage example:
|
|
1457
|
+
###################
|
|
1458
|
+
leap_integration.py
|
|
1459
|
+
###################
|
|
1460
|
+
from code_loader.inner_leap_binder.leapbinder_decorators import tensorleap_status_table
|
|
1461
|
+
...
|
|
1462
|
+
...
|
|
1463
|
+
...
|
|
1464
|
+
if __name__ == '__main__':
|
|
1465
|
+
tensorleap_status_table()
|
|
1466
|
+
...
|
|
1467
|
+
'''
|
|
1468
|
+
import atexit
|
|
1469
|
+
import sys
|
|
1470
|
+
import traceback
|
|
1471
|
+
CHECK = "✅"
|
|
1472
|
+
CROSS = "❌"
|
|
1473
|
+
code_mapping_failure = [0]
|
|
1474
|
+
table = [
|
|
1475
|
+
{"name": "tensorleap_preprocess", "Added to integration": CROSS},
|
|
1476
|
+
{"name": "tensorleap_integration_test", "Added to integration": CROSS},
|
|
1477
|
+
{"name": "tensorleap_input_encoder", "Added to integration": CROSS},
|
|
1478
|
+
{"name": "tensorleap_gt_encoder", "Added to integration": CROSS},
|
|
1479
|
+
{"name": "tensorleap_load_model", "Added to integration": CROSS},
|
|
1480
|
+
{"name": "tensorleap_custom_loss", "Added to integration": CROSS},
|
|
1481
|
+
{"name": "tensorleap_custom_metric (optional)", "Added to integration": CROSS},
|
|
1482
|
+
{"name": "tensorleap_metadata (optional)", "Added to integration": CROSS},
|
|
1483
|
+
{"name": "tensorleap_custom_visualizer (optional)", "Added to integration": CROSS},
|
|
1484
|
+
|
|
1485
|
+
]
|
|
1486
|
+
|
|
1487
|
+
_finalizer_called = {"done": False}
|
|
1488
|
+
|
|
1489
|
+
def _remove_suffix(s: str, suffix: str) -> str:
|
|
1490
|
+
# This is needed because str.remove_suffix was presented in python3.9+
|
|
1491
|
+
if suffix and s.endswith(suffix):
|
|
1492
|
+
return s[:-len(suffix)]
|
|
1493
|
+
return s
|
|
1494
|
+
|
|
1495
|
+
def _print_table():
|
|
1496
|
+
ready_mess = "\nAll parts have been successfully set. If no errors accured, you can now push the project to the Tensorleap system."
|
|
1497
|
+
not_ready_mess = "\nSome mandatory components have not yet been added to the Integration test. Recommended next interface to add is: "
|
|
1498
|
+
mandatory_ready_mess = "\nAll mandatory parts have been successfully set. If no errors accured, you can now push the project to the Tensorleap system or continue to the next optional reccomeded interface,adding: "
|
|
1499
|
+
code_mapping_failure_mes = "Tensorleap_integration_test code flow failed, check raised exception."
|
|
1500
|
+
|
|
1501
|
+
name_width = max(len(row["name"]) for row in table)
|
|
1502
|
+
status_width = max(len(row["Added to integration"]) for row in table)
|
|
1503
|
+
header = f"{'Decorator Name'.ljust(name_width)} | {'Added to integration'.ljust(status_width)}"
|
|
1504
|
+
sep = "-" * len(header)
|
|
1505
|
+
print("\n" + header)
|
|
1506
|
+
print(sep)
|
|
1507
|
+
ready = True
|
|
1508
|
+
for row in table:
|
|
1509
|
+
print(f"{row['name'].ljust(name_width)} | {row['Added to integration'].ljust(status_width)}")
|
|
1510
|
+
if row['Added to integration'] == CROSS and ready:
|
|
1511
|
+
ready = False
|
|
1512
|
+
next_step = row['name']
|
|
1513
|
+
|
|
1514
|
+
if code_mapping_failure[0]:
|
|
1515
|
+
print(f"\n{CROSS + code_mapping_failure_mes}.")
|
|
1516
|
+
else:
|
|
1517
|
+
print(ready_mess) if ready else print(
|
|
1518
|
+
mandatory_ready_mess + next_step) if "optional" in next_step else print(not_ready_mess + next_step)
|
|
1519
|
+
|
|
1520
|
+
def update_env_params(name: str, status: str = "✓"):
|
|
1521
|
+
for row in table:
|
|
1522
|
+
if _remove_suffix(row["name"], " (optional)") == name:
|
|
1523
|
+
row["Added to integration"] = CHECK if status == "v" else CROSS
|
|
1524
|
+
break
|
|
1525
|
+
if name == "code_mapping":
|
|
1526
|
+
code_mapping_failure[0] = 1
|
|
1527
|
+
|
|
1528
|
+
def run_on_exit():
|
|
1529
|
+
if _finalizer_called["done"]:
|
|
1530
|
+
return
|
|
1531
|
+
_finalizer_called["done"] = True
|
|
1532
|
+
_print_table()
|
|
1533
|
+
|
|
1534
|
+
def handle_exception(exc_type, exc_value, exc_traceback):
|
|
1535
|
+
traceback.print_exception(exc_type, exc_value, exc_traceback)
|
|
1536
|
+
run_on_exit()
|
|
1537
|
+
|
|
1538
|
+
atexit.register(run_on_exit)
|
|
1539
|
+
sys.excepthook = handle_exception
|
|
1540
|
+
return update_env_params
|
|
1541
|
+
|
|
1542
|
+
|
|
1543
|
+
if not _call_from_tl_platform:
|
|
1544
|
+
update_env_params_func = tensorleap_status_table()
|
|
1545
|
+
|
|
1546
|
+
|
|
1547
|
+
|
|
1548
|
+
|
|
1549
|
+
|
|
1550
|
+
|
|
1551
|
+
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/experiment_context.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{code_loader-1.0.147 → code_loader-1.0.148}/code_loader/experiment_api/workingspace_config_utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|