tracdap-runtime 0.9.0b1__py3-none-any.whl → 0.9.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tracdap/rt/_impl/core/config_parser.py +29 -3
- tracdap/rt/_impl/core/data.py +93 -51
- tracdap/rt/_impl/core/repos.py +15 -13
- tracdap/rt/_impl/core/storage.py +17 -12
- tracdap/rt/_impl/core/struct.py +254 -60
- tracdap/rt/_impl/core/util.py +94 -23
- tracdap/rt/_impl/exec/context.py +35 -8
- tracdap/rt/_impl/exec/dev_mode.py +60 -40
- tracdap/rt/_impl/exec/engine.py +44 -50
- tracdap/rt/_impl/exec/functions.py +12 -8
- tracdap/rt/_impl/exec/graph.py +3 -3
- tracdap/rt/_impl/exec/graph_builder.py +22 -5
- tracdap/rt/_impl/grpc/codec.py +4 -11
- tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.py +36 -34
- tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.pyi +37 -43
- tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.py +64 -64
- tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.py +22 -18
- tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.pyi +15 -2
- tracdap/rt/_impl/runtime.py +2 -16
- tracdap/rt/_impl/static_api.py +5 -6
- tracdap/rt/_plugins/format_csv.py +2 -2
- tracdap/rt/_plugins/storage_aws.py +165 -150
- tracdap/rt/_plugins/storage_azure.py +17 -11
- tracdap/rt/_plugins/storage_gcp.py +35 -18
- tracdap/rt/_version.py +1 -1
- tracdap/rt/api/model_api.py +45 -0
- tracdap/rt/config/__init__.py +8 -10
- tracdap/rt/config/common.py +0 -16
- tracdap/rt/config/job.py +4 -0
- tracdap/rt/config/platform.py +9 -32
- tracdap/rt/config/runtime.py +4 -11
- tracdap/rt/config/tenant.py +28 -0
- tracdap/rt/launch/cli.py +0 -8
- tracdap/rt/launch/launch.py +1 -3
- tracdap/rt/metadata/__init__.py +18 -19
- tracdap/rt/metadata/data.py +19 -31
- tracdap/rt/metadata/job.py +1 -1
- tracdap/rt/metadata/type.py +9 -5
- {tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/METADATA +3 -3
- {tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/RECORD +43 -42
- {tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/WHEEL +1 -1
- {tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/licenses/LICENSE +0 -0
- {tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/top_level.txt +0 -0
tracdap/rt/_impl/core/struct.py
CHANGED
@@ -37,6 +37,12 @@ import tracdap.rt._impl.core.type_system as _meta_types
|
|
37
37
|
|
38
38
|
class StructProcessor:
|
39
39
|
|
40
|
+
JSON_FORMAT = "text/json"
|
41
|
+
JSON_ALT_FORMATS = ["json", ".json"]
|
42
|
+
|
43
|
+
YAML_FORMAT = "text/yaml"
|
44
|
+
YAML_ALT_FORMATS = ["yaml", ".yaml", "yml"]
|
45
|
+
|
40
46
|
__primitive_types: dict[type, _meta.BasicType] = {
|
41
47
|
bool: _meta.BasicType.BOOLEAN,
|
42
48
|
int: _meta.BasicType.INTEGER,
|
@@ -68,13 +74,16 @@ class StructProcessor:
|
|
68
74
|
|
69
75
|
|
70
76
|
@classmethod
|
71
|
-
def define_struct(cls, python_type: type) -> _meta.
|
77
|
+
def define_struct(cls, python_type: type) -> _meta.SchemaDefinition:
|
78
|
+
|
79
|
+
named_types = dict()
|
80
|
+
named_enums = dict()
|
72
81
|
|
73
82
|
if _dc.is_dataclass(python_type):
|
74
|
-
return cls._define_struct_for_dataclass(python_type)
|
83
|
+
return cls._define_struct_for_dataclass(python_type, named_types, named_enums, type_stack=[])
|
75
84
|
|
76
85
|
if _pyd and issubclass(python_type, _pyd.BaseModel):
|
77
|
-
return cls._define_struct_for_pydantic(python_type)
|
86
|
+
return cls._define_struct_for_pydantic(python_type, named_types, named_enums, type_stack=[])
|
78
87
|
|
79
88
|
raise _ex.EUnexpected()
|
80
89
|
|
@@ -83,10 +92,10 @@ class StructProcessor:
|
|
83
92
|
|
84
93
|
try:
|
85
94
|
|
86
|
-
if src_format ==
|
95
|
+
if src_format == cls.YAML_FORMAT or src_format.lower() in cls.YAML_ALT_FORMATS:
|
87
96
|
config_dict = _yaml.safe_load(src)
|
88
97
|
|
89
|
-
elif src_format ==
|
98
|
+
elif src_format == cls.JSON_FORMAT or src_format.lower() in cls.JSON_ALT_FORMATS:
|
90
99
|
config_dict = _json.load(src)
|
91
100
|
|
92
101
|
else:
|
@@ -113,111 +122,297 @@ class StructProcessor:
|
|
113
122
|
StructQuoter.quote(struct, dst, dst_format)
|
114
123
|
|
115
124
|
@classmethod
|
116
|
-
def parse_struct(cls, data: dict, schema: _meta.
|
125
|
+
def parse_struct(cls, data: dict, schema: _meta.SchemaDefinition = None, python_type: type = None) -> object:
|
117
126
|
|
118
127
|
parser = StructParser()
|
119
128
|
return parser.parse(python_type, data)
|
120
129
|
|
121
130
|
@classmethod
|
122
|
-
def _define_struct_for_dataclass(
|
131
|
+
def _define_struct_for_dataclass(
|
132
|
+
cls, python_type: _dc.dataclass,
|
133
|
+
named_types: _tp.Dict[str, _meta.SchemaDefinition],
|
134
|
+
named_enums: _tp.Dict[str, _meta.EnumValues],
|
135
|
+
type_stack: _tp.List[str]) \
|
136
|
+
-> _meta.SchemaDefinition:
|
123
137
|
|
124
|
-
|
125
|
-
trac_fields = dict()
|
138
|
+
try:
|
126
139
|
|
127
|
-
|
140
|
+
type_stack.append(cls._qualified_type_name(python_type))
|
128
141
|
|
129
|
-
|
130
|
-
|
142
|
+
type_hints = _tp.get_type_hints(python_type)
|
143
|
+
trac_fields = list()
|
144
|
+
|
145
|
+
for field_index, dc_field in enumerate(_dc.fields(python_type)):
|
131
146
|
|
132
|
-
|
133
|
-
|
147
|
+
field_name = dc_field.name
|
148
|
+
python_type = type_hints[field_name]
|
134
149
|
|
135
|
-
|
150
|
+
trac_field = cls._define_field(
|
151
|
+
field_name, field_index, python_type, dc_field=dc_field,
|
152
|
+
named_types=named_types, named_enums=named_enums, type_stack=type_stack)
|
153
|
+
|
154
|
+
trac_fields.append(trac_field)
|
155
|
+
|
156
|
+
if len(type_stack) == 1:
|
157
|
+
return _meta.SchemaDefinition(
|
158
|
+
schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields,
|
159
|
+
namedTypes=named_types, namedEnums=named_enums)
|
160
|
+
else:
|
161
|
+
return _meta.SchemaDefinition(
|
162
|
+
schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields)
|
163
|
+
|
164
|
+
finally:
|
165
|
+
|
166
|
+
type_stack.pop()
|
136
167
|
|
137
168
|
@classmethod
|
138
|
-
def _define_struct_for_pydantic(
|
169
|
+
def _define_struct_for_pydantic(
|
170
|
+
cls, python_type: "type[_pyd.BaseModel]",
|
171
|
+
named_types: _tp.Dict[str, _meta.SchemaDefinition],
|
172
|
+
named_enums: _tp.Dict[str, _meta.EnumValues],
|
173
|
+
type_stack: _tp.List[str]) \
|
174
|
+
-> _meta.SchemaDefinition:
|
139
175
|
|
140
|
-
|
141
|
-
trac_fields = dict()
|
176
|
+
try:
|
142
177
|
|
143
|
-
|
178
|
+
type_stack.append(cls._qualified_type_name(python_type))
|
179
|
+
|
180
|
+
type_hints = _tp.get_type_hints(python_type)
|
181
|
+
trac_fields = list()
|
182
|
+
|
183
|
+
field_index = 0
|
184
|
+
|
185
|
+
for field_name, pyd_field in python_type.model_fields.items():
|
144
186
|
|
145
|
-
|
187
|
+
python_type = type_hints[field_name]
|
146
188
|
|
147
|
-
|
189
|
+
trac_field = cls._define_field(
|
190
|
+
field_name, field_index, python_type, pyd_field=pyd_field,
|
191
|
+
named_types=named_types, named_enums=named_enums, type_stack=type_stack)
|
192
|
+
|
193
|
+
if trac_field is not None:
|
194
|
+
trac_fields.append(trac_field)
|
195
|
+
field_index += 1
|
196
|
+
|
197
|
+
if len(type_stack) == 1:
|
198
|
+
return _meta.SchemaDefinition(
|
199
|
+
schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields,
|
200
|
+
namedTypes=named_types, namedEnums=named_enums)
|
201
|
+
else:
|
202
|
+
return _meta.SchemaDefinition(
|
203
|
+
schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields)
|
148
204
|
|
149
|
-
|
150
|
-
trac_fields[field_name] = trac_field
|
205
|
+
finally:
|
151
206
|
|
152
|
-
|
207
|
+
type_stack.pop()
|
153
208
|
|
154
209
|
@classmethod
|
155
210
|
def _define_field(
|
156
|
-
cls, python_type: type, *,
|
157
|
-
|
158
|
-
|
159
|
-
|
211
|
+
cls, name, index, python_type: type, optional=False, *,
|
212
|
+
named_types: _tp.Dict[str, _meta.SchemaDefinition],
|
213
|
+
named_enums: _tp.Dict[str, _meta.EnumValues],
|
214
|
+
type_stack: _tp.List[str],
|
215
|
+
dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
|
216
|
+
-> _meta.FieldSchema:
|
160
217
|
|
161
218
|
if python_type in cls.__primitive_types:
|
162
|
-
|
219
|
+
|
220
|
+
return cls._define_primitive_field(
|
221
|
+
name, index, python_type, optional,
|
222
|
+
dc_field=dc_field, pyd_field=pyd_field)
|
163
223
|
|
164
224
|
elif any(map(lambda _t: isinstance(python_type, _t), cls.__generic_types)):
|
165
|
-
return cls._define_generic_field(python_type, pyd_field=pyd_field)
|
166
225
|
|
167
|
-
|
168
|
-
|
226
|
+
return cls._define_generic_field(
|
227
|
+
name, index, python_type,
|
228
|
+
dc_field=dc_field, pyd_field=pyd_field,
|
229
|
+
named_types=named_types, named_enums=named_enums,
|
230
|
+
type_stack=type_stack)
|
231
|
+
|
232
|
+
elif isinstance(python_type, _enum.EnumMeta):
|
233
|
+
|
234
|
+
type_name = cls._qualified_type_name(python_type)
|
235
|
+
|
236
|
+
if type_name not in named_enums:
|
237
|
+
enum_values = cls._define_enum_values(python_type)
|
238
|
+
named_enums[type_name] = enum_values
|
239
|
+
|
240
|
+
return cls._define_enum_field(
|
241
|
+
name, index, python_type, optional,
|
242
|
+
dc_field=dc_field, pyd_field=pyd_field)
|
243
|
+
|
244
|
+
elif _dc.is_dataclass(python_type):
|
169
245
|
|
170
|
-
|
171
|
-
|
246
|
+
type_name = cls._qualified_type_name(python_type)
|
247
|
+
|
248
|
+
if type_name in type_stack:
|
249
|
+
raise _ex.EValidation("Recursive types are not supported")
|
250
|
+
|
251
|
+
if type_name not in named_types:
|
252
|
+
struct_type = cls._define_struct_for_dataclass(python_type, named_types, named_enums, type_stack)
|
253
|
+
named_types[type_name] = struct_type
|
254
|
+
|
255
|
+
return _meta.FieldSchema(
|
256
|
+
fieldName=name,
|
257
|
+
fieldOrder=index,
|
258
|
+
fieldType=_meta.BasicType.STRUCT,
|
259
|
+
notNull=not optional,
|
260
|
+
namedType=type_name)
|
261
|
+
|
262
|
+
elif issubclass(python_type, _pyd.BaseModel):
|
263
|
+
|
264
|
+
type_name = cls._qualified_type_name(python_type)
|
265
|
+
|
266
|
+
if type_name in type_stack:
|
267
|
+
raise _ex.EValidation("Recursive types are not supported")
|
268
|
+
|
269
|
+
if type_name not in named_types:
|
270
|
+
struct_type = cls._define_struct_for_pydantic(python_type, named_types, named_enums, type_stack)
|
271
|
+
named_types[type_name] = struct_type
|
272
|
+
|
273
|
+
return _meta.FieldSchema(
|
274
|
+
fieldName=name,
|
275
|
+
fieldOrder=index,
|
276
|
+
fieldType=_meta.BasicType.STRUCT,
|
277
|
+
notNull=not optional,
|
278
|
+
namedType=type_name)
|
172
279
|
|
173
280
|
else:
|
174
281
|
raise _ex.ETracInternal("Cannot encode field type: " + str(python_type))
|
175
282
|
|
176
283
|
@classmethod
|
177
284
|
def _define_primitive_field(
|
178
|
-
cls, python_type: type, optional=False, *,
|
179
|
-
dc_field: _dc.Field = None,
|
180
|
-
|
181
|
-
|
285
|
+
cls, name: str, index: int, python_type: type, optional=False, *,
|
286
|
+
dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
|
287
|
+
-> _meta.FieldSchema:
|
288
|
+
|
289
|
+
default_value = None
|
290
|
+
|
291
|
+
if dc_field is not None:
|
292
|
+
if dc_field.default not in [_dc.MISSING, None]:
|
293
|
+
default_value = _meta_types.MetadataCodec.encode_value(dc_field.default)
|
294
|
+
elif dc_field.default_factory not in [_dc.MISSING, None]:
|
295
|
+
native_value = dc_field.default_factory()
|
296
|
+
default_value = _meta_types.MetadataCodec.encode_value(native_value)
|
297
|
+
|
298
|
+
elif pyd_field is not None:
|
299
|
+
if pyd_field.default not in [_pyd.fields.PydanticUndefined, None]:
|
300
|
+
default_value = _meta_types.MetadataCodec.encode_value(pyd_field.default)
|
301
|
+
elif pyd_field.default_factory not in [_pyd.fields.PydanticUndefined, None]:
|
302
|
+
native_value = pyd_field.default_factory()
|
303
|
+
default_value = _meta_types.MetadataCodec.encode_value(native_value)
|
304
|
+
|
305
|
+
return _meta.FieldSchema(
|
306
|
+
fieldName=name,
|
307
|
+
fieldOrder=index,
|
308
|
+
fieldType=cls.__primitive_types[python_type],
|
309
|
+
notNull=not optional,
|
310
|
+
defaultValue=default_value)
|
311
|
+
|
312
|
+
@classmethod
|
313
|
+
def _define_enum_field(
|
314
|
+
cls, name: str, index: int, enum_type: _enum.EnumMeta, optional=False, *,
|
315
|
+
dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
|
316
|
+
-> _meta.FieldSchema:
|
182
317
|
|
183
|
-
|
184
|
-
struct_field.fieldType = _meta.TypeDescriptor(basicType=cls.__primitive_types[python_type])
|
185
|
-
struct_field.notNull = not optional
|
318
|
+
default_value = None
|
186
319
|
|
187
|
-
if dc_field is not None and dc_field.default
|
188
|
-
|
320
|
+
if dc_field is not None and dc_field.default not in [_dc.MISSING, None]:
|
321
|
+
default_value = _meta_types.MetadataCodec.encode_value(dc_field.default.name)
|
189
322
|
|
190
|
-
if pyd_field is not None and pyd_field.default
|
191
|
-
|
323
|
+
if pyd_field is not None and pyd_field.default not in [_pyd.fields.PydanticUndefined, None]:
|
324
|
+
default_value = _meta_types.MetadataCodec.encode_value(pyd_field.default.name)
|
192
325
|
|
193
|
-
return
|
326
|
+
return _meta.FieldSchema(
|
327
|
+
fieldName=name,
|
328
|
+
fieldOrder=index,
|
329
|
+
fieldType=_meta.BasicType.STRING,
|
330
|
+
categorical=True,
|
331
|
+
notNull=not optional,
|
332
|
+
namedEnum=cls._qualified_type_name(enum_type),
|
333
|
+
defaultValue=default_value)
|
334
|
+
|
335
|
+
@classmethod
|
336
|
+
def _define_enum_values(cls, enum_type: _enum.EnumMeta) -> _meta.EnumValues:
|
337
|
+
|
338
|
+
values = list(map(lambda value: value.name, enum_type))
|
339
|
+
return _meta.EnumValues(values=values)
|
194
340
|
|
195
341
|
@classmethod
|
196
342
|
def _define_generic_field(
|
197
|
-
cls, python_type: type, *,
|
198
|
-
|
199
|
-
|
343
|
+
cls, name, index, python_type: type, *,
|
344
|
+
named_types: _tp.Dict[str, _meta.SchemaDefinition],
|
345
|
+
named_enums: _tp.Dict[str, _meta.EnumValues],
|
346
|
+
type_stack: _tp.List[str],
|
347
|
+
dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
|
348
|
+
-> _meta.FieldSchema:
|
200
349
|
|
201
350
|
origin = _tp.get_origin(python_type)
|
202
351
|
args = _tp.get_args(python_type)
|
203
352
|
|
204
353
|
# types.NoneType not available in Python 3.9, so use type(None) instead
|
205
|
-
if origin in cls.__union_types and len(args) == 2 and
|
206
|
-
optional_type = args[0]
|
207
|
-
return cls.
|
354
|
+
if origin in cls.__union_types and len(args) == 2 and type(None) in args:
|
355
|
+
optional_type = args[0] if args[1] is type(None) else args[1]
|
356
|
+
return cls._define_field(
|
357
|
+
name, index, optional_type, optional=True,
|
358
|
+
dc_field=dc_field, pyd_field=pyd_field,
|
359
|
+
named_types=named_types, named_enums=named_enums,
|
360
|
+
type_stack=type_stack)
|
208
361
|
|
209
362
|
elif origin in [list, _tp.List]:
|
210
|
-
|
211
|
-
|
363
|
+
|
364
|
+
item_type = args[0]
|
365
|
+
item_field = cls._define_field(
|
366
|
+
"item", 0, item_type, optional=False,
|
367
|
+
named_types=named_types, named_enums=named_enums,
|
368
|
+
type_stack=type_stack)
|
369
|
+
|
370
|
+
return _meta.FieldSchema(
|
371
|
+
fieldName=name,
|
372
|
+
fieldOrder=index,
|
373
|
+
fieldType=_meta.BasicType.ARRAY,
|
374
|
+
notNull=True,
|
375
|
+
children=[item_field])
|
212
376
|
|
213
377
|
elif origin in [dict, _tp.Dict]:
|
378
|
+
|
214
379
|
key_type = args[0]
|
380
|
+
key_field = _meta.FieldSchema(
|
381
|
+
fieldName="key",
|
382
|
+
fieldOrder=0,
|
383
|
+
fieldType=_meta.BasicType.STRING,
|
384
|
+
notNull=True)
|
385
|
+
|
215
386
|
value_type = args[1]
|
216
|
-
|
387
|
+
value_field = cls._define_field(
|
388
|
+
"value", 1, value_type, optional=False,
|
389
|
+
named_types=named_types, named_enums=named_enums,
|
390
|
+
type_stack=type_stack)
|
391
|
+
|
392
|
+
return _meta.FieldSchema(
|
393
|
+
fieldName=name,
|
394
|
+
fieldOrder=index,
|
395
|
+
fieldType=_meta.BasicType.MAP,
|
396
|
+
notNull=True,
|
397
|
+
children=[key_field, value_field])
|
217
398
|
|
218
399
|
else:
|
219
400
|
raise _ex.ETracInternal("Cannot encode field type: " + str(python_type))
|
220
401
|
|
402
|
+
@classmethod
|
403
|
+
def _qualified_type_name(cls, python_type: type):
|
404
|
+
|
405
|
+
name = python_type.__name__
|
406
|
+
module = python_type.__module__
|
407
|
+
|
408
|
+
if module.startswith(cls.__SHIM_PREFIX):
|
409
|
+
shim_root_index = module.index(".", len(cls.__SHIM_PREFIX)) + 1
|
410
|
+
module = module[shim_root_index:]
|
411
|
+
|
412
|
+
return f"{module}.{name}"
|
413
|
+
|
414
|
+
__SHIM_PREFIX = "tracdap.shim."
|
415
|
+
|
221
416
|
|
222
417
|
class StructParser:
|
223
418
|
|
@@ -476,19 +671,18 @@ class StructQuoter:
|
|
476
671
|
# New implementation of STRUCT quoting, copied from config_parser
|
477
672
|
# After a period of stabilization, config_parser will switch to this implementation
|
478
673
|
|
479
|
-
JSON_FORMAT = "json"
|
480
|
-
YAML_FORMAT = "yaml"
|
481
674
|
INDENT = 3
|
482
675
|
|
483
676
|
@classmethod
|
484
677
|
def quote(cls, obj: _tp.Any, dst: _tp.TextIO, dst_format: str):
|
485
678
|
|
486
|
-
if dst_format.lower() == cls.JSON_FORMAT:
|
487
|
-
return cls.quote_json(obj, dst)
|
488
679
|
|
489
|
-
if dst_format.lower()
|
680
|
+
if dst_format == StructProcessor.YAML_FORMAT or dst_format.lower() in StructProcessor.YAML_ALT_FORMATS:
|
490
681
|
return cls.quote_yaml(obj, dst)
|
491
682
|
|
683
|
+
if dst_format == StructProcessor.JSON_FORMAT or dst_format.lower() in StructProcessor.JSON_ALT_FORMATS:
|
684
|
+
return cls.quote_json(obj, dst)
|
685
|
+
|
492
686
|
# TODO :This is probably an error in the user-supplied parameters
|
493
687
|
raise _ex.ETracInternal(f"Unsupported output format [{dst_format}]")
|
494
688
|
|
tracdap/rt/_impl/core/util.py
CHANGED
@@ -21,6 +21,7 @@ import re
|
|
21
21
|
import typing as tp
|
22
22
|
import uuid
|
23
23
|
|
24
|
+
import tracdap.rt.api as api
|
24
25
|
import tracdap.rt.exceptions as ex
|
25
26
|
import tracdap.rt.metadata as meta
|
26
27
|
import tracdap.rt.config as cfg
|
@@ -121,55 +122,103 @@ def selector_for_latest(object_id: meta.TagHeader) -> meta.TagSelector:
|
|
121
122
|
latestTag=True)
|
122
123
|
|
123
124
|
|
125
|
+
def get_job_mapping(
|
126
|
+
selector: tp.Union[meta.TagHeader, meta.TagSelector],
|
127
|
+
job_config: cfg.JobConfig) \
|
128
|
+
-> meta.TagHeader:
|
129
|
+
|
130
|
+
obj_key = object_key(selector)
|
131
|
+
obj_id = job_config.objectMapping.get(obj_key)
|
132
|
+
|
133
|
+
if obj_id is not None:
|
134
|
+
return obj_id
|
135
|
+
|
136
|
+
obj_key_match = __OBJ_KEY_PATTERN.match(obj_key)
|
137
|
+
|
138
|
+
if not obj_key_match:
|
139
|
+
err = f"Missing required {selector.objectType.name} ID for [{object_key(selector)}]"
|
140
|
+
raise ex.ERuntimeValidation(err)
|
141
|
+
|
142
|
+
obj_type = obj_key_match.group(1)
|
143
|
+
obj_id = obj_key_match.group(2)
|
144
|
+
obj_ver = obj_key_match.group(3)
|
145
|
+
obj_ts = job_config.jobId.objectTimestamp
|
146
|
+
|
147
|
+
return meta.TagHeader(
|
148
|
+
meta.ObjectType.__members__[obj_type], obj_id,
|
149
|
+
int(obj_ver), obj_ts, 1, obj_ts)
|
150
|
+
|
151
|
+
|
124
152
|
def get_job_metadata(
|
125
153
|
selector: tp.Union[meta.TagHeader, meta.TagSelector],
|
126
154
|
job_config: cfg.JobConfig,
|
127
155
|
optional: bool = False) \
|
128
156
|
-> tp.Optional[meta.ObjectDefinition]:
|
129
157
|
|
158
|
+
return __get_job_metadata_item(selector, job_config, job_config.objects, "object", optional)
|
159
|
+
|
160
|
+
|
161
|
+
def get_job_metadata_tag(
|
162
|
+
selector: tp.Union[meta.TagHeader, meta.TagSelector],
|
163
|
+
job_config: cfg.JobConfig,
|
164
|
+
optional: bool = False) \
|
165
|
+
-> tp.Optional[meta.Tag]:
|
166
|
+
|
167
|
+
return __get_job_metadata_item(selector, job_config, job_config.tags, "tag", optional)
|
168
|
+
|
169
|
+
|
170
|
+
__METADATA_TYPE = tp.TypeVar("__METADATA_TYPE")
|
171
|
+
|
172
|
+
|
173
|
+
def __get_job_metadata_item(
|
174
|
+
selector: tp.Union[meta.TagHeader, meta.TagSelector], job_config: cfg.JobConfig,
|
175
|
+
metadata: tp.Dict[str, __METADATA_TYPE], metadata_type: str,
|
176
|
+
optional: bool = False) \
|
177
|
+
-> tp.Optional[__METADATA_TYPE]:
|
178
|
+
|
130
179
|
obj_key = object_key(selector)
|
131
180
|
obj_id = job_config.objectMapping.get(obj_key)
|
132
181
|
|
133
182
|
if obj_id is not None:
|
134
183
|
obj_key = object_key(obj_id)
|
135
184
|
|
136
|
-
|
185
|
+
item = metadata.get(obj_key)
|
137
186
|
|
138
|
-
if
|
139
|
-
return
|
187
|
+
if item is not None:
|
188
|
+
return item
|
140
189
|
|
141
190
|
if optional:
|
142
191
|
return None
|
143
192
|
|
144
|
-
err = f"Missing required {selector.objectType.name}
|
193
|
+
err = f"Missing required {selector.objectType.name} {metadata_type} for [{object_key(selector)}]"
|
145
194
|
raise ex.ERuntimeValidation(err)
|
146
195
|
|
147
196
|
|
148
|
-
def
|
149
|
-
selector: tp.Union[meta.TagHeader, meta.TagSelector],
|
150
|
-
job_config: cfg.JobConfig) \
|
151
|
-
-> meta.TagHeader:
|
197
|
+
def attach_runtime_metadata(obj: tp.Any, metadata: api.RuntimeMetadata):
|
152
198
|
|
153
|
-
|
154
|
-
|
199
|
+
if hasattr(obj, "with_metadata"):
|
200
|
+
attach_func = getattr(obj, "with_metadata")
|
201
|
+
if isinstance(attach_func, tp.Callable):
|
202
|
+
return attach_func(metadata)
|
155
203
|
|
156
|
-
|
157
|
-
return obj_id
|
204
|
+
setattr(obj, "_metadata", metadata)
|
158
205
|
|
159
|
-
|
206
|
+
return obj
|
160
207
|
|
161
|
-
if not obj_key_match:
|
162
|
-
err = f"Missing required {selector.objectType.name} ID for [{object_key(selector)}]"
|
163
|
-
raise ex.ERuntimeValidation(err)
|
164
208
|
|
165
|
-
|
166
|
-
obj_id = obj_key_match.group(2)
|
167
|
-
obj_ver = obj_key_match.group(3)
|
168
|
-
obj_ts = job_config.jobId.objectTimestamp
|
209
|
+
def retrieve_runtime_metadata(obj: tp.Any) -> tp.Optional[api.RuntimeMetadata]:
|
169
210
|
|
170
|
-
|
171
|
-
|
172
|
-
|
211
|
+
if hasattr(obj, "metadata"):
|
212
|
+
metadata = getattr(obj, "metadata")
|
213
|
+
if isinstance(metadata, api.RuntimeMetadata):
|
214
|
+
return metadata
|
215
|
+
|
216
|
+
if hasattr(obj, "_metadata"):
|
217
|
+
metadata = getattr(obj, "_metadata")
|
218
|
+
if isinstance(metadata, api.RuntimeMetadata):
|
219
|
+
return metadata
|
220
|
+
|
221
|
+
return None
|
173
222
|
|
174
223
|
|
175
224
|
def get_origin(metaclass: type):
|
@@ -296,3 +345,25 @@ def filter_model_stack_trace(full_stack: tb.StackSummary, checkout_directory: pa
|
|
296
345
|
last_model_frame = first_model_frame + frame_index
|
297
346
|
|
298
347
|
return full_stack[first_model_frame:last_model_frame+1]
|
348
|
+
|
349
|
+
|
350
|
+
__T = tp.TypeVar("__T")
|
351
|
+
|
352
|
+
def read_property(properties: tp.Dict[str, str], key: str, default: tp.Optional[__T] = None, convert: tp.Optional[tp.Type[__T]] = str) -> __T:
|
353
|
+
|
354
|
+
value = properties.get(key)
|
355
|
+
|
356
|
+
if value is None:
|
357
|
+
if default is not None:
|
358
|
+
value = default
|
359
|
+
else:
|
360
|
+
raise ex.EConfigParse(f"Missing required property: [{key}]")
|
361
|
+
|
362
|
+
try:
|
363
|
+
if convert is bool and isinstance(value, str):
|
364
|
+
return True if value.lower() == "true" else False
|
365
|
+
else:
|
366
|
+
return convert(value)
|
367
|
+
|
368
|
+
except (ValueError, TypeError):
|
369
|
+
raise ex.EConfigParse(f"Wrong property type: [{key}] = [{value}], expected type is [{convert}]")
|
tracdap/rt/_impl/exec/context.py
CHANGED
@@ -118,6 +118,19 @@ class TracContextImpl(_api.TracContext):
|
|
118
118
|
|
119
119
|
return not data_view.is_empty()
|
120
120
|
|
121
|
+
def get_metadata(self, item_name: str) -> tp.Optional[_api.RuntimeMetadata]:
|
122
|
+
|
123
|
+
_val.validate_signature(self.get_metadata, item_name)
|
124
|
+
|
125
|
+
self.__val.check_item_valid_identifier(item_name, TracContextValidator.ITEM)
|
126
|
+
self.__val.check_item_defined_in_model(item_name, TracContextValidator.ITEM)
|
127
|
+
self.__val.check_item_available_in_context(item_name, TracContextValidator.ITEM)
|
128
|
+
|
129
|
+
obj = self.__local_ctx.get(item_name)
|
130
|
+
|
131
|
+
# Can be none if no metadata is attached
|
132
|
+
return _util.retrieve_runtime_metadata(obj)
|
133
|
+
|
121
134
|
def get_schema(self, dataset_name: str) -> _meta.SchemaDefinition:
|
122
135
|
|
123
136
|
_val.validate_signature(self.get_schema, dataset_name)
|
@@ -201,8 +214,12 @@ class TracContextImpl(_api.TracContext):
|
|
201
214
|
self.__val.check_context_data_view_type(struct_name, data_view, _meta.ObjectType.DATA)
|
202
215
|
self.__val.check_dataset_schema_defined(struct_name, data_view)
|
203
216
|
|
204
|
-
struct_data
|
205
|
-
|
217
|
+
struct_data = data_view.parts[part_key][0].content
|
218
|
+
|
219
|
+
if isinstance(struct_data, python_class):
|
220
|
+
return struct_data
|
221
|
+
else:
|
222
|
+
return _struct.StructProcessor.parse_struct(struct_data, None, python_class)
|
206
223
|
|
207
224
|
def get_file(self, file_name: str) -> bytes:
|
208
225
|
|
@@ -371,16 +388,25 @@ class TracContextImpl(_api.TracContext):
|
|
371
388
|
self.__val.check_item_valid_identifier(file_name, TracContextValidator.FILE)
|
372
389
|
self.__val.check_item_is_model_output(file_name, TracContextValidator.FILE)
|
373
390
|
|
391
|
+
class DelayedClose(io.BytesIO):
|
392
|
+
|
393
|
+
def __init__(self):
|
394
|
+
super().__init__()
|
395
|
+
|
396
|
+
def close(self):
|
397
|
+
super().flush()
|
398
|
+
|
374
399
|
@contextlib.contextmanager
|
375
400
|
def memory_stream(stream: io.BytesIO):
|
376
401
|
try:
|
377
402
|
yield stream
|
378
|
-
buffer = stream.getbuffer().tobytes()
|
379
|
-
self.put_file(file_name, buffer)
|
380
403
|
finally:
|
381
|
-
stream.
|
404
|
+
with stream.getbuffer() as buffer:
|
405
|
+
self.put_file(file_name, bytes(buffer))
|
406
|
+
if not stream.closed:
|
407
|
+
io.BytesIO.close(stream)
|
382
408
|
|
383
|
-
return memory_stream(
|
409
|
+
return memory_stream(DelayedClose())
|
384
410
|
|
385
411
|
def log(self) -> logging.Logger:
|
386
412
|
|
@@ -816,6 +842,7 @@ class TracContextErrorReporter:
|
|
816
842
|
|
817
843
|
class TracContextValidator(TracContextErrorReporter):
|
818
844
|
|
845
|
+
ITEM = "Item"
|
819
846
|
PARAMETER = "Parameter"
|
820
847
|
DATASET = "Dataset"
|
821
848
|
FILE = "File"
|
@@ -891,10 +918,10 @@ class TracContextValidator(TracContextErrorReporter):
|
|
891
918
|
if schema is None:
|
892
919
|
self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
|
893
920
|
|
894
|
-
if schema.schemaType == _meta.SchemaType.
|
921
|
+
if schema.schemaType == _meta.SchemaType.TABLE_SCHEMA and (schema.table is None or not schema.table.fields):
|
895
922
|
self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
|
896
923
|
|
897
|
-
if schema.schemaType == _meta.SchemaType.
|
924
|
+
if schema.schemaType == _meta.SchemaType.STRUCT_SCHEMA and not schema.fields:
|
898
925
|
self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
|
899
926
|
|
900
927
|
def check_dataset_schema_not_defined(self, dataset_name: str, data_view: _data.DataView):
|