tracdap-runtime 0.9.0b1__py3-none-any.whl → 0.9.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. tracdap/rt/_impl/core/config_parser.py +29 -3
  2. tracdap/rt/_impl/core/data.py +93 -51
  3. tracdap/rt/_impl/core/repos.py +15 -13
  4. tracdap/rt/_impl/core/storage.py +17 -12
  5. tracdap/rt/_impl/core/struct.py +254 -60
  6. tracdap/rt/_impl/core/util.py +94 -23
  7. tracdap/rt/_impl/exec/context.py +35 -8
  8. tracdap/rt/_impl/exec/dev_mode.py +60 -40
  9. tracdap/rt/_impl/exec/engine.py +44 -50
  10. tracdap/rt/_impl/exec/functions.py +12 -8
  11. tracdap/rt/_impl/exec/graph.py +3 -3
  12. tracdap/rt/_impl/exec/graph_builder.py +22 -5
  13. tracdap/rt/_impl/grpc/codec.py +4 -11
  14. tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.py +36 -34
  15. tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.pyi +37 -43
  16. tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.py +64 -64
  17. tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.py +22 -18
  18. tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.pyi +15 -2
  19. tracdap/rt/_impl/runtime.py +2 -16
  20. tracdap/rt/_impl/static_api.py +5 -6
  21. tracdap/rt/_plugins/format_csv.py +2 -2
  22. tracdap/rt/_plugins/storage_aws.py +165 -150
  23. tracdap/rt/_plugins/storage_azure.py +17 -11
  24. tracdap/rt/_plugins/storage_gcp.py +35 -18
  25. tracdap/rt/_version.py +1 -1
  26. tracdap/rt/api/model_api.py +45 -0
  27. tracdap/rt/config/__init__.py +8 -10
  28. tracdap/rt/config/common.py +0 -16
  29. tracdap/rt/config/job.py +4 -0
  30. tracdap/rt/config/platform.py +9 -32
  31. tracdap/rt/config/runtime.py +4 -11
  32. tracdap/rt/config/tenant.py +28 -0
  33. tracdap/rt/launch/cli.py +0 -8
  34. tracdap/rt/launch/launch.py +1 -3
  35. tracdap/rt/metadata/__init__.py +18 -19
  36. tracdap/rt/metadata/data.py +19 -31
  37. tracdap/rt/metadata/job.py +1 -1
  38. tracdap/rt/metadata/type.py +9 -5
  39. {tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/METADATA +3 -3
  40. {tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/RECORD +43 -42
  41. {tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/WHEEL +1 -1
  42. {tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/licenses/LICENSE +0 -0
  43. {tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/top_level.txt +0 -0
@@ -37,6 +37,12 @@ import tracdap.rt._impl.core.type_system as _meta_types
37
37
 
38
38
  class StructProcessor:
39
39
 
40
+ JSON_FORMAT = "text/json"
41
+ JSON_ALT_FORMATS = ["json", ".json"]
42
+
43
+ YAML_FORMAT = "text/yaml"
44
+ YAML_ALT_FORMATS = ["yaml", ".yaml", "yml"]
45
+
40
46
  __primitive_types: dict[type, _meta.BasicType] = {
41
47
  bool: _meta.BasicType.BOOLEAN,
42
48
  int: _meta.BasicType.INTEGER,
@@ -68,13 +74,16 @@ class StructProcessor:
68
74
 
69
75
 
70
76
  @classmethod
71
- def define_struct(cls, python_type: type) -> _meta.StructSchema:
77
+ def define_struct(cls, python_type: type) -> _meta.SchemaDefinition:
78
+
79
+ named_types = dict()
80
+ named_enums = dict()
72
81
 
73
82
  if _dc.is_dataclass(python_type):
74
- return cls._define_struct_for_dataclass(python_type)
83
+ return cls._define_struct_for_dataclass(python_type, named_types, named_enums, type_stack=[])
75
84
 
76
85
  if _pyd and issubclass(python_type, _pyd.BaseModel):
77
- return cls._define_struct_for_pydantic(python_type)
86
+ return cls._define_struct_for_pydantic(python_type, named_types, named_enums, type_stack=[])
78
87
 
79
88
  raise _ex.EUnexpected()
80
89
 
@@ -83,10 +92,10 @@ class StructProcessor:
83
92
 
84
93
  try:
85
94
 
86
- if src_format == "yaml" or src_format == "yml":
95
+ if src_format == cls.YAML_FORMAT or src_format.lower() in cls.YAML_ALT_FORMATS:
87
96
  config_dict = _yaml.safe_load(src)
88
97
 
89
- elif src_format == "json":
98
+ elif src_format == cls.JSON_FORMAT or src_format.lower() in cls.JSON_ALT_FORMATS:
90
99
  config_dict = _json.load(src)
91
100
 
92
101
  else:
@@ -113,111 +122,297 @@ class StructProcessor:
113
122
  StructQuoter.quote(struct, dst, dst_format)
114
123
 
115
124
  @classmethod
116
- def parse_struct(cls, data: dict, schema: _meta.StructSchema = None, python_type: type = None) -> object:
125
+ def parse_struct(cls, data: dict, schema: _meta.SchemaDefinition = None, python_type: type = None) -> object:
117
126
 
118
127
  parser = StructParser()
119
128
  return parser.parse(python_type, data)
120
129
 
121
130
  @classmethod
122
- def _define_struct_for_dataclass(cls, python_type: _dc.dataclass) -> _meta.StructSchema:
131
+ def _define_struct_for_dataclass(
132
+ cls, python_type: _dc.dataclass,
133
+ named_types: _tp.Dict[str, _meta.SchemaDefinition],
134
+ named_enums: _tp.Dict[str, _meta.EnumValues],
135
+ type_stack: _tp.List[str]) \
136
+ -> _meta.SchemaDefinition:
123
137
 
124
- type_hints = _tp.get_type_hints(python_type)
125
- trac_fields = dict()
138
+ try:
126
139
 
127
- for dc_field in _dc.fields(python_type):
140
+ type_stack.append(cls._qualified_type_name(python_type))
128
141
 
129
- field_name = dc_field.name
130
- python_type = type_hints[field_name]
142
+ type_hints = _tp.get_type_hints(python_type)
143
+ trac_fields = list()
144
+
145
+ for field_index, dc_field in enumerate(_dc.fields(python_type)):
131
146
 
132
- trac_field = cls._define_field(python_type, dc_field=dc_field)
133
- trac_fields[field_name] = trac_field
147
+ field_name = dc_field.name
148
+ python_type = type_hints[field_name]
134
149
 
135
- return _meta.StructSchema(fields=trac_fields)
150
+ trac_field = cls._define_field(
151
+ field_name, field_index, python_type, dc_field=dc_field,
152
+ named_types=named_types, named_enums=named_enums, type_stack=type_stack)
153
+
154
+ trac_fields.append(trac_field)
155
+
156
+ if len(type_stack) == 1:
157
+ return _meta.SchemaDefinition(
158
+ schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields,
159
+ namedTypes=named_types, namedEnums=named_enums)
160
+ else:
161
+ return _meta.SchemaDefinition(
162
+ schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields)
163
+
164
+ finally:
165
+
166
+ type_stack.pop()
136
167
 
137
168
  @classmethod
138
- def _define_struct_for_pydantic(cls, python_type: "type[_pyd.BaseModel]") -> _meta.StructSchema:
169
+ def _define_struct_for_pydantic(
170
+ cls, python_type: "type[_pyd.BaseModel]",
171
+ named_types: _tp.Dict[str, _meta.SchemaDefinition],
172
+ named_enums: _tp.Dict[str, _meta.EnumValues],
173
+ type_stack: _tp.List[str]) \
174
+ -> _meta.SchemaDefinition:
139
175
 
140
- type_hints = _tp.get_type_hints(python_type)
141
- trac_fields = dict()
176
+ try:
142
177
 
143
- for field_name, pyd_field in python_type.model_fields.items():
178
+ type_stack.append(cls._qualified_type_name(python_type))
179
+
180
+ type_hints = _tp.get_type_hints(python_type)
181
+ trac_fields = list()
182
+
183
+ field_index = 0
184
+
185
+ for field_name, pyd_field in python_type.model_fields.items():
144
186
 
145
- python_type = type_hints[field_name]
187
+ python_type = type_hints[field_name]
146
188
 
147
- trac_field = cls._define_field(python_type, pyd_field=pyd_field)
189
+ trac_field = cls._define_field(
190
+ field_name, field_index, python_type, pyd_field=pyd_field,
191
+ named_types=named_types, named_enums=named_enums, type_stack=type_stack)
192
+
193
+ if trac_field is not None:
194
+ trac_fields.append(trac_field)
195
+ field_index += 1
196
+
197
+ if len(type_stack) == 1:
198
+ return _meta.SchemaDefinition(
199
+ schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields,
200
+ namedTypes=named_types, namedEnums=named_enums)
201
+ else:
202
+ return _meta.SchemaDefinition(
203
+ schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields)
148
204
 
149
- if trac_field is not None:
150
- trac_fields[field_name] = trac_field
205
+ finally:
151
206
 
152
- return _meta.StructSchema(fields=trac_fields)
207
+ type_stack.pop()
153
208
 
154
209
  @classmethod
155
210
  def _define_field(
156
- cls, python_type: type, *,
157
- dc_field: _dc.Field = None,
158
- pyd_field: "_pyd.fields.FieldInfo" = None) \
159
- -> _meta.StructField:
211
+ cls, name, index, python_type: type, optional=False, *,
212
+ named_types: _tp.Dict[str, _meta.SchemaDefinition],
213
+ named_enums: _tp.Dict[str, _meta.EnumValues],
214
+ type_stack: _tp.List[str],
215
+ dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
216
+ -> _meta.FieldSchema:
160
217
 
161
218
  if python_type in cls.__primitive_types:
162
- return cls._define_primitive_field(python_type, dc_field=dc_field, pyd_field=pyd_field)
219
+
220
+ return cls._define_primitive_field(
221
+ name, index, python_type, optional,
222
+ dc_field=dc_field, pyd_field=pyd_field)
163
223
 
164
224
  elif any(map(lambda _t: isinstance(python_type, _t), cls.__generic_types)):
165
- return cls._define_generic_field(python_type, pyd_field=pyd_field)
166
225
 
167
- elif dc_field is not None and _dc.is_dataclass(python_type):
168
- pass
226
+ return cls._define_generic_field(
227
+ name, index, python_type,
228
+ dc_field=dc_field, pyd_field=pyd_field,
229
+ named_types=named_types, named_enums=named_enums,
230
+ type_stack=type_stack)
231
+
232
+ elif isinstance(python_type, _enum.EnumMeta):
233
+
234
+ type_name = cls._qualified_type_name(python_type)
235
+
236
+ if type_name not in named_enums:
237
+ enum_values = cls._define_enum_values(python_type)
238
+ named_enums[type_name] = enum_values
239
+
240
+ return cls._define_enum_field(
241
+ name, index, python_type, optional,
242
+ dc_field=dc_field, pyd_field=pyd_field)
243
+
244
+ elif _dc.is_dataclass(python_type):
169
245
 
170
- elif pyd_field is not None and issubclass(python_type, _pyd.BaseModel):
171
- pass
246
+ type_name = cls._qualified_type_name(python_type)
247
+
248
+ if type_name in type_stack:
249
+ raise _ex.EValidation("Recursive types are not supported")
250
+
251
+ if type_name not in named_types:
252
+ struct_type = cls._define_struct_for_dataclass(python_type, named_types, named_enums, type_stack)
253
+ named_types[type_name] = struct_type
254
+
255
+ return _meta.FieldSchema(
256
+ fieldName=name,
257
+ fieldOrder=index,
258
+ fieldType=_meta.BasicType.STRUCT,
259
+ notNull=not optional,
260
+ namedType=type_name)
261
+
262
+ elif issubclass(python_type, _pyd.BaseModel):
263
+
264
+ type_name = cls._qualified_type_name(python_type)
265
+
266
+ if type_name in type_stack:
267
+ raise _ex.EValidation("Recursive types are not supported")
268
+
269
+ if type_name not in named_types:
270
+ struct_type = cls._define_struct_for_pydantic(python_type, named_types, named_enums, type_stack)
271
+ named_types[type_name] = struct_type
272
+
273
+ return _meta.FieldSchema(
274
+ fieldName=name,
275
+ fieldOrder=index,
276
+ fieldType=_meta.BasicType.STRUCT,
277
+ notNull=not optional,
278
+ namedType=type_name)
172
279
 
173
280
  else:
174
281
  raise _ex.ETracInternal("Cannot encode field type: " + str(python_type))
175
282
 
176
283
  @classmethod
177
284
  def _define_primitive_field(
178
- cls, python_type: type, optional=False, *,
179
- dc_field: _dc.Field = None,
180
- pyd_field: "_pyd.fields.FieldInfo" = None) \
181
- -> _meta.StructField:
285
+ cls, name: str, index: int, python_type: type, optional=False, *,
286
+ dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
287
+ -> _meta.FieldSchema:
288
+
289
+ default_value = None
290
+
291
+ if dc_field is not None:
292
+ if dc_field.default not in [_dc.MISSING, None]:
293
+ default_value = _meta_types.MetadataCodec.encode_value(dc_field.default)
294
+ elif dc_field.default_factory not in [_dc.MISSING, None]:
295
+ native_value = dc_field.default_factory()
296
+ default_value = _meta_types.MetadataCodec.encode_value(native_value)
297
+
298
+ elif pyd_field is not None:
299
+ if pyd_field.default not in [_pyd.fields.PydanticUndefined, None]:
300
+ default_value = _meta_types.MetadataCodec.encode_value(pyd_field.default)
301
+ elif pyd_field.default_factory not in [_pyd.fields.PydanticUndefined, None]:
302
+ native_value = pyd_field.default_factory()
303
+ default_value = _meta_types.MetadataCodec.encode_value(native_value)
304
+
305
+ return _meta.FieldSchema(
306
+ fieldName=name,
307
+ fieldOrder=index,
308
+ fieldType=cls.__primitive_types[python_type],
309
+ notNull=not optional,
310
+ defaultValue=default_value)
311
+
312
+ @classmethod
313
+ def _define_enum_field(
314
+ cls, name: str, index: int, enum_type: _enum.EnumMeta, optional=False, *,
315
+ dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
316
+ -> _meta.FieldSchema:
182
317
 
183
- struct_field = _meta.StructField()
184
- struct_field.fieldType = _meta.TypeDescriptor(basicType=cls.__primitive_types[python_type])
185
- struct_field.notNull = not optional
318
+ default_value = None
186
319
 
187
- if dc_field is not None and dc_field.default is not _dc.MISSING:
188
- struct_field.defaultValue = _meta_types.MetadataCodec.encode_value(dc_field.default)
320
+ if dc_field is not None and dc_field.default not in [_dc.MISSING, None]:
321
+ default_value = _meta_types.MetadataCodec.encode_value(dc_field.default.name)
189
322
 
190
- if pyd_field is not None and pyd_field.default is not _pyd.fields.PydanticUndefined:
191
- struct_field.defaultValue = _meta_types.MetadataCodec.encode_value(pyd_field.default)
323
+ if pyd_field is not None and pyd_field.default not in [_pyd.fields.PydanticUndefined, None]:
324
+ default_value = _meta_types.MetadataCodec.encode_value(pyd_field.default.name)
192
325
 
193
- return struct_field
326
+ return _meta.FieldSchema(
327
+ fieldName=name,
328
+ fieldOrder=index,
329
+ fieldType=_meta.BasicType.STRING,
330
+ categorical=True,
331
+ notNull=not optional,
332
+ namedEnum=cls._qualified_type_name(enum_type),
333
+ defaultValue=default_value)
334
+
335
+ @classmethod
336
+ def _define_enum_values(cls, enum_type: _enum.EnumMeta) -> _meta.EnumValues:
337
+
338
+ values = list(map(lambda value: value.name, enum_type))
339
+ return _meta.EnumValues(values=values)
194
340
 
195
341
  @classmethod
196
342
  def _define_generic_field(
197
- cls, python_type: type, *,
198
- dc_field: _dc.Field = None,
199
- pyd_field: "_pyd.fields.FieldInfo" = None) -> _meta.StructField:
343
+ cls, name, index, python_type: type, *,
344
+ named_types: _tp.Dict[str, _meta.SchemaDefinition],
345
+ named_enums: _tp.Dict[str, _meta.EnumValues],
346
+ type_stack: _tp.List[str],
347
+ dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
348
+ -> _meta.FieldSchema:
200
349
 
201
350
  origin = _tp.get_origin(python_type)
202
351
  args = _tp.get_args(python_type)
203
352
 
204
353
  # types.NoneType not available in Python 3.9, so use type(None) instead
205
- if origin in cls.__union_types and len(args) == 2 and args[1] is type(None):
206
- optional_type = args[0]
207
- return cls._define_primitive_field(optional_type, optional=True, dc_field=dc_field, pyd_field=pyd_field)
354
+ if origin in cls.__union_types and len(args) == 2 and type(None) in args:
355
+ optional_type = args[0] if args[1] is type(None) else args[1]
356
+ return cls._define_field(
357
+ name, index, optional_type, optional=True,
358
+ dc_field=dc_field, pyd_field=pyd_field,
359
+ named_types=named_types, named_enums=named_enums,
360
+ type_stack=type_stack)
208
361
 
209
362
  elif origin in [list, _tp.List]:
210
- list_type = args[0]
211
- pass
363
+
364
+ item_type = args[0]
365
+ item_field = cls._define_field(
366
+ "item", 0, item_type, optional=False,
367
+ named_types=named_types, named_enums=named_enums,
368
+ type_stack=type_stack)
369
+
370
+ return _meta.FieldSchema(
371
+ fieldName=name,
372
+ fieldOrder=index,
373
+ fieldType=_meta.BasicType.ARRAY,
374
+ notNull=True,
375
+ children=[item_field])
212
376
 
213
377
  elif origin in [dict, _tp.Dict]:
378
+
214
379
  key_type = args[0]
380
+ key_field = _meta.FieldSchema(
381
+ fieldName="key",
382
+ fieldOrder=0,
383
+ fieldType=_meta.BasicType.STRING,
384
+ notNull=True)
385
+
215
386
  value_type = args[1]
216
- pass
387
+ value_field = cls._define_field(
388
+ "value", 1, value_type, optional=False,
389
+ named_types=named_types, named_enums=named_enums,
390
+ type_stack=type_stack)
391
+
392
+ return _meta.FieldSchema(
393
+ fieldName=name,
394
+ fieldOrder=index,
395
+ fieldType=_meta.BasicType.MAP,
396
+ notNull=True,
397
+ children=[key_field, value_field])
217
398
 
218
399
  else:
219
400
  raise _ex.ETracInternal("Cannot encode field type: " + str(python_type))
220
401
 
402
+ @classmethod
403
+ def _qualified_type_name(cls, python_type: type):
404
+
405
+ name = python_type.__name__
406
+ module = python_type.__module__
407
+
408
+ if module.startswith(cls.__SHIM_PREFIX):
409
+ shim_root_index = module.index(".", len(cls.__SHIM_PREFIX)) + 1
410
+ module = module[shim_root_index:]
411
+
412
+ return f"{module}.{name}"
413
+
414
+ __SHIM_PREFIX = "tracdap.shim."
415
+
221
416
 
222
417
  class StructParser:
223
418
 
@@ -476,19 +671,18 @@ class StructQuoter:
476
671
  # New implementation of STRUCT quoting, copied from config_parser
477
672
  # After a period of stabilization, config_parser will switch to this implementation
478
673
 
479
- JSON_FORMAT = "json"
480
- YAML_FORMAT = "yaml"
481
674
  INDENT = 3
482
675
 
483
676
  @classmethod
484
677
  def quote(cls, obj: _tp.Any, dst: _tp.TextIO, dst_format: str):
485
678
 
486
- if dst_format.lower() == cls.JSON_FORMAT:
487
- return cls.quote_json(obj, dst)
488
679
 
489
- if dst_format.lower() == cls.YAML_FORMAT:
680
+ if dst_format == StructProcessor.YAML_FORMAT or dst_format.lower() in StructProcessor.YAML_ALT_FORMATS:
490
681
  return cls.quote_yaml(obj, dst)
491
682
 
683
+ if dst_format == StructProcessor.JSON_FORMAT or dst_format.lower() in StructProcessor.JSON_ALT_FORMATS:
684
+ return cls.quote_json(obj, dst)
685
+
492
686
  # TODO :This is probably an error in the user-supplied parameters
493
687
  raise _ex.ETracInternal(f"Unsupported output format [{dst_format}]")
494
688
 
@@ -21,6 +21,7 @@ import re
21
21
  import typing as tp
22
22
  import uuid
23
23
 
24
+ import tracdap.rt.api as api
24
25
  import tracdap.rt.exceptions as ex
25
26
  import tracdap.rt.metadata as meta
26
27
  import tracdap.rt.config as cfg
@@ -121,55 +122,103 @@ def selector_for_latest(object_id: meta.TagHeader) -> meta.TagSelector:
121
122
  latestTag=True)
122
123
 
123
124
 
125
+ def get_job_mapping(
126
+ selector: tp.Union[meta.TagHeader, meta.TagSelector],
127
+ job_config: cfg.JobConfig) \
128
+ -> meta.TagHeader:
129
+
130
+ obj_key = object_key(selector)
131
+ obj_id = job_config.objectMapping.get(obj_key)
132
+
133
+ if obj_id is not None:
134
+ return obj_id
135
+
136
+ obj_key_match = __OBJ_KEY_PATTERN.match(obj_key)
137
+
138
+ if not obj_key_match:
139
+ err = f"Missing required {selector.objectType.name} ID for [{object_key(selector)}]"
140
+ raise ex.ERuntimeValidation(err)
141
+
142
+ obj_type = obj_key_match.group(1)
143
+ obj_id = obj_key_match.group(2)
144
+ obj_ver = obj_key_match.group(3)
145
+ obj_ts = job_config.jobId.objectTimestamp
146
+
147
+ return meta.TagHeader(
148
+ meta.ObjectType.__members__[obj_type], obj_id,
149
+ int(obj_ver), obj_ts, 1, obj_ts)
150
+
151
+
124
152
  def get_job_metadata(
125
153
  selector: tp.Union[meta.TagHeader, meta.TagSelector],
126
154
  job_config: cfg.JobConfig,
127
155
  optional: bool = False) \
128
156
  -> tp.Optional[meta.ObjectDefinition]:
129
157
 
158
+ return __get_job_metadata_item(selector, job_config, job_config.objects, "object", optional)
159
+
160
+
161
+ def get_job_metadata_tag(
162
+ selector: tp.Union[meta.TagHeader, meta.TagSelector],
163
+ job_config: cfg.JobConfig,
164
+ optional: bool = False) \
165
+ -> tp.Optional[meta.Tag]:
166
+
167
+ return __get_job_metadata_item(selector, job_config, job_config.tags, "tag", optional)
168
+
169
+
170
+ __METADATA_TYPE = tp.TypeVar("__METADATA_TYPE")
171
+
172
+
173
+ def __get_job_metadata_item(
174
+ selector: tp.Union[meta.TagHeader, meta.TagSelector], job_config: cfg.JobConfig,
175
+ metadata: tp.Dict[str, __METADATA_TYPE], metadata_type: str,
176
+ optional: bool = False) \
177
+ -> tp.Optional[__METADATA_TYPE]:
178
+
130
179
  obj_key = object_key(selector)
131
180
  obj_id = job_config.objectMapping.get(obj_key)
132
181
 
133
182
  if obj_id is not None:
134
183
  obj_key = object_key(obj_id)
135
184
 
136
- obj = job_config.objects.get(obj_key)
185
+ item = metadata.get(obj_key)
137
186
 
138
- if obj is not None:
139
- return obj
187
+ if item is not None:
188
+ return item
140
189
 
141
190
  if optional:
142
191
  return None
143
192
 
144
- err = f"Missing required {selector.objectType.name} object for [{object_key(selector)}]"
193
+ err = f"Missing required {selector.objectType.name} {metadata_type} for [{object_key(selector)}]"
145
194
  raise ex.ERuntimeValidation(err)
146
195
 
147
196
 
148
- def get_job_mapping(
149
- selector: tp.Union[meta.TagHeader, meta.TagSelector],
150
- job_config: cfg.JobConfig) \
151
- -> meta.TagHeader:
197
+ def attach_runtime_metadata(obj: tp.Any, metadata: api.RuntimeMetadata):
152
198
 
153
- obj_key = object_key(selector)
154
- obj_id = job_config.objectMapping.get(obj_key)
199
+ if hasattr(obj, "with_metadata"):
200
+ attach_func = getattr(obj, "with_metadata")
201
+ if isinstance(attach_func, tp.Callable):
202
+ return attach_func(metadata)
155
203
 
156
- if obj_id is not None:
157
- return obj_id
204
+ setattr(obj, "_metadata", metadata)
158
205
 
159
- obj_key_match = __OBJ_KEY_PATTERN.match(obj_key)
206
+ return obj
160
207
 
161
- if not obj_key_match:
162
- err = f"Missing required {selector.objectType.name} ID for [{object_key(selector)}]"
163
- raise ex.ERuntimeValidation(err)
164
208
 
165
- obj_type = obj_key_match.group(1)
166
- obj_id = obj_key_match.group(2)
167
- obj_ver = obj_key_match.group(3)
168
- obj_ts = job_config.jobId.objectTimestamp
209
+ def retrieve_runtime_metadata(obj: tp.Any) -> tp.Optional[api.RuntimeMetadata]:
169
210
 
170
- return meta.TagHeader(
171
- meta.ObjectType.__members__[obj_type], obj_id,
172
- int(obj_ver), obj_ts, 1, obj_ts)
211
+ if hasattr(obj, "metadata"):
212
+ metadata = getattr(obj, "metadata")
213
+ if isinstance(metadata, api.RuntimeMetadata):
214
+ return metadata
215
+
216
+ if hasattr(obj, "_metadata"):
217
+ metadata = getattr(obj, "_metadata")
218
+ if isinstance(metadata, api.RuntimeMetadata):
219
+ return metadata
220
+
221
+ return None
173
222
 
174
223
 
175
224
  def get_origin(metaclass: type):
@@ -296,3 +345,25 @@ def filter_model_stack_trace(full_stack: tb.StackSummary, checkout_directory: pa
296
345
  last_model_frame = first_model_frame + frame_index
297
346
 
298
347
  return full_stack[first_model_frame:last_model_frame+1]
348
+
349
+
350
+ __T = tp.TypeVar("__T")
351
+
352
+ def read_property(properties: tp.Dict[str, str], key: str, default: tp.Optional[__T] = None, convert: tp.Optional[tp.Type[__T]] = str) -> __T:
353
+
354
+ value = properties.get(key)
355
+
356
+ if value is None:
357
+ if default is not None:
358
+ value = default
359
+ else:
360
+ raise ex.EConfigParse(f"Missing required property: [{key}]")
361
+
362
+ try:
363
+ if convert is bool and isinstance(value, str):
364
+ return True if value.lower() == "true" else False
365
+ else:
366
+ return convert(value)
367
+
368
+ except (ValueError, TypeError):
369
+ raise ex.EConfigParse(f"Wrong property type: [{key}] = [{value}], expected type is [{convert}]")
@@ -118,6 +118,19 @@ class TracContextImpl(_api.TracContext):
118
118
 
119
119
  return not data_view.is_empty()
120
120
 
121
+ def get_metadata(self, item_name: str) -> tp.Optional[_api.RuntimeMetadata]:
122
+
123
+ _val.validate_signature(self.get_metadata, item_name)
124
+
125
+ self.__val.check_item_valid_identifier(item_name, TracContextValidator.ITEM)
126
+ self.__val.check_item_defined_in_model(item_name, TracContextValidator.ITEM)
127
+ self.__val.check_item_available_in_context(item_name, TracContextValidator.ITEM)
128
+
129
+ obj = self.__local_ctx.get(item_name)
130
+
131
+ # Can be none if no metadata is attached
132
+ return _util.retrieve_runtime_metadata(obj)
133
+
121
134
  def get_schema(self, dataset_name: str) -> _meta.SchemaDefinition:
122
135
 
123
136
  _val.validate_signature(self.get_schema, dataset_name)
@@ -201,8 +214,12 @@ class TracContextImpl(_api.TracContext):
201
214
  self.__val.check_context_data_view_type(struct_name, data_view, _meta.ObjectType.DATA)
202
215
  self.__val.check_dataset_schema_defined(struct_name, data_view)
203
216
 
204
- struct_data: dict = data_view.parts[part_key][0].content
205
- return _struct.StructProcessor.parse_struct(struct_data, None, python_class)
217
+ struct_data = data_view.parts[part_key][0].content
218
+
219
+ if isinstance(struct_data, python_class):
220
+ return struct_data
221
+ else:
222
+ return _struct.StructProcessor.parse_struct(struct_data, None, python_class)
206
223
 
207
224
  def get_file(self, file_name: str) -> bytes:
208
225
 
@@ -371,16 +388,25 @@ class TracContextImpl(_api.TracContext):
371
388
  self.__val.check_item_valid_identifier(file_name, TracContextValidator.FILE)
372
389
  self.__val.check_item_is_model_output(file_name, TracContextValidator.FILE)
373
390
 
391
+ class DelayedClose(io.BytesIO):
392
+
393
+ def __init__(self):
394
+ super().__init__()
395
+
396
+ def close(self):
397
+ super().flush()
398
+
374
399
  @contextlib.contextmanager
375
400
  def memory_stream(stream: io.BytesIO):
376
401
  try:
377
402
  yield stream
378
- buffer = stream.getbuffer().tobytes()
379
- self.put_file(file_name, buffer)
380
403
  finally:
381
- stream.close()
404
+ with stream.getbuffer() as buffer:
405
+ self.put_file(file_name, bytes(buffer))
406
+ if not stream.closed:
407
+ io.BytesIO.close(stream)
382
408
 
383
- return memory_stream(io.BytesIO())
409
+ return memory_stream(DelayedClose())
384
410
 
385
411
  def log(self) -> logging.Logger:
386
412
 
@@ -816,6 +842,7 @@ class TracContextErrorReporter:
816
842
 
817
843
  class TracContextValidator(TracContextErrorReporter):
818
844
 
845
+ ITEM = "Item"
819
846
  PARAMETER = "Parameter"
820
847
  DATASET = "Dataset"
821
848
  FILE = "File"
@@ -891,10 +918,10 @@ class TracContextValidator(TracContextErrorReporter):
891
918
  if schema is None:
892
919
  self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
893
920
 
894
- if schema.schemaType == _meta.SchemaType.TABLE and (schema.table is None or not schema.table.fields):
921
+ if schema.schemaType == _meta.SchemaType.TABLE_SCHEMA and (schema.table is None or not schema.table.fields):
895
922
  self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
896
923
 
897
- if schema.schemaType == _meta.SchemaType.STRUCT and (schema.struct is None or not schema.struct.fields):
924
+ if schema.schemaType == _meta.SchemaType.STRUCT_SCHEMA and not schema.fields:
898
925
  self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
899
926
 
900
927
  def check_dataset_schema_not_defined(self, dataset_name: str, data_view: _data.DataView):