tracdap-runtime 0.8.0rc2__py3-none-any.whl → 0.9.0b2__py3-none-any.whl

tracdap/rt/_impl/core/util.py

@@ -16,10 +16,12 @@
 import datetime as dt
 import pathlib
 import platform
+import re
 
 import typing as tp
 import uuid
 
+import tracdap.rt.api as api
 import tracdap.rt.exceptions as ex
 import tracdap.rt.metadata as meta
 import tracdap.rt.config as cfg
@@ -30,6 +32,7 @@ import traceback as tb
 __IS_WINDOWS = platform.system() == "Windows"
 __FIRST_MODEL_FRAME_NAME = "run_model"
 __FIRST_MODEL_FRAME_TEST_NAME = "_callTestMethod"
+__OBJ_KEY_PATTERN = re.compile(r"([A-Z]+)-(.*)-v(\d+)")
 
 
 def is_windows():
@@ -60,7 +63,7 @@ def format_file_size(size: int) -> str:
 
 def new_object_id(object_type: meta.ObjectType) -> meta.TagHeader:
 
-    timestamp = dt.datetime.utcnow()
+    timestamp = dt.datetime.now(dt.timezone.utc)
 
     return meta.TagHeader(
         objectType=object_type,
@@ -71,6 +74,19 @@ def new_object_id(object_type: meta.ObjectType) -> meta.TagHeader:
         tagTimestamp=meta.DatetimeValue(timestamp.isoformat()))
 
 
+def new_object_version(prior_id: meta.TagHeader) -> meta.TagHeader:
+
+    timestamp = dt.datetime.now(dt.timezone.utc)
+
+    return meta.TagHeader(
+        objectType=prior_id.objectType,
+        objectId=prior_id.objectId,
+        objectVersion=prior_id.objectVersion + 1,
+        objectTimestamp=meta.DatetimeValue(timestamp.isoformat()),
+        tagVersion=1,
+        tagTimestamp=meta.DatetimeValue(timestamp.isoformat()))
+
+
 def object_key(object_id: tp.Union[meta.TagHeader, meta.TagSelector]) -> str:
 
     if isinstance(object_id, meta.TagHeader):
@@ -106,29 +122,105 @@ def selector_for_latest(object_id: meta.TagHeader) -> meta.TagSelector:
         latestTag=True)
 
 
-def get_job_resource(
+def get_job_mapping(
+        selector: tp.Union[meta.TagHeader, meta.TagSelector],
+        job_config: cfg.JobConfig) \
+        -> meta.TagHeader:
+
+    obj_key = object_key(selector)
+    obj_id = job_config.objectMapping.get(obj_key)
+
+    if obj_id is not None:
+        return obj_id
+
+    obj_key_match = __OBJ_KEY_PATTERN.match(obj_key)
+
+    if not obj_key_match:
+        err = f"Missing required {selector.objectType.name} ID for [{object_key(selector)}]"
+        raise ex.ERuntimeValidation(err)
+
+    obj_type = obj_key_match.group(1)
+    obj_id = obj_key_match.group(2)
+    obj_ver = obj_key_match.group(3)
+    obj_ts = job_config.jobId.objectTimestamp
+
+    return meta.TagHeader(
+        meta.ObjectType.__members__[obj_type], obj_id,
+        int(obj_ver), obj_ts, 1, obj_ts)
+
+
+def get_job_metadata(
         selector: tp.Union[meta.TagHeader, meta.TagSelector],
         job_config: cfg.JobConfig,
-        optional: bool = False):
+        optional: bool = False) \
+        -> tp.Optional[meta.ObjectDefinition]:
+
+    return __get_job_metadata_item(selector, job_config, job_config.objects, "object", optional)
+
+
+def get_job_metadata_tag(
+        selector: tp.Union[meta.TagHeader, meta.TagSelector],
+        job_config: cfg.JobConfig,
+        optional: bool = False) \
+        -> tp.Optional[meta.Tag]:
+
+    return __get_job_metadata_item(selector, job_config, job_config.tags, "tag", optional)
 
-    resource_key = object_key(selector)
-    resource_id = job_config.resourceMapping.get(resource_key)
 
-    if resource_id is not None:
-        resource_key = object_key(resource_id)
+__METADATA_TYPE = tp.TypeVar("__METADATA_TYPE")
 
-    resource = job_config.resources.get(resource_key)
 
-    if resource is not None:
-        return resource
+def __get_job_metadata_item(
+        selector: tp.Union[meta.TagHeader, meta.TagSelector], job_config: cfg.JobConfig,
+        metadata: tp.Dict[str, __METADATA_TYPE], metadata_type: str,
+        optional: bool = False) \
+        -> tp.Optional[__METADATA_TYPE]:
+
+    obj_key = object_key(selector)
+    obj_id = job_config.objectMapping.get(obj_key)
+
+    if obj_id is not None:
+        obj_key = object_key(obj_id)
+
+    item = metadata.get(obj_key)
+
+    if item is not None:
+        return item
 
     if optional:
         return None
 
-    err = f"Missing required {selector.objectType.name} resource [{object_key(selector)}]"
+    err = f"Missing required {selector.objectType.name} {metadata_type} for [{object_key(selector)}]"
     raise ex.ERuntimeValidation(err)
 
 
+def attach_runtime_metadata(obj: tp.Any, metadata: api.RuntimeMetadata):
+
+    if hasattr(obj, "with_metadata"):
+        attach_func = getattr(obj, "with_metadata")
+        if isinstance(attach_func, tp.Callable):
+            return attach_func(metadata)
+
+    setattr(obj, "_metadata", metadata)
+
+    return obj
+
+
+def retrieve_runtime_metadata(obj: tp.Any) -> tp.Optional[api.RuntimeMetadata]:
+
+    if hasattr(obj, "metadata"):
+        metadata = getattr(obj, "metadata")
+        if isinstance(metadata, api.RuntimeMetadata):
+            return metadata
+
+    if hasattr(obj, "_metadata"):
+        metadata = getattr(obj, "_metadata")
+        if isinstance(metadata, api.RuntimeMetadata):
+            return metadata
+
+    return None
+
+
 def get_origin(metaclass: type):
 
     # Minimum supported Python is 3.7, which does not provide get_origin and get_args
@@ -253,3 +345,25 @@ def filter_model_stack_trace(full_stack: tb.StackSummary, checkout_directory: pa
         last_model_frame = first_model_frame + frame_index
 
     return full_stack[first_model_frame:last_model_frame+1]
+
+
+__T = tp.TypeVar("__T")
+
+def read_property(properties: tp.Dict[str, str], key: str, default: tp.Optional[__T] = None, convert: tp.Optional[tp.Type[__T]] = str) -> __T:
+
+    value = properties.get(key)
+
+    if value is None:
+        if default is not None:
+            value = default
+        else:
+            raise ex.EConfigParse(f"Missing required property: [{key}]")
+
+    try:
+        if convert is bool and isinstance(value, str):
+            return True if value.lower() == "true" else False
+        else:
+            return convert(value)
+
+    except (ValueError, TypeError):
+        raise ex.EConfigParse(f"Wrong property type: [{key}] = [{value}], expected type is [{convert}]")

tracdap/rt/_impl/exec/context.py

@@ -118,6 +118,19 @@ class TracContextImpl(_api.TracContext):
 
         return not data_view.is_empty()
 
+    def get_metadata(self, item_name: str) -> tp.Optional[_api.RuntimeMetadata]:
+
+        _val.validate_signature(self.get_metadata, item_name)
+
+        self.__val.check_item_valid_identifier(item_name, TracContextValidator.ITEM)
+        self.__val.check_item_defined_in_model(item_name, TracContextValidator.ITEM)
+        self.__val.check_item_available_in_context(item_name, TracContextValidator.ITEM)
+
+        obj = self.__local_ctx.get(item_name)
+
+        # Can be none if no metadata is attached
+        return _util.retrieve_runtime_metadata(obj)
+
     def get_schema(self, dataset_name: str) -> _meta.SchemaDefinition:
 
         _val.validate_signature(self.get_schema, dataset_name)
@@ -201,8 +214,12 @@ class TracContextImpl(_api.TracContext):
         self.__val.check_context_data_view_type(struct_name, data_view, _meta.ObjectType.DATA)
         self.__val.check_dataset_schema_defined(struct_name, data_view)
 
-        struct_data: dict = data_view.parts[part_key][0].content
-        return _struct.StructProcessor.parse_struct(struct_data, None, python_class)
+        struct_data = data_view.parts[part_key][0].content
+
+        if isinstance(struct_data, python_class):
+            return struct_data
+        else:
+            return _struct.StructProcessor.parse_struct(struct_data, None, python_class)
 
     def get_file(self, file_name: str) -> bytes:
 
@@ -371,16 +388,25 @@ class TracContextImpl(_api.TracContext):
         self.__val.check_item_valid_identifier(file_name, TracContextValidator.FILE)
         self.__val.check_item_is_model_output(file_name, TracContextValidator.FILE)
 
+        class DelayedClose(io.BytesIO):
+
+            def __init__(self):
+                super().__init__()
+
+            def close(self):
+                super().flush()
+
         @contextlib.contextmanager
         def memory_stream(stream: io.BytesIO):
             try:
                 yield stream
-                buffer = stream.getbuffer().tobytes()
-                self.put_file(file_name, buffer)
             finally:
-                stream.close()
+                with stream.getbuffer() as buffer:
+                    self.put_file(file_name, bytes(buffer))
+                if not stream.closed:
+                    io.BytesIO.close(stream)
 
-        return memory_stream(io.BytesIO())
+        return memory_stream(DelayedClose())
 
     def log(self) -> logging.Logger:
 
@@ -816,6 +842,7 @@ class TracContextErrorReporter:
 
 class TracContextValidator(TracContextErrorReporter):
 
+    ITEM = "Item"
     PARAMETER = "Parameter"
     DATASET = "Dataset"
     FILE = "File"
@@ -891,10 +918,10 @@ class TracContextValidator(TracContextErrorReporter):
         if schema is None:
             self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
 
-        if schema.schemaType == _meta.SchemaType.TABLE and (schema.table is None or not schema.table.fields):
+        if schema.schemaType == _meta.SchemaType.TABLE_SCHEMA and (schema.table is None or not schema.table.fields):
             self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
 
-        if schema.schemaType == _meta.SchemaType.STRUCT and (schema.struct is None or not schema.struct.fields):
+        if schema.schemaType == _meta.SchemaType.STRUCT_SCHEMA and not schema.fields:
             self._report_error(f"Schema not defined for dataset {dataset_name} in the current context")
 
     def check_dataset_schema_not_defined(self, dataset_name: str, data_view: _data.DataView):