tracdap-runtime 0.9.0b1__py3-none-any.whl → 0.9.0b2__py3-none-any.whl

tracdap/rt/_impl/exec/dev_mode.py

@@ -58,11 +58,6 @@ class DevModeTranslator:
 
         cls._log.info(f"Applying dev mode config translation to system config")
 
-        # TODO: In code gen, default object types to a new object unless the field is marked as optional
-        # This would match the general semantics of protobuf
-        if sys_config.storage is None:
-            sys_config.storage = _cfg.StorageConfig()
-
         sys_config = cls._add_integrated_repo(sys_config)
         sys_config = cls._process_storage(sys_config, config_mgr)
 
@@ -73,11 +68,12 @@ class DevModeTranslator:
 
         # Add the integrated model repo trac_integrated
 
-        integrated_repo_config = _cfg.PluginConfig(
+        integrated_repo_config = _meta.ResourceDefinition(
+            resourceType=_meta.ResourceType.MODEL_REPOSITORY,
             protocol="integrated",
             properties={})
 
-        sys_config.repositories["trac_integrated"] = integrated_repo_config
+        sys_config.resources["trac_integrated"] = integrated_repo_config
 
         return sys_config
 
@@ -86,24 +82,17 @@ class DevModeTranslator:
             cls, sys_config: _cfg.RuntimeConfig,
             config_mgr: _cfg_p.ConfigManager):
 
-        storage_config = copy.deepcopy(sys_config.storage)
-        storage_config.defaultLayout = _meta.StorageLayout.DEVELOPER_LAYOUT
-
-        for bucket_key, bucket_config in storage_config.buckets.items():
-            storage_config.buckets[bucket_key] = cls._resolve_storage_location(
-                bucket_key, bucket_config, config_mgr)
-
-        for bucket_key, bucket_config in storage_config.external.items():
-            storage_config.external[bucket_key] = cls._resolve_storage_location(
-                bucket_key, bucket_config, config_mgr)
+        sys_config.properties[_cfg_p.ConfigKeys.STORAGE_DEFAULT_LAYOUT] = _meta.StorageLayout.DEVELOPER_LAYOUT.name
 
-        sys_config = copy.copy(sys_config)
-        sys_config.storage = storage_config
+        for resource_key, resource in sys_config.resources.items():
+            if resource.resourceType in [_meta.ResourceType.INTERNAL_STORAGE, _meta.ResourceType.EXTERNAL_STORAGE]:
+                sys_config.resources[resource_key] = cls._resolve_storage_location(
+                resource_key, resource, config_mgr)
 
         return sys_config
 
     @classmethod
-    def _resolve_storage_location(cls, bucket_key, bucket_config, config_mgr: _cfg_p.ConfigManager):
+    def _resolve_storage_location(cls, bucket_key, bucket_config: _meta.ResourceDefinition, config_mgr: _cfg_p.ConfigManager):
 
         if bucket_config.protocol != "LOCAL":
             return bucket_config
@@ -167,6 +156,9 @@ class DevModeTranslator:
             job_config, job_def = self.translate_job_def(job_config, job_def, model_class)
             job_config.job = job_def
 
+            # Include some basic tags and attributes in the generated metadata
+            job_config = self.generate_dev_mode_tags(job_config)
+
             return job_config
 
         finally:
@@ -225,6 +217,8 @@ class DevModeTranslator:
             -> _cfg.JobConfig:
 
         obj_key = _util.object_key(obj_id)
+
+        job_config.objectMapping[obj_key] = obj_id
         job_config.objects[obj_key] = obj
 
         return job_config
@@ -799,7 +793,6 @@ class DevModeTranslator:
         else:
             return job_config, job_def
 
-        job_metadata = job_config.objects
         job_inputs = job_detail.inputs
         job_outputs = job_detail.outputs
         job_prior_outputs = job_detail.priorOutputs
@@ -810,7 +803,7 @@ class DevModeTranslator:
                     raise _ex.EJobValidation(f"Missing required input [{key}]")
                 continue
             supplied_input = job_inputs.pop(key) if key in job_inputs else None
-            input_selector = self._process_socket(key, schema, supplied_input, job_metadata, is_output=False)
+            input_selector = self._process_socket(key, schema, supplied_input, job_config, is_output=False)
             if input_selector is not None:
                 job_inputs[key] = input_selector
 
@@ -818,28 +811,28 @@ class DevModeTranslator:
             if key not in job_outputs:
                 raise _ex.EJobValidation(f"Missing required output [{key}]")
             supplied_output = job_outputs.pop(key)
-            output_selector = self._process_socket(key, schema, supplied_output, job_metadata, is_output=True)
+            output_selector = self._process_socket(key, schema, supplied_output, job_config, is_output=True)
             if output_selector is not None:
                 job_prior_outputs[key] = output_selector
 
         return job_config, job_def
 
-    def _process_socket(self, key, socket, supplied_value, job_metadata, is_output) -> _meta.TagSelector:
+    def _process_socket(self, key, socket, supplied_value, job_config, is_output) -> _meta.TagSelector:
 
         if socket.objectType == _meta.ObjectType.DATA:
             schema = socket.schema if socket and not socket.dynamic else None
-            return self._process_data_socket(key, supplied_value, schema, job_metadata, is_output)
+            return self._process_data_socket(key, supplied_value, schema, job_config, is_output)
 
         elif socket.objectType == _meta.ObjectType.FILE:
             file_type = socket.fileType
-            return self._process_file_socket(key, supplied_value, file_type, job_metadata, is_output)
+            return self._process_file_socket(key, supplied_value, file_type, job_config, is_output)
 
         else:
             raise _ex.EUnexpected()
 
     def _process_data_socket(
             self, data_key, data_value, schema: tp.Optional[_meta.SchemaDefinition],
-            job_metadata: tp.Dict[str, _meta.ObjectDefinition], is_output: bool)\
+            job_config: _cfg.JobConfig, is_output: bool)\
             -> _meta.TagSelector:
 
         data_id = _util.new_object_id(_meta.ObjectType.DATA)
@@ -847,8 +840,8 @@ class DevModeTranslator:
 
         if isinstance(data_value, str):
             storage_path = data_value
-            storage_key = self._sys_config.storage.defaultBucket
-            storage_format = self.infer_format(storage_path, self._sys_config.storage, schema)
+            storage_key = _util.read_property(self._sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
+            storage_format = self.infer_format(storage_path, self._sys_config, schema)
 
         elif isinstance(data_value, dict):
 
@@ -857,8 +850,8 @@ class DevModeTranslator:
             if not storage_path:
                 raise _ex.EConfigParse(f"Invalid configuration for input [{data_key}] (missing required value 'path'")
 
-            storage_key = data_value.get("storageKey") or self._sys_config.storage.defaultBucket
-            storage_format = data_value.get("format") or self.infer_format(storage_path, self._sys_config.storage, schema)
+            storage_key = data_value.get("storageKey") or _util.read_property(self._sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
+            storage_format = data_value.get("format") or self.infer_format(storage_path, self._sys_config, schema)
 
         else:
             raise _ex.EConfigParse(f"Invalid configuration for input '{data_key}'")
@@ -895,14 +888,14 @@ class DevModeTranslator:
             storage_id, storage_key, storage_path, storage_format,
             data_item, incarnation_index)
 
-        job_metadata[_util.object_key(data_id)] = data_obj
-        job_metadata[_util.object_key(storage_id)] = storage_obj
+        self._add_job_metadata(job_config, data_id, data_obj)
+        self._add_job_metadata(job_config, storage_id, storage_obj)
 
         return _util.selector_for(data_id)
 
     def _process_file_socket(
             self, file_key, file_value, file_type: _meta.FileType,
-            job_metadata: tp.Dict[str, _meta.ObjectDefinition], is_output: bool) \
+            job_config: _cfg.JobConfig, is_output: bool) \
             -> tp.Optional[_meta.TagSelector]:
 
         file_id = _util.new_object_id(_meta.ObjectType.FILE)
@@ -910,12 +903,12 @@ class DevModeTranslator:
 
         if isinstance(file_value, str):
 
-            storage_key = self._sys_config.storage.defaultBucket
+            storage_key = _util.read_property(self._sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
             storage_path = file_value
 
         elif isinstance(file_value, dict):
 
-            storage_key = file_value.get("storageKey") or self._sys_config.storage.defaultBucket
+            storage_key = file_value.get("storageKey") or _util.read_property(self._sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
             storage_path = file_value.get("path")
 
             if not storage_path:
@@ -956,13 +949,13 @@ class DevModeTranslator:
             storage_id, storage_key, storage_path, storage_format,
             data_item, incarnation_index=0)
 
-        job_metadata[_util.object_key(file_id)] = file_obj
-        job_metadata[_util.object_key(storage_id)] = storage_obj
+        self._add_job_metadata(job_config, file_id, file_obj)
+        self._add_job_metadata(job_config, storage_id, storage_obj)
 
         return _util.selector_for(file_id)
 
     @staticmethod
-    def infer_format(storage_path: str, storage_config: _cfg.StorageConfig, schema: tp.Optional[_meta.SchemaDefinition]):
+    def infer_format(storage_path: str, sys_config: _cfg.RuntimeConfig, schema: tp.Optional[_meta.SchemaDefinition]):
 
         schema_type = schema.schemaType if schema and schema.schemaType else _meta.SchemaType.TABLE
 
@@ -976,7 +969,7 @@ class DevModeTranslator:
                 return extension[1:] if extension.startswith(".") else extension
 
         else:
-            return storage_config.defaultFormat
+            return _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_FORMAT, "CSV")
 
     def _find_latest_version(self, storage_key, storage_path):
 
@@ -1071,6 +1064,33 @@ class DevModeTranslator:
 
         return _meta.ObjectDefinition(objectType=_meta.ObjectType.STORAGE, storage=storage_def)
 
+    @classmethod
+    def generate_dev_mode_tags(cls, job_config: _cfg.JobConfig) -> _cfg.JobConfig:
+
+        job_config = copy.copy(job_config)
+
+        for key, object_id in job_config.objectMapping.items():
+            if key not in job_config.tags:
+
+                tag = _meta.Tag(header=object_id)
+
+                timestamp = _types.MetadataCodec.convert_datetime_value(object_id.objectTimestamp.isoDatetime)
+                user_id = _types.MetadataCodec.encode_value("local_user")
+                user_name = _types.MetadataCodec.encode_value("Local User")
+
+                tag.attrs["trac_dev_mode"] = _types.MetadataCodec.encode_value(True)
+
+                tag.attrs["trac_create_time"] = timestamp
+                tag.attrs["trac_create_user_id"] = user_id
+                tag.attrs["trac_create_user_name"] = user_name
+
+                tag.attrs["trac_update_time"] = timestamp
+                tag.attrs["trac_update_user_id"] = user_id
+                tag.attrs["trac_update_user_name"] = user_name
+
+                job_config.tags[key] = tag
+
+        return job_config
 
 
 DevModeTranslator._log = _logging.logger_for_class(DevModeTranslator)

tracdap/rt/_impl/exec/engine.py

@@ -17,7 +17,6 @@ import copy as cp
 import dataclasses as dc
 import enum
 import io
-import pathlib
 import typing as tp
 
 import tracdap.rt.metadata as _meta
@@ -83,14 +82,6 @@ class _EngineContext:
             pending_nodes, active_nodes, succeeded_nodes, failed_nodes)
 
 
-@dc.dataclass
-class _JobResultSpec:
-
-    save_result: bool = False
-    result_dir: tp.Union[str, pathlib.Path] = None
-    result_format: str = None
-
-
 @dc.dataclass
 class _JobLog:
 
@@ -117,19 +108,16 @@ class _JobLog:
 class _JobState:
 
     job_id: _meta.TagHeader
+    job_config: _cfg.JobConfig = None
+    parent_key: str = None
 
     actor_id: _actors.ActorId = None
     monitors: tp.List[_actors.ActorId] = dc.field(default_factory=list)
+    job_log: _JobLog = None
 
-    job_config: _cfg.JobConfig = None
     job_result: _cfg.JobResult = None
     job_error: Exception = None
 
-    parent_key: str = None
-    result_spec: _JobResultSpec = None
-
-    job_log: _JobLog = None
-
 
 class TracEngine(_actors.Actor):
 
@@ -197,25 +185,25 @@ class TracEngine(_actors.Actor):
         return super().on_signal(signal)
 
     @_actors.Message
-    def submit_job(
-            self, job_config: _cfg.JobConfig,
-            job_result_dir: str,
-            job_result_format: str):
+    def submit_job(self, job_config: _cfg.JobConfig):
 
         job_key = _util.object_key(job_config.jobId)
 
         self._log.info(f"Received a new job: [{job_key}]")
 
-        result_needed = bool(job_result_dir)
-        result_spec = _JobResultSpec(result_needed, job_result_dir, job_result_format)
-        job_log = _JobLog(log_file_needed=result_needed)
-
         job_state = _JobState(job_config.jobId)
-        job_state.job_log = job_log
+        job_state.job_config = job_config
+
+        job_logs_enabled = _util.read_property(
+            job_config.properties,
+            _cfg_p.ConfigKeys.RESULT_LOGS_ENABLED,
+            False, bool)
+
+        job_log = _JobLog(log_file_needed=job_logs_enabled)
 
         job_processor = JobProcessor(
             self._sys_config, self._models, self._storage,
-            job_key, job_config, graph_spec=None, job_log=job_state.job_log)
+            job_key, job_config, graph_spec=None, job_log=job_log)
 
         job_actor_id = self.actors().spawn(job_processor)
 
@@ -226,8 +214,7 @@ class TracEngine(_actors.Actor):
 
         job_state.actor_id = job_actor_id
         job_state.monitors.append(job_monitor_id)
-        job_state.job_config = job_config
-        job_state.result_spec = result_spec
+        job_state.job_log = job_log
 
         self._jobs[job_key] = job_state
 
@@ -245,6 +232,10 @@ class TracEngine(_actors.Actor):
 
         self._log.info(f"Received a child job: [{child_key}] for parent [{parent_key}]")
 
+        # Copy job config properties from parent job
+        child_config = _cfg.JobConfig()
+        child_config.properties.update(parent_state.job_config.properties)
+
         child_job_log = _JobLog(parent_state.job_log)
 
         child_processor = JobProcessor(
@@ -254,11 +245,11 @@ class TracEngine(_actors.Actor):
         child_actor_id = self.actors().spawn(child_processor)
 
         child_state = _JobState(child_id)
-        child_state.job_log = child_job_log
+        child_state.job_config = child_config
+        child_state.parent_key = parent_key
         child_state.actor_id = child_actor_id
         child_state.monitors.append(monitor_id)
-        child_state.parent_key = parent_key
-        child_state.result_spec = _JobResultSpec(False)  # Do not output separate results for child jobs
+        child_state.job_log = child_job_log
 
         self._jobs[child_key] = child_state
 
@@ -332,8 +323,13 @@ class TracEngine(_actors.Actor):
 
         job_state = self._jobs.get(job_key)
 
-        # Record output metadata if required (not needed for local runs or when using API server)
-        if job_state.parent_key is None and job_state.result_spec.save_result:
+        result_enabled = _util.read_property(
+            job_state.job_config.properties,
+            _cfg_p.ConfigKeys.RESULT_ENABLED,
+            False, bool)
+
+        # Record output metadata if required
+        if result_enabled and job_state.parent_key is None:
             self._save_job_result(job_key, job_state)
 
         # Stop any monitors that were created directly by the engine
@@ -352,18 +348,13 @@ class TracEngine(_actors.Actor):
 
         self._log.info(f"Saving job result for [{job_key}]")
 
-        # It might be better abstract reporting of results, job status etc., perhaps with a job monitor
-
-        if job_state.result_spec.save_result:
+        storage_key = _util.read_property(job_state.job_config.properties, _cfg_p.ConfigKeys.RESULT_STORAGE_LOCATION)
+        storage_path = _util.read_property(job_state.job_config.properties, _cfg_p.ConfigKeys.RESULT_STORAGE_PATH)
+        result_format = _util.read_property(job_state.job_config.properties, _cfg_p.ConfigKeys.RESULT_FORMAT, "JSON")
+        result_content = _cfg_p.ConfigQuoter.quote(job_state.job_result, result_format)
 
-            result_format = job_state.result_spec.result_format
-            result_dir = job_state.result_spec.result_dir
-            result_file = f"job_result_{job_key}.{result_format}"
-            result_path = pathlib.Path(result_dir).joinpath(result_file)
-
-            with open(result_path, "xt") as result_stream:
-                result_content = _cfg_p.ConfigQuoter.quote(job_state.job_result, result_format)
-                result_stream.write(result_content)
+        storage = self._storage.get_file_storage(storage_key)
+        storage.write_bytes(storage_path, result_content.encode('utf-8'))
 
     def _get_job_info(self, job_key: str, details: bool = False) -> tp.Optional[_cfg.JobResult]:
 
@@ -455,7 +446,7 @@ class JobProcessor(_actors.Actor):
         self._models = models
         self._storage = storage
 
-        self._job_log = job_log or _JobLog()
+        self._job_log = job_log if job_log is not None else _JobLog()
         self._log_provider = self._job_log.log_provider
         self._log = self._job_log.log_provider.logger_for_object(self)
 
@@ -532,7 +523,8 @@ class JobProcessor(_actors.Actor):
         # This will be the last message in the job log file
         self._log.info(f"Job succeeded [{self.job_key}]")
 
-        self._save_job_log_file(job_result)
+        if self._job_log.log_file_needed:
+            self._save_job_log_file(job_result)
 
         self.actors().stop(self.actors().sender)
         self.actors().send_parent("job_succeeded", self.job_key, job_result)
@@ -556,7 +548,8 @@ class JobProcessor(_actors.Actor):
             result_def.statusMessage = str(error)
             job_result = _cfg.JobResult(job_id, result_id, result_def)
 
-            self._save_job_log_file(job_result)
+            if self._job_log.log_file_needed:
+                self._save_job_log_file(job_result)
 
             self.actors().send_parent("job_failed", self.job_key, error, job_result)
 
@@ -567,10 +560,9 @@ class JobProcessor(_actors.Actor):
 
     def _save_job_log_file(self, job_result: _cfg.JobResult):
 
-        # Do not save log files for child jobs, or if a log is not available
+        # Do not fail the job if log content is not available
         if self._job_log.log_buffer is None:
-            if self._job_log.log_file_needed:
-                self._log.warning(f"Job log not available for [{self.job_key}]")
+            self._log.warning(f"Job log not available for [{self.job_key}]")
             return
 
         # Saving log files could go into a separate actor
@@ -578,13 +570,15 @@ class JobProcessor(_actors.Actor):
         file_id = self._allocate_id(_meta.ObjectType.FILE)
         storage_id = self._allocate_id(_meta.ObjectType.STORAGE)
 
+        self._log.info(f"Saving job log [{_util.object_key(file_id)}]")
+
         file_name = "trac_job_log_file"
         file_type = _meta.FileType("TXT", "text/plain")
 
         file_spec = _data.build_file_spec(
             file_id, storage_id,
             file_name, file_type,
-            self._sys_config.storage)
+            self._sys_config)
 
         file_def = file_spec.definition
         storage_def = file_spec.storage

tracdap/rt/_impl/exec/functions.py

@@ -231,7 +231,7 @@ class DataSpecFunc(NodeFunction[_data.DataSpec]):
         return _data.build_data_spec(
             self.node.data_obj_id, self.node.storage_obj_id,
             self.node.context_key, trac_schema,
-            self.node.storage_config,
+            self.node.sys_config,
             self.node.prior_data_spec)
 
 
@@ -256,7 +256,7 @@ class DataViewFunc(NodeFunction[_data.DataView]):
 
         # TODO: Generalize processing across DataView / DataItem types
 
-        if root_item.schema_type == _meta.SchemaType.STRUCT:
+        if root_item.schema_type == _meta.SchemaType.STRUCT_SCHEMA:
             view = _data.DataView.for_trac_schema(self.node.schema)
             view.parts[root_part_key] = [root_item]
             return view
@@ -269,7 +269,9 @@ class DataViewFunc(NodeFunction[_data.DataView]):
             arrow_schema = root_item.schema
             data_view = _data.DataView.for_arrow_schema(arrow_schema)
 
-        data_view = _data.DataMapping.add_item_to_view(data_view, root_part_key, root_item)
+        data_view = _data.DataMapping \
+            .add_item_to_view(data_view, root_part_key, root_item) \
+            .with_metadata(root_item.metadata)
 
         return data_view
 
@@ -359,10 +361,10 @@ class LoadDataFunc( _LoadSaveDataFunc, NodeFunction[_data.DataItem],):
         if data_spec.object_type == _api.ObjectType.FILE:
             return self._load_file(data_copy)
 
-        elif data_spec.schema_type == _api.SchemaType.TABLE:
+        elif data_spec.schema_type == _api.SchemaType.TABLE_SCHEMA:
             return self._load_table(data_spec, data_copy)
 
-        elif data_spec.schema_type == _api.SchemaType.STRUCT:
+        elif data_spec.schema_type == _api.SchemaType.STRUCT_SCHEMA:
             return self._load_struct(data_copy)
 
         # TODO: Handle dynamic inputs, they should work for any schema type
@@ -394,7 +396,9 @@ class LoadDataFunc( _LoadSaveDataFunc, NodeFunction[_data.DataItem],):
             data_copy.storagePath, data_copy.storageFormat, arrow_schema,
             storage_options=storage_options)
 
-        return _data.DataItem.for_table(table, table.schema, trac_schema)
+        return _data.DataItem \
+            .for_table(table, table.schema, trac_schema) \
+            .with_metadata(data_spec.metadata)
 
     def _load_struct(self, data_copy):
 
@@ -429,10 +433,10 @@ class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[_data.DataSpec]):
         if data_item.object_type == _api.ObjectType.FILE:
             return self._save_file(data_item, data_spec, data_copy)
 
-        elif data_item.schema_type == _api.SchemaType.TABLE:
+        elif data_item.schema_type == _api.SchemaType.TABLE_SCHEMA:
             return self._save_table(data_item, data_spec, data_copy)
 
-        elif data_item.schema_type == _api.SchemaType.STRUCT:
+        elif data_item.schema_type == _api.SchemaType.STRUCT_SCHEMA:
             return self._save_struct(data_item, data_spec, data_copy)
 
         else:

tracdap/rt/_impl/exec/graph.py

@@ -47,7 +47,7 @@ class NodeNamespace:
     def __repr__(self):
         return repr(self.components())
 
-    def components(self) -> [str]:
+    def components(self) -> _tp.List[str]:
         if self == self.__ROOT:
             return ["ROOT"]
         elif self.parent is self.__ROOT or self.parent is None:
@@ -161,7 +161,7 @@ class GraphContext:
     job_namespace: NodeNamespace
     ctx_namespace: NodeNamespace
 
-    storage_config: _cfg.StorageConfig
+    sys_config: _cfg.RuntimeConfig
 
 
 @_dc.dataclass(frozen=True)
@@ -297,7 +297,7 @@ class DataSpecNode(Node[_data.DataSpec]):
     storage_obj_id: _meta.TagHeader
     context_key: str
 
-    storage_config: _cfg.StorageConfig
+    sys_config: _cfg.RuntimeConfig
 
     prior_data_spec: _tp.Optional[_data.DataSpec]
 

tracdap/rt/_impl/exec/graph_builder.py

@@ -20,7 +20,9 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.type_system as _type_system
 import tracdap.rt._impl.core.util as _util
+import tracdap.rt.api as _api
 
 from .graph import *
 
@@ -40,7 +42,7 @@ class GraphBuilder:
     @classmethod
     def dynamic(cls, context: GraphContext) -> "GraphBuilder":
 
-        sys_config = _cfg.RuntimeConfig(storage=context.storage_config)
+        sys_config = context.sys_config
         job_config = _cfg.JobConfig(context.job_id)
 
         return GraphBuilder(sys_config, job_config)
@@ -484,6 +486,7 @@ class GraphBuilder:
     def _build_data_input(self, input_name, input_selector, nodes, outputs, explicit_deps):
 
         data_spec = self._build_data_spec(input_selector)
+        data_spec = self._attach_metadata(data_spec, input_selector)
 
         # Physical load of data items from disk
         # Currently one item per input, since inputs are single part/delta
@@ -523,7 +526,7 @@ class GraphBuilder:
             nodes[data_spec_id] = DataSpecNode(
                 data_spec_id, data_view_id,
                 data_id, storage_id, output_name,
-                self._sys_config.storage,
+                self._sys_config,
                 prior_data_spec=prior_spec,
                 explicit_deps=explicit_deps)
 
@@ -536,7 +539,7 @@ class GraphBuilder:
             data_spec = _data.build_data_spec(
                 data_id, storage_id, output_name,
                 output_schema.schema,
-                self._sys_config.storage,
+                self._sys_config,
                 prior_spec=prior_spec)
 
             # Save operation uses the statically produced schema info
@@ -567,6 +570,7 @@ class GraphBuilder:
     def _build_file_input(self, input_name, input_selector, nodes, outputs, explicit_deps):
 
         file_spec = self._build_file_spec(input_selector)
+        file_spec = self._attach_metadata(file_spec, input_selector)
 
         file_load_id = NodeId.of(f"{input_name}:LOAD", self._job_namespace, _data.DataItem)
         nodes[file_load_id] = LoadDataNode(file_load_id, spec=file_spec, explicit_deps=explicit_deps)
@@ -597,7 +601,7 @@ class GraphBuilder:
         file_spec = _data.build_file_spec(
             file_id, storage_id,
             output_name, output_schema.fileType,
-            self._sys_config.storage,
+            self._sys_config,
             prior_spec=prior_spec)
 
         # Graph node for the save operation
@@ -709,7 +713,7 @@ class GraphBuilder:
         context = GraphContext(
             self._job_config.jobId,
             self._job_namespace, namespace,
-            self._sys_config.storage)
+            self._sys_config)
 
         model_node = RunModelNode(
             model_id, model_def, model_scope,
@@ -1028,6 +1032,19 @@ class GraphBuilder:
         else:
             return _util.new_object_id(object_type)
 
+    def _attach_metadata(self, obj: _tp.Any, selector: _meta.TagSelector):
+
+        item_id = _util.get_job_mapping(selector, self._job_config)
+        tag = _util.get_job_metadata_tag(selector, self._job_config, optional=True)
+
+        attributes = dict() if tag is None else dict(
+            (attr_name, _type_system.MetadataCodec.decode_value(attr_value))
+            for attr_name, attr_value in tag.attrs.items())
+
+        metadata = _api.RuntimeMetadata(objectId=item_id, attributes=attributes)
+
+        return _util.attach_runtime_metadata(obj, metadata)
+
     def _join_sections(self, *sections: GraphSection, allow_partial_inputs: bool = False):
 
         n_sections = len(sections)

tracdap/rt/_impl/grpc/codec.py

@@ -61,16 +61,6 @@ __METADATA_MAPPING = {
 }
 
 
-_T_MSG = tp.TypeVar('_T_MSG', bound=_message.Message)
-
-
-def encode_message(msg_class: _T_MSG.__class__, obj: tp.Any) -> _T_MSG:
-
-    attrs = dict((k, encode(v)) for k, v in obj.__dict__.items())
-
-    return msg_class(**attrs)
-
-
 def encode(obj: tp.Any) -> tp.Any:
 
     # Translate TRAC domain objects into generic dict / list structures
@@ -97,6 +87,9 @@ def encode(obj: tp.Any) -> tp.Any:
     if msg_class is None:
         raise ex.ETracInternal(f"No gRPC metadata mapping is available for type {type(obj).__name__}")
 
-    attrs = dict((k, encode(v)) for k, v in obj.__dict__.items() if v is not None)
+    if hasattr(obj, "__slots__"):
+        attrs = dict((k, encode(getattr(obj, k))) for k in obj.__slots__)
+    else:
+        attrs = dict((k, encode(v)) for k, v in obj.__dict__.items() if v is not None)
 
     return msg_class(**attrs)