tracdap-runtime 0.7.0__py3-none-any.whl → 0.8.0b1__py3-none-any.whl

tracdap/rt/_exec/dev_mode.py

@@ -137,11 +137,14 @@ class DevModeTranslator:
         raise _ex.EConfigParse(msg)
 
 
-    def __init__(self, sys_config: _cfg.RuntimeConfig, config_mgr: _cfg_p.ConfigManager, scratch_dir: pathlib.Path):
+    def __init__(
+            self, sys_config: _cfg.RuntimeConfig, config_mgr: _cfg_p.ConfigManager, scratch_dir: pathlib.Path = None,
+            model_loader: _models.ModelLoader = None, storage_manager: _storage.StorageManager = None):
+
         self._sys_config = sys_config
         self._config_mgr = config_mgr
-        self._scratch_dir = scratch_dir
-        self._model_loader: tp.Optional[_models.ModelLoader] = None
+        self._model_loader = model_loader or _models.ModelLoader(self._sys_config, scratch_dir)
+        self._storage_manager = storage_manager or _storage.StorageManager(self._sys_config)
 
     def translate_job_config(
             self, job_config: _cfg.JobConfig,
@@ -150,8 +153,6 @@ class DevModeTranslator:
 
         try:
             self._log.info(f"Applying dev mode config translation to job config")
-
-            self._model_loader = _models.ModelLoader(self._sys_config, self._scratch_dir)
             self._model_loader.create_scope("DEV_MODE_TRANSLATION")
 
             job_config = copy.deepcopy(job_config)
@@ -168,7 +169,6 @@ class DevModeTranslator:
 
         finally:
             self._model_loader.destroy_scope("DEV_MODE_TRANSLATION")
-            self._model_loader = None
 
     def translate_job_def(
             self, job_config: _cfg.JobConfig, job_def: _meta.JobDefinition,
@@ -694,7 +694,7 @@ class DevModeTranslator:
 
             model_selector = job_def.runFlow.models.get(source.node)
             model_obj = _util.get_job_resource(model_selector, job_config)
-            model_input = model_obj.model.inputs.get(source.socket)
+            model_input = model_obj.model.outputs.get(source.socket)
             model_outputs.append(model_input)
 
         if len(model_outputs) == 0:
@@ -764,7 +764,7 @@ class DevModeTranslator:
             else:
                 p_spec = param_specs[p_name]
 
-                cls._log.info(f"Encoding parameter [{p_name}] as {p_spec.paramType.basicType}")
+                cls._log.info(f"Encoding parameter [{p_name}] as {p_spec.paramType.basicType.name}")
 
                 encoded_value = _types.MetadataCodec.convert_value(p_value, p_spec.paramType)
                 encoded_values[p_name] = encoded_value
@@ -798,38 +798,46 @@ class DevModeTranslator:
             if not (isinstance(input_value, str) and input_value in job_resources):
 
                 model_input = required_inputs[input_key]
-                input_schema = model_input.schema if model_input and not model_input.dynamic else None
 
-                input_id = self._process_input_or_output(
-                    input_key, input_value, job_resources,
-                    new_unique_file=False, schema=input_schema)
+                if model_input.objectType == _meta.ObjectType.DATA:
+                    schema = model_input.schema if model_input and not model_input.dynamic else None
+                    input_id = self._process_data_socket(input_key, input_value, schema, job_resources, new_unique_file=False)
+                elif model_input.objectType == _meta.ObjectType.FILE:
+                    file_type = model_input.fileType
+                    input_id = self._process_file_socket(input_key, input_value, file_type, job_resources, new_unique_file=False)
+                else:
+                    raise _ex.EUnexpected()
 
                 job_inputs[input_key] = _util.selector_for(input_id)
 
         for output_key, output_value in job_outputs.items():
             if not (isinstance(output_value, str) and output_value in job_resources):
 
-                model_output= required_outputs[output_key]
-                output_schema = model_output.schema if model_output and not model_output.dynamic else None
+                model_output = required_outputs[output_key]
 
-                output_id = self._process_input_or_output(
-                   output_key, output_value, job_resources,
-                    new_unique_file=True, schema=output_schema)
+                if model_output.objectType == _meta.ObjectType.DATA:
+                    schema = model_output.schema if model_output and not model_output.dynamic else None
+                    output_id = self._process_data_socket(output_key, output_value, schema, job_resources, new_unique_file=True)
+                elif model_output.objectType == _meta.ObjectType.FILE:
+                    file_type = model_output.fileType
+                    output_id = self._process_file_socket(output_key, output_value, file_type, job_resources, new_unique_file=True)
+                else:
+                    raise _ex.EUnexpected()
 
                 job_outputs[output_key] = _util.selector_for(output_id)
 
         return job_config, job_def
 
-    def _process_input_or_output(
-            self, data_key, data_value,
-            resources: tp.Dict[str, _meta.ObjectDefinition],
-            new_unique_file=False,
-            schema: tp.Optional[_meta.SchemaDefinition] = None) \
+    def _process_data_socket(
+            self, data_key, data_value, schema: tp.Optional[_meta.SchemaDefinition],
+            resources: tp.Dict[str, _meta.ObjectDefinition], new_unique_file=False) \
             -> _meta.TagHeader:
 
         data_id = _util.new_object_id(_meta.ObjectType.DATA)
         storage_id = _util.new_object_id(_meta.ObjectType.STORAGE)
 
+        self._log.info(f"Generating data definition for [{data_key}] with ID = [{_util.object_key(data_id)}]")
+
         if isinstance(data_value, str):
             storage_path = data_value
             storage_key = self._sys_config.storage.defaultBucket
@@ -850,43 +858,85 @@ class DevModeTranslator:
         else:
             raise _ex.EConfigParse(f"Invalid configuration for input '{data_key}'")
 
-        self._log.info(f"Generating data definition for [{data_key}] with ID = [{_util.object_key(data_id)}]")
-
         # For unique outputs, increment the snap number to find a new unique snap
         # These are not incarnations, bc likely in dev mode model code and inputs are changing
         # Incarnations are for recreation of a dataset using the exact same code path and inputs
 
         if new_unique_file:
+            storage_path, snap_version = self._new_unique_file(data_key, storage_key, storage_path, snap_version)
 
-            x_storage_mgr = _storage.StorageManager(self._sys_config)
-            x_storage = x_storage_mgr.get_file_storage(storage_key)
-            x_orig_path = pathlib.PurePath(storage_path)
-            x_name = x_orig_path.name
-
-            if x_storage.exists(str(x_orig_path.parent)):
-                listing = x_storage.ls(str(x_orig_path.parent))
-                existing_files = list(map(lambda stat: stat.file_name, listing))
-            else:
-                existing_files = []
-
-            while x_name in existing_files:
+        part_key = _meta.PartKey(opaqueKey="part-root", partType=_meta.PartType.PART_ROOT)
+        delta_index = 1
+        incarnation_index = 1
 
-                snap_version += 1
-                x_name = f"{x_orig_path.stem}-{snap_version}"
-                storage_path = str(x_orig_path.parent.joinpath(x_name))
+        # This is also defined in functions.DynamicDataSpecFunc, maybe centralize?
+        data_item = f"data/table/{data_id.objectId}/{part_key.opaqueKey}/snap-{snap_version}/delta-{delta_index}"
 
-            self._log.info(f"Output for [{data_key}] will be snap version {snap_version}")
+        data_obj = self._generate_data_definition(
+            part_key, snap_version, delta_index, data_item,
+            schema, storage_id)
 
-        data_obj, storage_obj = self._generate_input_definition(
-            data_id, storage_id, storage_key, storage_path, storage_format,
-            snap_index=snap_version, delta_index=1, incarnation_index=1,
-            schema=schema)
+        storage_obj = self._generate_storage_definition(
+            storage_id, storage_key, storage_path, storage_format,
+            data_item, incarnation_index)
 
         resources[_util.object_key(data_id)] = data_obj
         resources[_util.object_key(storage_id)] = storage_obj
 
         return data_id
 
+    def _process_file_socket(
+            self, file_key, file_value, file_type: _meta.FileType,
+            resources: tp.Dict[str, _meta.ObjectDefinition], new_unique_file=False) \
+            -> _meta.TagHeader:
+
+        file_id = _util.new_object_id(_meta.ObjectType.FILE)
+        storage_id = _util.new_object_id(_meta.ObjectType.STORAGE)
+
+        self._log.info(f"Generating file definition for [{file_key}] with ID = [{_util.object_key(file_id)}]")
+
+        if isinstance(file_value, str):
+
+            storage_key = self._sys_config.storage.defaultBucket
+            storage_path = file_value
+
+        elif isinstance(file_value, dict):
+
+            storage_key = file_value.get("storageKey") or self._sys_config.storage.defaultBucket
+            storage_path = file_value.get("path")
+
+            if not storage_path:
+                raise _ex.EConfigParse(f"Invalid configuration for input [{file_key}] (missing required value 'path'")
+
+        else:
+            raise _ex.EConfigParse(f"Invalid configuration for input '{file_key}'")
+
+        storage_format = "application/x-binary"
+        file_version = 1
+
+        if new_unique_file:
+            storage_path, file_version = self._new_unique_file(file_key, storage_key, storage_path, file_version)
+            file_size = 0
+        else:
+            storage = self._storage_manager.get_file_storage(storage_key)
+            file_size = storage.size(storage_path)
+
+        data_item = f"file/{file_id.objectId}/version-{file_version}"
+        file_name = f"{file_key}.{file_type.extension}"
+
+        file_obj = self._generate_file_definition(
+            file_name, file_type, file_size,
+            storage_id, data_item)
+
+        storage_obj = self._generate_storage_definition(
+            storage_id, storage_key, storage_path, storage_format,
+            data_item, incarnation_index=1)
+
+        resources[_util.object_key(file_id)] = file_obj
+        resources[_util.object_key(storage_id)] = storage_obj
+
+        return file_id
+
     @staticmethod
     def infer_format(storage_path: str, storage_config: _cfg.StorageConfig):
 
@@ -898,20 +948,33 @@ class DevModeTranslator:
         else:
             return storage_config.defaultFormat
 
-    @classmethod
-    def _generate_input_definition(
-            cls, data_id: _meta.TagHeader, storage_id: _meta.TagHeader,
-            storage_key: str, storage_path: str, storage_format: str,
-            snap_index: int, delta_index: int, incarnation_index: int,
-            schema: tp.Optional[_meta.SchemaDefinition] = None) \
-            -> (_meta.ObjectDefinition, _meta.ObjectDefinition):
+    def _new_unique_file(self, socket_name, storage_key, storage_path, version):
 
-        part_key = _meta.PartKey(
-            opaqueKey="part-root",
-            partType=_meta.PartType.PART_ROOT)
+        x_storage = self._storage_manager.get_file_storage(storage_key)
+        x_orig_path = pathlib.PurePath(storage_path)
+        x_name = x_orig_path.name
 
-        # This is also defined in functions.DynamicDataSpecFunc, maybe centralize?
-        data_item = f"data/table/{data_id.objectId}/{part_key.opaqueKey}/snap-{snap_index}/delta-{delta_index}"
+        if x_storage.exists(str(x_orig_path.parent)):
+            listing = x_storage.ls(str(x_orig_path.parent))
+            existing_files = list(map(lambda stat: stat.file_name, listing))
+        else:
+            existing_files = []
+
+        while x_name in existing_files:
+
+            version += 1
+            x_name = f"{x_orig_path.stem}-{version}{x_orig_path.suffix}"
+            storage_path = str(x_orig_path.parent.joinpath(x_name))
+
+        self._log.info(f"Output for [{socket_name}] will be version {version}")
+
+        return storage_path, version
+
+    @classmethod
+    def _generate_data_definition(
+            cls, part_key: _meta.PartKey, snap_index: int, delta_index: int, data_item: str,
+            schema: tp.Optional[_meta.SchemaDefinition], storage_id: _meta.TagHeader) \
+            -> (_meta.ObjectDefinition, _meta.ObjectDefinition):
 
         delta = _meta.DataDefinition.Delta(
             deltaIndex=delta_index,
@@ -925,17 +988,31 @@ class DevModeTranslator:
             partKey=part_key,
             snap=snap)
 
-        data_def = _meta.DataDefinition(parts={})
+        data_def = _meta.DataDefinition()
         data_def.parts[part_key.opaqueKey] = part
+        data_def.schema = schema
+        data_def.storageId = _util.selector_for(storage_id)
 
-        if schema is not None:
-            data_def.schema = schema
-        else:
-            data_def.schema = None
+        return _meta.ObjectDefinition(objectType=_meta.ObjectType.DATA, data=data_def)
 
-        data_def.storageId = _meta.TagSelector(
-            _meta.ObjectType.STORAGE, storage_id.objectId,
-            objectVersion=storage_id.objectVersion, latestTag=True)
+    @classmethod
+    def _generate_file_definition(
+            cls, file_name: str, file_type: _meta.FileType, file_size: int,
+            storage_id: _meta.TagHeader, data_item: str) \
+            -> _meta.ObjectDefinition:
+
+        file_def = _meta.FileDefinition(
+            name=file_name, extension=file_type.extension, mimeType=file_type.mimeType,
+            storageId=_util.selector_for(storage_id), dataItem=data_item, size=file_size)
+
+        return _meta.ObjectDefinition(objectType=_meta.ObjectType.FILE, file=file_def)
+
+    @classmethod
+    def _generate_storage_definition(
+            cls, storage_id: _meta.TagHeader,
+            storage_key: str, storage_path: str, storage_format: str,
+            data_item: str, incarnation_index: int) \
+            -> _meta.ObjectDefinition:
 
         storage_copy = _meta.StorageCopy(
             storageKey=storage_key,
@@ -952,16 +1029,14 @@ class DevModeTranslator:
         storage_item = _meta.StorageItem(
             incarnations=[storage_incarnation])
 
-        storage_def = _meta.StorageDefinition(dataItems={})
-        storage_def.dataItems[delta.dataItem] = storage_item
+        storage_def = _meta.StorageDefinition()
+        storage_def.dataItems[data_item] = storage_item
 
         if storage_format.lower() == "csv":
             storage_def.storageOptions["lenient_csv_parser"] = _types.MetadataCodec.encode_value(True)
 
-        data_obj = _meta.ObjectDefinition(objectType=_meta.ObjectType.DATA, data=data_def)
-        storage_obj = _meta.ObjectDefinition(objectType=_meta.ObjectType.STORAGE, storage=storage_def)
+        return _meta.ObjectDefinition(objectType=_meta.ObjectType.STORAGE, storage=storage_def)
 
-        return data_obj, storage_obj
 
 
 DevModeTranslator._log = _util.logger_for_class(DevModeTranslator)

tracdap/rt/_exec/engine.py

@@ -170,7 +170,7 @@ class TracEngine(_actors.Actor):
 
         self._log.info(f"Job submitted: [{job_key}]")
 
-        job_processor = JobProcessor(self._models, self._storage, job_key, job_config, result_spec, graph_spec=None)
+        job_processor = JobProcessor(self._sys_config, self._models, self._storage, job_key, job_config, result_spec, graph_spec=None)
         job_actor_id = self.actors().spawn(job_processor)
 
         job_monitor_success = lambda ctx, key, result: self._notify_callback(key, result, None)
@@ -190,7 +190,7 @@ class TracEngine(_actors.Actor):
 
         child_key = _util.object_key(child_id)
 
-        child_processor = JobProcessor(self._models, self._storage, child_key, None, None, graph_spec=child_graph)  # noqa
+        child_processor = JobProcessor(self._sys_config, self._models, self._storage, child_key, None, None, graph_spec=child_graph)  # noqa
         child_actor_id = self.actors().spawn(child_processor)
 
         child_state = _JobState(child_id)
@@ -336,7 +336,8 @@ class JobProcessor(_actors.Actor):
     """
 
     def __init__(
-            self, models: _models.ModelLoader, storage: _storage.StorageManager,
+            self, sys_config: _cfg.RuntimeConfig,
+            models: _models.ModelLoader, storage: _storage.StorageManager,
             job_key: str, job_config: _cfg.JobConfig, result_spec: _graph.JobResultSpec,
             graph_spec: tp.Optional[_graph.Graph]):
 
@@ -345,6 +346,7 @@ class JobProcessor(_actors.Actor):
         self.job_config = job_config
         self.result_spec = result_spec
         self.graph_spec = graph_spec
+        self._sys_config = sys_config
         self._models = models
         self._storage = storage
         self._resolver = _func.FunctionResolver(models, storage)
@@ -358,7 +360,7 @@ class JobProcessor(_actors.Actor):
         if self.graph_spec is not None:
             self.actors().send(self.actors().id, "build_graph_succeeded", self.graph_spec)
         else:
-            self.actors().spawn(GraphBuilder(self.job_config, self.result_spec))
+            self.actors().spawn(GraphBuilder(self._sys_config, self.job_config, self.result_spec))
 
     def on_stop(self):
 
@@ -426,8 +428,9 @@ class GraphBuilder(_actors.Actor):
     GraphBuilder is a worker (actor) to wrap the GraphBuilder logic from graph_builder.py
     """
 
-    def __init__(self, job_config: _cfg.JobConfig, result_spec: _graph.JobResultSpec):
+    def __init__(self, sys_config: _cfg.RuntimeConfig, job_config: _cfg.JobConfig, result_spec: _graph.JobResultSpec):
         super().__init__()
+        self.sys_config = sys_config
         self.job_config = job_config
         self.result_spec = result_spec
         self._log = _util.logger_for_object(self)
@@ -440,8 +443,7 @@ class GraphBuilder(_actors.Actor):
 
         self._log.info("Building execution graph")
 
-        # TODO: Get sys config, or find a way to pass storage settings
-        graph_builder = _graph.GraphBuilder(job_config, self.result_spec)
+        graph_builder = _graph.GraphBuilder(self.sys_config, job_config, self.result_spec)
         graph_spec = graph_builder.build_job(job_config.job)
 
         self.actors().reply("build_graph_succeeded", graph_spec)

tracdap/rt/_exec/functions.py

@@ -15,6 +15,7 @@
 
 from __future__ import annotations
 
+import copy
 import datetime
 import abc
 import random
@@ -296,8 +297,13 @@ class DataViewFunc(NodeFunction[_data.DataView]):
 
         # Map empty item -> emtpy view (for optional inputs not supplied)
         if root_item.is_empty():
-            return _data.DataView.create_empty()
+            return _data.DataView.create_empty(root_item.object_type)
 
+        # Handle file data views
+        if root_item.object_type == meta.ObjectType.FILE:
+            return _data.DataView.for_file_item(root_item)
+
+        # Everything else is a regular data view
         if self.node.schema is not None and len(self.node.schema.table.fields) > 0:
             trac_schema = self.node.schema
         else:
@@ -322,7 +328,11 @@ class DataItemFunc(NodeFunction[_data.DataItem]):
 
         # Map empty view -> emtpy item (for optional outputs not supplied)
         if data_view.is_empty():
-            return _data.DataItem.create_empty()
+            return _data.DataItem.create_empty(data_view.object_type)
+
+        # Handle file data views
+        if data_view.object_type == meta.ObjectType.FILE:
+            return data_view.file_item
 
         # TODO: Support selecting data item described by self.node
 
@@ -342,25 +352,24 @@ class DataResultFunc(NodeFunction[ObjectBundle]):
 
     def _execute(self, ctx: NodeContext) -> ObjectBundle:
 
-        data_item = _ctx_lookup(self.node.data_item_id, ctx)
+        data_spec = _ctx_lookup(self.node.data_save_id, ctx)
 
-        # Do not record output metadata for optional outputs that are empty
-        if data_item.is_empty():
-            return {}
+        result_bundle = dict()
 
-        data_spec = _ctx_lookup(self.node.data_spec_id, ctx)
+        # Do not record output metadata for optional outputs that are empty
+        if data_spec.is_empty():
+            return result_bundle
 
-        # TODO: Check result of save operation
-        # save_result = _ctx_lookup(self.node.data_save_id, ctx)
+        if self.node.data_key is not None:
+            result_bundle[self.node.data_key] = meta.ObjectDefinition(objectType=meta.ObjectType.DATA, data=data_spec.data_def)
 
-        data_result = meta.ObjectDefinition(objectType=meta.ObjectType.DATA, data=data_spec.data_def)
-        storage_result = meta.ObjectDefinition(objectType=meta.ObjectType.STORAGE, storage=data_spec.storage_def)
+        if self.node.file_key is not None:
+            result_bundle[self.node.file_key] = meta.ObjectDefinition(objectType=meta.ObjectType.FILE, file=data_spec.file_def)
 
-        bundle = {
-            self.node.data_key: data_result,
-            self.node.storage_key: storage_result}
+        if self.node.storage_key is not None:
+            result_bundle[self.node.storage_key] = meta.ObjectDefinition(objectType=meta.ObjectType.STORAGE, storage=data_spec.storage_def)
 
-        return bundle
+        return result_bundle
 
 
 class DynamicDataSpecFunc(NodeFunction[_data.DataSpec]):
@@ -443,11 +452,7 @@ class DynamicDataSpecFunc(NodeFunction[_data.DataSpec]):
 
         # Dynamic data def will always use an embedded schema (this is no ID for an external schema)
 
-        return _data.DataSpec(
-            data_item,
-            data_def,
-            storage_def,
-            schema_def=None)
+        return _data.DataSpec.create_data_spec(data_item, data_def, storage_def, schema_def=None)
 
 
 class _LoadSaveDataFunc(abc.ABC):
@@ -455,6 +460,16 @@ class _LoadSaveDataFunc(abc.ABC):
     def __init__(self, storage: _storage.StorageManager):
         self.storage = storage
 
+    @classmethod
+    def _choose_data_spec(cls, spec_id, spec, ctx: NodeContext):
+
+        if spec_id is not None:
+            return _ctx_lookup(spec_id, ctx)
+        elif spec is not None:
+            return spec
+        else:
+            raise _ex.EUnexpected()
+
     def _choose_copy(self, data_item: str, storage_def: meta.StorageDefinition) -> meta.StorageCopy:
 
         # Metadata should be checked for consistency before a job is accepted
@@ -491,9 +506,19 @@ class LoadDataFunc( _LoadSaveDataFunc, NodeFunction[_data.DataItem],):
 
     def _execute(self, ctx: NodeContext) -> _data.DataItem:
 
-        data_spec = _ctx_lookup(self.node.spec_id, ctx)
+        data_spec = self._choose_data_spec(self.node.spec_id, self.node.spec, ctx)
         data_copy = self._choose_copy(data_spec.data_item, data_spec.storage_def)
-        data_storage = self.storage.get_data_storage(data_copy.storageKey)
+
+        if data_spec.object_type == _api.ObjectType.DATA:
+            return self._load_data(data_spec, data_copy)
+
+        elif data_spec.object_type == _api.ObjectType.FILE:
+            return self._load_file(data_copy)
+
+        else:
+            raise _ex.EUnexpected()
+
+    def _load_data(self, data_spec, data_copy):
 
         trac_schema = data_spec.schema_def if data_spec.schema_def else data_spec.data_def.schema
         arrow_schema = _data.DataMapping.trac_to_arrow_schema(trac_schema) if trac_schema else None
@@ -503,36 +528,52 @@ class LoadDataFunc( _LoadSaveDataFunc, NodeFunction[_data.DataItem],):
         for opt_key, opt_value in data_spec.storage_def.storageOptions.items():
             options[opt_key] = _types.MetadataCodec.decode_value(opt_value)
 
-        table = data_storage.read_table(
+        storage = self.storage.get_data_storage(data_copy.storageKey)
+        table = storage.read_table(
             data_copy.storagePath,
             data_copy.storageFormat,
             arrow_schema,
             storage_options=options)
 
-        return _data.DataItem(table.schema, table)
+        return _data.DataItem(_api.ObjectType.DATA, table.schema, table)
+
+    def _load_file(self, data_copy):
+
+        storage = self.storage.get_file_storage(data_copy.storageKey)
+        raw_bytes = storage.read_bytes(data_copy.storagePath)
 
+        return _data.DataItem(_api.ObjectType.FILE, raw_bytes=raw_bytes)
 
-class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[None]):
+
+class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[_data.DataSpec]):
 
     def __init__(self, node: SaveDataNode, storage: _storage.StorageManager):
         super().__init__(storage)
         self.node = node
 
-    def _execute(self, ctx: NodeContext):
+    def _execute(self, ctx: NodeContext) -> _data.DataSpec:
 
         # Item to be saved should exist in the current context
         data_item = _ctx_lookup(self.node.data_item_id, ctx)
 
+        # Metadata already exists as data_spec but may not contain schema, row count, file size etc.
+        data_spec = self._choose_data_spec(self.node.spec_id, self.node.spec, ctx)
+        data_copy = self._choose_copy(data_spec.data_item, data_spec.storage_def)
+
         # Do not save empty outputs (optional outputs that were not produced)
         if data_item.is_empty():
-            return
+            return _data.DataSpec.create_empty_spec(data_item.object_type)
 
-        # This function assumes that metadata has already been generated as the data_spec
-        # i.e. it is already known which incarnation / copy of the data will be created
+        if data_item.object_type == _api.ObjectType.DATA:
+            return self._save_data(data_item, data_spec, data_copy)
 
-        data_spec = _ctx_lookup(self.node.spec_id, ctx)
-        data_copy = self._choose_copy(data_spec.data_item, data_spec.storage_def)
-        data_storage = self.storage.get_data_storage(data_copy.storageKey)
+        elif data_item.object_type == _api.ObjectType.FILE:
+            return self._save_file(data_item, data_spec, data_copy)
+
+        else:
+            raise _ex.EUnexpected()
+
+    def _save_data(self, data_item, data_spec, data_copy):
 
         # Current implementation will always put an Arrow table in the data item
         # Empty tables are allowed, so explicitly check if table is None
@@ -546,11 +587,32 @@ class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[None]):
         for opt_key, opt_value in data_spec.storage_def.storageOptions.items():
             options[opt_key] = _types.MetadataCodec.decode_value(opt_value)
 
-        data_storage.write_table(
+        storage = self.storage.get_data_storage(data_copy.storageKey)
+        storage.write_table(
             data_copy.storagePath, data_copy.storageFormat,
             data_item.table,
             storage_options=options, overwrite=False)
 
+        data_spec = copy.deepcopy(data_spec)
+        # TODO: Save row count in metadata
+
+        if data_spec.data_def.schema is None and data_spec.data_def.schemaId is None:
+            data_spec.data_def.schema = _data.DataMapping.arrow_to_trac_schema(data_item.table.schema)
+
+        return data_spec
+
+    def _save_file(self, data_item, data_spec, data_copy):
+
+        if data_item.raw_bytes is None:
+            raise _ex.EUnexpected()
+
+        storage = self.storage.get_file_storage(data_copy.storageKey)
+        storage.write_bytes(data_copy.storagePath, data_item.raw_bytes)
+
+        data_spec = copy.deepcopy(data_spec)
+        data_spec.file_def.size = len(data_item.raw_bytes)
+
+        return data_spec
 
 def _model_def_for_import(import_details: meta.ImportModelJob):