tracdap-runtime 0.6.1.dev3__py3-none-any.whl → 0.6.3__py3-none-any.whl

tracdap/rt/_exec/dev_mode.py

@@ -34,8 +34,9 @@ DEV_MODE_JOB_CONFIG = [
     re.compile(r"job\.run(Model|Flow)\.parameters\.\w+"),
     re.compile(r"job\.run(Model|Flow)\.inputs\.\w+"),
     re.compile(r"job\.run(Model|Flow)\.outputs\.\w+"),
-    re.compile(r"job\.run(Model|Flow)\.models\.\w+"),
-    re.compile(r"job\.run(Model|Flow)\.flow+")]
+    re.compile(r"job\.runModel\.model"),
+    re.compile(r"job\.runFlow\.flow"),
+    re.compile(r"job\.runFlow\.models\.\w+")]
 
 DEV_MODE_SYS_CONFIG = []
 
@@ -45,7 +46,7 @@ class DevModeTranslator:
     _log: tp.Optional[_util.logging.Logger] = None
 
     @classmethod
-    def translate_sys_config(cls, sys_config: _cfg.RuntimeConfig, config_dir: tp.Optional[pathlib.Path]):
+    def translate_sys_config(cls, sys_config: _cfg.RuntimeConfig, config_mgr: _cfg_p.ConfigManager):
 
         cls._log.info(f"Applying dev mode config translation to system config")
 
@@ -55,7 +56,7 @@ class DevModeTranslator:
             sys_config.storage = _cfg.StorageConfig()
 
         sys_config = cls._add_integrated_repo(sys_config)
-        sys_config = cls._resolve_relative_storage_root(sys_config, config_dir)
+        sys_config = cls._resolve_relative_storage_root(sys_config, config_mgr)
 
         return sys_config
 
@@ -65,98 +66,30 @@ class DevModeTranslator:
             sys_config: _cfg.RuntimeConfig,
             job_config: _cfg.JobConfig,
             scratch_dir: pathlib.Path,
-            config_dir: tp.Optional[pathlib.Path],
+            config_mgr: _cfg_p.ConfigManager,
             model_class: tp.Optional[_api.TracModel.__class__]) \
             -> _cfg.JobConfig:
 
         cls._log.info(f"Applying dev mode config translation to job config")
 
-        model_loader = _models.ModelLoader(sys_config, scratch_dir)
-        model_loader.create_scope("DEV_MODE_TRANSLATION")
-
         if not job_config.jobId:
             job_config = cls._process_job_id(job_config)
 
         if job_config.job.jobType is None or job_config.job.jobType == _meta.JobType.JOB_TYPE_NOT_SET:
             job_config = cls._process_job_type(job_config)
 
-        if model_class is not None:
-
-            model_id, model_obj = cls._generate_model_for_class(model_loader, model_class)
-            job_config = cls._add_job_resource(job_config, model_id, model_obj)
-            job_config.job.runModel.model = _util.selector_for(model_id)
+        # Load and populate any models provided as a Python class or class name
+        if job_config.job.jobType in [_meta.JobType.RUN_MODEL, _meta.JobType.RUN_FLOW]:
+            job_config = cls._process_models(sys_config, job_config, scratch_dir, model_class)
 
+        # Fow flows, load external flow definitions then perform auto-wiring and type inference
         if job_config.job.jobType == _meta.JobType.RUN_FLOW:
+            job_config = cls._process_flow_definition(job_config, config_mgr)
 
-            original_models = job_config.job.runFlow.models.copy()
-            for model_key, model_detail in original_models.items():
-                model_id, model_obj = cls._generate_model_for_entry_point(model_loader, model_detail)
-                job_config = cls._add_job_resource(job_config, model_id, model_obj)
-                job_config.job.runFlow.models[model_key] = _util.selector_for(model_id)
-
-            flow_id, flow_obj = cls._expand_flow_definition(job_config, config_dir)
-            job_config = cls._add_job_resource(job_config, flow_id, flow_obj)
-            job_config.job.runFlow.flow = _util.selector_for(flow_id)
-
-        model_loader.destroy_scope("DEV_MODE_TRANSLATION")
-
+        # For run (model|flow) jobs, apply processing to the parameters, inputs and outputs
         if job_config.job.jobType in [_meta.JobType.RUN_MODEL, _meta.JobType.RUN_FLOW]:
             job_config = cls._process_parameters(job_config)
-
-        if job_config.job.jobType not in [_meta.JobType.RUN_MODEL, _meta.JobType.RUN_FLOW]:
-            return job_config
-
-        run_info = job_config.job.runModel \
-            if job_config.job.jobType == _meta.JobType.RUN_MODEL \
-            else job_config.job.runFlow
-
-        original_inputs = run_info.inputs
-        original_outputs = run_info.outputs
-        original_resources = job_config.resources
-
-        translated_inputs = copy.copy(original_inputs)
-        translated_outputs = copy.copy(original_outputs)
-        translated_resources = copy.copy(job_config.resources)
-
-        def process_input_or_output(data_key, data_value, is_input: bool):
-
-            data_id = _util.new_object_id(_meta.ObjectType.DATA)
-            storage_id = _util.new_object_id(_meta.ObjectType.STORAGE)
-
-            if is_input:
-                if job_config.job.jobType == _meta.JobType.RUN_MODEL:
-                    model_def = job_config.resources[_util.object_key(job_config.job.runModel.model)]
-                    schema = model_def.model.inputs[data_key].schema
-                else:
-                    flow_def = job_config.resources[_util.object_key(job_config.job.runFlow.flow)]
-                    schema = flow_def.flow.inputs[data_key].schema
-            else:
-                schema = None
-
-            data_obj, storage_obj = cls._process_job_io(
-                sys_config, data_key, data_value, data_id, storage_id,
-                new_unique_file=not is_input, schema=schema)
-
-            translated_resources[_util.object_key(data_id)] = data_obj
-            translated_resources[_util.object_key(storage_id)] = storage_obj
-
-            if is_input:
-                translated_inputs[data_key] = _util.selector_for(data_id)
-            else:
-                translated_outputs[data_key] = _util.selector_for(data_id)
-
-        for input_key, input_value in original_inputs.items():
-            if not (isinstance(input_value, str) and input_value in original_resources):
-                process_input_or_output(input_key, input_value, is_input=True)
-
-        for output_key, output_value in original_outputs.items():
-            if not (isinstance(output_value, str) and output_value in original_outputs):
-                process_input_or_output(output_key, output_value, is_input=False)
-
-        job_config = copy.copy(job_config)
-        job_config.resources = translated_resources
-        run_info.inputs = translated_inputs
-        run_info.outputs = translated_outputs
+            job_config = cls._process_inputs_and_outputs(sys_config, job_config)
 
         return job_config
 
@@ -176,7 +109,7 @@ class DevModeTranslator:
     @classmethod
     def _resolve_relative_storage_root(
             cls, sys_config: _cfg.RuntimeConfig,
-            sys_config_path: tp.Optional[pathlib.Path]):
+            config_mgr: _cfg_p.ConfigManager):
 
         storage_config = copy.deepcopy(sys_config.storage)
 
@@ -195,6 +128,7 @@ class DevModeTranslator:
 
             cls._log.info(f"Resolving relative path for [{bucket_key}] local storage...")
 
+            sys_config_path = config_mgr.config_dir_path()
             if sys_config_path is not None:
                 absolute_path = sys_config_path.joinpath(root_path).resolve()
                 if absolute_path.exists():
@@ -268,6 +202,58 @@ class DevModeTranslator:
 
         return job_config
 
+    @classmethod
+    def _process_models(
+            cls,
+            sys_config: _cfg.RuntimeConfig,
+            job_config: _cfg.JobConfig,
+            scratch_dir: pathlib.Path,
+            model_class: tp.Optional[_api.TracModel.__class__]) \
+            -> _cfg.JobConfig:
+
+        model_loader = _models.ModelLoader(sys_config, scratch_dir)
+        model_loader.create_scope("DEV_MODE_TRANSLATION")
+
+        original_config = job_config
+
+        job_config = copy.copy(job_config)
+        job_config.job = copy.copy(job_config.job)
+        job_config.resources = copy.copy(job_config.resources)
+
+        if job_config.job.jobType == _meta.JobType.RUN_MODEL:
+
+            job_config.job.runModel = copy.copy(job_config.job.runModel)
+
+            # If a model class is supplied in code, use that to generate the model def
+            if model_class is not None:
+                model_id, model_obj = cls._generate_model_for_class(model_loader, model_class)
+                job_config = cls._add_job_resource(job_config, model_id, model_obj)
+                job_config.job.runModel.model = _util.selector_for(model_id)
+
+            # Otherwise if model specified as a string instead of a selector, apply the translation
+            elif isinstance(original_config.job.runModel.model, str):
+                model_detail = original_config.job.runModel.model
+                model_id, model_obj = cls._generate_model_for_entry_point(model_loader, model_detail)  # noqa
+                job_config = cls._add_job_resource(job_config, model_id, model_obj)
+                job_config.job.runModel.model = _util.selector_for(model_id)
+
+        if job_config.job.jobType == _meta.JobType.RUN_FLOW:
+
+            job_config.job.runFlow = copy.copy(job_config.job.runFlow)
+            job_config.job.runFlow.models = copy.copy(job_config.job.runFlow.models)
+
+            for model_key, model_detail in original_config.job.runFlow.models.items():
+
+                # Only apply translation if the model is specified as a string instead of a selector
+                if isinstance(model_detail, str):
+                    model_id, model_obj = cls._generate_model_for_entry_point(model_loader, model_detail)
+                    job_config = cls._add_job_resource(job_config, model_id, model_obj)
+                    job_config.job.runFlow.models[model_key] = _util.selector_for(model_id)
+
+        model_loader.destroy_scope("DEV_MODE_TRANSLATION")
+
+        return job_config
+
     @classmethod
     def _generate_model_for_class(
             cls, model_loader: _models.ModelLoader, model_class: _api.TracModel.__class__) \
@@ -306,17 +292,13 @@ class DevModeTranslator:
         return model_id, model_object
 
     @classmethod
-    def _expand_flow_definition(
-            cls, job_config: _cfg.JobConfig, config_dir: pathlib.Path) \
-            -> (_meta.TagHeader, _meta.ObjectDefinition):
+    def _process_flow_definition(cls, job_config: _cfg.JobConfig, config_mgr: _cfg_p.ConfigManager) -> _cfg.JobConfig:
 
         flow_details = job_config.job.runFlow.flow
 
-        # The full specification for a flow is as a tag selector for a valid job resource
-        # This is still allowed in dev mode, in which case dev mode translation is not applied
+        # Do not apply translation if flow is specified as an object ID / selector (assume full config is supplied)
         if isinstance(flow_details, _meta.TagHeader) or isinstance(flow_details, _meta.TagSelector):
-            flow_obj = _util.get_job_resource(flow_details, job_config, optional=False)
-            return flow_details, flow_obj
+            return job_config
 
         # Otherwise, flow is specified as the path to dev-mode flow definition
         if not isinstance(flow_details, str):
@@ -327,36 +309,47 @@ class DevModeTranslator:
         flow_id = _util.new_object_id(_meta.ObjectType.FLOW)
         flow_key = _util.object_key(flow_id)
 
-        cls._log.info(f"Generating flow definition for [{flow_details}] with ID = [{flow_key}]")
+        cls._log.info(f"Generating flow definition from [{flow_details}] with ID = [{flow_key}]")
 
-        flow_path = config_dir.joinpath(flow_details) if config_dir is not None else pathlib.Path(flow_details)
-        flow_parser = _cfg_p.ConfigParser(_meta.FlowDefinition)
-        flow_raw_data = flow_parser.load_raw_config(flow_path, flow_path.name)
-        flow_def = flow_parser.parse(flow_raw_data, flow_path.name)
+        flow_def = config_mgr.load_config_object(flow_details, _meta.FlowDefinition)
 
-        flow_def = cls._autowire_flow(flow_def)
-        flow_def = cls._generate_flow_parameters(flow_def, job_config)
-        flow_def = cls._generate_flow_inputs(flow_def, job_config)
-        flow_def = cls._generate_flow_outputs(flow_def, job_config)
+        # Auto-wiring and inference only applied to externally loaded flows for now
+        flow_def = cls._autowire_flow(flow_def, job_config)
+        flow_def = cls._apply_type_inference(flow_def, job_config)
 
-        flow_object = _meta.ObjectDefinition(
+        flow_obj = _meta.ObjectDefinition(
             objectType=_meta.ObjectType.FLOW,
             flow=flow_def)
 
-        return flow_id, flow_object
+        job_config = copy.copy(job_config)
+        job_config.job = copy.copy(job_config.job)
+        job_config.job.runFlow = copy.copy(job_config.job.runFlow)
+        job_config.resources = copy.copy(job_config.resources)
+
+        job_config = cls._add_job_resource(job_config, flow_id, flow_obj)
+        job_config.job.runFlow.flow = _util.selector_for(flow_id)
+
+        return job_config
 
     @classmethod
-    def _autowire_flow(cls, flow: _meta.FlowDefinition):
+    def _autowire_flow(cls, flow: _meta.FlowDefinition, job_config: _cfg.JobConfig):
+
+        job = job_config.job.runFlow
+        nodes = copy.copy(flow.nodes)
+        edges: tp.Dict[str, _meta.FlowEdge] = dict()
 
         sources: tp.Dict[str, _meta.FlowSocket] = dict()
         duplicates: tp.Dict[str, tp.List[_meta.FlowSocket]] = dict()
-
-        edges: tp.Dict[str, _meta.FlowEdge] = dict()
         errors: tp.Dict[str, str] = dict()
 
         def socket_key(socket: _meta.FlowSocket):
             return f"{socket.node}.{socket.socket}" if socket.socket else socket.node
 
+        # Before starting, add any edges defined explicitly in the flow
+        # These take precedence over auto-wired edges
+        for edge in flow.edges:
+            edges[socket_key(edge.target)] = edge
+
         def add_source(name: str, socket: _meta.FlowSocket):
             if name in duplicates:
                 duplicates[name].append(socket)
@@ -366,6 +359,14 @@ class DevModeTranslator:
             else:
                 sources[name] = socket
 
+        def add_param_to_flow(nodel_node: str, param: str):
+            target = f"{nodel_node}.{param}"
+            if target not in edges and param not in nodes:
+                param_node = _meta.FlowNode(_meta.FlowNodeType.PARAMETER_NODE)
+                nodes[param] = param_node
+                socket = _meta.FlowSocket(param)
+                add_source(param, socket)
+
         def add_edge(target: _meta.FlowSocket):
             target_key = socket_key(target)
             if target_key in edges:
@@ -380,23 +381,29 @@ class DevModeTranslator:
                 errors[target_key] = f"Flow target {target_name} is not provided by any node"
 
         for node_name, node in flow.nodes.items():
-            if node.nodeType == _meta.FlowNodeType.INPUT_NODE:
+            if node.nodeType == _meta.FlowNodeType.INPUT_NODE or node.nodeType == _meta.FlowNodeType.PARAMETER_NODE:
                 add_source(node_name, _meta.FlowSocket(node_name))
             if node.nodeType == _meta.FlowNodeType.MODEL_NODE:
                 for model_output in node.outputs:
                     add_source(model_output, _meta.FlowSocket(node_name, model_output))
-
-        # Include any edges defined explicitly in the flow
-        # These take precedence over auto-wired edges
-        for edge in flow.edges:
-            edges[socket_key(edge.target)] = edge
-
-        for node_name, node in flow.nodes.items():
+                # Generate node param sockets needed by the model
+                if node_name in job.models:
+                    model_selector = job.models[node_name]
+                    model_obj = _util.get_job_resource(model_selector, job_config)
+                    for param_name in model_obj.model.parameters:
+                        add_param_to_flow(node_name, param_name)
+                        if param_name not in node.parameters:
+                            node.parameters.append(param_name)
+
+        # Look at the new set of nodes, which includes any added by auto-wiring
+        for node_name, node in nodes.items():
             if node.nodeType == _meta.FlowNodeType.OUTPUT_NODE:
                 add_edge(_meta.FlowSocket(node_name))
             if node.nodeType == _meta.FlowNodeType.MODEL_NODE:
                 for model_input in node.inputs:
                     add_edge(_meta.FlowSocket(node_name, model_input))
+                for model_param in node.parameters:
+                    add_edge(_meta.FlowSocket(node_name, model_param))
 
         if any(errors):
 
@@ -408,140 +415,149 @@ class DevModeTranslator:
             raise _ex.EConfigParse(err)
 
         autowired_flow = copy.copy(flow)
+        autowired_flow.nodes = nodes
         autowired_flow.edges = list(edges.values())
 
         return autowired_flow
 
     @classmethod
-    def _generate_flow_parameters(cls, flow: _meta.FlowDefinition, job_config: _cfg.JobConfig) -> _meta.FlowDefinition:
-
-        params: tp.Dict[str, _meta.ModelParameter] = dict()
+    def _apply_type_inference(cls, flow: _meta.FlowDefinition, job_config: _cfg.JobConfig) -> _meta.FlowDefinition:
 
-        for node_name, node in flow.nodes.items():
-
-            if node.nodeType != _meta.FlowNodeType.MODEL_NODE:
-                continue
+        updated_flow = copy.copy(flow)
+        updated_flow.parameters = copy.copy(flow.parameters)
+        updated_flow.inputs = copy.copy(flow.inputs)
+        updated_flow.outputs = copy.copy(flow.outputs)
 
-            if node_name not in job_config.job.runFlow.models:
-                err = f"No model supplied for flow model node [{node_name}]"
-                cls._log.error(err)
-                raise _ex.EConfigParse(err)
-
-            model_selector = job_config.job.runFlow.models[node_name]
-            model_obj = _util.get_job_resource(model_selector, job_config)
-
-            for param_name, param in model_obj.model.parameters.items():
+        def socket_key(socket):
+            return f"{socket.node}.{socket.socket}" if socket.socket else socket.node
 
-                if param_name not in params:
-                    params[param_name] = param
+        # Build a map of edges by source socket, mapping to all edges flowing from that source
+        edges_by_source = {socket_key(edge.source): [] for edge in flow.edges}
+        edges_by_target = {socket_key(edge.target): [] for edge in flow.edges}
+        for edge in flow.edges:
+            edges_by_source[socket_key(edge.source)].append(edge.target)
+            edges_by_target[socket_key(edge.target)].append(edge.source)
 
-                else:
-                    existing_param = params[param_name]
+        for node_name, node in flow.nodes.items():
 
-                    if param.paramType != existing_param.paramType:
-                        err = f"Model parameter [{param_name}] has different types in different models"
-                        cls._log.error(err)
-                        raise _ex.EConfigParse(err)
+            if node.nodeType == _meta.FlowNodeType.PARAMETER_NODE and node_name not in flow.parameters:
+                targets = edges_by_source.get(node_name) or []
+                model_parameter = cls._infer_parameter(node_name, targets, job_config)
+                updated_flow.parameters[node_name] = model_parameter
 
-                    if param.defaultValue != existing_param.defaultValue:
-                        if existing_param.defaultValue is None:
-                            params[param_name] = param
-                        elif param.defaultValue is not None:
-                            warn = f"Model parameter [{param_name}] has different default values in different models" \
-                                 + f" (using [{_types.MetadataCodec.decode_value(existing_param.defaultValue)}])"
-                            cls._log.warning(warn)
+            if node.nodeType == _meta.FlowNodeType.INPUT_NODE and node_name not in flow.inputs:
+                targets = edges_by_source.get(node_name) or []
+                model_input = cls._infer_input_schema(node_name, targets, job_config)
+                updated_flow.inputs[node_name] = model_input
 
-        flow.parameters = params
+            if node.nodeType == _meta.FlowNodeType.OUTPUT_NODE and node_name not in flow.outputs:
+                sources = edges_by_target.get(node_name) or []
+                model_output = cls._infer_output_schema(node_name, sources, job_config)
+                updated_flow.outputs[node_name] = model_output
 
-        return flow
+        return updated_flow
 
     @classmethod
-    def _generate_flow_inputs(cls, flow: _meta.FlowDefinition, job_config: _cfg.JobConfig) -> _meta.FlowDefinition:
+    def _infer_parameter(
+            cls, param_name: str, targets: tp.List[_meta.FlowSocket],
+            job_config: _cfg.JobConfig) -> _meta.ModelParameter:
 
-        inputs: tp.Dict[str, _meta.ModelInputSchema] = dict()
+        model_params = []
 
-        def socket_key(socket):
-            return f"{socket.node}.{socket.socket}" if socket.socket else socket.node
+        for target in targets:
 
-        # Build a map of edges by source socket, mapping to all edges flowing from that source
-        edges = {socket_key(edge.source): [] for edge in flow.edges}
-        for edge in flow.edges:
-            edges[socket_key(edge.source)].append(edge)
-
-        for node_name, node in flow.nodes.items():
-
-            if node.nodeType != _meta.FlowNodeType.INPUT_NODE:
-                continue
-
-            input_edges = edges.get(node_name)
+            model_selector = job_config.job.runFlow.models.get(target.node)
+            model_obj = _util.get_job_resource(model_selector, job_config)
+            model_param = model_obj.model.parameters.get(target.socket)
+            model_params.append(model_param)
 
-            if not input_edges:
-                err = f"Flow input [{node_name}] is not connected, so the input schema cannot be inferred" \
-                    + f" (either remove the input or connect it to a model)"
-                cls._log.error(err)
-                raise _ex.EConfigParse(err)
+        if len(model_params) == 0:
+            err = f"Flow parameter [{param_name}] is not connected to any models, type information cannot be inferred" \
+                  + f" (either remove the parameter or connect it to a model)"
+            cls._log.error(err)
+            raise _ex.EJobValidation(err)
 
-            input_schemas = []
+        if len(model_params) == 1:
+            return model_params[0]
 
-            for edge in input_edges:
+        model_param = model_params[0]
 
-                target_node = flow.nodes.get(edge.target.node) # or cls._report_error(cls._MISSING_FLOW_NODE, node_name)
-                # cls._require(target_node.nodeType == _meta.FlowNodeType.MODEL_NODE)
+        for i in range(1, len(targets)):
+            next_param = model_params[i]
+            if next_param.paramType != model_param.paramType:
+                err = f"Parameter is ambiguous for [{param_name}]: " + \
+                      f"Types are different for [{cls._socket_key(targets[0])}] and [{cls._socket_key(targets[i])}]"
+                raise _ex.EJobValidation(err)
+            if next_param.defaultValue is None or next_param.defaultValue != model_param.defaultValue:
+                model_param.defaultValue = None
 
-                model_selector = job_config.job.runFlow.models.get(edge.target.node)
-                model_obj = _util.get_job_resource(model_selector, job_config)
-                model_input = model_obj.model.inputs[edge.target.socket]
-                input_schemas.append(model_input)
+        return model_param
 
-            if len(input_schemas) == 1:
-                inputs[node_name] = input_schemas[0]
-            else:
-                first_schema = input_schemas[0]
-                if all(map(lambda s: s == first_schema, input_schemas[1:])):
-                    inputs[node_name] = first_schema
-                else:
-                    raise _ex.EJobValidation(f"Multiple models use input [{node_name}] but expect different schemas")
+    @classmethod
+    def _infer_input_schema(
+            cls, input_name: str, targets: tp.List[_meta.FlowSocket],
+            job_config: _cfg.JobConfig) -> _meta.ModelInputSchema:
 
-        flow.inputs = inputs
+        model_inputs = []
 
-        return flow
+        for target in targets:
 
-    @classmethod
-    def _generate_flow_outputs(cls, flow: _meta.FlowDefinition, job_config: _cfg.JobConfig) -> _meta.FlowDefinition:
+            model_selector = job_config.job.runFlow.models.get(target.node)
+            model_obj = _util.get_job_resource(model_selector, job_config)
+            model_input = model_obj.model.inputs.get(target.socket)
+            model_inputs.append(model_input)
 
-        outputs: tp.Dict[str, _meta.ModelOutputSchema] = dict()
+        if len(model_inputs) == 0:
+            err = f"Flow input [{input_name}] is not connected to any models, schema cannot be inferred" \
+                  + f" (either remove the input or connect it to a model)"
+            cls._log.error(err)
+            raise _ex.EJobValidation(err)
 
-        def socket_key(socket):
-            return f"{socket.node}.{socket.socket}" if socket.socket else socket.node
+        if len(model_inputs) == 1:
+            return model_inputs[0]
 
-        # Build a map of edges by target socket, there can only be one edge per target in a valid flow
-        edges = {socket_key(edge.target): edge for edge in flow.edges}
+        model_input = model_inputs[0]
 
-        for node_name, node in flow.nodes.items():
+        for i in range(1, len(targets)):
+            next_input = model_inputs[i]
+            # Very strict rules on inputs, they must have the exact same schema
+            # The Java code includes a combineSchema() method which could be used here as well
+            if next_input != model_input:
+                raise _ex.EJobValidation(f"Multiple models use input [{input_name}] but expect different schemas")
 
-            if node.nodeType != _meta.FlowNodeType.OUTPUT_NODE:
-                continue
+        return model_input
 
-            edge = edges.get(node_name)
+    @classmethod
+    def _infer_output_schema(
+            cls, output_name: str, sources: tp.List[_meta.FlowSocket],
+            job_config: _cfg.JobConfig) -> _meta.ModelOutputSchema:
 
-            if not edge:
-                err = f"Flow output [{node_name}] is not connected, so the output schema cannot be inferred" \
-                      + f" (either remove the output or connect it to a model)"
-                cls._log.error(err)
-                raise _ex.EConfigParse(err)
+        model_outputs = []
 
-            source_node = flow.nodes.get(edge.source.node) # or cls._report_error(cls._MISSING_FLOW_NODE, node_name)
-            # cls._require(target_node.nodeType == _meta.FlowNodeType.MODEL_NODE)
+        for source in sources:
 
-            model_selector = job_config.job.runFlow.models.get(edge.source.node)
+            model_selector = job_config.job.runFlow.models.get(source.node)
             model_obj = _util.get_job_resource(model_selector, job_config)
-            model_output = model_obj.model.outputs[edge.source.socket]
+            model_input = model_obj.model.inputs.get(source.socket)
+            model_outputs.append(model_input)
 
-            outputs[node_name] = model_output
+        if len(model_outputs) == 0:
+            err = f"Flow output [{output_name}] is not connected to any models, schema cannot be inferred" \
+                  + f" (either remove the output or connect it to a model)"
+            cls._log.error(err)
+            raise _ex.EJobValidation(err)
 
-        flow.outputs = outputs
+        if len(model_outputs) > 1:
+            err = f"Flow output [{output_name}] is not to multiple models" \
+                  + f" (only one model can supply one output)"
+            cls._log.error(err)
+            raise _ex.EJobValidation(err)
 
-        return flow
+        return model_outputs[0]
+
+    @classmethod
+    def _socket_key(cls, socket):
+        return f"{socket.node}.{socket.socket}" if socket.socket else socket.node
 
     @classmethod
     def _process_parameters(cls, job_config: _cfg.JobConfig) -> _cfg.JobConfig:
@@ -564,19 +580,8 @@ class DevModeTranslator:
         param_specs = model_or_flow.parameters
         param_values = job_details.parameters
 
-        encoded_params = cls._process_parameters_dict(param_specs, param_values)
-
-        job_details = copy.copy(job_details)
-        job_def = copy.copy(job_config.job)
-        job_config = copy.copy(job_config)
-
-        if job_config.job.jobType == _meta.JobType.RUN_MODEL:
-            job_def.runModel = job_details
-        else:
-            job_def.runFlow = job_details
-
-        job_details.parameters = encoded_params
-        job_config.job = job_def
+        # Set encoded params on runModel or runFlow depending on the job type
+        job_details.parameters = cls._process_parameters_dict(param_specs, param_values)
 
         return job_config
 
@@ -610,9 +615,57 @@ class DevModeTranslator:
         return encoded_values
 
     @classmethod
-    def _process_job_io(
-            cls, sys_config, data_key, data_value, data_id, storage_id,
-            new_unique_file=False, schema: tp.Optional[_meta.SchemaDefinition] = None):
+    def _process_inputs_and_outputs(cls, sys_config: _cfg.RuntimeConfig, job_config: _cfg.JobConfig) -> _cfg.JobConfig:
+
+        if job_config.job.jobType == _meta.JobType.RUN_MODEL:
+            job_details = job_config.job.runModel
+            model_obj = _util.get_job_resource(job_details.model, job_config)
+            required_inputs = model_obj.model.inputs
+
+        elif job_config.job.jobType == _meta.JobType.RUN_FLOW:
+            job_details = job_config.job.runFlow
+            flow_obj = _util.get_job_resource(job_details.flow, job_config)
+            required_inputs = flow_obj.flow.inputs
+
+        else:
+            return job_config
+
+        job_inputs = job_details.inputs
+        job_outputs = job_details.outputs
+        job_resources = job_config.resources
+
+        for input_key, input_value in job_inputs.items():
+            if not (isinstance(input_value, str) and input_value in job_resources):
+
+                input_schema = required_inputs[input_key].schema
+
+                input_id = cls._process_input_or_output(
+                    sys_config, input_key, input_value, job_resources,
+                    new_unique_file=False, schema=input_schema)
+
+                job_inputs[input_key] = _util.selector_for(input_id)
+
+        for output_key, output_value in job_outputs.items():
+            if not (isinstance(output_value, str) and output_value in job_resources):
+
+                output_id = cls._process_input_or_output(
+                    sys_config, output_key, output_value, job_resources,
+                    new_unique_file=True, schema=None)
+
+                job_outputs[output_key] = _util.selector_for(output_id)
+
+        return job_config
+
+    @classmethod
+    def _process_input_or_output(
+            cls, sys_config, data_key, data_value,
+            resources: tp.Dict[str, _meta.ObjectDefinition],
+            new_unique_file=False,
+            schema: tp.Optional[_meta.SchemaDefinition] = None) \
+            -> _meta.TagHeader:
+
+        data_id = _util.new_object_id(_meta.ObjectType.DATA)
+        storage_id = _util.new_object_id(_meta.ObjectType.STORAGE)
 
         if isinstance(data_value, str):
             storage_path = data_value
@@ -666,7 +719,10 @@ class DevModeTranslator:
             snap_index=snap_version, delta_index=1, incarnation_index=1,
             schema=schema)
 
-        return data_obj, storage_obj
+        resources[_util.object_key(data_id)] = data_obj
+        resources[_util.object_key(storage_id)] = storage_obj
+
+        return data_id
 
     @staticmethod
     def infer_format(storage_path: str, storage_config: _cfg.StorageConfig):