tracdap-runtime 0.7.0rc1__py3-none-any.whl → 0.8.0__py3-none-any.whl

tracdap/rt/{_exec → _impl/exec}/graph_builder.py

@@ -13,10 +13,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import datetime as _dt
+
 import tracdap.rt.config as config
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.data as _data  # noqa
-import tracdap.rt._impl.util as _util  # noqa
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.util as _util
 
 from .graph import *
 
@@ -33,10 +35,10 @@ class GraphBuilder:
 
     __JOB_BUILD_FUNC = tp.Callable[[meta.JobDefinition, NodeId], GraphSection]
 
-    def __init__(self, job_config: config.JobConfig, result_spec: JobResultSpec):
+    def __init__(self, sys_config: config.RuntimeConfig, job_config: config.JobConfig):
 
+        self._sys_config = sys_config
         self._job_config = job_config
-        self._result_spec = result_spec
 
         self._job_key = _util.object_key(job_config.jobId)
         self._job_namespace = NodeNamespace(self._job_key)
@@ -45,7 +47,7 @@ class GraphBuilder:
 
     def _child_builder(self, job_id: meta.TagHeader) -> "GraphBuilder":
 
-        builder = GraphBuilder(self._job_config, JobResultSpec(save_result=False))
+        builder = GraphBuilder(self._sys_config, self._job_config)
         builder._job_key = _util.object_key(job_id)
         builder._job_namespace = NodeNamespace(builder._job_key)
 
@@ -355,58 +357,76 @@ class GraphBuilder:
 
         nodes = dict()
         outputs = set()
-        must_run = list()
 
-        for input_name, input_schema in required_inputs.items():
+        for input_name, input_def in required_inputs.items():
+
+            # Backwards compatibility with pre 0.8 versions
+            input_type = meta.ObjectType.DATA \
+                if input_def.objectType == meta.ObjectType.OBJECT_TYPE_NOT_SET \
+                else input_def.objectType
+
+            input_selector = supplied_inputs.get(input_name)
 
-            data_selector = supplied_inputs.get(input_name)
+            if input_selector is None:
 
-            if data_selector is None:
-                if input_schema.optional:
+                if input_def.optional:
                     data_view_id = NodeId.of(input_name, self._job_namespace, _data.DataView)
-                    nodes[data_view_id] = StaticValueNode(data_view_id, _data.DataView.create_empty())
+                    data_view = _data.DataView.create_empty(input_type)
+                    nodes[data_view_id] = StaticValueNode(data_view_id, data_view, explicit_deps=explicit_deps)
                     outputs.add(data_view_id)
-                    continue
                 else:
                     self._error(_ex.EJobValidation(f"Missing required input: [{input_name}]"))
-                    continue
 
-            # Build a data spec using metadata from the job config
-            # For now we are always loading the root part, snap 0, delta 0
-            data_def = _util.get_job_resource(data_selector, self._job_config).data
-            storage_def = _util.get_job_resource(data_def.storageId, self._job_config).storage
+            elif input_type == meta.ObjectType.DATA:
+                self._build_data_input(input_name, input_selector, nodes, outputs, explicit_deps)
+
+            elif input_type == meta.ObjectType.FILE:
+                self._build_file_input(input_name, input_selector, nodes, outputs, explicit_deps)
 
-            if data_def.schemaId:
-                schema_def = _util.get_job_resource(data_def.schemaId, self._job_config).schema
             else:
-                schema_def = data_def.schema
+                self._error(_ex.EJobValidation(f"Invalid input type [{input_type.name}] for input [{input_name}]"))
 
-            root_part_opaque_key = 'part-root'  # TODO: Central part names / constants
-            data_item = data_def.parts[root_part_opaque_key].snap.deltas[0].dataItem
-            data_spec = _data.DataSpec(data_item, data_def, storage_def, schema_def)
+        return GraphSection(nodes, outputs=outputs)
 
-            # Data spec node is static, using the assembled data spec
-            data_spec_id = NodeId.of(f"{input_name}:SPEC", self._job_namespace, _data.DataSpec)
-            data_spec_node = StaticValueNode(data_spec_id, data_spec, explicit_deps=explicit_deps)
+    def _build_data_input(self, input_name, input_selector, nodes, outputs, explicit_deps):
 
-            # Physical load of data items from disk
-            # Currently one item per input, since inputs are single part/delta
-            data_load_id = NodeId.of(f"{input_name}:LOAD", self._job_namespace, _data.DataItem)
-            data_load_node = LoadDataNode(data_load_id, data_spec_id, explicit_deps=explicit_deps)
+        # Build a data spec using metadata from the job config
+        # For now we are always loading the root part, snap 0, delta 0
+        data_def = _util.get_job_resource(input_selector, self._job_config).data
+        storage_def = _util.get_job_resource(data_def.storageId, self._job_config).storage
 
-            # Input views assembled by mapping one root part to each view
-            data_view_id = NodeId.of(input_name, self._job_namespace, _data.DataView)
-            data_view_node = DataViewNode(data_view_id, schema_def, data_load_id)
+        if data_def.schemaId:
+            schema_def = _util.get_job_resource(data_def.schemaId, self._job_config).schema
+        else:
+            schema_def = data_def.schema
 
-            nodes[data_spec_id] = data_spec_node
-            nodes[data_load_id] = data_load_node
-            nodes[data_view_id] = data_view_node
+        root_part_opaque_key = 'part-root'  # TODO: Central part names / constants
+        data_item = data_def.parts[root_part_opaque_key].snap.deltas[0].dataItem
+        data_spec = _data.DataSpec.create_data_spec(data_item, data_def, storage_def, schema_def)
+
+        # Physical load of data items from disk
+        # Currently one item per input, since inputs are single part/delta
+        data_load_id = NodeId.of(f"{input_name}:LOAD", self._job_namespace, _data.DataItem)
+        nodes[data_load_id] = LoadDataNode(data_load_id, spec=data_spec, explicit_deps=explicit_deps)
+
+        # Input views assembled by mapping one root part to each view
+        data_view_id = NodeId.of(input_name, self._job_namespace, _data.DataView)
+        nodes[data_view_id] = DataViewNode(data_view_id, schema_def, data_load_id)
+        outputs.add(data_view_id)
+
+    def _build_file_input(self, input_name, input_selector, nodes, outputs, explicit_deps):
+
+        file_def = _util.get_job_resource(input_selector, self._job_config).file
+        storage_def = _util.get_job_resource(file_def.storageId, self._job_config).storage
 
-            # Job-level data view is an output of the load operation
-            outputs.add(data_view_id)
-            must_run.append(data_spec_id)
+        file_spec = _data.DataSpec.create_file_spec(file_def.dataItem, file_def, storage_def)
+        file_load_id = NodeId.of(f"{input_name}:LOAD", self._job_namespace, _data.DataItem)
+        nodes[file_load_id] = LoadDataNode(file_load_id, spec=file_spec, explicit_deps=explicit_deps)
 
-        return GraphSection(nodes, outputs=outputs, must_run=must_run)
+        # Input views assembled by mapping one root part to each view
+        file_view_id = NodeId.of(input_name, self._job_namespace, _data.DataView)
+        nodes[file_view_id] = DataViewNode(file_view_id, None, file_load_id)
+        outputs.add(file_view_id)
 
     def build_job_outputs(
             self,
@@ -418,12 +438,21 @@ class GraphBuilder:
         nodes = {}
         inputs = set()
 
-        for output_name, output_schema in required_outputs.items():
+        for output_name, output_def in required_outputs.items():
 
-            data_selector = supplied_outputs.get(output_name)
+            # Output data view must already exist in the namespace, it is an input to the save operation
+            data_view_id = NodeId.of(output_name, self._job_namespace, _data.DataView)
+            inputs.add(data_view_id)
+
+            # Backwards compatibility with pre 0.8 versions
+            output_type = meta.ObjectType.DATA \
+                if output_def.objectType == meta.ObjectType.OBJECT_TYPE_NOT_SET \
+                else output_def.objectType
 
-            if data_selector is None:
-                if output_schema.optional:
+            output_selector = supplied_outputs.get(output_name)
+
+            if output_selector is None:
+                if output_def.optional:
                     optional_info = "(configuration is required for all optional outputs, in case they are produced)"
                     self._error(_ex.EJobValidation(f"Missing optional output: [{output_name}] {optional_info}"))
                     continue
@@ -431,75 +460,150 @@ class GraphBuilder:
                     self._error(_ex.EJobValidation(f"Missing required output: [{output_name}]"))
                     continue
 
-            # Output data view must already exist in the namespace
-            data_view_id = NodeId.of(output_name, self._job_namespace, _data.DataView)
-            data_spec_id = NodeId.of(f"{output_name}:SPEC", self._job_namespace, _data.DataSpec)
+            elif output_type == meta.ObjectType.DATA:
+                self._build_data_output(output_name, output_selector, data_view_id, nodes, explicit_deps)
+
+            elif output_type == meta.ObjectType.FILE:
+                self._build_file_output(output_name, output_def, output_selector, data_view_id, nodes, explicit_deps)
 
-            data_obj = _util.get_job_resource(data_selector, self._job_config, optional=True)
+            else:
+                self._error(_ex.EJobValidation(f"Invalid output type [{output_type.name}] for input [{output_name}]"))
 
-            if data_obj is not None:
+        return GraphSection(nodes, inputs=inputs)
 
-                # If data def for the output has been built in advance, use a static data spec
+    def _build_data_output(self, output_name, output_selector, data_view_id, nodes, explicit_deps):
 
-                data_def = data_obj.data
-                storage_def = _util.get_job_resource(data_def.storageId, self._job_config).storage
+        # Map one data item from each view, since outputs are single part/delta
+        data_item_id = NodeId(f"{output_name}:ITEM", self._job_namespace, _data.DataItem)
+        nodes[data_item_id] = DataItemNode(data_item_id, data_view_id)
 
-                if data_def.schemaId:
-                    schema_def = _util.get_job_resource(data_def.schemaId, self._job_config).schema
-                else:
-                    schema_def = data_def.schema
+        data_obj = _util.get_job_resource(output_selector, self._job_config, optional=True)
 
-                root_part_opaque_key = 'part-root'  # TODO: Central part names / constants
-                data_item = data_def.parts[root_part_opaque_key].snap.deltas[0].dataItem
-                data_spec = _data.DataSpec(data_item, data_def, storage_def, schema_def)
+        if data_obj is not None:
 
-                data_spec_node = StaticValueNode(data_spec_id, data_spec, explicit_deps=explicit_deps)
+            # If data def for the output has been built in advance, use a static data spec
 
-                output_data_key = output_name + ":DATA"
-                output_storage_key = output_name + ":STORAGE"
+            data_def = data_obj.data
+            storage_def = _util.get_job_resource(data_def.storageId, self._job_config).storage
 
+            if data_def.schemaId:
+                schema_def = _util.get_job_resource(data_def.schemaId, self._job_config).schema
             else:
+                schema_def = data_def.schema
 
-                # If output data def for an output was not supplied in the job, create a dynamic data spec
-                # Dynamic data def will always use an embedded schema (this is no ID for an external schema)
+            root_part_opaque_key = 'part-root'  # TODO: Central part names / constants
+            data_item = data_def.parts[root_part_opaque_key].snap.deltas[0].dataItem
+            data_spec = _data.DataSpec.create_data_spec(data_item, data_def, storage_def, schema_def)
 
-                data_key = output_name + ":DATA"
-                data_id = self._job_config.resultMapping[data_key]
-                storage_key = output_name + ":STORAGE"
-                storage_id = self._job_config.resultMapping[storage_key]
+            # Create a physical save operation for the data item
+            data_save_id = NodeId.of(f"{output_name}:SAVE", self._job_namespace, _data.DataSpec)
+            nodes[data_save_id] = SaveDataNode(data_save_id, data_item_id, spec=data_spec)
 
-                data_spec_node = DynamicDataSpecNode(
-                    data_spec_id, data_view_id,
-                    data_id, storage_id,
-                    prior_data_spec=None,
-                    explicit_deps=explicit_deps)
+            output_key = output_name
+            storage_key = output_name + ":STORAGE"
 
-                output_data_key = _util.object_key(data_id)
-                output_storage_key = _util.object_key(storage_id)
+        else:
 
-            # Map one data item from each view, since outputs are single part/delta
-            data_item_id = NodeId(f"{output_name}:ITEM", self._job_namespace, _data.DataItem)
-            data_item_node = DataItemNode(data_item_id, data_view_id)
+            # If output data def for an output was not supplied in the job, create a dynamic data spec
+            # Dynamic data def will always use an embedded schema (this is no ID for an external schema)
+
+            mapped_output_key = output_name
+            mapped_storage_key = output_name + ":STORAGE"
+
+            data_id = self._job_config.resultMapping[mapped_output_key]
+            storage_id = self._job_config.resultMapping[mapped_storage_key]
+
+            data_spec_id = NodeId.of(f"{output_name}:SPEC", self._job_namespace, _data.DataSpec)
+            nodes[data_spec_id] = DynamicDataSpecNode(
+                data_spec_id, data_view_id,
+                data_id, storage_id,
+                prior_data_spec=None,
+                explicit_deps=explicit_deps)
 
             # Create a physical save operation for the data item
-            data_save_id = NodeId.of(f"{output_name}:SAVE", self._job_namespace, None)
-            data_save_node = SaveDataNode(data_save_id, data_spec_id, data_item_id)
+            data_save_id = NodeId.of(f"{output_name}:SAVE", self._job_namespace, _data.DataSpec)
+            nodes[data_save_id] = SaveDataNode(data_save_id, data_item_id, spec_id=data_spec_id)
 
-            data_result_id = NodeId.of(f"{output_name}:RESULT", self._job_namespace, ObjectBundle)
-            data_result_node = DataResultNode(
-                data_result_id, output_name,
-                data_item_id, data_spec_id, data_save_id,
-                output_data_key, output_storage_key)
+            output_key = _util.object_key(data_id)
+            storage_key = _util.object_key(storage_id)
 
-            nodes[data_spec_id] = data_spec_node
-            nodes[data_item_id] = data_item_node
-            nodes[data_save_id] = data_save_node
-            nodes[data_result_id] = data_result_node
+        data_result_id = NodeId.of(f"{output_name}:RESULT", self._job_namespace, ObjectBundle)
+        nodes[data_result_id] = DataResultNode(
+            data_result_id, output_name, data_save_id,
+            data_key=output_key,
+            storage_key=storage_key)
 
-            # Job-level data view is an input to the save operation
-            inputs.add(data_view_id)
+    def _build_file_output(self, output_name, output_def, output_selector, file_view_id, nodes, explicit_deps):
 
-        return GraphSection(nodes, inputs=inputs)
+        mapped_output_key = output_name
+        mapped_storage_key = output_name + ":STORAGE"
+
+        file_obj = _util.get_job_resource(output_selector, self._job_config, optional=True)
+
+        if file_obj is not None:
+
+            # Definitions already exist (generated by dev mode translator)
+
+            file_def = _util.get_job_resource(output_selector, self._job_config).file
+            storage_def = _util.get_job_resource(file_def.storageId, self._job_config).storage
+
+            resolved_output_key = mapped_output_key
+            resolved_storage_key = mapped_storage_key
+
+        else:
+
+            # Create new definitions (default behavior for jobs sent from the platform)
+
+            output_id = self._job_config.resultMapping[mapped_output_key]
+            storage_id = self._job_config.resultMapping[mapped_storage_key]
+
+            file_type = output_def.fileType
+            timestamp = _dt.datetime.fromisoformat(output_id.objectTimestamp.isoDatetime)
+            data_item = f"file/{output_id.objectId}/version-{output_id.objectVersion}"
+            storage_key = self._sys_config.storage.defaultBucket
+            storage_path = f"file/FILE-{output_id.objectId}/version-{output_id.objectVersion}/{output_name}.{file_type.extension}"
+
+            file_def = self.build_file_def(output_name, file_type, storage_id, data_item)
+            storage_def = self.build_storage_def(data_item, storage_key, storage_path, file_type.mimeType, timestamp)
+
+            resolved_output_key = _util.object_key(output_id)
+            resolved_storage_key = _util.object_key(storage_id)
+
+        # Required object defs are available, now build the graph nodes
+
+        file_item_id = NodeId(f"{output_name}:ITEM", self._job_namespace, _data.DataItem)
+        nodes[file_item_id] = DataItemNode(file_item_id, file_view_id, explicit_deps=explicit_deps)
+
+        file_spec = _data.DataSpec.create_file_spec(file_def.dataItem, file_def, storage_def)
+        file_save_id = NodeId.of(f"{output_name}:SAVE", self._job_namespace, _data.DataSpec)
+        nodes[file_save_id] = SaveDataNode(file_save_id, file_item_id, spec=file_spec)
+
+        data_result_id = NodeId.of(f"{output_name}:RESULT", self._job_namespace, ObjectBundle)
+        nodes[data_result_id] = DataResultNode(
+            data_result_id, output_name, file_save_id,
+            file_key=resolved_output_key,
+            storage_key=resolved_storage_key)
+
+    @classmethod
+    def build_output_file_and_storage(cls, output_key, file_type: meta.FileType, sys_config: cfg.RuntimeConfig, job_config: cfg.JobConfig):
+
+        # TODO: Review and de-dupe building of output metadata
+        # Responsibility for assigning outputs could perhaps move from orchestrator to runtime
+
+        output_storage_key = f"{output_key}:STORAGE"
+
+        output_id = job_config.resultMapping[output_key]
+        output_storage_id = job_config.resultMapping[output_storage_key]
+
+        timestamp = _dt.datetime.fromisoformat(output_id.objectTimestamp.isoDatetime)
+        data_item = f"file/{output_id.objectId}/version-{output_id.objectVersion}"
+        storage_key = sys_config.storage.defaultBucket
+        storage_path = f"file/FILE-{output_id.objectId}/version-{output_id.objectVersion}/{output_key}.{file_type.extension}"
+
+        file_def = cls.build_file_def(output_key, file_type, output_storage_id, data_item)
+        storage_def = cls.build_storage_def(data_item, storage_key, storage_path, file_type.mimeType, timestamp)
+
+        return file_def, storage_def
 
     @classmethod
     def build_runtime_outputs(cls, output_names: tp.List[str], job_namespace: NodeNamespace):
@@ -519,9 +623,10 @@ class GraphBuilder:
             data_view_id = NodeId.of(output_name, job_namespace, _data.DataView)
             data_spec_id = NodeId.of(f"{output_name}:SPEC", job_namespace, _data.DataSpec)
 
-            data_key = output_name + ":DATA"
+            mapped_output_key = output_name
+            mapped_storage_key = output_name + ":STORAGE"
+
             data_id = _util.new_object_id(meta.ObjectType.DATA)
-            storage_key = output_name + ":STORAGE"
             storage_id = _util.new_object_id(meta.ObjectType.STORAGE)
 
             data_spec_node = DynamicDataSpecNode(
@@ -529,22 +634,21 @@ class GraphBuilder:
                 data_id, storage_id,
                 prior_data_spec=None)
 
-            output_data_key = _util.object_key(data_id)
-            output_storage_key = _util.object_key(storage_id)
+            output_key = _util.object_key(data_id)
+            storage_key = _util.object_key(storage_id)
 
             # Map one data item from each view, since outputs are single part/delta
             data_item_id = NodeId(f"{output_name}:ITEM", job_namespace, _data.DataItem)
             data_item_node = DataItemNode(data_item_id, data_view_id)
 
             # Create a physical save operation for the data item
-            data_save_id = NodeId.of(f"{output_name}:SAVE", job_namespace, None)
-            data_save_node = SaveDataNode(data_save_id, data_spec_id, data_item_id)
+            data_save_id = NodeId.of(f"{output_name}:SAVE", job_namespace, _data.DataSpec)
+            data_save_node = SaveDataNode(data_save_id, data_item_id, spec_id=data_spec_id)
 
             data_result_id = NodeId.of(f"{output_name}:RESULT", job_namespace, ObjectBundle)
             data_result_node = DataResultNode(
-                data_result_id, output_name,
-                data_item_id, data_spec_id, data_save_id,
-                output_data_key, output_storage_key)
+                data_result_id, output_name, data_save_id,
+                output_key, storage_key)
 
             nodes[data_spec_id] = data_spec_node
             nodes[data_item_id] = data_item_node
@@ -563,6 +667,45 @@ class GraphBuilder:
 
         return GraphSection(nodes, inputs=inputs, outputs={runtime_outputs_id})
 
+    @classmethod
+    def build_file_def(cls, file_name, file_type, storage_id, data_item):
+
+        file_def = meta.FileDefinition()
+        file_def.name = f"{file_name}.{file_type.extension}"
+        file_def.extension = file_type.extension
+        file_def.mimeType = file_type.mimeType
+        file_def.storageId = _util.selector_for_latest(storage_id)
+        file_def.dataItem = data_item
+        file_def.size = 0
+
+        return file_def
+
+    @classmethod
+    def build_storage_def(
+            cls, data_item: str,
+            storage_key, storage_path, storage_format,
+            timestamp: _dt.datetime):
+
+        first_incarnation = 0
+
+        storage_copy = meta.StorageCopy(
+            storage_key, storage_path, storage_format,
+            copyStatus=meta.CopyStatus.COPY_AVAILABLE,
+            copyTimestamp=meta.DatetimeValue(timestamp.isoformat()))
+
+        storage_incarnation = meta.StorageIncarnation(
+            [storage_copy],
+            incarnationIndex=first_incarnation,
+            incarnationTimestamp=meta.DatetimeValue(timestamp.isoformat()),
+            incarnationStatus=meta.IncarnationStatus.INCARNATION_AVAILABLE)
+
+        storage_item = meta.StorageItem([storage_incarnation])
+
+        storage_def = meta.StorageDefinition()
+        storage_def.dataItems[data_item] = storage_item
+
+        return storage_def
+
     def build_job_results(
             self,
             objects: tp.Dict[str, NodeId[meta.ObjectDefinition]] = None,
@@ -570,15 +713,16 @@ class GraphBuilder:
             explicit_deps: tp.Optional[tp.List[NodeId]] = None) \
             -> GraphSection:
 
-        build_result_id = NodeId.of("trac_job_result", self._job_namespace, cfg.JobResult)
+        result_id = self._job_config.resultMapping.get("trac_job_result")
+        result_node_id = NodeId.of("trac_job_result", self._job_namespace, cfg.JobResult)
 
         if objects is not None:
 
             results_inputs = set(objects.values())
 
             build_result_node = BuildJobResultNode(
-                build_result_id, self._job_config.jobId,
-                outputs = JobOutputs(objects=objects),
+                result_node_id, result_id, self._job_config.jobId,
+                outputs=JobOutputs(objects=objects),
                 explicit_deps=explicit_deps)
 
         elif bundles is not None:
@@ -586,23 +730,16 @@ class GraphBuilder:
             results_inputs = set(bundles)
 
             build_result_node = BuildJobResultNode(
-                build_result_id, self._job_config.jobId,
-                outputs = JobOutputs(bundles=bundles),
+                result_node_id, result_id, self._job_config.jobId,
+                outputs=JobOutputs(bundles=bundles),
                 explicit_deps=explicit_deps)
 
         else:
             raise _ex.EUnexpected()
 
-        if self._result_spec.save_result:
-            save_result_id = NodeId("trac_save_result", self._job_namespace)
-            save_result_node = SaveJobResultNode(save_result_id, build_result_id, self._result_spec)
-            result_nodes = {build_result_id: build_result_node, save_result_id: save_result_node}
-            job_result_id = save_result_id
-        else:
-            result_nodes = {build_result_id: build_result_node}
-            job_result_id = build_result_id
+        result_nodes = {result_node_id: build_result_node}
 
-        return GraphSection(result_nodes, inputs=results_inputs, must_run=[job_result_id])
+        return GraphSection(result_nodes, inputs=results_inputs, must_run=[result_node_id])
 
     def build_model_or_flow_with_context(
             self, namespace: NodeNamespace, model_or_flow_name: str,

tracdap/rt/_impl/grpc/codec.py

@@ -24,7 +24,7 @@ import tracdap.rt._impl.grpc.tracdap.metadata.object_id_pb2 as object_id_pb2
 import tracdap.rt._impl.grpc.tracdap.metadata.object_pb2 as object_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import model_pb2
 import tracdap.rt._impl.grpc.tracdap.metadata.data_pb2 as data_pb2
-import tracdap.rt._impl.grpc.tracdap.metadata.stoarge_pb2 as storage_pb2
+import tracdap.rt._impl.grpc.tracdap.metadata.storage_pb2 as storage_pb2
 
 from google.protobuf import message as _message
 

tracdap/rt/{_exec → _impl/grpc}/server.py

@@ -19,9 +19,10 @@ import typing as tp
 
 import tracdap.rt.config as config
 import tracdap.rt.exceptions as ex
-import tracdap.rt._exec.actors as actors
-import tracdap.rt._impl.grpc.codec as codec  # noqa
-import tracdap.rt._impl.util as util  # noqa
+import tracdap.rt._impl.exec.actors as actors
+import tracdap.rt._impl.grpc.codec as codec
+import tracdap.rt._impl.core.logging as logging
+import tracdap.rt._impl.core.util as util
 
 # Check whether gRPC is installed before trying to load any of the generated modules
 try:
@@ -44,7 +45,7 @@ class RuntimeApiServer(runtime_grpc.TracRuntimeApiServicer):
 
     def __init__(self, system: actors.ActorSystem, port: int):
 
-        self.__log = util.logger_for_object(self)
+        self.__log = logging.logger_for_object(self)
 
         self.__system = system
         self.__engine_id = system.main_id()
@@ -158,7 +159,7 @@ class ApiAgent(actors.ThreadsafeActor):
 
     def __init__(self):
         super().__init__()
-        self._log = util.logger_for_object(self)
+        self._log = logging.logger_for_object(self)
         self._event_loop = asyncio.get_event_loop()
         self.__start_signal = asyncio.Event()
 
@@ -258,7 +259,7 @@ class ApiRequest(actors.ThreadsafeActor, tp.Generic[_T_REQUEST, _T_RESPONSE]):
             self.threadsafe().stop()
 
 
-ApiRequest._log = util.logger_for_class(ApiRequest)
+ApiRequest._log = logging.logger_for_class(ApiRequest)
 
 
 class ListJobsRequest(ApiRequest[runtime_pb2.RuntimeListJobsRequest, runtime_pb2.RuntimeListJobsResponse]):

tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/api/internal/runtime.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -18,14 +18,14 @@ from tracdap.rt._impl.grpc.tracdap.metadata import object_pb2 as tracdap_dot_rt_
 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8tracdap/rt/_impl/grpc/tracdap/api/internal/runtime.proto\x12\x14tracdap.api.internal\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x1a\x1cgoogle/api/annotations.proto\"6\n\x16RuntimeListJobsRequest\x12\x12\n\x05limit\x18\x01 \x01(\rH\x00\x88\x01\x01\x42\x08\n\x06_limit\"O\n\x17RuntimeListJobsResponse\x12\x34\n\x04jobs\x18\x01 \x03(\x0b\x32&.tracdap.api.internal.RuntimeJobStatus\"f\n\x15RuntimeJobInfoRequest\x12\x34\n\x0bjobSelector\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x10\n\x06jobKey\x18\x02 \x01(\tH\x00\x42\x05\n\x03job\"\x9f\x01\n\x10RuntimeJobStatus\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x13\n\x0b\x65rrorDetail\x18\x04 \x01(\t\"\xa4\x02\n\x10RuntimeJobResult\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x44\n\x07results\x18\x04 \x03(\x0b\x32\x33.tracdap.api.internal.RuntimeJobResult.ResultsEntry\x1aR\n\x0cResultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".tracdap.metadata.ObjectDefinition:\x02\x38\x01\x32\x95\x03\n\x0eTracRuntimeApi\x12{\n\x08listJobs\x12,.tracdap.api.internal.RuntimeListJobsRequest\x1a-.tracdap.api.internal.RuntimeListJobsResponse\"\x12\x82\xd3\xe4\x93\x02\x0c\x12\n/list-jobs\x12\x81\x01\n\x0cgetJobStatus\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobStatus\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-status/{jobKey}\x12\x81\x01\n\x0cgetJobResult\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobResult\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-result/{jobKey}B\"\n\x1eorg.finos.tracdap.api.internalP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8tracdap/rt/_impl/grpc/tracdap/api/internal/runtime.proto\x12\x14tracdap.api.internal\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x1a\x1cgoogle/api/annotations.proto\"6\n\x16RuntimeListJobsRequest\x12\x12\n\x05limit\x18\x01 \x01(\rH\x00\x88\x01\x01\x42\x08\n\x06_limit\"O\n\x17RuntimeListJobsResponse\x12\x34\n\x04jobs\x18\x01 \x03(\x0b\x32&.tracdap.api.internal.RuntimeJobStatus\"f\n\x15RuntimeJobInfoRequest\x12\x34\n\x0bjobSelector\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x10\n\x06jobKey\x18\x02 \x01(\tH\x00\x42\x05\n\x03job\"\x9f\x01\n\x10RuntimeJobStatus\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x13\n\x0b\x65rrorDetail\x18\x04 \x01(\t\"\xa4\x02\n\x10RuntimeJobResult\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x44\n\x07results\x18\x04 \x03(\x0b\x32\x33.tracdap.api.internal.RuntimeJobResult.ResultsEntry\x1aR\n\x0cResultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".tracdap.metadata.ObjectDefinition:\x02\x38\x01\x32\x95\x03\n\x0eTracRuntimeApi\x12{\n\x08listJobs\x12,.tracdap.api.internal.RuntimeListJobsRequest\x1a-.tracdap.api.internal.RuntimeListJobsResponse\"\x12\x82\xd3\xe4\x93\x02\x0c\x12\n/list-jobs\x12\x81\x01\n\x0cgetJobStatus\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobStatus\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-status/{jobKey}\x12\x81\x01\n\x0cgetJobResult\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobResult\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-result/{jobKey}B7\n\x1eorg.finos.tracdap.api.internalB\x13RuntimeServiceProtoP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
 _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracdap.api.internal.runtime_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._options = None
-  _globals['DESCRIPTOR']._serialized_options = b'\n\036org.finos.tracdap.api.internalP\001'
+  _globals['DESCRIPTOR']._serialized_options = b'\n\036org.finos.tracdap.api.internalB\023RuntimeServiceProtoP\001'
   _globals['_RUNTIMEJOBRESULT_RESULTSENTRY']._options = None
   _globals['_RUNTIMEJOBRESULT_RESULTSENTRY']._serialized_options = b'8\001'
   _globals['_TRACRUNTIMEAPI'].methods_by_name['listJobs']._options = None

tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2_grpc.py

@@ -5,7 +5,7 @@ import warnings
 
 from tracdap.rt._impl.grpc.tracdap.api.internal import runtime_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_api_dot_internal_dot_runtime__pb2
 
-GRPC_GENERATED_VERSION = '1.66.1'
+GRPC_GENERATED_VERSION = '1.70.0'
 GRPC_VERSION = grpc.__version__
 _version_not_supported = False
 

tracdap/rt/_impl/grpc/tracdap/metadata/common_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/common.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool