tracdap-runtime 0.8.0b2__py3-none-any.whl → 0.8.0b4__py3-none-any.whl

tracdap/rt/{_exec → _impl/exec}/engine.py

@@ -23,15 +23,15 @@ import typing as tp
 import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._exec.actors as _actors
-import tracdap.rt._exec.graph_builder as _graph
-import tracdap.rt._exec.functions as _func
-import tracdap.rt._impl.config_parser as _cfg_p  # noqa
-import tracdap.rt._impl.data as _data  # noqa
-import tracdap.rt._impl.logging as _logging  # noqa
-import tracdap.rt._impl.models as _models  # noqa
-import tracdap.rt._impl.storage as _storage  # noqa
-import tracdap.rt._impl.util as _util  # noqa
+import tracdap.rt._impl.exec.actors as _actors
+import tracdap.rt._impl.exec.graph_builder as _graph
+import tracdap.rt._impl.exec.functions as _func
+import tracdap.rt._impl.core.config_parser as _cfg_p
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.models as _models
+import tracdap.rt._impl.core.storage as _storage
+import tracdap.rt._impl.core.util as _util
 
 from .graph import NodeId
 

tracdap/rt/{_exec → _impl/exec}/functions.py

@@ -16,6 +16,7 @@
 import copy
 import datetime
 import abc
+import io
 import pathlib
 import random
 import dataclasses as dc  # noqa
@@ -23,18 +24,18 @@ import dataclasses as dc  # noqa
 import tracdap.rt.api as _api
 import tracdap.rt.config as _config
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._exec.context as _ctx
-import tracdap.rt._exec.graph_builder as _graph
-import tracdap.rt._impl.config_parser as _cfg_p  # noqa
-import tracdap.rt._impl.type_system as _types  # noqa
-import tracdap.rt._impl.data as _data  # noqa
-import tracdap.rt._impl.logging as _logging  # noqa
-import tracdap.rt._impl.storage as _storage  # noqa
-import tracdap.rt._impl.models as _models  # noqa
-import tracdap.rt._impl.util as _util  # noqa
+import tracdap.rt._impl.exec.context as _ctx
+import tracdap.rt._impl.exec.graph_builder as _graph
+import tracdap.rt._impl.core.type_system as _types
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.storage as _storage
+import tracdap.rt._impl.core.struct as _struct
+import tracdap.rt._impl.core.models as _models
+import tracdap.rt._impl.core.util as _util
 
-from tracdap.rt._exec.graph import *
-from tracdap.rt._exec.graph import _T
+from tracdap.rt._impl.exec.graph import *
+from tracdap.rt._impl.exec.graph import _T
 
 
 class NodeContext:
@@ -283,6 +284,13 @@ class DataViewFunc(NodeFunction[_data.DataView]):
         if root_item.object_type == meta.ObjectType.FILE:
             return _data.DataView.for_file_item(root_item)
 
+        # TODO: Generalize processing across DataView / DataItem types
+
+        if root_item.schema_type == meta.SchemaType.STRUCT:
+            view = _data.DataView.for_trac_schema(self.node.schema)
+            view.parts[root_part_key] = [root_item]
+            return view
+
         # Everything else is a regular data view
         if self.node.schema is not None and len(self.node.schema.table.fields) > 0:
             trac_schema = self.node.schema
@@ -489,40 +497,55 @@ class LoadDataFunc( _LoadSaveDataFunc, NodeFunction[_data.DataItem],):
         data_spec = self._choose_data_spec(self.node.spec_id, self.node.spec, ctx)
         data_copy = self._choose_copy(data_spec.data_item, data_spec.storage_def)
 
-        if data_spec.object_type == _api.ObjectType.DATA:
-            return self._load_data(data_spec, data_copy)
-
-        elif data_spec.object_type == _api.ObjectType.FILE:
+        if data_spec.object_type == _api.ObjectType.FILE:
             return self._load_file(data_copy)
 
+        elif data_spec.schema_type == _api.SchemaType.TABLE:
+            return self._load_table(data_spec, data_copy)
+
+        elif data_spec.schema_type == _api.SchemaType.STRUCT:
+            return self._load_struct(data_copy)
+
+        # TODO: Handle dynamic inputs, they should work for any schema type
+        elif data_spec.schema_type == _api.SchemaType.SCHEMA_TYPE_NOT_SET:
+            return self._load_table(data_spec, data_copy)
+
         else:
             raise _ex.EUnexpected()
 
-    def _load_data(self, data_spec, data_copy):
+    def _load_file(self, data_copy):
+
+        storage = self.storage.get_file_storage(data_copy.storageKey)
+        content = storage.read_bytes(data_copy.storagePath)
+
+        return _data.DataItem.for_file_content(content)
+
+    def _load_table(self, data_spec, data_copy):
 
         trac_schema = data_spec.schema_def if data_spec.schema_def else data_spec.data_def.schema
         arrow_schema = _data.DataMapping.trac_to_arrow_schema(trac_schema) if trac_schema else None
 
-        # Decode options (metadata values) from the storage definition
-        options = dict()
-        for opt_key, opt_value in data_spec.storage_def.storageOptions.items():
-            options[opt_key] = _types.MetadataCodec.decode_value(opt_value)
+        storage_options = dict(
+            (opt_key, _types.MetadataCodec.decode_value(opt_value))
+            for opt_key, opt_value in data_spec.storage_def.storageOptions.items())
 
         storage = self.storage.get_data_storage(data_copy.storageKey)
+
         table = storage.read_table(
-            data_copy.storagePath,
-            data_copy.storageFormat,
-            arrow_schema,
-            storage_options=options)
+            data_copy.storagePath, data_copy.storageFormat, arrow_schema,
+            storage_options=storage_options)
 
-        return _data.DataItem(_api.ObjectType.DATA, table.schema, table)
+        return _data.DataItem.for_table(table, table.schema, trac_schema)
 
-    def _load_file(self, data_copy):
+    def _load_struct(self, data_copy):
 
         storage = self.storage.get_file_storage(data_copy.storageKey)
-        raw_bytes = storage.read_bytes(data_copy.storagePath)
 
-        return _data.DataItem(_api.ObjectType.FILE, raw_bytes=raw_bytes)
+        with storage.read_byte_stream(data_copy.storagePath) as stream:
+            with io.TextIOWrapper(stream, "utf-8") as text_stream:
+                struct = _struct.StructProcessor.load_struct(text_stream, data_copy.storageFormat)
+
+        return _data.DataItem.for_struct(struct)
 
 
 class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[_data.DataSpec]):
@@ -542,24 +565,40 @@ class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[_data.DataSpec]):
 
         # Do not save empty outputs (optional outputs that were not produced)
         if data_item.is_empty():
-            return _data.DataSpec.create_empty_spec(data_item.object_type)
+            return _data.DataSpec.create_empty_spec(data_item.object_type, data_item.schema_type)
 
-        if data_item.object_type == _api.ObjectType.DATA:
-            return self._save_data(data_item, data_spec, data_copy)
-
-        elif data_item.object_type == _api.ObjectType.FILE:
+        if data_item.object_type == _api.ObjectType.FILE:
             return self._save_file(data_item, data_spec, data_copy)
 
+        elif data_item.schema_type == _api.SchemaType.TABLE:
+            return self._save_table(data_item, data_spec, data_copy)
+
+        elif data_item.schema_type == _api.SchemaType.STRUCT:
+            return self._save_struct(data_item, data_spec, data_copy)
+
         else:
             raise _ex.EUnexpected()
 
-    def _save_data(self, data_item, data_spec, data_copy):
+    def _save_file(self, data_item, data_spec, data_copy):
+
+        if data_item.content is None:
+            raise _ex.EUnexpected()
+
+        storage = self.storage.get_file_storage(data_copy.storageKey)
+        storage.write_bytes(data_copy.storagePath, data_item.content)
+
+        data_spec = copy.deepcopy(data_spec)
+        data_spec.file_def.size = len(data_item.content)
+
+        return data_spec
+
+    def _save_table(self, data_item, data_spec, data_copy):
 
         # Current implementation will always put an Arrow table in the data item
         # Empty tables are allowed, so explicitly check if table is None
         # Testing "if not data_item.table" will fail for empty tables
 
-        if data_item.table is None:
+        if data_item.content is None:
             raise _ex.EUnexpected()
 
         # Decode options (metadata values) from the storage definition
@@ -570,7 +609,7 @@ class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[_data.DataSpec]):
         storage = self.storage.get_data_storage(data_copy.storageKey)
         storage.write_table(
             data_copy.storagePath, data_copy.storageFormat,
-            data_item.table,
+            data_item.content,
             storage_options=options, overwrite=False)
 
         data_spec = copy.deepcopy(data_spec)
@@ -581,16 +620,26 @@ class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[_data.DataSpec]):
 
         return data_spec
 
-    def _save_file(self, data_item, data_spec, data_copy):
+    def _save_struct(self, data_item, data_spec, data_copy):
 
-        if data_item.raw_bytes is None:
+        if data_item.content is None:
             raise _ex.EUnexpected()
 
+        struct_data = data_item.content
+        storage_format = data_copy.storageFormat
+
         storage = self.storage.get_file_storage(data_copy.storageKey)
-        storage.write_bytes(data_copy.storagePath, data_item.raw_bytes)
+
+        # Using the text wrapper closes the stream early, which is inefficient in the data layer
+        # Supporting text IO directly from the storage API would allow working with text streams more naturally
+        with storage.write_byte_stream(data_copy.storagePath) as stream:
+            with io.TextIOWrapper(stream, "utf-8") as text_stream:
+                _struct.StructProcessor.save_struct(struct_data, text_stream, storage_format)
 
         data_spec = copy.deepcopy(data_spec)
-        data_spec.file_def.size = len(data_item.raw_bytes)
+
+        if data_spec.data_def.schema is None and data_spec.data_def.schemaId is None:
+            data_spec.data_def.schema = data_item.trac_schema
 
         return data_spec
 

tracdap/rt/{_exec → _impl/exec}/graph.py

@@ -16,7 +16,7 @@
 import typing as tp
 import dataclasses as dc
 
-import tracdap.rt._impl.data as _data  # noqa
+import tracdap.rt._impl.core.data as _data
 import tracdap.rt.metadata as meta
 import tracdap.rt.config as cfg
 

tracdap/rt/{_exec → _impl/exec}/graph_builder.py

@@ -17,8 +17,8 @@ import datetime as _dt
 
 import tracdap.rt.config as config
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.data as _data  # noqa
-import tracdap.rt._impl.util as _util  # noqa
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.util as _util
 
 from .graph import *
 

tracdap/rt/{_exec → _impl/grpc}/server.py

@@ -19,10 +19,10 @@ import typing as tp
 
 import tracdap.rt.config as config
 import tracdap.rt.exceptions as ex
-import tracdap.rt._exec.actors as actors
-import tracdap.rt._impl.grpc.codec as codec  # noqa
-import tracdap.rt._impl.logging as logging  # noqa
-import tracdap.rt._impl.util as util  # noqa
+import tracdap.rt._impl.exec.actors as actors
+import tracdap.rt._impl.grpc.codec as codec
+import tracdap.rt._impl.core.logging as logging
+import tracdap.rt._impl.core.util as util
 
 # Check whether gRPC is installed before trying to load any of the generated modules
 try:

tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2.py

@@ -18,14 +18,14 @@ from tracdap.rt._impl.grpc.tracdap.metadata import object_pb2 as tracdap_dot_rt_
 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8tracdap/rt/_impl/grpc/tracdap/api/internal/runtime.proto\x12\x14tracdap.api.internal\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x1a\x1cgoogle/api/annotations.proto\"6\n\x16RuntimeListJobsRequest\x12\x12\n\x05limit\x18\x01 \x01(\rH\x00\x88\x01\x01\x42\x08\n\x06_limit\"O\n\x17RuntimeListJobsResponse\x12\x34\n\x04jobs\x18\x01 \x03(\x0b\x32&.tracdap.api.internal.RuntimeJobStatus\"f\n\x15RuntimeJobInfoRequest\x12\x34\n\x0bjobSelector\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x10\n\x06jobKey\x18\x02 \x01(\tH\x00\x42\x05\n\x03job\"\x9f\x01\n\x10RuntimeJobStatus\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x13\n\x0b\x65rrorDetail\x18\x04 \x01(\t\"\xa4\x02\n\x10RuntimeJobResult\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x44\n\x07results\x18\x04 \x03(\x0b\x32\x33.tracdap.api.internal.RuntimeJobResult.ResultsEntry\x1aR\n\x0cResultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".tracdap.metadata.ObjectDefinition:\x02\x38\x01\x32\x95\x03\n\x0eTracRuntimeApi\x12{\n\x08listJobs\x12,.tracdap.api.internal.RuntimeListJobsRequest\x1a-.tracdap.api.internal.RuntimeListJobsResponse\"\x12\x82\xd3\xe4\x93\x02\x0c\x12\n/list-jobs\x12\x81\x01\n\x0cgetJobStatus\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobStatus\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-status/{jobKey}\x12\x81\x01\n\x0cgetJobResult\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobResult\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-result/{jobKey}B\"\n\x1eorg.finos.tracdap.api.internalP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8tracdap/rt/_impl/grpc/tracdap/api/internal/runtime.proto\x12\x14tracdap.api.internal\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x1a\x1cgoogle/api/annotations.proto\"6\n\x16RuntimeListJobsRequest\x12\x12\n\x05limit\x18\x01 \x01(\rH\x00\x88\x01\x01\x42\x08\n\x06_limit\"O\n\x17RuntimeListJobsResponse\x12\x34\n\x04jobs\x18\x01 \x03(\x0b\x32&.tracdap.api.internal.RuntimeJobStatus\"f\n\x15RuntimeJobInfoRequest\x12\x34\n\x0bjobSelector\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x10\n\x06jobKey\x18\x02 \x01(\tH\x00\x42\x05\n\x03job\"\x9f\x01\n\x10RuntimeJobStatus\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x13\n\x0b\x65rrorDetail\x18\x04 \x01(\t\"\xa4\x02\n\x10RuntimeJobResult\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x44\n\x07results\x18\x04 \x03(\x0b\x32\x33.tracdap.api.internal.RuntimeJobResult.ResultsEntry\x1aR\n\x0cResultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".tracdap.metadata.ObjectDefinition:\x02\x38\x01\x32\x95\x03\n\x0eTracRuntimeApi\x12{\n\x08listJobs\x12,.tracdap.api.internal.RuntimeListJobsRequest\x1a-.tracdap.api.internal.RuntimeListJobsResponse\"\x12\x82\xd3\xe4\x93\x02\x0c\x12\n/list-jobs\x12\x81\x01\n\x0cgetJobStatus\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobStatus\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-status/{jobKey}\x12\x81\x01\n\x0cgetJobResult\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobResult\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-result/{jobKey}B7\n\x1eorg.finos.tracdap.api.internalB\x13RuntimeServiceProtoP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
 _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracdap.api.internal.runtime_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._options = None
-  _globals['DESCRIPTOR']._serialized_options = b'\n\036org.finos.tracdap.api.internalP\001'
+  _globals['DESCRIPTOR']._serialized_options = b'\n\036org.finos.tracdap.api.internalB\023RuntimeServiceProtoP\001'
   _globals['_RUNTIMEJOBRESULT_RESULTSENTRY']._options = None
   _globals['_RUNTIMEJOBRESULT_RESULTSENTRY']._serialized_options = b'8\001'
   _globals['_TRACRUNTIMEAPI'].methods_by_name['listJobs']._options = None

tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2_grpc.py

@@ -5,7 +5,7 @@ import warnings
 
 from tracdap.rt._impl.grpc.tracdap.api.internal import runtime_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_api_dot_internal_dot_runtime__pb2
 
-GRPC_GENERATED_VERSION = '1.66.1'
+GRPC_GENERATED_VERSION = '1.70.0'
 GRPC_VERSION = grpc.__version__
 _version_not_supported = False
 

tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.py

@@ -16,7 +16,7 @@ from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as tracdap_dot_rt_do
 from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__id__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\"\xe7\x01\n\x0b\x46ieldSchema\x12\x11\n\tfieldName\x18\x01 \x01(\t\x12\x12\n\nfieldOrder\x18\x02 \x01(\x11\x12.\n\tfieldType\x18\x03 \x01(\x0e\x32\x1b.tracdap.metadata.BasicType\x12\r\n\x05label\x18\x04 \x01(\t\x12\x13\n\x0b\x62usinessKey\x18\x05 \x01(\x08\x12\x13\n\x0b\x63\x61tegorical\x18\x06 \x01(\x08\x12\x14\n\x07notNull\x18\x08 \x01(\x08H\x00\x88\x01\x01\x12\x17\n\nformatCode\x18\x07 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_notNullB\r\n\x0b_formatCode\"<\n\x0bTableSchema\x12-\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.FieldSchema\"\xb3\x01\n\x10SchemaDefinition\x12\x30\n\nschemaType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.SchemaType\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12.\n\x05table\x18\x03 \x01(\x0b\x32\x1d.tracdap.metadata.TableSchemaH\x00\x42\x0f\n\rschemaDetails\"\x81\x02\n\x07PartKey\x12\x11\n\topaqueKey\x18\x01 \x01(\t\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12+\n\npartValues\x18\x03 \x03(\x0b\x32\x17.tracdap.metadata.Value\x12\x32\n\x0cpartRangeMin\x18\x04 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x00\x88\x01\x01\x12\x32\n\x0cpartRangeMax\x18\x05 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x01\x88\x01\x01\x42\x0f\n\r_partRangeMinB\x0f\n\r_partRangeMax\"\xba\x04\n\x0e\x44\x61taDefinition\x12\x31\n\x08schemaId\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x34\n\x06schema\x18\x02 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12:\n\x05parts\x18\x03 \x03(\x0b\x32+.tracdap.metadata.DataDefinition.PartsEntry\x12\x30\n\tstorageId\x18\x04 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x1a-\n\x05\x44\x65lta\x12\x12\n\ndeltaIndex\x18\x01 \x01(\r\x12\x10\n\x08\x64\x61taItem\x18\x02 \x01(\t\x1aQ\n\x04Snap\x12\x11\n\tsnapIndex\x18\x01 \x01(\r\x12\x36\n\x06\x64\x65ltas\x18\x02 \x03(\x0b\x32&.tracdap.metadata.DataDefinition.Delta\x1ag\n\x04Part\x12*\n\x07partKey\x18\x01 \x01(\x0b\x32\x19.tracdap.metadata.PartKey\x12\x33\n\x04snap\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Snap\x1aS\n\nPartsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x34\n\x05value\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Part:\x02\x38\x01\x42\x11\n\x0fschemaSpecifier*0\n\nSchemaType\x12\x17\n\x13SCHEMA_TYPE_NOT_SET\x10\x00\x12\t\n\x05TABLE\x10\x01*?\n\x08PartType\x12\r\n\tPART_ROOT\x10\x00\x12\x11\n\rPART_BY_RANGE\x10\x01\x12\x11\n\rPART_BY_VALUE\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\"\xe7\x01\n\x0b\x46ieldSchema\x12\x11\n\tfieldName\x18\x01 \x01(\t\x12\x12\n\nfieldOrder\x18\x02 \x01(\x11\x12.\n\tfieldType\x18\x03 \x01(\x0e\x32\x1b.tracdap.metadata.BasicType\x12\r\n\x05label\x18\x04 \x01(\t\x12\x13\n\x0b\x62usinessKey\x18\x05 \x01(\x08\x12\x13\n\x0b\x63\x61tegorical\x18\x06 \x01(\x08\x12\x14\n\x07notNull\x18\x08 \x01(\x08H\x00\x88\x01\x01\x12\x17\n\nformatCode\x18\x07 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_notNullB\r\n\x0b_formatCode\"<\n\x0bTableSchema\x12-\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.FieldSchema\"\x9c\x02\n\x0bStructField\x12\x33\n\tfieldType\x18\x03 \x01(\x0b\x32 .tracdap.metadata.TypeDescriptor\x12\r\n\x05label\x18\x04 \x01(\t\x12\x13\n\x0b\x62usinessKey\x18\x05 \x01(\x08\x12\x13\n\x0b\x63\x61tegorical\x18\x06 \x01(\x08\x12\x14\n\x07notNull\x18\x08 \x01(\x08H\x00\x88\x01\x01\x12\x17\n\nformatCode\x18\x07 \x01(\tH\x01\x88\x01\x01\x12-\n\x0c\x64\x65\x66\x61ultValue\x18\t \x01(\x0b\x32\x17.tracdap.metadata.Value\x12\x17\n\nstructType\x18\n \x01(\tH\x02\x88\x01\x01\x42\n\n\x08_notNullB\r\n\x0b_formatCodeB\r\n\x0b_structType\"\xaf\x02\n\x0cStructSchema\x12:\n\x06\x66ields\x18\x01 \x03(\x0b\x32*.tracdap.metadata.StructSchema.FieldsEntry\x12\x42\n\nnamedTypes\x18\x02 \x03(\x0b\x32..tracdap.metadata.StructSchema.NamedTypesEntry\x1aL\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.StructField:\x02\x38\x01\x1aQ\n\x0fNamedTypesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.tracdap.metadata.StructSchema:\x02\x38\x01\"\xe5\x01\n\x10SchemaDefinition\x12\x30\n\nschemaType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.SchemaType\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12.\n\x05table\x18\x03 \x01(\x0b\x32\x1d.tracdap.metadata.TableSchemaH\x00\x12\x30\n\x06struct\x18\x04 \x01(\x0b\x32\x1e.tracdap.metadata.StructSchemaH\x00\x42\x0f\n\rschemaDetails\"\x81\x02\n\x07PartKey\x12\x11\n\topaqueKey\x18\x01 \x01(\t\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12+\n\npartValues\x18\x03 \x03(\x0b\x32\x17.tracdap.metadata.Value\x12\x32\n\x0cpartRangeMin\x18\x04 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x00\x88\x01\x01\x12\x32\n\x0cpartRangeMax\x18\x05 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x01\x88\x01\x01\x42\x0f\n\r_partRangeMinB\x0f\n\r_partRangeMax\"\xba\x04\n\x0e\x44\x61taDefinition\x12\x31\n\x08schemaId\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x34\n\x06schema\x18\x02 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12:\n\x05parts\x18\x03 \x03(\x0b\x32+.tracdap.metadata.DataDefinition.PartsEntry\x12\x30\n\tstorageId\x18\x04 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x1a-\n\x05\x44\x65lta\x12\x12\n\ndeltaIndex\x18\x01 \x01(\r\x12\x10\n\x08\x64\x61taItem\x18\x02 \x01(\t\x1aQ\n\x04Snap\x12\x11\n\tsnapIndex\x18\x01 \x01(\r\x12\x36\n\x06\x64\x65ltas\x18\x02 \x03(\x0b\x32&.tracdap.metadata.DataDefinition.Delta\x1ag\n\x04Part\x12*\n\x07partKey\x18\x01 \x01(\x0b\x32\x19.tracdap.metadata.PartKey\x12\x33\n\x04snap\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Snap\x1aS\n\nPartsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x34\n\x05value\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Part:\x02\x38\x01\x42\x11\n\x0fschemaSpecifier*<\n\nSchemaType\x12\x17\n\x13SCHEMA_TYPE_NOT_SET\x10\x00\x12\t\n\x05TABLE\x10\x01\x12\n\n\x06STRUCT\x10\x02*?\n\x08PartType\x12\r\n\tPART_ROOT\x10\x00\x12\x11\n\rPART_BY_RANGE\x10\x01\x12\x11\n\rPART_BY_VALUE\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -24,28 +24,40 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracd
 if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._options = None
   _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
+  _globals['_STRUCTSCHEMA_FIELDSENTRY']._options = None
+  _globals['_STRUCTSCHEMA_FIELDSENTRY']._serialized_options = b'8\001'
+  _globals['_STRUCTSCHEMA_NAMEDTYPESENTRY']._options = None
+  _globals['_STRUCTSCHEMA_NAMEDTYPESENTRY']._serialized_options = b'8\001'
   _globals['_DATADEFINITION_PARTSENTRY']._options = None
   _globals['_DATADEFINITION_PARTSENTRY']._serialized_options = b'8\001'
-  _globals['_SCHEMATYPE']._serialized_start=1489
-  _globals['_SCHEMATYPE']._serialized_end=1537
-  _globals['_PARTTYPE']._serialized_start=1539
-  _globals['_PARTTYPE']._serialized_end=1602
+  _globals['_SCHEMATYPE']._serialized_start=2132
+  _globals['_SCHEMATYPE']._serialized_end=2192
+  _globals['_PARTTYPE']._serialized_start=2194
+  _globals['_PARTTYPE']._serialized_end=2257
   _globals['_FIELDSCHEMA']._serialized_start=179
   _globals['_FIELDSCHEMA']._serialized_end=410
   _globals['_TABLESCHEMA']._serialized_start=412
   _globals['_TABLESCHEMA']._serialized_end=472
-  _globals['_SCHEMADEFINITION']._serialized_start=475
-  _globals['_SCHEMADEFINITION']._serialized_end=654
-  _globals['_PARTKEY']._serialized_start=657
-  _globals['_PARTKEY']._serialized_end=914
-  _globals['_DATADEFINITION']._serialized_start=917
-  _globals['_DATADEFINITION']._serialized_end=1487
-  _globals['_DATADEFINITION_DELTA']._serialized_start=1150
-  _globals['_DATADEFINITION_DELTA']._serialized_end=1195
-  _globals['_DATADEFINITION_SNAP']._serialized_start=1197
-  _globals['_DATADEFINITION_SNAP']._serialized_end=1278
-  _globals['_DATADEFINITION_PART']._serialized_start=1280
-  _globals['_DATADEFINITION_PART']._serialized_end=1383
-  _globals['_DATADEFINITION_PARTSENTRY']._serialized_start=1385
-  _globals['_DATADEFINITION_PARTSENTRY']._serialized_end=1468
+  _globals['_STRUCTFIELD']._serialized_start=475
+  _globals['_STRUCTFIELD']._serialized_end=759
+  _globals['_STRUCTSCHEMA']._serialized_start=762
+  _globals['_STRUCTSCHEMA']._serialized_end=1065
+  _globals['_STRUCTSCHEMA_FIELDSENTRY']._serialized_start=906
+  _globals['_STRUCTSCHEMA_FIELDSENTRY']._serialized_end=982
+  _globals['_STRUCTSCHEMA_NAMEDTYPESENTRY']._serialized_start=984
+  _globals['_STRUCTSCHEMA_NAMEDTYPESENTRY']._serialized_end=1065
+  _globals['_SCHEMADEFINITION']._serialized_start=1068
+  _globals['_SCHEMADEFINITION']._serialized_end=1297
+  _globals['_PARTKEY']._serialized_start=1300
+  _globals['_PARTKEY']._serialized_end=1557
+  _globals['_DATADEFINITION']._serialized_start=1560
+  _globals['_DATADEFINITION']._serialized_end=2130
+  _globals['_DATADEFINITION_DELTA']._serialized_start=1793
+  _globals['_DATADEFINITION_DELTA']._serialized_end=1838
+  _globals['_DATADEFINITION_SNAP']._serialized_start=1840
+  _globals['_DATADEFINITION_SNAP']._serialized_end=1921
+  _globals['_DATADEFINITION_PART']._serialized_start=1923
+  _globals['_DATADEFINITION_PART']._serialized_end=2026
+  _globals['_DATADEFINITION_PARTSENTRY']._serialized_start=2028
+  _globals['_DATADEFINITION_PARTSENTRY']._serialized_end=2111
 # @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.pyi

@@ -12,6 +12,7 @@ class SchemaType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
     SCHEMA_TYPE_NOT_SET: _ClassVar[SchemaType]
     TABLE: _ClassVar[SchemaType]
+    STRUCT: _ClassVar[SchemaType]
 
 class PartType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
@@ -20,6 +21,7 @@ class PartType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     PART_BY_VALUE: _ClassVar[PartType]
 SCHEMA_TYPE_NOT_SET: SchemaType
 TABLE: SchemaType
+STRUCT: SchemaType
 PART_ROOT: PartType
 PART_BY_RANGE: PartType
 PART_BY_VALUE: PartType
@@ -50,15 +52,59 @@ class TableSchema(_message.Message):
     fields: _containers.RepeatedCompositeFieldContainer[FieldSchema]
     def __init__(self, fields: _Optional[_Iterable[_Union[FieldSchema, _Mapping]]] = ...) -> None: ...
 
+class StructField(_message.Message):
+    __slots__ = ("fieldType", "label", "businessKey", "categorical", "notNull", "formatCode", "defaultValue", "structType")
+    FIELDTYPE_FIELD_NUMBER: _ClassVar[int]
+    LABEL_FIELD_NUMBER: _ClassVar[int]
+    BUSINESSKEY_FIELD_NUMBER: _ClassVar[int]
+    CATEGORICAL_FIELD_NUMBER: _ClassVar[int]
+    NOTNULL_FIELD_NUMBER: _ClassVar[int]
+    FORMATCODE_FIELD_NUMBER: _ClassVar[int]
+    DEFAULTVALUE_FIELD_NUMBER: _ClassVar[int]
+    STRUCTTYPE_FIELD_NUMBER: _ClassVar[int]
+    fieldType: _type_pb2.TypeDescriptor
+    label: str
+    businessKey: bool
+    categorical: bool
+    notNull: bool
+    formatCode: str
+    defaultValue: _type_pb2.Value
+    structType: str
+    def __init__(self, fieldType: _Optional[_Union[_type_pb2.TypeDescriptor, _Mapping]] = ..., label: _Optional[str] = ..., businessKey: bool = ..., categorical: bool = ..., notNull: bool = ..., formatCode: _Optional[str] = ..., defaultValue: _Optional[_Union[_type_pb2.Value, _Mapping]] = ..., structType: _Optional[str] = ...) -> None: ...
+
+class StructSchema(_message.Message):
+    __slots__ = ("fields", "namedTypes")
+    class FieldsEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: StructField
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[StructField, _Mapping]] = ...) -> None: ...
+    class NamedTypesEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: StructSchema
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[StructSchema, _Mapping]] = ...) -> None: ...
+    FIELDS_FIELD_NUMBER: _ClassVar[int]
+    NAMEDTYPES_FIELD_NUMBER: _ClassVar[int]
+    fields: _containers.MessageMap[str, StructField]
+    namedTypes: _containers.MessageMap[str, StructSchema]
+    def __init__(self, fields: _Optional[_Mapping[str, StructField]] = ..., namedTypes: _Optional[_Mapping[str, StructSchema]] = ...) -> None: ...
+
 class SchemaDefinition(_message.Message):
-    __slots__ = ("schemaType", "partType", "table")
+    __slots__ = ("schemaType", "partType", "table", "struct")
     SCHEMATYPE_FIELD_NUMBER: _ClassVar[int]
     PARTTYPE_FIELD_NUMBER: _ClassVar[int]
     TABLE_FIELD_NUMBER: _ClassVar[int]
+    STRUCT_FIELD_NUMBER: _ClassVar[int]
     schemaType: SchemaType
     partType: PartType
     table: TableSchema
-    def __init__(self, schemaType: _Optional[_Union[SchemaType, str]] = ..., partType: _Optional[_Union[PartType, str]] = ..., table: _Optional[_Union[TableSchema, _Mapping]] = ...) -> None: ...
+    struct: StructSchema
+    def __init__(self, schemaType: _Optional[_Union[SchemaType, str]] = ..., partType: _Optional[_Union[PartType, str]] = ..., table: _Optional[_Union[TableSchema, _Mapping]] = ..., struct: _Optional[_Union[StructSchema, _Mapping]] = ...) -> None: ...
 
 class PartKey(_message.Message):
     __slots__ = ("opaqueKey", "partType", "partValues", "partRangeMin", "partRangeMax")

tracdap/rt/{_exec → _impl}/runtime.py

@@ -29,16 +29,16 @@ import tracdap.rt.api as _api
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as _plugins
-import tracdap.rt._exec.actors as _actors
-import tracdap.rt._exec.engine as _engine
-import tracdap.rt._exec.dev_mode as _dev_mode
-import tracdap.rt._impl.config_parser as _cparse  # noqa
-import tracdap.rt._impl.guard_rails as _guard  # noqa
-import tracdap.rt._impl.logging as _log  # noqa
-import tracdap.rt._impl.models as _models  # noqa
-import tracdap.rt._impl.storage as _storage  # noqa
-import tracdap.rt._impl.static_api as _static_api  # noqa
-import tracdap.rt._impl.util as _util  # noqa
+import tracdap.rt._impl.core.config_parser as _cparse
+import tracdap.rt._impl.core.guard_rails as _guard
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.models as _models
+import tracdap.rt._impl.core.storage as _storage
+import tracdap.rt._impl.core.util as _util
+import tracdap.rt._impl.exec.actors as _actors
+import tracdap.rt._impl.exec.engine as _engine
+import tracdap.rt._impl.exec.dev_mode as _dev_mode
+import tracdap.rt._impl.static_api as _static_api
 import tracdap.rt._version as _version
 
 
@@ -84,8 +84,8 @@ class TracRuntime:
         if isinstance(scratch_dir, str):
             scratch_dir = pathlib.Path(scratch_dir)
 
-        _log.configure_logging()
-        self._log = _log.logger_for_object(self)
+        _logging.configure_logging()
+        self._log = _logging.logger_for_object(self)
         self._log.info(f"TRAC D.A.P. Python Runtime {trac_version}")
 
         self._sys_config = sys_config if isinstance(sys_config, _cfg.RuntimeConfig) else None
@@ -229,7 +229,7 @@ class TracRuntime:
                 self._log.info("Starting the runtime API server...")
 
                 # The server module pulls in all the gRPC dependencies, don't import it unless we have to
-                import tracdap.rt._exec.server as _server
+                import tracdap.rt._impl.grpc.server as _server
 
                 self._server = _server.RuntimeApiServer(self._system, self._server_port)
                 self._server.start()

tracdap/rt/_impl/static_api.py

@@ -19,10 +19,11 @@ import types as _ts
 import tracdap.rt.api.experimental as _api
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.data as _data
-import tracdap.rt._impl.schemas as _schemas
-import tracdap.rt._impl.type_system as _type_system
-import tracdap.rt._impl.validation as _val
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.schemas as _schemas
+import tracdap.rt._impl.core.struct as _struct
+import tracdap.rt._impl.core.type_system as _type_system
+import tracdap.rt._impl.core.validation as _val
 
 # Import hook interfaces into this module namespace
 from tracdap.rt.api.hook import _StaticApiHook  # noqa
@@ -55,7 +56,7 @@ class StaticApiImpl(_StaticApiHook):
         if not _val.is_primitive_type(entry_type):
             raise _ex.EModelValidation(f"Maps can only contain primitive types, [{entry_type}] is not primitive")
 
-        return _meta.TypeDescriptor(_meta.BasicType.MAP, arrayType=_meta.TypeDescriptor(entry_type))
+        return _meta.TypeDescriptor(_meta.BasicType.MAP, mapType=_meta.TypeDescriptor(entry_type))
 
     def define_attribute(
             self, attr_name: str, attr_value: _tp.Any,
@@ -150,6 +151,13 @@ class StaticApiImpl(_StaticApiHook):
             notNull=not_null,
             formatCode=format_code)
 
+    def define_struct(self, python_type: type[_api.STRUCT_TYPE]):
+
+        _val.validate_signature(self.define_struct, python_type)
+
+        struct_schema = _struct.StructProcessor.define_struct(python_type)
+        return _meta.SchemaDefinition(schemaType=_meta.SchemaType.STRUCT, struct=struct_schema)
+
     def define_schema(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
             schema_type: _meta.SchemaType = _meta.SchemaType.TABLE, dynamic: bool = False) \

tracdap/rt/_plugins/format_csv.py

@@ -34,7 +34,7 @@ from tracdap.rt.ext.storage import IDataFormat
 from . import _helpers
 
 # TODO: Remove dependency on internal code
-import tracdap.rt._impl.data as _data
+import tracdap.rt._impl.core.data as _data
 
 
 class CsvStorageFormat(IDataFormat):

tracdap/rt/_plugins/storage_sql.py

@@ -30,7 +30,7 @@ from tracdap.rt._impl.ext.sql import *  # noqa
 import tracdap.rt._plugins._helpers as _helpers
 
 # TODO: Remove internal references
-import tracdap.rt._impl.data as _data
+import tracdap.rt._impl.core.data as _data
 
 
 class SqlDataStorage(IDataStorageBase[pa.Table, pa.Schema]):
@@ -268,10 +268,18 @@ plugins.PluginManager.register_plugin(IStorageProvider, SqlStorageProvider, ["SQ
 
 
 try:
-
     import sqlalchemy as sqla  # noqa
     import sqlalchemy.exc as sqla_exc  # noqa
 
+    # Only 2.x versions of SQL Alchemy are currently supported
+    sqla_supported = sqla.__version__.startswith("2.")
+
+except ModuleNotFoundError:
+    sqla = None
+    sqla_supported = False
+
+if sqla_supported:
+
     class SqlAlchemyDriver(ISqlDriver):
 
         def __init__(self, properties: tp.Dict[str, str]):
@@ -336,7 +344,7 @@ try:
 
         class ConnectionWrapper(DbApiWrapper.Connection):
 
-            def __init__(self, conn: sqla.Connection):
+            def __init__(self, conn: "sqla.Connection"):
                 self.__conn = conn
 
             def close(self):
@@ -355,7 +363,7 @@ try:
 
             arraysize: int = 1000
 
-            def __init__(self, conn: sqla.Connection):
+            def __init__(self, conn: "sqla.Connection"):
                 self.__conn = conn
                 self.__result: tp.Optional[sqla.CursorResult] = None
 
@@ -414,5 +422,4 @@ try:
 
     plugins.PluginManager.register_plugin(ISqlDriver, SqlAlchemyDriver, ["alchemy"])
 
-except ModuleNotFoundError:
-    pass
+

tracdap/rt/_version.py

@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.8.0b2"
+__version__ = "0.8.0b4"