tracdap-runtime 0.8.0b3__py3-none-any.whl → 0.8.0rc1__py3-none-any.whl

tracdap/rt/_impl/exec/functions.py

@@ -16,6 +16,7 @@
 import copy
 import datetime
 import abc
+import io
 import pathlib
 import random
 import dataclasses as dc  # noqa
@@ -29,6 +30,7 @@ import tracdap.rt._impl.core.type_system as _types
 import tracdap.rt._impl.core.data as _data
 import tracdap.rt._impl.core.logging as _logging
 import tracdap.rt._impl.core.storage as _storage
+import tracdap.rt._impl.core.struct as _struct
 import tracdap.rt._impl.core.models as _models
 import tracdap.rt._impl.core.util as _util
 
@@ -282,6 +284,13 @@ class DataViewFunc(NodeFunction[_data.DataView]):
         if root_item.object_type == meta.ObjectType.FILE:
             return _data.DataView.for_file_item(root_item)
 
+        # TODO: Generalize processing across DataView / DataItem types
+
+        if root_item.schema_type == meta.SchemaType.STRUCT:
+            view = _data.DataView.for_trac_schema(self.node.schema)
+            view.parts[root_part_key] = [root_item]
+            return view
+
         # Everything else is a regular data view
         if self.node.schema is not None and len(self.node.schema.table.fields) > 0:
             trac_schema = self.node.schema
@@ -488,40 +497,55 @@ class LoadDataFunc( _LoadSaveDataFunc, NodeFunction[_data.DataItem],):
         data_spec = self._choose_data_spec(self.node.spec_id, self.node.spec, ctx)
         data_copy = self._choose_copy(data_spec.data_item, data_spec.storage_def)
 
-        if data_spec.object_type == _api.ObjectType.DATA:
-            return self._load_data(data_spec, data_copy)
-
-        elif data_spec.object_type == _api.ObjectType.FILE:
+        if data_spec.object_type == _api.ObjectType.FILE:
             return self._load_file(data_copy)
 
+        elif data_spec.schema_type == _api.SchemaType.TABLE:
+            return self._load_table(data_spec, data_copy)
+
+        elif data_spec.schema_type == _api.SchemaType.STRUCT:
+            return self._load_struct(data_copy)
+
+        # TODO: Handle dynamic inputs, they should work for any schema type
+        elif data_spec.schema_type == _api.SchemaType.SCHEMA_TYPE_NOT_SET:
+            return self._load_table(data_spec, data_copy)
+
         else:
             raise _ex.EUnexpected()
 
-    def _load_data(self, data_spec, data_copy):
+    def _load_file(self, data_copy):
+
+        storage = self.storage.get_file_storage(data_copy.storageKey)
+        content = storage.read_bytes(data_copy.storagePath)
+
+        return _data.DataItem.for_file_content(content)
+
+    def _load_table(self, data_spec, data_copy):
 
         trac_schema = data_spec.schema_def if data_spec.schema_def else data_spec.data_def.schema
         arrow_schema = _data.DataMapping.trac_to_arrow_schema(trac_schema) if trac_schema else None
 
-        # Decode options (metadata values) from the storage definition
-        options = dict()
-        for opt_key, opt_value in data_spec.storage_def.storageOptions.items():
-            options[opt_key] = _types.MetadataCodec.decode_value(opt_value)
+        storage_options = dict(
+            (opt_key, _types.MetadataCodec.decode_value(opt_value))
+            for opt_key, opt_value in data_spec.storage_def.storageOptions.items())
 
         storage = self.storage.get_data_storage(data_copy.storageKey)
+
         table = storage.read_table(
-            data_copy.storagePath,
-            data_copy.storageFormat,
-            arrow_schema,
-            storage_options=options)
+            data_copy.storagePath, data_copy.storageFormat, arrow_schema,
+            storage_options=storage_options)
 
-        return _data.DataItem(_api.ObjectType.DATA, table.schema, table)
+        return _data.DataItem.for_table(table, table.schema, trac_schema)
 
-    def _load_file(self, data_copy):
+    def _load_struct(self, data_copy):
 
         storage = self.storage.get_file_storage(data_copy.storageKey)
-        raw_bytes = storage.read_bytes(data_copy.storagePath)
 
-        return _data.DataItem(_api.ObjectType.FILE, raw_bytes=raw_bytes)
+        with storage.read_byte_stream(data_copy.storagePath) as stream:
+            with io.TextIOWrapper(stream, "utf-8") as text_stream:
+                struct = _struct.StructProcessor.load_struct(text_stream, data_copy.storageFormat)
+
+        return _data.DataItem.for_struct(struct)
 
 
 class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[_data.DataSpec]):
@@ -541,24 +565,40 @@ class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[_data.DataSpec]):
 
         # Do not save empty outputs (optional outputs that were not produced)
         if data_item.is_empty():
-            return _data.DataSpec.create_empty_spec(data_item.object_type)
+            return _data.DataSpec.create_empty_spec(data_item.object_type, data_item.schema_type)
 
-        if data_item.object_type == _api.ObjectType.DATA:
-            return self._save_data(data_item, data_spec, data_copy)
-
-        elif data_item.object_type == _api.ObjectType.FILE:
+        if data_item.object_type == _api.ObjectType.FILE:
             return self._save_file(data_item, data_spec, data_copy)
 
+        elif data_item.schema_type == _api.SchemaType.TABLE:
+            return self._save_table(data_item, data_spec, data_copy)
+
+        elif data_item.schema_type == _api.SchemaType.STRUCT:
+            return self._save_struct(data_item, data_spec, data_copy)
+
         else:
             raise _ex.EUnexpected()
 
-    def _save_data(self, data_item, data_spec, data_copy):
+    def _save_file(self, data_item, data_spec, data_copy):
+
+        if data_item.content is None:
+            raise _ex.EUnexpected()
+
+        storage = self.storage.get_file_storage(data_copy.storageKey)
+        storage.write_bytes(data_copy.storagePath, data_item.content)
+
+        data_spec = copy.deepcopy(data_spec)
+        data_spec.file_def.size = len(data_item.content)
+
+        return data_spec
+
+    def _save_table(self, data_item, data_spec, data_copy):
 
         # Current implementation will always put an Arrow table in the data item
         # Empty tables are allowed, so explicitly check if table is None
         # Testing "if not data_item.table" will fail for empty tables
 
-        if data_item.table is None:
+        if data_item.content is None:
             raise _ex.EUnexpected()
 
         # Decode options (metadata values) from the storage definition
@@ -569,7 +609,7 @@ class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[_data.DataSpec]):
         storage = self.storage.get_data_storage(data_copy.storageKey)
         storage.write_table(
             data_copy.storagePath, data_copy.storageFormat,
-            data_item.table,
+            data_item.content,
             storage_options=options, overwrite=False)
 
         data_spec = copy.deepcopy(data_spec)
@@ -580,16 +620,26 @@ class SaveDataFunc(_LoadSaveDataFunc, NodeFunction[_data.DataSpec]):
 
         return data_spec
 
-    def _save_file(self, data_item, data_spec, data_copy):
+    def _save_struct(self, data_item, data_spec, data_copy):
 
-        if data_item.raw_bytes is None:
+        if data_item.content is None:
             raise _ex.EUnexpected()
 
+        struct_data = data_item.content
+        storage_format = data_copy.storageFormat
+
         storage = self.storage.get_file_storage(data_copy.storageKey)
-        storage.write_bytes(data_copy.storagePath, data_item.raw_bytes)
+
+        # Using the text wrapper closes the stream early, which is inefficient in the data layer
+        # Supporting text IO directly from the storage API would allow working with text streams more naturally
+        with storage.write_byte_stream(data_copy.storagePath) as stream:
+            with io.TextIOWrapper(stream, "utf-8") as text_stream:
+                _struct.StructProcessor.save_struct(struct_data, text_stream, storage_format)
 
         data_spec = copy.deepcopy(data_spec)
-        data_spec.file_def.size = len(data_item.raw_bytes)
+
+        if data_spec.data_def.schema is None and data_spec.data_def.schemaId is None:
+            data_spec.data_def.schema = data_item.trac_schema
 
         return data_spec
 

tracdap/rt/_impl/grpc/codec.py

@@ -24,7 +24,7 @@ import tracdap.rt._impl.grpc.tracdap.metadata.object_id_pb2 as object_id_pb2
 import tracdap.rt._impl.grpc.tracdap.metadata.object_pb2 as object_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import model_pb2
 import tracdap.rt._impl.grpc.tracdap.metadata.data_pb2 as data_pb2
-import tracdap.rt._impl.grpc.tracdap.metadata.stoarge_pb2 as storage_pb2
+import tracdap.rt._impl.grpc.tracdap.metadata.storage_pb2 as storage_pb2
 
 from google.protobuf import message as _message
 

tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2.py

@@ -18,14 +18,14 @@ from tracdap.rt._impl.grpc.tracdap.metadata import object_pb2 as tracdap_dot_rt_
 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8tracdap/rt/_impl/grpc/tracdap/api/internal/runtime.proto\x12\x14tracdap.api.internal\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x1a\x1cgoogle/api/annotations.proto\"6\n\x16RuntimeListJobsRequest\x12\x12\n\x05limit\x18\x01 \x01(\rH\x00\x88\x01\x01\x42\x08\n\x06_limit\"O\n\x17RuntimeListJobsResponse\x12\x34\n\x04jobs\x18\x01 \x03(\x0b\x32&.tracdap.api.internal.RuntimeJobStatus\"f\n\x15RuntimeJobInfoRequest\x12\x34\n\x0bjobSelector\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x10\n\x06jobKey\x18\x02 \x01(\tH\x00\x42\x05\n\x03job\"\x9f\x01\n\x10RuntimeJobStatus\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x13\n\x0b\x65rrorDetail\x18\x04 \x01(\t\"\xa4\x02\n\x10RuntimeJobResult\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x44\n\x07results\x18\x04 \x03(\x0b\x32\x33.tracdap.api.internal.RuntimeJobResult.ResultsEntry\x1aR\n\x0cResultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".tracdap.metadata.ObjectDefinition:\x02\x38\x01\x32\x95\x03\n\x0eTracRuntimeApi\x12{\n\x08listJobs\x12,.tracdap.api.internal.RuntimeListJobsRequest\x1a-.tracdap.api.internal.RuntimeListJobsResponse\"\x12\x82\xd3\xe4\x93\x02\x0c\x12\n/list-jobs\x12\x81\x01\n\x0cgetJobStatus\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobStatus\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-status/{jobKey}\x12\x81\x01\n\x0cgetJobResult\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobResult\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-result/{jobKey}B\"\n\x1eorg.finos.tracdap.api.internalP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8tracdap/rt/_impl/grpc/tracdap/api/internal/runtime.proto\x12\x14tracdap.api.internal\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x1a\x1cgoogle/api/annotations.proto\"6\n\x16RuntimeListJobsRequest\x12\x12\n\x05limit\x18\x01 \x01(\rH\x00\x88\x01\x01\x42\x08\n\x06_limit\"O\n\x17RuntimeListJobsResponse\x12\x34\n\x04jobs\x18\x01 \x03(\x0b\x32&.tracdap.api.internal.RuntimeJobStatus\"f\n\x15RuntimeJobInfoRequest\x12\x34\n\x0bjobSelector\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x10\n\x06jobKey\x18\x02 \x01(\tH\x00\x42\x05\n\x03job\"\x9f\x01\n\x10RuntimeJobStatus\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x13\n\x0b\x65rrorDetail\x18\x04 \x01(\t\"\xa4\x02\n\x10RuntimeJobResult\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12\x33\n\nstatusCode\x18\x02 \x01(\x0e\x32\x1f.tracdap.metadata.JobStatusCode\x12\x15\n\rstatusMessage\x18\x03 \x01(\t\x12\x44\n\x07results\x18\x04 \x03(\x0b\x32\x33.tracdap.api.internal.RuntimeJobResult.ResultsEntry\x1aR\n\x0cResultsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".tracdap.metadata.ObjectDefinition:\x02\x38\x01\x32\x95\x03\n\x0eTracRuntimeApi\x12{\n\x08listJobs\x12,.tracdap.api.internal.RuntimeListJobsRequest\x1a-.tracdap.api.internal.RuntimeListJobsResponse\"\x12\x82\xd3\xe4\x93\x02\x0c\x12\n/list-jobs\x12\x81\x01\n\x0cgetJobStatus\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobStatus\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-status/{jobKey}\x12\x81\x01\n\x0cgetJobResult\x12+.tracdap.api.internal.RuntimeJobInfoRequest\x1a&.tracdap.api.internal.RuntimeJobResult\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/job-result/{jobKey}B7\n\x1eorg.finos.tracdap.api.internalB\x13RuntimeServiceProtoP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
 _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracdap.api.internal.runtime_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._options = None
-  _globals['DESCRIPTOR']._serialized_options = b'\n\036org.finos.tracdap.api.internalP\001'
+  _globals['DESCRIPTOR']._serialized_options = b'\n\036org.finos.tracdap.api.internalB\023RuntimeServiceProtoP\001'
   _globals['_RUNTIMEJOBRESULT_RESULTSENTRY']._options = None
   _globals['_RUNTIMEJOBRESULT_RESULTSENTRY']._serialized_options = b'8\001'
   _globals['_TRACRUNTIMEAPI'].methods_by_name['listJobs']._options = None

tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2_grpc.py

@@ -5,7 +5,7 @@ import warnings
 
 from tracdap.rt._impl.grpc.tracdap.api.internal import runtime_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_api_dot_internal_dot_runtime__pb2
 
-GRPC_GENERATED_VERSION = '1.66.1'
+GRPC_GENERATED_VERSION = '1.70.0'
 GRPC_VERSION = grpc.__version__
 _version_not_supported = False
 

tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.py

@@ -16,7 +16,7 @@ from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as tracdap_dot_rt_do
 from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__id__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\"\xe7\x01\n\x0b\x46ieldSchema\x12\x11\n\tfieldName\x18\x01 \x01(\t\x12\x12\n\nfieldOrder\x18\x02 \x01(\x11\x12.\n\tfieldType\x18\x03 \x01(\x0e\x32\x1b.tracdap.metadata.BasicType\x12\r\n\x05label\x18\x04 \x01(\t\x12\x13\n\x0b\x62usinessKey\x18\x05 \x01(\x08\x12\x13\n\x0b\x63\x61tegorical\x18\x06 \x01(\x08\x12\x14\n\x07notNull\x18\x08 \x01(\x08H\x00\x88\x01\x01\x12\x17\n\nformatCode\x18\x07 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_notNullB\r\n\x0b_formatCode\"<\n\x0bTableSchema\x12-\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.FieldSchema\"\xb3\x01\n\x10SchemaDefinition\x12\x30\n\nschemaType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.SchemaType\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12.\n\x05table\x18\x03 \x01(\x0b\x32\x1d.tracdap.metadata.TableSchemaH\x00\x42\x0f\n\rschemaDetails\"\x81\x02\n\x07PartKey\x12\x11\n\topaqueKey\x18\x01 \x01(\t\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12+\n\npartValues\x18\x03 \x03(\x0b\x32\x17.tracdap.metadata.Value\x12\x32\n\x0cpartRangeMin\x18\x04 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x00\x88\x01\x01\x12\x32\n\x0cpartRangeMax\x18\x05 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x01\x88\x01\x01\x42\x0f\n\r_partRangeMinB\x0f\n\r_partRangeMax\"\xba\x04\n\x0e\x44\x61taDefinition\x12\x31\n\x08schemaId\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x34\n\x06schema\x18\x02 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12:\n\x05parts\x18\x03 \x03(\x0b\x32+.tracdap.metadata.DataDefinition.PartsEntry\x12\x30\n\tstorageId\x18\x04 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x1a-\n\x05\x44\x65lta\x12\x12\n\ndeltaIndex\x18\x01 \x01(\r\x12\x10\n\x08\x64\x61taItem\x18\x02 \x01(\t\x1aQ\n\x04Snap\x12\x11\n\tsnapIndex\x18\x01 \x01(\r\x12\x36\n\x06\x64\x65ltas\x18\x02 \x03(\x0b\x32&.tracdap.metadata.DataDefinition.Delta\x1ag\n\x04Part\x12*\n\x07partKey\x18\x01 \x01(\x0b\x32\x19.tracdap.metadata.PartKey\x12\x33\n\x04snap\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Snap\x1aS\n\nPartsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x34\n\x05value\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Part:\x02\x38\x01\x42\x11\n\x0fschemaSpecifier*0\n\nSchemaType\x12\x17\n\x13SCHEMA_TYPE_NOT_SET\x10\x00\x12\t\n\x05TABLE\x10\x01*?\n\x08PartType\x12\r\n\tPART_ROOT\x10\x00\x12\x11\n\rPART_BY_RANGE\x10\x01\x12\x11\n\rPART_BY_VALUE\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\"\xe7\x01\n\x0b\x46ieldSchema\x12\x11\n\tfieldName\x18\x01 \x01(\t\x12\x12\n\nfieldOrder\x18\x02 \x01(\x11\x12.\n\tfieldType\x18\x03 \x01(\x0e\x32\x1b.tracdap.metadata.BasicType\x12\r\n\x05label\x18\x04 \x01(\t\x12\x13\n\x0b\x62usinessKey\x18\x05 \x01(\x08\x12\x13\n\x0b\x63\x61tegorical\x18\x06 \x01(\x08\x12\x14\n\x07notNull\x18\x08 \x01(\x08H\x00\x88\x01\x01\x12\x17\n\nformatCode\x18\x07 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_notNullB\r\n\x0b_formatCode\"<\n\x0bTableSchema\x12-\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.FieldSchema\"\x9c\x02\n\x0bStructField\x12\x33\n\tfieldType\x18\x03 \x01(\x0b\x32 .tracdap.metadata.TypeDescriptor\x12\r\n\x05label\x18\x04 \x01(\t\x12\x13\n\x0b\x62usinessKey\x18\x05 \x01(\x08\x12\x13\n\x0b\x63\x61tegorical\x18\x06 \x01(\x08\x12\x14\n\x07notNull\x18\x08 \x01(\x08H\x00\x88\x01\x01\x12\x17\n\nformatCode\x18\x07 \x01(\tH\x01\x88\x01\x01\x12-\n\x0c\x64\x65\x66\x61ultValue\x18\t \x01(\x0b\x32\x17.tracdap.metadata.Value\x12\x17\n\nstructType\x18\n \x01(\tH\x02\x88\x01\x01\x42\n\n\x08_notNullB\r\n\x0b_formatCodeB\r\n\x0b_structType\"\xaf\x02\n\x0cStructSchema\x12:\n\x06\x66ields\x18\x01 \x03(\x0b\x32*.tracdap.metadata.StructSchema.FieldsEntry\x12\x42\n\nnamedTypes\x18\x02 \x03(\x0b\x32..tracdap.metadata.StructSchema.NamedTypesEntry\x1aL\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.StructField:\x02\x38\x01\x1aQ\n\x0fNamedTypesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.tracdap.metadata.StructSchema:\x02\x38\x01\"\xe5\x01\n\x10SchemaDefinition\x12\x30\n\nschemaType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.SchemaType\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12.\n\x05table\x18\x03 \x01(\x0b\x32\x1d.tracdap.metadata.TableSchemaH\x00\x12\x30\n\x06struct\x18\x04 \x01(\x0b\x32\x1e.tracdap.metadata.StructSchemaH\x00\x42\x0f\n\rschemaDetails\"\x81\x02\n\x07PartKey\x12\x11\n\topaqueKey\x18\x01 \x01(\t\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12+\n\npartValues\x18\x03 \x03(\x0b\x32\x17.tracdap.metadata.Value\x12\x32\n\x0cpartRangeMin\x18\x04 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x00\x88\x01\x01\x12\x32\n\x0cpartRangeMax\x18\x05 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x01\x88\x01\x01\x42\x0f\n\r_partRangeMinB\x0f\n\r_partRangeMax\"\xba\x04\n\x0e\x44\x61taDefinition\x12\x31\n\x08schemaId\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x34\n\x06schema\x18\x02 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12:\n\x05parts\x18\x03 \x03(\x0b\x32+.tracdap.metadata.DataDefinition.PartsEntry\x12\x30\n\tstorageId\x18\x04 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x1a-\n\x05\x44\x65lta\x12\x12\n\ndeltaIndex\x18\x01 \x01(\r\x12\x10\n\x08\x64\x61taItem\x18\x02 \x01(\t\x1aQ\n\x04Snap\x12\x11\n\tsnapIndex\x18\x01 \x01(\r\x12\x36\n\x06\x64\x65ltas\x18\x02 \x03(\x0b\x32&.tracdap.metadata.DataDefinition.Delta\x1ag\n\x04Part\x12*\n\x07partKey\x18\x01 \x01(\x0b\x32\x19.tracdap.metadata.PartKey\x12\x33\n\x04snap\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Snap\x1aS\n\nPartsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x34\n\x05value\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Part:\x02\x38\x01\x42\x11\n\x0fschemaSpecifier*<\n\nSchemaType\x12\x17\n\x13SCHEMA_TYPE_NOT_SET\x10\x00\x12\t\n\x05TABLE\x10\x01\x12\n\n\x06STRUCT\x10\x02*?\n\x08PartType\x12\r\n\tPART_ROOT\x10\x00\x12\x11\n\rPART_BY_RANGE\x10\x01\x12\x11\n\rPART_BY_VALUE\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -24,28 +24,40 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracd
 if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._options = None
   _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
+  _globals['_STRUCTSCHEMA_FIELDSENTRY']._options = None
+  _globals['_STRUCTSCHEMA_FIELDSENTRY']._serialized_options = b'8\001'
+  _globals['_STRUCTSCHEMA_NAMEDTYPESENTRY']._options = None
+  _globals['_STRUCTSCHEMA_NAMEDTYPESENTRY']._serialized_options = b'8\001'
   _globals['_DATADEFINITION_PARTSENTRY']._options = None
   _globals['_DATADEFINITION_PARTSENTRY']._serialized_options = b'8\001'
-  _globals['_SCHEMATYPE']._serialized_start=1489
-  _globals['_SCHEMATYPE']._serialized_end=1537
-  _globals['_PARTTYPE']._serialized_start=1539
-  _globals['_PARTTYPE']._serialized_end=1602
+  _globals['_SCHEMATYPE']._serialized_start=2132
+  _globals['_SCHEMATYPE']._serialized_end=2192
+  _globals['_PARTTYPE']._serialized_start=2194
+  _globals['_PARTTYPE']._serialized_end=2257
   _globals['_FIELDSCHEMA']._serialized_start=179
   _globals['_FIELDSCHEMA']._serialized_end=410
   _globals['_TABLESCHEMA']._serialized_start=412
   _globals['_TABLESCHEMA']._serialized_end=472
-  _globals['_SCHEMADEFINITION']._serialized_start=475
-  _globals['_SCHEMADEFINITION']._serialized_end=654
-  _globals['_PARTKEY']._serialized_start=657
-  _globals['_PARTKEY']._serialized_end=914
-  _globals['_DATADEFINITION']._serialized_start=917
-  _globals['_DATADEFINITION']._serialized_end=1487
-  _globals['_DATADEFINITION_DELTA']._serialized_start=1150
-  _globals['_DATADEFINITION_DELTA']._serialized_end=1195
-  _globals['_DATADEFINITION_SNAP']._serialized_start=1197
-  _globals['_DATADEFINITION_SNAP']._serialized_end=1278
-  _globals['_DATADEFINITION_PART']._serialized_start=1280
-  _globals['_DATADEFINITION_PART']._serialized_end=1383
-  _globals['_DATADEFINITION_PARTSENTRY']._serialized_start=1385
-  _globals['_DATADEFINITION_PARTSENTRY']._serialized_end=1468
+  _globals['_STRUCTFIELD']._serialized_start=475
+  _globals['_STRUCTFIELD']._serialized_end=759
+  _globals['_STRUCTSCHEMA']._serialized_start=762
+  _globals['_STRUCTSCHEMA']._serialized_end=1065
+  _globals['_STRUCTSCHEMA_FIELDSENTRY']._serialized_start=906
+  _globals['_STRUCTSCHEMA_FIELDSENTRY']._serialized_end=982
+  _globals['_STRUCTSCHEMA_NAMEDTYPESENTRY']._serialized_start=984
+  _globals['_STRUCTSCHEMA_NAMEDTYPESENTRY']._serialized_end=1065
+  _globals['_SCHEMADEFINITION']._serialized_start=1068
+  _globals['_SCHEMADEFINITION']._serialized_end=1297
+  _globals['_PARTKEY']._serialized_start=1300
+  _globals['_PARTKEY']._serialized_end=1557
+  _globals['_DATADEFINITION']._serialized_start=1560
+  _globals['_DATADEFINITION']._serialized_end=2130
+  _globals['_DATADEFINITION_DELTA']._serialized_start=1793
+  _globals['_DATADEFINITION_DELTA']._serialized_end=1838
+  _globals['_DATADEFINITION_SNAP']._serialized_start=1840
+  _globals['_DATADEFINITION_SNAP']._serialized_end=1921
+  _globals['_DATADEFINITION_PART']._serialized_start=1923
+  _globals['_DATADEFINITION_PART']._serialized_end=2026
+  _globals['_DATADEFINITION_PARTSENTRY']._serialized_start=2028
+  _globals['_DATADEFINITION_PARTSENTRY']._serialized_end=2111
 # @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.pyi

@@ -12,6 +12,7 @@ class SchemaType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
     SCHEMA_TYPE_NOT_SET: _ClassVar[SchemaType]
     TABLE: _ClassVar[SchemaType]
+    STRUCT: _ClassVar[SchemaType]
 
 class PartType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
@@ -20,6 +21,7 @@ class PartType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     PART_BY_VALUE: _ClassVar[PartType]
 SCHEMA_TYPE_NOT_SET: SchemaType
 TABLE: SchemaType
+STRUCT: SchemaType
 PART_ROOT: PartType
 PART_BY_RANGE: PartType
 PART_BY_VALUE: PartType
@@ -50,15 +52,59 @@ class TableSchema(_message.Message):
     fields: _containers.RepeatedCompositeFieldContainer[FieldSchema]
     def __init__(self, fields: _Optional[_Iterable[_Union[FieldSchema, _Mapping]]] = ...) -> None: ...
 
+class StructField(_message.Message):
+    __slots__ = ("fieldType", "label", "businessKey", "categorical", "notNull", "formatCode", "defaultValue", "structType")
+    FIELDTYPE_FIELD_NUMBER: _ClassVar[int]
+    LABEL_FIELD_NUMBER: _ClassVar[int]
+    BUSINESSKEY_FIELD_NUMBER: _ClassVar[int]
+    CATEGORICAL_FIELD_NUMBER: _ClassVar[int]
+    NOTNULL_FIELD_NUMBER: _ClassVar[int]
+    FORMATCODE_FIELD_NUMBER: _ClassVar[int]
+    DEFAULTVALUE_FIELD_NUMBER: _ClassVar[int]
+    STRUCTTYPE_FIELD_NUMBER: _ClassVar[int]
+    fieldType: _type_pb2.TypeDescriptor
+    label: str
+    businessKey: bool
+    categorical: bool
+    notNull: bool
+    formatCode: str
+    defaultValue: _type_pb2.Value
+    structType: str
+    def __init__(self, fieldType: _Optional[_Union[_type_pb2.TypeDescriptor, _Mapping]] = ..., label: _Optional[str] = ..., businessKey: bool = ..., categorical: bool = ..., notNull: bool = ..., formatCode: _Optional[str] = ..., defaultValue: _Optional[_Union[_type_pb2.Value, _Mapping]] = ..., structType: _Optional[str] = ...) -> None: ...
+
+class StructSchema(_message.Message):
+    __slots__ = ("fields", "namedTypes")
+    class FieldsEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: StructField
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[StructField, _Mapping]] = ...) -> None: ...
+    class NamedTypesEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: StructSchema
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[StructSchema, _Mapping]] = ...) -> None: ...
+    FIELDS_FIELD_NUMBER: _ClassVar[int]
+    NAMEDTYPES_FIELD_NUMBER: _ClassVar[int]
+    fields: _containers.MessageMap[str, StructField]
+    namedTypes: _containers.MessageMap[str, StructSchema]
+    def __init__(self, fields: _Optional[_Mapping[str, StructField]] = ..., namedTypes: _Optional[_Mapping[str, StructSchema]] = ...) -> None: ...
+
 class SchemaDefinition(_message.Message):
-    __slots__ = ("schemaType", "partType", "table")
+    __slots__ = ("schemaType", "partType", "table", "struct")
     SCHEMATYPE_FIELD_NUMBER: _ClassVar[int]
     PARTTYPE_FIELD_NUMBER: _ClassVar[int]
     TABLE_FIELD_NUMBER: _ClassVar[int]
+    STRUCT_FIELD_NUMBER: _ClassVar[int]
     schemaType: SchemaType
     partType: PartType
     table: TableSchema
-    def __init__(self, schemaType: _Optional[_Union[SchemaType, str]] = ..., partType: _Optional[_Union[PartType, str]] = ..., table: _Optional[_Union[TableSchema, _Mapping]] = ...) -> None: ...
+    struct: StructSchema
+    def __init__(self, schemaType: _Optional[_Union[SchemaType, str]] = ..., partType: _Optional[_Union[PartType, str]] = ..., table: _Optional[_Union[TableSchema, _Mapping]] = ..., struct: _Optional[_Union[StructSchema, _Mapping]] = ...) -> None: ...
 
 class PartKey(_message.Message):
     __slots__ = ("opaqueKey", "partType", "partValues", "partRangeMin", "partRangeMax")

tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.py

@@ -20,10 +20,10 @@ from tracdap.rt._impl.grpc.tracdap.metadata import flow_pb2 as tracdap_dot_rt_do
 from tracdap.rt._impl.grpc.tracdap.metadata import job_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_job__pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import file_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_file__pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import custom_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_custom__pb2
-from tracdap.rt._impl.grpc.tracdap.metadata import stoarge_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_stoarge__pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import storage_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_storage__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x32tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto\x1a\x34tracdap/rt/_impl/grpc/tracdap/metadata/stoarge.proto\"\xbd\x05\n\x10ObjectDefinition\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .tracdap.metadata.DataDefinitionH\x00\x12\x32\n\x05model\x18\x03 \x01(\x0b\x32!.tracdap.metadata.ModelDefinitionH\x00\x12\x30\n\x04\x66low\x18\x04 \x01(\x0b\x32 .tracdap.metadata.FlowDefinitionH\x00\x12.\n\x03job\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinitionH\x00\x12\x30\n\x04\x66ile\x18\x06 \x01(\x0b\x32 .tracdap.metadata.FileDefinitionH\x00\x12\x34\n\x06\x63ustom\x18\x07 \x01(\x0b\x32\".tracdap.metadata.CustomDefinitionH\x00\x12\x36\n\x07storage\x18\x08 \x01(\x0b\x32#.tracdap.metadata.StorageDefinitionH\x00\x12\x34\n\x06schema\x18\t \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12\x34\n\x06result\x18\n \x01(\x0b\x32\".tracdap.metadata.ResultDefinitionH\x00\x12H\n\x0bobjectProps\x18\x64 \x03(\x0b\x32\x33.tracdap.metadata.ObjectDefinition.ObjectPropsEntry\x1aK\n\x10ObjectPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0c\n\ndefinitionB2\n\x1aorg.finos.tracdap.metadataB\x12ObjectProtoWrapperP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x32tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto\x1a\x34tracdap/rt/_impl/grpc/tracdap/metadata/storage.proto\"\xbd\x05\n\x10ObjectDefinition\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .tracdap.metadata.DataDefinitionH\x00\x12\x32\n\x05model\x18\x03 \x01(\x0b\x32!.tracdap.metadata.ModelDefinitionH\x00\x12\x30\n\x04\x66low\x18\x04 \x01(\x0b\x32 .tracdap.metadata.FlowDefinitionH\x00\x12.\n\x03job\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinitionH\x00\x12\x30\n\x04\x66ile\x18\x06 \x01(\x0b\x32 .tracdap.metadata.FileDefinitionH\x00\x12\x34\n\x06\x63ustom\x18\x07 \x01(\x0b\x32\".tracdap.metadata.CustomDefinitionH\x00\x12\x36\n\x07storage\x18\x08 \x01(\x0b\x32#.tracdap.metadata.StorageDefinitionH\x00\x12\x34\n\x06schema\x18\t \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12\x34\n\x06result\x18\n \x01(\x0b\x32\".tracdap.metadata.ResultDefinitionH\x00\x12H\n\x0bobjectProps\x18\x64 \x03(\x0b\x32\x33.tracdap.metadata.ObjectDefinition.ObjectPropsEntry\x1aK\n\x10ObjectPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0c\n\ndefinitionB2\n\x1aorg.finos.tracdap.metadataB\x12ObjectProtoWrapperP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)

tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.pyi

@@ -6,7 +6,7 @@ from tracdap.rt._impl.grpc.tracdap.metadata import flow_pb2 as _flow_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import job_pb2 as _job_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import file_pb2 as _file_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import custom_pb2 as _custom_pb2
-from tracdap.rt._impl.grpc.tracdap.metadata import stoarge_pb2 as _stoarge_pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import storage_pb2 as _storage_pb2
 from google.protobuf.internal import containers as _containers
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
@@ -41,8 +41,8 @@ class ObjectDefinition(_message.Message):
     job: _job_pb2.JobDefinition
     file: _file_pb2.FileDefinition
     custom: _custom_pb2.CustomDefinition
-    storage: _stoarge_pb2.StorageDefinition
+    storage: _storage_pb2.StorageDefinition
     schema: _data_pb2.SchemaDefinition
     result: _job_pb2.ResultDefinition
     objectProps: _containers.MessageMap[str, _type_pb2.Value]
-    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., data: _Optional[_Union[_data_pb2.DataDefinition, _Mapping]] = ..., model: _Optional[_Union[_model_pb2.ModelDefinition, _Mapping]] = ..., flow: _Optional[_Union[_flow_pb2.FlowDefinition, _Mapping]] = ..., job: _Optional[_Union[_job_pb2.JobDefinition, _Mapping]] = ..., file: _Optional[_Union[_file_pb2.FileDefinition, _Mapping]] = ..., custom: _Optional[_Union[_custom_pb2.CustomDefinition, _Mapping]] = ..., storage: _Optional[_Union[_stoarge_pb2.StorageDefinition, _Mapping]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., result: _Optional[_Union[_job_pb2.ResultDefinition, _Mapping]] = ..., objectProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
+    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., data: _Optional[_Union[_data_pb2.DataDefinition, _Mapping]] = ..., model: _Optional[_Union[_model_pb2.ModelDefinition, _Mapping]] = ..., flow: _Optional[_Union[_flow_pb2.FlowDefinition, _Mapping]] = ..., job: _Optional[_Union[_job_pb2.JobDefinition, _Mapping]] = ..., file: _Optional[_Union[_file_pb2.FileDefinition, _Mapping]] = ..., custom: _Optional[_Union[_custom_pb2.CustomDefinition, _Mapping]] = ..., storage: _Optional[_Union[_storage_pb2.StorageDefinition, _Mapping]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., result: _Optional[_Union[_job_pb2.ResultDefinition, _Mapping]] = ..., objectProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...

tracdap/rt/_impl/grpc/tracdap/metadata/{stoarge_pb2.py → storage_pb2.py}

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: tracdap/rt/_impl/grpc/tracdap/metadata/stoarge.proto
+# source: tracdap/rt/_impl/grpc/tracdap/metadata/storage.proto
 # Protobuf Python Version: 4.25.3
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
@@ -15,11 +15,11 @@ _sym_db = _symbol_database.Default()
 from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_type__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n4tracdap/rt/_impl/grpc/tracdap/metadata/stoarge.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\"\xd2\x02\n\x0bStorageCopy\x12\x12\n\nstorageKey\x18\x01 \x01(\t\x12\x13\n\x0bstoragePath\x18\x02 \x01(\t\x12\x15\n\rstorageFormat\x18\x03 \x01(\t\x12\x30\n\ncopyStatus\x18\x04 \x01(\x0e\x32\x1c.tracdap.metadata.CopyStatus\x12\x36\n\rcopyTimestamp\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12I\n\x0estorageOptions\x18\x06 \x03(\x0b\x32\x31.tracdap.metadata.StorageCopy.StorageOptionsEntry\x1aN\n\x13StorageOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\"\xdc\x01\n\x12StorageIncarnation\x12-\n\x06\x63opies\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.StorageCopy\x12\x18\n\x10incarnationIndex\x18\x02 \x01(\x05\x12=\n\x14incarnationTimestamp\x18\x03 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12>\n\x11incarnationStatus\x18\x04 \x01(\x0e\x32#.tracdap.metadata.IncarnationStatus\"I\n\x0bStorageItem\x12:\n\x0cincarnations\x18\x01 \x03(\x0b\x32$.tracdap.metadata.StorageIncarnation\"\xe1\x02\n\x11StorageDefinition\x12\x45\n\tdataItems\x18\x01 \x03(\x0b\x32\x32.tracdap.metadata.StorageDefinition.DataItemsEntry\x12O\n\x0estorageOptions\x18\x03 \x03(\x0b\x32\x37.tracdap.metadata.StorageDefinition.StorageOptionsEntry\x1aO\n\x0e\x44\x61taItemsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.StorageItem:\x02\x38\x01\x1aN\n\x13StorageOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01J\x04\x08\x02\x10\x03R\rstorageFormat*L\n\nCopyStatus\x12\x17\n\x13\x43OPY_STATUS_NOT_SET\x10\x00\x12\x12\n\x0e\x43OPY_AVAILABLE\x10\x01\x12\x11\n\rCOPY_EXPUNGED\x10\x02*h\n\x11IncarnationStatus\x12\x1e\n\x1aINCARNATION_STATUS_NOT_SET\x10\x00\x12\x19\n\x15INCARNATION_AVAILABLE\x10\x01\x12\x18\n\x14INCARNATION_EXPUNGED\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n4tracdap/rt/_impl/grpc/tracdap/metadata/storage.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\"\xd2\x02\n\x0bStorageCopy\x12\x12\n\nstorageKey\x18\x01 \x01(\t\x12\x13\n\x0bstoragePath\x18\x02 \x01(\t\x12\x15\n\rstorageFormat\x18\x03 \x01(\t\x12\x30\n\ncopyStatus\x18\x04 \x01(\x0e\x32\x1c.tracdap.metadata.CopyStatus\x12\x36\n\rcopyTimestamp\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12I\n\x0estorageOptions\x18\x06 \x03(\x0b\x32\x31.tracdap.metadata.StorageCopy.StorageOptionsEntry\x1aN\n\x13StorageOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\"\xdc\x01\n\x12StorageIncarnation\x12-\n\x06\x63opies\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.StorageCopy\x12\x18\n\x10incarnationIndex\x18\x02 \x01(\x05\x12=\n\x14incarnationTimestamp\x18\x03 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12>\n\x11incarnationStatus\x18\x04 \x01(\x0e\x32#.tracdap.metadata.IncarnationStatus\"I\n\x0bStorageItem\x12:\n\x0cincarnations\x18\x01 \x03(\x0b\x32$.tracdap.metadata.StorageIncarnation\"\xe1\x02\n\x11StorageDefinition\x12\x45\n\tdataItems\x18\x01 \x03(\x0b\x32\x32.tracdap.metadata.StorageDefinition.DataItemsEntry\x12O\n\x0estorageOptions\x18\x03 \x03(\x0b\x32\x37.tracdap.metadata.StorageDefinition.StorageOptionsEntry\x1aO\n\x0e\x44\x61taItemsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.StorageItem:\x02\x38\x01\x1aN\n\x13StorageOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01J\x04\x08\x02\x10\x03R\rstorageFormat*L\n\nCopyStatus\x12\x17\n\x13\x43OPY_STATUS_NOT_SET\x10\x00\x12\x12\n\x0e\x43OPY_AVAILABLE\x10\x01\x12\x11\n\rCOPY_EXPUNGED\x10\x02*h\n\x11IncarnationStatus\x12\x1e\n\x1aINCARNATION_STATUS_NOT_SET\x10\x00\x12\x19\n\x15INCARNATION_AVAILABLE\x10\x01\x12\x18\n\x14INCARNATION_EXPUNGED\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracdap.metadata.stoarge_pb2', _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracdap.metadata.storage_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._options = None
   _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'

tracdap/rt/_impl/static_api.py

@@ -21,6 +21,7 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
 import tracdap.rt._impl.core.data as _data
 import tracdap.rt._impl.core.schemas as _schemas
+import tracdap.rt._impl.core.struct as _struct
 import tracdap.rt._impl.core.type_system as _type_system
 import tracdap.rt._impl.core.validation as _val
 
@@ -55,7 +56,7 @@ class StaticApiImpl(_StaticApiHook):
         if not _val.is_primitive_type(entry_type):
             raise _ex.EModelValidation(f"Maps can only contain primitive types, [{entry_type}] is not primitive")
 
-        return _meta.TypeDescriptor(_meta.BasicType.MAP, arrayType=_meta.TypeDescriptor(entry_type))
+        return _meta.TypeDescriptor(_meta.BasicType.MAP, mapType=_meta.TypeDescriptor(entry_type))
 
     def define_attribute(
             self, attr_name: str, attr_value: _tp.Any,
@@ -150,6 +151,13 @@ class StaticApiImpl(_StaticApiHook):
             notNull=not_null,
             formatCode=format_code)
 
+    def define_struct(self, python_type: type[_api.STRUCT_TYPE]):
+
+        _val.validate_signature(self.define_struct, python_type)
+
+        struct_schema = _struct.StructProcessor.define_struct(python_type)
+        return _meta.SchemaDefinition(schemaType=_meta.SchemaType.STRUCT, struct=struct_schema)
+
     def define_schema(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
             schema_type: _meta.SchemaType = _meta.SchemaType.TABLE, dynamic: bool = False) \

tracdap/rt/_plugins/storage_sql.py

@@ -268,10 +268,18 @@ plugins.PluginManager.register_plugin(IStorageProvider, SqlStorageProvider, ["SQ
 
 
 try:
-
     import sqlalchemy as sqla  # noqa
     import sqlalchemy.exc as sqla_exc  # noqa
 
+    # Only 2.x versions of SQL Alchemy are currently supported
+    sqla_supported = sqla.__version__.startswith("2.")
+
+except ModuleNotFoundError:
+    sqla = None
+    sqla_supported = False
+
+if sqla_supported:
+
     class SqlAlchemyDriver(ISqlDriver):
 
         def __init__(self, properties: tp.Dict[str, str]):
@@ -336,7 +344,7 @@ try:
 
         class ConnectionWrapper(DbApiWrapper.Connection):
 
-            def __init__(self, conn: sqla.Connection):
+            def __init__(self, conn: "sqla.Connection"):
                 self.__conn = conn
 
             def close(self):
@@ -355,7 +363,7 @@ try:
 
             arraysize: int = 1000
 
-            def __init__(self, conn: sqla.Connection):
+            def __init__(self, conn: "sqla.Connection"):
                 self.__conn = conn
                 self.__result: tp.Optional[sqla.CursorResult] = None
 
@@ -414,5 +422,4 @@ try:
 
     plugins.PluginManager.register_plugin(ISqlDriver, SqlAlchemyDriver, ["alchemy"])
 
-except ModuleNotFoundError:
-    pass
+

tracdap/rt/_version.py

@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.8.0b3"
+__version__ = "0.8.0rc1"

tracdap/rt/api/__init__.py

@@ -23,29 +23,7 @@ from tracdap.rt.metadata import *  # noqa DOCGEN_REMOVE
 
 # static_api overrides some metadata types for backwards compatibility with pre-0.8 versions
 # Make sure it is last in the list
-from .file_types import *
+from .constants import *
 from .model_api import *
 from .static_api import *
 
-# Map basic types into the root of the API package
-
-BOOLEAN = BasicType.BOOLEAN
-"""Synonym for :py:attr:`BasicType.BOOLEAN <tracdap.rt.metadata.BasicType.BOOLEAN>`"""
-
-INTEGER = BasicType.INTEGER
-"""Synonym for :py:attr:`BasicType.INTEGER <tracdap.rt.metadata.BasicType.INTEGER>`"""
-
-FLOAT = BasicType.FLOAT
-"""Synonym for :py:attr:`BasicType.FLOAT <tracdap.rt.metadata.BasicType.FLOAT>`"""
-
-DECIMAL = BasicType.DECIMAL
-"""Synonym for :py:attr:`BasicType.DECIMAL <tracdap.rt.metadata.BasicType.DECIMAL>`"""
-
-STRING = BasicType.STRING
-"""Synonym for :py:attr:`BasicType.STRING <tracdap.rt.metadata.BasicType.STRING>`"""
-
-DATE = BasicType.DATE
-"""Synonym for :py:attr:`BasicType.DATE <tracdap.rt.metadata.BasicType.DATE>`"""
-
-DATETIME = BasicType.DATETIME
-"""Synonym for :py:attr:`BasicType.DATETIME <tracdap.rt.metadata.BasicType.DATETIME>`"""

tracdap/rt/api/constants.py

@@ -0,0 +1,57 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import tracdap.rt.metadata
+
+
+class CommonFileTypes:
+
+    """
+    A collection of common :py:class:`FileTypes <tracdap.rt.metadata.FileType>` to use as model inputs and outputs
+    """
+
+    TXT = tracdap.rt.metadata.FileType("txt", "text/plain")
+
+    JPG = tracdap.rt.metadata.FileType("jpg", "image/jpeg")
+    PNG = tracdap.rt.metadata.FileType("png", "image/png")
+    SVG  = tracdap.rt.metadata.FileType("svg", "image/svg+xml")
+
+    WORD = tracdap.rt.metadata.FileType("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
+    EXCEL = tracdap.rt.metadata.FileType("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
+    POWERPOINT = tracdap.rt.metadata.FileType("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation")
+
+
+# Map basic types into the root of the API package
+
+BOOLEAN = tracdap.rt.metadata.BasicType.BOOLEAN
+"""Synonym for :py:attr:`BasicType.BOOLEAN <tracdap.rt.metadata.BasicType.BOOLEAN>`"""
+
+INTEGER = tracdap.rt.metadata.BasicType.INTEGER
+"""Synonym for :py:attr:`BasicType.INTEGER <tracdap.rt.metadata.BasicType.INTEGER>`"""
+
+FLOAT = tracdap.rt.metadata.BasicType.FLOAT
+"""Synonym for :py:attr:`BasicType.FLOAT <tracdap.rt.metadata.BasicType.FLOAT>`"""
+
+DECIMAL = tracdap.rt.metadata.BasicType.DECIMAL
+"""Synonym for :py:attr:`BasicType.DECIMAL <tracdap.rt.metadata.BasicType.DECIMAL>`"""
+
+STRING = tracdap.rt.metadata.BasicType.STRING
+"""Synonym for :py:attr:`BasicType.STRING <tracdap.rt.metadata.BasicType.STRING>`"""
+
+DATE = tracdap.rt.metadata.BasicType.DATE
+"""Synonym for :py:attr:`BasicType.DATE <tracdap.rt.metadata.BasicType.DATE>`"""
+
+DATETIME = tracdap.rt.metadata.BasicType.DATETIME
+"""Synonym for :py:attr:`BasicType.DATETIME <tracdap.rt.metadata.BasicType.DATETIME>`"""