tracdap-runtime 0.7.0rc1__py3-none-any.whl → 0.8.0b1__py3-none-any.whl

tracdap/rt/_impl/data.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 
 import abc
+import copy
 import dataclasses as dc
 import typing as tp
 import datetime as dt
@@ -42,11 +43,48 @@ import tracdap.rt._impl.util as _util
 @dc.dataclass(frozen=True)
 class DataSpec:
 
+    object_type: _meta.ObjectType
     data_item: str
+
     data_def: _meta.DataDefinition
+    file_def: _meta.FileDefinition
     storage_def: _meta.StorageDefinition
     schema_def: tp.Optional[_meta.SchemaDefinition]
 
+    @staticmethod
+    def create_data_spec(
+            data_item: str,
+            data_def: _meta.DataDefinition,
+            storage_def: _meta.StorageDefinition,
+            schema_def: tp.Optional[_meta.SchemaDefinition] = None) -> "DataSpec":
+
+        return DataSpec(
+            _meta.ObjectType.DATA, data_item,
+            data_def,
+            storage_def=storage_def,
+            schema_def=schema_def,
+            file_def=None)
+
+    @staticmethod
+    def create_file_spec(
+            data_item: str,
+            file_def: _meta.FileDefinition,
+            storage_def: _meta.StorageDefinition) -> "DataSpec":
+
+        return DataSpec(
+            _meta.ObjectType.FILE, data_item,
+            file_def=file_def,
+            storage_def=storage_def,
+            data_def=None,
+            schema_def=None)
+
+    @staticmethod
+    def create_empty_spec(object_type: _meta.ObjectType):
+        return DataSpec(object_type, None, None, None, None, None)
+
+    def is_empty(self):
+        return self.data_item is None or len(self.data_item) == 0
+
 
 @dc.dataclass(frozen=True)
 class DataPartKey:
@@ -61,44 +99,79 @@ class DataPartKey:
 @dc.dataclass(frozen=True)
 class DataItem:
 
-    schema: pa.Schema
+    object_type: _meta.ObjectType
+
+    schema: pa.Schema = None
     table: tp.Optional[pa.Table] = None
     batches: tp.Optional[tp.List[pa.RecordBatch]] = None
 
     pandas: "tp.Optional[pandas.DataFrame]" = None
     pyspark: tp.Any = None
 
+    raw_bytes: bytes = None
+
     def is_empty(self) -> bool:
-        return self.table is None and (self.batches is None or len(self.batches) == 0)
+        if self.object_type == _meta.ObjectType.FILE:
+            return self.raw_bytes is None or len(self.raw_bytes) == 0
+        else:
+            return self.table is None and (self.batches is None or len(self.batches) == 0)
 
     @staticmethod
-    def create_empty() -> "DataItem":
-        return DataItem(pa.schema([]))
+    def create_empty(object_type: _meta.ObjectType = _meta.ObjectType.DATA) -> "DataItem":
+        if object_type == _meta.ObjectType.DATA:
+            return DataItem(_meta.ObjectType.DATA, pa.schema([]))
+        else:
+            return DataItem(object_type)
+
+    @staticmethod
+    def for_file_content(raw_bytes: bytes):
+        return DataItem(_meta.ObjectType.FILE, raw_bytes=raw_bytes)
 
 
 @dc.dataclass(frozen=True)
 class DataView:
 
-    trac_schema: _meta.SchemaDefinition
-    arrow_schema: pa.Schema
+    object_type: _meta.ObjectType
 
-    parts: tp.Dict[DataPartKey, tp.List[DataItem]]
+    trac_schema: _meta.SchemaDefinition = None
+    arrow_schema: pa.Schema = None
+
+    parts: tp.Dict[DataPartKey, tp.List[DataItem]] = None
+    file_item: tp.Optional[DataItem] = None
 
     @staticmethod
-    def create_empty() -> "DataView":
-        return DataView(_meta.SchemaDefinition(), pa.schema([]), dict())
+    def create_empty(object_type: _meta.ObjectType = _meta.ObjectType.DATA) -> "DataView":
+        if object_type == _meta.ObjectType.DATA:
+            return DataView(object_type, _meta.SchemaDefinition(), pa.schema([]), dict())
+        else:
+            return DataView(object_type)
 
     @staticmethod
     def for_trac_schema(trac_schema: _meta.SchemaDefinition):
         arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
-        return DataView(trac_schema, arrow_schema, dict())
+        return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, dict())
+
+    @staticmethod
+    def for_file_item(file_item: DataItem):
+        return DataView(file_item.object_type, file_item=file_item)
 
     def with_trac_schema(self, trac_schema: _meta.SchemaDefinition):
         arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
-        return DataView(trac_schema, arrow_schema, self.parts)
+        return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, self.parts)
+
+    def with_part(self, part_key: DataPartKey, part: DataItem):
+        new_parts = copy.copy(self.parts)
+        new_parts[part_key] = [part]
+        return DataView(self.object_type, self.trac_schema, self.arrow_schema, new_parts)
+
+    def with_file_item(self, file_item: DataItem):
+        return DataView(self.object_type, file_item=file_item)
 
     def is_empty(self) -> bool:
-        return self.parts is None or not any(self.parts.values())
+        if self.object_type == _meta.ObjectType.FILE:
+            return self.file_item is None
+        else:
+            return self.parts is None or not any(self.parts.values())
 
 
 class _DataInternal:
@@ -293,7 +366,7 @@ class DataMapping:
         deltas = [*prior_deltas, item]
         parts = {**view.parts, part: deltas}
 
-        return DataView(view.trac_schema, view.arrow_schema, parts)
+        return DataView(view.object_type, view.trac_schema, view.arrow_schema, parts=parts)
 
     @classmethod
     def view_to_arrow(cls, view: DataView, part: DataPartKey) -> pa.Table:

tracdap/rt/_impl/grpc/tracdap/metadata/file_pb2.py

@@ -15,7 +15,7 @@ _sym_db = _symbol_database.Default()
 from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__id__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\"\x95\x01\n\x0e\x46ileDefinition\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\textension\x18\x02 \x01(\t\x12\x10\n\x08mimeType\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x04\x12\x30\n\tstorageId\x18\x05 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x12\x10\n\x08\x64\x61taItem\x18\x06 \x01(\tB\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\"\x95\x01\n\x0e\x46ileDefinition\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\textension\x18\x02 \x01(\t\x12\x10\n\x08mimeType\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x04\x12\x30\n\tstorageId\x18\x05 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x12\x10\n\x08\x64\x61taItem\x18\x06 \x01(\t\"/\n\x08\x46ileType\x12\x11\n\textension\x18\x01 \x01(\t\x12\x10\n\x08mimeType\x18\x02 \x01(\tB\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -25,4 +25,6 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
   _globals['_FILEDEFINITION']._serialized_start=128
   _globals['_FILEDEFINITION']._serialized_end=277
+  _globals['_FILETYPE']._serialized_start=279
+  _globals['_FILETYPE']._serialized_end=326
 # @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/file_pb2.pyi

@@ -20,3 +20,11 @@ class FileDefinition(_message.Message):
     storageId: _object_id_pb2.TagSelector
     dataItem: str
     def __init__(self, name: _Optional[str] = ..., extension: _Optional[str] = ..., mimeType: _Optional[str] = ..., size: _Optional[int] = ..., storageId: _Optional[_Union[_object_id_pb2.TagSelector, _Mapping]] = ..., dataItem: _Optional[str] = ...) -> None: ...
+
+class FileType(_message.Message):
+    __slots__ = ("extension", "mimeType")
+    EXTENSION_FIELD_NUMBER: _ClassVar[int]
+    MIMETYPE_FIELD_NUMBER: _ClassVar[int]
+    extension: str
+    mimeType: str
+    def __init__(self, extension: _Optional[str] = ..., mimeType: _Optional[str] = ...) -> None: ...

tracdap/rt/_impl/grpc/tracdap/metadata/model_pb2.py

@@ -13,10 +13,12 @@ _sym_db = _symbol_database.Default()
 
 
 from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_type__pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__id__pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import data_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_data__pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import file_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_file__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\"\xab\x02\n\x0eModelParameter\x12\x33\n\tparamType\x18\x01 \x01(\x0b\x32 .tracdap.metadata.TypeDescriptor\x12\r\n\x05label\x18\x02 \x01(\t\x12\x32\n\x0c\x64\x65\x66\x61ultValue\x18\x03 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x00\x88\x01\x01\x12\x44\n\nparamProps\x18\x04 \x03(\x0b\x32\x30.tracdap.metadata.ModelParameter.ParamPropsEntry\x1aJ\n\x0fParamPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0f\n\r_defaultValue\"\x9b\x02\n\x10ModelInputSchema\x12\x32\n\x06schema\x18\x01 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinition\x12\x12\n\x05label\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x10\n\x08optional\x18\x03 \x01(\x08\x12\x0f\n\x07\x64ynamic\x18\x05 \x01(\x08\x12\x46\n\ninputProps\x18\x04 \x03(\x0b\x32\x32.tracdap.metadata.ModelInputSchema.InputPropsEntry\x1aJ\n\x0fInputPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x08\n\x06_label\"\xa0\x02\n\x11ModelOutputSchema\x12\x32\n\x06schema\x18\x01 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinition\x12\x12\n\x05label\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x10\n\x08optional\x18\x03 \x01(\x08\x12\x0f\n\x07\x64ynamic\x18\x05 \x01(\x08\x12I\n\x0boutputProps\x18\x04 \x03(\x0b\x32\x34.tracdap.metadata.ModelOutputSchema.OutputPropsEntry\x1aK\n\x10OutputPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x08\n\x06_label\"\xce\x06\n\x0fModelDefinition\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nrepository\x18\x02 \x01(\t\x12\x19\n\x0cpackageGroup\x18\n \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07package\x18\x0b \x01(\t\x12\x0f\n\x07version\x18\x06 \x01(\t\x12\x12\n\nentryPoint\x18\x05 \x01(\t\x12\x11\n\x04path\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x45\n\nparameters\x18\x07 \x03(\x0b\x32\x31.tracdap.metadata.ModelDefinition.ParametersEntry\x12=\n\x06inputs\x18\x08 \x03(\x0b\x32-.tracdap.metadata.ModelDefinition.InputsEntry\x12?\n\x07outputs\x18\t \x03(\x0b\x32..tracdap.metadata.ModelDefinition.OutputsEntry\x12Q\n\x10staticAttributes\x18\x0c \x03(\x0b\x32\x37.tracdap.metadata.ModelDefinition.StaticAttributesEntry\x12.\n\tmodelType\x18\r \x01(\x0e\x32\x1b.tracdap.metadata.ModelType\x1aS\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .tracdap.metadata.ModelParameter:\x02\x38\x01\x1aQ\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".tracdap.metadata.ModelInputSchema:\x02\x38\x01\x1aS\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x32\n\x05value\x18\x02 \x01(\x0b\x32#.tracdap.metadata.ModelOutputSchema:\x02\x38\x01\x1aP\n\x15StaticAttributesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0f\n\r_packageGroupB\x07\n\x05_path*M\n\tModelType\x12\x12\n\x0eSTANDARD_MODEL\x10\x00\x12\x15\n\x11\x44\x41TA_IMPORT_MODEL\x10\x01\x12\x15\n\x11\x44\x41TA_EXPORT_MODEL\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\"\xab\x02\n\x0eModelParameter\x12\x33\n\tparamType\x18\x01 \x01(\x0b\x32 .tracdap.metadata.TypeDescriptor\x12\r\n\x05label\x18\x02 \x01(\t\x12\x32\n\x0c\x64\x65\x66\x61ultValue\x18\x03 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x00\x88\x01\x01\x12\x44\n\nparamProps\x18\x04 \x03(\x0b\x32\x30.tracdap.metadata.ModelParameter.ParamPropsEntry\x1aJ\n\x0fParamPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0f\n\r_defaultValue\"\x8e\x03\n\x10ModelInputSchema\x12\x30\n\nobjectType\x18\x06 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x34\n\x06schema\x18\x01 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12.\n\x08\x66ileType\x18\x07 \x01(\x0b\x32\x1a.tracdap.metadata.FileTypeH\x00\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08optional\x18\x03 \x01(\x08\x12\x0f\n\x07\x64ynamic\x18\x05 \x01(\x08\x12\x46\n\ninputProps\x18\x04 \x03(\x0b\x32\x32.tracdap.metadata.ModelInputSchema.InputPropsEntry\x1aJ\n\x0fInputPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\r\n\x0brequirementB\x08\n\x06_label\"\x93\x03\n\x11ModelOutputSchema\x12\x30\n\nobjectType\x18\x06 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x34\n\x06schema\x18\x01 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12.\n\x08\x66ileType\x18\x07 \x01(\x0b\x32\x1a.tracdap.metadata.FileTypeH\x00\x12\x12\n\x05label\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08optional\x18\x03 \x01(\x08\x12\x0f\n\x07\x64ynamic\x18\x05 \x01(\x08\x12I\n\x0boutputProps\x18\x04 \x03(\x0b\x32\x34.tracdap.metadata.ModelOutputSchema.OutputPropsEntry\x1aK\n\x10OutputPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\r\n\x0brequirementB\x08\n\x06_label\"\xce\x06\n\x0fModelDefinition\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nrepository\x18\x02 \x01(\t\x12\x19\n\x0cpackageGroup\x18\n \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07package\x18\x0b \x01(\t\x12\x0f\n\x07version\x18\x06 \x01(\t\x12\x12\n\nentryPoint\x18\x05 \x01(\t\x12\x11\n\x04path\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x45\n\nparameters\x18\x07 \x03(\x0b\x32\x31.tracdap.metadata.ModelDefinition.ParametersEntry\x12=\n\x06inputs\x18\x08 \x03(\x0b\x32-.tracdap.metadata.ModelDefinition.InputsEntry\x12?\n\x07outputs\x18\t \x03(\x0b\x32..tracdap.metadata.ModelDefinition.OutputsEntry\x12Q\n\x10staticAttributes\x18\x0c \x03(\x0b\x32\x37.tracdap.metadata.ModelDefinition.StaticAttributesEntry\x12.\n\tmodelType\x18\r \x01(\x0e\x32\x1b.tracdap.metadata.ModelType\x1aS\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .tracdap.metadata.ModelParameter:\x02\x38\x01\x1aQ\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".tracdap.metadata.ModelInputSchema:\x02\x38\x01\x1aS\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x32\n\x05value\x18\x02 \x01(\x0b\x32#.tracdap.metadata.ModelOutputSchema:\x02\x38\x01\x1aP\n\x15StaticAttributesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0f\n\r_packageGroupB\x07\n\x05_path*M\n\tModelType\x12\x12\n\x0eSTANDARD_MODEL\x10\x00\x12\x15\n\x11\x44\x41TA_IMPORT_MODEL\x10\x01\x12\x15\n\x11\x44\x41TA_EXPORT_MODEL\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -38,28 +40,28 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_MODELDEFINITION_OUTPUTSENTRY']._serialized_options = b'8\001'
   _globals['_MODELDEFINITION_STATICATTRIBUTESENTRY']._options = None
   _globals['_MODELDEFINITION_STATICATTRIBUTESENTRY']._serialized_options = b'8\001'
-  _globals['_MODELTYPE']._serialized_start=1902
-  _globals['_MODELTYPE']._serialized_end=1979
-  _globals['_MODELPARAMETER']._serialized_start=175
-  _globals['_MODELPARAMETER']._serialized_end=474
-  _globals['_MODELPARAMETER_PARAMPROPSENTRY']._serialized_start=383
-  _globals['_MODELPARAMETER_PARAMPROPSENTRY']._serialized_end=457
-  _globals['_MODELINPUTSCHEMA']._serialized_start=477
-  _globals['_MODELINPUTSCHEMA']._serialized_end=760
-  _globals['_MODELINPUTSCHEMA_INPUTPROPSENTRY']._serialized_start=676
-  _globals['_MODELINPUTSCHEMA_INPUTPROPSENTRY']._serialized_end=750
-  _globals['_MODELOUTPUTSCHEMA']._serialized_start=763
-  _globals['_MODELOUTPUTSCHEMA']._serialized_end=1051
-  _globals['_MODELOUTPUTSCHEMA_OUTPUTPROPSENTRY']._serialized_start=966
-  _globals['_MODELOUTPUTSCHEMA_OUTPUTPROPSENTRY']._serialized_end=1041
-  _globals['_MODELDEFINITION']._serialized_start=1054
-  _globals['_MODELDEFINITION']._serialized_end=1900
-  _globals['_MODELDEFINITION_PARAMETERSENTRY']._serialized_start=1541
-  _globals['_MODELDEFINITION_PARAMETERSENTRY']._serialized_end=1624
-  _globals['_MODELDEFINITION_INPUTSENTRY']._serialized_start=1626
-  _globals['_MODELDEFINITION_INPUTSENTRY']._serialized_end=1707
-  _globals['_MODELDEFINITION_OUTPUTSENTRY']._serialized_start=1709
-  _globals['_MODELDEFINITION_OUTPUTSENTRY']._serialized_end=1792
-  _globals['_MODELDEFINITION_STATICATTRIBUTESENTRY']._serialized_start=1794
-  _globals['_MODELDEFINITION_STATICATTRIBUTESENTRY']._serialized_end=1874
+  _globals['_MODELTYPE']._serialized_start=2239
+  _globals['_MODELTYPE']._serialized_end=2316
+  _globals['_MODELPARAMETER']._serialized_start=282
+  _globals['_MODELPARAMETER']._serialized_end=581
+  _globals['_MODELPARAMETER_PARAMPROPSENTRY']._serialized_start=490
+  _globals['_MODELPARAMETER_PARAMPROPSENTRY']._serialized_end=564
+  _globals['_MODELINPUTSCHEMA']._serialized_start=584
+  _globals['_MODELINPUTSCHEMA']._serialized_end=982
+  _globals['_MODELINPUTSCHEMA_INPUTPROPSENTRY']._serialized_start=883
+  _globals['_MODELINPUTSCHEMA_INPUTPROPSENTRY']._serialized_end=957
+  _globals['_MODELOUTPUTSCHEMA']._serialized_start=985
+  _globals['_MODELOUTPUTSCHEMA']._serialized_end=1388
+  _globals['_MODELOUTPUTSCHEMA_OUTPUTPROPSENTRY']._serialized_start=1288
+  _globals['_MODELOUTPUTSCHEMA_OUTPUTPROPSENTRY']._serialized_end=1363
+  _globals['_MODELDEFINITION']._serialized_start=1391
+  _globals['_MODELDEFINITION']._serialized_end=2237
+  _globals['_MODELDEFINITION_PARAMETERSENTRY']._serialized_start=1878
+  _globals['_MODELDEFINITION_PARAMETERSENTRY']._serialized_end=1961
+  _globals['_MODELDEFINITION_INPUTSENTRY']._serialized_start=1963
+  _globals['_MODELDEFINITION_INPUTSENTRY']._serialized_end=2044
+  _globals['_MODELDEFINITION_OUTPUTSENTRY']._serialized_start=2046
+  _globals['_MODELDEFINITION_OUTPUTSENTRY']._serialized_end=2129
+  _globals['_MODELDEFINITION_STATICATTRIBUTESENTRY']._serialized_start=2131
+  _globals['_MODELDEFINITION_STATICATTRIBUTESENTRY']._serialized_end=2211
 # @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/model_pb2.pyi

@@ -1,5 +1,7 @@
 from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as _type_pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as _object_id_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import data_pb2 as _data_pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import file_pb2 as _file_pb2
 from google.protobuf.internal import containers as _containers
 from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
 from google.protobuf import descriptor as _descriptor
@@ -37,7 +39,7 @@ class ModelParameter(_message.Message):
     def __init__(self, paramType: _Optional[_Union[_type_pb2.TypeDescriptor, _Mapping]] = ..., label: _Optional[str] = ..., defaultValue: _Optional[_Union[_type_pb2.Value, _Mapping]] = ..., paramProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
 
 class ModelInputSchema(_message.Message):
-    __slots__ = ("schema", "label", "optional", "dynamic", "inputProps")
+    __slots__ = ("objectType", "schema", "fileType", "label", "optional", "dynamic", "inputProps")
     class InputPropsEntry(_message.Message):
         __slots__ = ("key", "value")
         KEY_FIELD_NUMBER: _ClassVar[int]
@@ -45,20 +47,24 @@ class ModelInputSchema(_message.Message):
         key: str
         value: _type_pb2.Value
         def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[_type_pb2.Value, _Mapping]] = ...) -> None: ...
+    OBJECTTYPE_FIELD_NUMBER: _ClassVar[int]
     SCHEMA_FIELD_NUMBER: _ClassVar[int]
+    FILETYPE_FIELD_NUMBER: _ClassVar[int]
     LABEL_FIELD_NUMBER: _ClassVar[int]
     OPTIONAL_FIELD_NUMBER: _ClassVar[int]
     DYNAMIC_FIELD_NUMBER: _ClassVar[int]
     INPUTPROPS_FIELD_NUMBER: _ClassVar[int]
+    objectType: _object_id_pb2.ObjectType
     schema: _data_pb2.SchemaDefinition
+    fileType: _file_pb2.FileType
     label: str
     optional: bool
     dynamic: bool
     inputProps: _containers.MessageMap[str, _type_pb2.Value]
-    def __init__(self, schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., label: _Optional[str] = ..., optional: bool = ..., dynamic: bool = ..., inputProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
+    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., fileType: _Optional[_Union[_file_pb2.FileType, _Mapping]] = ..., label: _Optional[str] = ..., optional: bool = ..., dynamic: bool = ..., inputProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
 
 class ModelOutputSchema(_message.Message):
-    __slots__ = ("schema", "label", "optional", "dynamic", "outputProps")
+    __slots__ = ("objectType", "schema", "fileType", "label", "optional", "dynamic", "outputProps")
     class OutputPropsEntry(_message.Message):
         __slots__ = ("key", "value")
         KEY_FIELD_NUMBER: _ClassVar[int]
@@ -66,17 +72,21 @@ class ModelOutputSchema(_message.Message):
         key: str
         value: _type_pb2.Value
         def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[_type_pb2.Value, _Mapping]] = ...) -> None: ...
+    OBJECTTYPE_FIELD_NUMBER: _ClassVar[int]
     SCHEMA_FIELD_NUMBER: _ClassVar[int]
+    FILETYPE_FIELD_NUMBER: _ClassVar[int]
     LABEL_FIELD_NUMBER: _ClassVar[int]
     OPTIONAL_FIELD_NUMBER: _ClassVar[int]
     DYNAMIC_FIELD_NUMBER: _ClassVar[int]
     OUTPUTPROPS_FIELD_NUMBER: _ClassVar[int]
+    objectType: _object_id_pb2.ObjectType
     schema: _data_pb2.SchemaDefinition
+    fileType: _file_pb2.FileType
     label: str
     optional: bool
     dynamic: bool
     outputProps: _containers.MessageMap[str, _type_pb2.Value]
-    def __init__(self, schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., label: _Optional[str] = ..., optional: bool = ..., dynamic: bool = ..., outputProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
+    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., fileType: _Optional[_Union[_file_pb2.FileType, _Mapping]] = ..., label: _Optional[str] = ..., optional: bool = ..., dynamic: bool = ..., outputProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
 
 class ModelDefinition(_message.Message):
     __slots__ = ("language", "repository", "packageGroup", "package", "version", "entryPoint", "path", "parameters", "inputs", "outputs", "staticAttributes", "modelType")

tracdap/rt/_impl/models.py

@@ -226,13 +226,15 @@ class ModelLoader:
                 self.__log.info(f"Parameter [{name}] - {param.paramType.basicType.name}")
                 param.paramProps = self._encoded_props(param.paramProps, "parameter", name)
 
-            for name, schema in model_def.inputs.items():
-                self.__log.info(f"Input [{name}] - {schema.schema.schemaType.name}")
-                schema.inputProps = self._encoded_props(schema.inputProps, "input", name)
-
-            for name, schema in model_def.outputs.items():
-                self.__log.info(f"Output [{name}] - {schema.schema.schemaType.name}")
-                schema.outputProps = self._encoded_props(schema.outputProps, "input", name)
+            for name, input_def in model_def.inputs.items():
+                input_type = input_def.schema.schemaType.name if input_def.objectType == _meta.ObjectType.DATA else input_def.objectType.name
+                self.__log.info(f"Input [{name}] - {input_type}")
+                input_def.inputProps = self._encoded_props(input_def.inputProps, "input", name)
+
+            for name, output_def in model_def.outputs.items():
+                output_type = output_def.schema.schemaType.name if output_def.objectType == _meta.ObjectType.DATA else output_def.objectType.name
+                self.__log.info(f"Output [{name}] - {output_type}")
+                output_def.outputProps = self._encoded_props(output_def.outputProps, "input", name)
 
             return model_def
 

tracdap/rt/_impl/static_api.py

@@ -152,14 +152,18 @@ class StaticApiImpl(_StaticApiHook):
 
     def define_schema(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE) \
+            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE, dynamic: bool = False) \
             -> _meta.SchemaDefinition:
 
-        _val.validate_signature(self.define_schema, *fields, schema_type=schema_type)
+        _val.validate_signature(self.define_schema, *fields, schema_type=schema_type, dynamic=dynamic)
 
         if schema_type == _meta.SchemaType.TABLE:
 
-            table_schema = self._build_table_schema(*fields)
+            if dynamic and not fields:
+                table_schema = None
+            else:
+                table_schema = self._build_table_schema(*fields)
+
             return _meta.SchemaDefinition(_meta.SchemaType.TABLE, table=table_schema)
 
         raise _ex.ERuntimeValidation(f"Invalid schema type [{schema_type.name}]")
@@ -182,51 +186,67 @@ class StaticApiImpl(_StaticApiHook):
 
         return converter.infer_schema(dataset)
 
-    def define_input_table(
-            self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
-            input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
-            -> _meta.ModelInputSchema:
+    def define_file_type(self, extension: str, mime_type: str) -> _meta.FileType:
+
+        _val.validate_signature(self.define_file_type, extension, mime_type)
+
+        return _meta.FileType(extension=extension, mimeType=mime_type)
+
+    def define_input(
+            self, requirement: _tp.Union[_meta.SchemaDefinition, _meta.FileType], *,
+            label: _tp.Optional[str] = None,
+            optional: bool = False, dynamic: bool = False,
+            input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None):
 
         _val.validate_signature(
-            self.define_input_table, *fields,
+            self.define_input, requirement,
             label=label, optional=optional, dynamic=dynamic,
             input_props=input_props)
 
-        # Do not define details for dynamic schemas
+        if isinstance(requirement, _meta.SchemaDefinition):
 
-        if dynamic:
-            schema_def = _meta.SchemaDefinition(_meta.SchemaType.TABLE)
-        else:
-            schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
+            return _meta.ModelInputSchema(
+                objectType=_meta.ObjectType.DATA, schema=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                inputProps=input_props)
 
-        return _meta.ModelInputSchema(
-            schema=schema_def, label=label,
-            optional=optional, dynamic=dynamic,
-            inputProps=input_props)
+        elif isinstance(requirement, _meta.FileType):
 
-    def define_output_table(
-            self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
-            output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
-            -> _meta.ModelOutputSchema:
+            return _meta.ModelInputSchema(
+                objectType=_meta.ObjectType.FILE, fileType=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                inputProps=input_props)
+
+        else:
+            raise _ex.EUnexpected()
+
+    def define_output(
+            self, requirement: _tp.Union[_meta.SchemaDefinition, _meta.FileType], *,
+            label: _tp.Optional[str] = None,
+            optional: bool = False, dynamic: bool = False,
+            output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None):
 
         _val.validate_signature(
-            self.define_output_table, *fields,
+            self.define_output, requirement,
             label=label, optional=optional, dynamic=dynamic,
             output_props=output_props)
 
-        # Do not define details for dynamic schemas
+        if isinstance(requirement, _meta.SchemaDefinition):
 
-        if dynamic:
-            schema_def = _meta.SchemaDefinition(_meta.SchemaType.TABLE)
-        else:
-            schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
+            return _meta.ModelOutputSchema(
+                objectType=_meta.ObjectType.DATA, schema=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                outputProps=output_props)
 
-        return _meta.ModelOutputSchema(
-            schema=schema_def, label=label,
-            optional=optional, dynamic=dynamic,
-            outputProps=output_props)
+        elif isinstance(requirement, _meta.FileType):
+
+            return _meta.ModelOutputSchema(
+                objectType=_meta.ObjectType.FILE, fileType=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                outputProps=output_props)
+
+        else:
+            raise _ex.EUnexpected()
 
     @staticmethod
     def _build_named_dict(

tracdap/rt/_impl/util.py

@@ -235,7 +235,7 @@ def get_job_resource(
     if optional:
         return None
 
-    err = f"Missing required {selector.objectType} resource [{object_key(selector)}]"
+    err = f"Missing required {selector.objectType.name} resource [{object_key(selector)}]"
     raise ex.ERuntimeValidation(err)
 
 

tracdap/rt/_impl/validation.py

@@ -306,6 +306,9 @@ class StaticValidator:
     __reserved_identifier_pattern = re.compile("\\A(_|trac_)", re.ASCII)
     __label_length_limit = 4096
 
+    __file_extension_pattern = re.compile('\\A[a-zA-Z0-9]+\\Z')
+    __mime_type_pattern = re.compile('\\A\\w+/[-.\\w]+(?:\\+[-.\\w]+)?\\Z')
+
     __PRIMITIVE_TYPES = [
         meta.BasicType.BOOLEAN,
         meta.BasicType.INTEGER,
@@ -418,49 +421,72 @@ class StaticValidator:
                 cls._valid_identifiers(param.paramProps.keys(), "entry in param props")
 
     @classmethod
-    def _check_inputs_or_outputs(cls, inputs_or_outputs):
+    def _check_inputs_or_outputs(cls, sockets):
 
-        for input_name, input_schema in inputs_or_outputs.items():
+        for socket_name, socket in sockets.items():
 
-            cls._log.info(f"Checking {input_name}")
+            if socket.objectType == meta.ObjectType.DATA:
+                cls._check_socket_schema(socket_name, socket)
+            elif socket.objectType == meta.ObjectType.FILE:
+                cls._check_socket_file_type(socket_name, socket)
+            else:
+                raise ex.EModelValidation(f"Invalid object type [{socket.objectType.name}] for [{socket_name}]")
 
-            if input_schema.dynamic:
-                if input_schema.schema and input_schema.schema.table:
-                    error = "Dynamic schemas must have schema.table = None"
-                    cls._fail(f"Invalid schema for [{input_name}]: {error}")
-                else:
-                    continue
+            label = socket.label
+            cls._check_label(label, socket_name)
+
+            if isinstance(socket, meta.ModelInputSchema):
+                if socket.inputProps is not None:
+                    cls._valid_identifiers(socket.inputProps.keys(), "entry in input props")
+            else:
+                if socket.outputProps is not None:
+                    cls._valid_identifiers(socket.outputProps.keys(), "entry in output props")
 
-            fields = input_schema.schema.table.fields
-            field_names = list(map(lambda f: f.fieldName, fields))
-            property_type = f"field in [{input_name}]"
+    @classmethod
+    def _check_socket_schema(cls, socket_name, socket):
 
-            if len(fields) == 0:
-                cls._fail(f"Invalid schema for [{input_name}]: No fields defined")
+        if socket.schema is None:
+            cls._fail(f"Missing schema requirement for [{socket_name}]")
+            return
 
-            cls._valid_identifiers(field_names, property_type)
-            cls._case_insensitive_duplicates(field_names, property_type)
+        if socket.dynamic:
+            if socket.schema and socket.schema.table:
+                error = "Dynamic schemas must have schema.table = None"
+                cls._fail(f"Invalid schema for [{socket_name}]: {error}")
+            else:
+                return
 
-            for field in fields:
-                cls._check_single_field(field, property_type)
+        fields = socket.schema.table.fields
+        field_names = list(map(lambda f: f.fieldName, fields))
+        property_type = f"field in [{socket_name}]"
 
-            label = input_schema.label
-            cls._check_label(label, input_name)
+        if len(fields) == 0:
+            cls._fail(f"Invalid schema for [{socket_name}]: No fields defined")
 
-            if isinstance(input_schema, meta.ModelInputSchema):
-                if input_schema.inputProps is not None:
-                    cls._valid_identifiers(input_schema.inputProps.keys(), "entry in input props")
-            else:
-                if input_schema.outputProps is not None:
-                    cls._valid_identifiers(input_schema.outputProps.keys(), "entry in output props")
+        cls._valid_identifiers(field_names, property_type)
+        cls._case_insensitive_duplicates(field_names, property_type)
+
+        for field in fields:
+            cls._check_single_field(field, property_type)
+
+    @classmethod
+    def _check_socket_file_type(cls, socket_name, socket):
+
+        if socket.fileType is None:
+            cls._fail(f"Missing file type requirement for [{socket_name}]")
+            return
+
+        if not cls.__file_extension_pattern.match(socket.fileType.extension):
+            cls._fail(f"Invalid extension [{socket.fileType.extension}] for [{socket_name}]")
+
+        if not cls.__mime_type_pattern.match(socket.fileType.mimeType):
+            cls._fail(f"Invalid mime type [{socket.fileType.mimeType}] for [{socket_name}]")
 
     @classmethod
     def _check_single_field(cls, field: meta.FieldSchema, property_type):
 
         # Valid identifier and not trac reserved checked separately
 
-        cls._log.info(field.fieldName)
-
         if field.fieldOrder < 0:
             cls._fail(f"Invalid {property_type}: [{field.fieldName}] fieldOrder < 0")
 

tracdap/rt/_version.py

@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.7.0rc1"
+__version__ = "0.8.0b1"

tracdap/rt/api/__init__.py

@@ -17,13 +17,16 @@
 TRAC model API for Python
 """
 
-from .model_api import *
-from .static_api import *
-
 # Make metadata classes available to client code when importing the API package
 # Remove this import when generating docs, so metadata classes are only documented once
 from tracdap.rt.metadata import *  # noqa DOCGEN_REMOVE
 
+# static_api overrides some metadata types for backwards compatibility with pre-0.8 versions
+# Make sure it is last in the list
+from .file_types import *
+from .model_api import *
+from .static_api import *
+
 # Map basic types into the root of the API package
 
 BOOLEAN = BasicType.BOOLEAN