tracdap-runtime 0.5.30__py3-none-any.whl → 0.6.0__py3-none-any.whl

tracdap/rt/exceptions.py

@@ -200,6 +200,15 @@ class EStorageConfig(EStorage):
     pass
 
 
+class EStorageValidation(EStorage):
+
+    """
+    Storage request is invalid or not allowed (e.g. illegal chars in storage path or path escapes storage root)
+    """
+
+    pass
+
+
 class EStorageCommunication(EStorage):
 
     """

tracdap/rt/ext/plugins.py

@@ -11,18 +11,6 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 import typing as _tp
 import logging as _log

tracdap/rt/ext/storage.py

@@ -19,6 +19,7 @@ import enum as _enum
 import typing as _tp
 
 import pyarrow as _pa
+import pyarrow.fs as _afs
 
 
 class FileType(_enum.Enum):
@@ -36,75 +37,71 @@ class FileStat:
     Timestamps are held in UTC
     """
 
+    file_name: str
     file_type: FileType
+    storage_path: str
     size: int
 
-    ctime: _tp.Optional[_dt.datetime] = None
     mtime: _tp.Optional[_dt.datetime] = None
     atime: _tp.Optional[_dt.datetime] = None
 
-    uid: _tp.Optional[int] = None
-    gid: _tp.Optional[int] = None
-    mode: _tp.Optional[int] = None
-
 
 class IFileStorage:
 
     @_abc.abstractmethod
     def exists(self, storage_path: str) -> bool:
+        """The exists method can be used for both files and directories"""
         pass
 
     @_abc.abstractmethod
     def size(self, storage_path: str) -> int:
+        """The rm method only works on regular files, it cannot be used for directories"""
         pass
 
     @_abc.abstractmethod
     def stat(self, storage_path: str) -> FileStat:
+        """The stat method can be used for both files and directories, so long as they exist"""
         pass
 
     @_abc.abstractmethod
-    def ls(self, storage_path: str) -> _tp.List[str]:
-        pass
-
-    @_abc.abstractmethod
-    def mkdir(self, storage_path: str, recursive: bool = False, exists_ok: bool = False):
+    def ls(self, storage_path: str, recursive: bool = False) -> _tp.List[FileStat]:
+        """The ls method only works on directories, it cannot be used for regular files"""
         pass
 
     @_abc.abstractmethod
-    def rm(self, storage_path: str, recursive: bool = False):
-        pass
-
-    @_abc.abstractmethod
-    def read_bytes(self, storage_path: str) -> bytes:
+    def mkdir(self, storage_path: str, recursive: bool = False):
+        """The mkdir method will succeed silently if the directory already exists"""
         pass
 
     @_abc.abstractmethod
-    def read_byte_stream(self, storage_path: str) -> _tp.BinaryIO:
+    def rm(self, storage_path: str):
+        """The rm method only works on regular files, it cannot be used for directories and is not recursive"""
         pass
 
     @_abc.abstractmethod
-    def write_bytes(self, storage_path: str, data: bytes, overwrite: bool = False):
+    def rmdir(self, storage_path: str):
+        """The rmdir method only works on directories and is always recursive"""
         pass
 
     @_abc.abstractmethod
-    def write_byte_stream(self, storage_path: str, overwrite: bool = False) -> _tp.BinaryIO:
+    def read_byte_stream(self, storage_path: str) -> _tp.ContextManager[_tp.BinaryIO]:
+        """The read_byte_stream method only works for existing files"""
         pass
 
     @_abc.abstractmethod
-    def read_text(self, storage_path: str, encoding: str = 'utf-8') -> str:
+    def write_byte_stream(self, storage_path: str) -> _tp.ContextManager[_tp.BinaryIO]:
+        """The write_byte_stream method will always overwrite an existing file if it exists"""
         pass
 
-    @_abc.abstractmethod
-    def read_text_stream(self, storage_path: str, encoding: str = 'utf-8') -> _tp.TextIO:
-        pass
-
-    @_abc.abstractmethod
-    def write_text(self, storage_path: str, data: str, encoding: str = 'utf-8', overwrite: bool = False):
-        pass
+    def read_bytes(self, storage_path: str) -> bytes:
+        """The read_bytes method only works for existing files"""
+        with self.read_byte_stream(storage_path) as stream:
+            return stream.read()
 
-    @_abc.abstractmethod
-    def write_text_stream(self, storage_path: str, encoding: str = 'utf-8', overwrite: bool = False) -> _tp.TextIO:
-        pass
+    def write_bytes(self, storage_path: str, data: bytes):
+        """The write_bytes method will always overwrite an existing file if it exists"""
+        with self.write_byte_stream(storage_path) as stream:
+            stream.write(data)
 
 
 class IDataStorage:
@@ -128,6 +125,27 @@ class IDataStorage:
         pass
 
 
+class IStorageProvider(_abc.ABC):
+
+    def has_arrow_native(self) -> bool:
+        pass
+
+    def has_file_storage(self) -> bool:
+        pass
+
+    def has_data_storage(self) -> bool:
+        pass
+
+    def get_arrow_native(self) -> _afs.SubTreeFileSystem:
+        pass
+
+    def get_file_storage(self) -> IFileStorage:
+        pass
+
+    def get_data_storage(self) -> IDataStorage:
+        pass
+
+
 class IDataFormat:
 
     @_abc.abstractmethod

tracdap/rt/metadata/common_pb2.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/metadata/common.proto
+# Protobuf Python Version: 4.25.3
 """Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -15,18 +16,18 @@ _sym_db = _symbol_database.Default()
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dtracdap/metadata/common.proto\x12\x10tracdap.metadata\"5\n\nTenantInfo\x12\x12\n\ntenantCode\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t*L\n\x0eMetadataFormat\x12\x1b\n\x17METADATA_FORMAT_NOT_SET\x10\x00\x12\t\n\x05PROTO\x10\x01\x12\x08\n\x04JSON\x10\x02\x12\x08\n\x04YAML\x10\x03*H\n\x0fMetadataVersion\x12\x1c\n\x18METADATA_VERSION_NOT_SET\x10\x00\x12\x06\n\x02V1\x10\x01\x12\x0b\n\x07\x43URRENT\x10\x01\x1a\x02\x10\x01\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.common_pb2', globals())
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.common_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-
-  DESCRIPTOR._options = None
-  DESCRIPTOR._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
-  _METADATAVERSION._options = None
-  _METADATAVERSION._serialized_options = b'\020\001'
-  _METADATAFORMAT._serialized_start=106
-  _METADATAFORMAT._serialized_end=182
-  _METADATAVERSION._serialized_start=184
-  _METADATAVERSION._serialized_end=256
-  _TENANTINFO._serialized_start=51
-  _TENANTINFO._serialized_end=104
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
+  _globals['_METADATAVERSION']._options = None
+  _globals['_METADATAVERSION']._serialized_options = b'\020\001'
+  _globals['_METADATAFORMAT']._serialized_start=106
+  _globals['_METADATAFORMAT']._serialized_end=182
+  _globals['_METADATAVERSION']._serialized_start=184
+  _globals['_METADATAVERSION']._serialized_end=256
+  _globals['_TENANTINFO']._serialized_start=51
+  _globals['_TENANTINFO']._serialized_end=104
 # @@protoc_insertion_point(module_scope)

tracdap/rt/metadata/custom_pb2.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/metadata/custom.proto
+# Protobuf Python Version: 4.25.3
 """Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -15,12 +16,12 @@ _sym_db = _symbol_database.Default()
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dtracdap/metadata/custom.proto\x12\x10tracdap.metadata\"]\n\x10\x43ustomDefinition\x12\x18\n\x10\x63ustomSchemaType\x18\x01 \x01(\t\x12\x1b\n\x13\x63ustomSchemaVersion\x18\x02 \x01(\r\x12\x12\n\ncustomData\x18\x03 \x01(\x0c\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.custom_pb2', globals())
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.custom_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-
-  DESCRIPTOR._options = None
-  DESCRIPTOR._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
-  _CUSTOMDEFINITION._serialized_start=51
-  _CUSTOMDEFINITION._serialized_end=144
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
+  _globals['_CUSTOMDEFINITION']._serialized_start=51
+  _globals['_CUSTOMDEFINITION']._serialized_end=144
 # @@protoc_insertion_point(module_scope)

tracdap/rt/metadata/data_pb2.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/metadata/data.proto
+# Protobuf Python Version: 4.25.3
 """Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -17,34 +18,34 @@ from tracdap.metadata import object_id_pb2 as tracdap_dot_metadata_dot_object__i
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1btracdap/metadata/data.proto\x12\x10tracdap.metadata\x1a\x1btracdap/metadata/type.proto\x1a tracdap/metadata/object_id.proto\"\xe7\x01\n\x0b\x46ieldSchema\x12\x11\n\tfieldName\x18\x01 \x01(\t\x12\x12\n\nfieldOrder\x18\x02 \x01(\x11\x12.\n\tfieldType\x18\x03 \x01(\x0e\x32\x1b.tracdap.metadata.BasicType\x12\r\n\x05label\x18\x04 \x01(\t\x12\x13\n\x0b\x62usinessKey\x18\x05 \x01(\x08\x12\x13\n\x0b\x63\x61tegorical\x18\x06 \x01(\x08\x12\x14\n\x07notNull\x18\x08 \x01(\x08H\x00\x88\x01\x01\x12\x17\n\nformatCode\x18\x07 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_notNullB\r\n\x0b_formatCode\"<\n\x0bTableSchema\x12-\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.FieldSchema\"\xba\x01\n\x10SchemaDefinition\x12\x30\n\nschemaType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.SchemaType\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12.\n\x05table\x18\x03 \x01(\x0b\x32\x1d.tracdap.metadata.TableSchemaH\x00\x42\x16\n\x14schemaTypeDefinition\"\x81\x02\n\x07PartKey\x12\x11\n\topaqueKey\x18\x01 \x01(\t\x12,\n\x08partType\x18\x02 \x01(\x0e\x32\x1a.tracdap.metadata.PartType\x12+\n\npartValues\x18\x03 \x03(\x0b\x32\x17.tracdap.metadata.Value\x12\x32\n\x0cpartRangeMin\x18\x04 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x00\x88\x01\x01\x12\x32\n\x0cpartRangeMax\x18\x05 \x01(\x0b\x32\x17.tracdap.metadata.ValueH\x01\x88\x01\x01\x42\x0f\n\r_partRangeMinB\x0f\n\r_partRangeMax\"\xba\x04\n\x0e\x44\x61taDefinition\x12\x31\n\x08schemaId\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelectorH\x00\x12\x34\n\x06schema\x18\x02 \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12:\n\x05parts\x18\x03 \x03(\x0b\x32+.tracdap.metadata.DataDefinition.PartsEntry\x12\x30\n\tstorageId\x18\x04 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x1a-\n\x05\x44\x65lta\x12\x12\n\ndeltaIndex\x18\x01 \x01(\r\x12\x10\n\x08\x64\x61taItem\x18\x02 \x01(\t\x1aQ\n\x04Snap\x12\x11\n\tsnapIndex\x18\x01 \x01(\r\x12\x36\n\x06\x64\x65ltas\x18\x02 \x03(\x0b\x32&.tracdap.metadata.DataDefinition.Delta\x1ag\n\x04Part\x12*\n\x07partKey\x18\x01 \x01(\x0b\x32\x19.tracdap.metadata.PartKey\x12\x33\n\x04snap\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Snap\x1aS\n\nPartsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x34\n\x05value\x18\x02 \x01(\x0b\x32%.tracdap.metadata.DataDefinition.Part:\x02\x38\x01\x42\x11\n\x0fschemaSpecifier*0\n\nSchemaType\x12\x17\n\x13SCHEMA_TYPE_NOT_SET\x10\x00\x12\t\n\x05TABLE\x10\x01*?\n\x08PartType\x12\r\n\tPART_ROOT\x10\x00\x12\x11\n\rPART_BY_RANGE\x10\x01\x12\x11\n\rPART_BY_VALUE\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.data_pb2', globals())
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.data_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-
-  DESCRIPTOR._options = None
-  DESCRIPTOR._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
-  _DATADEFINITION_PARTSENTRY._options = None
-  _DATADEFINITION_PARTSENTRY._serialized_options = b'8\001'
-  _SCHEMATYPE._serialized_start=1430
-  _SCHEMATYPE._serialized_end=1478
-  _PARTTYPE._serialized_start=1480
-  _PARTTYPE._serialized_end=1543
-  _FIELDSCHEMA._serialized_start=113
-  _FIELDSCHEMA._serialized_end=344
-  _TABLESCHEMA._serialized_start=346
-  _TABLESCHEMA._serialized_end=406
-  _SCHEMADEFINITION._serialized_start=409
-  _SCHEMADEFINITION._serialized_end=595
-  _PARTKEY._serialized_start=598
-  _PARTKEY._serialized_end=855
-  _DATADEFINITION._serialized_start=858
-  _DATADEFINITION._serialized_end=1428
-  _DATADEFINITION_DELTA._serialized_start=1091
-  _DATADEFINITION_DELTA._serialized_end=1136
-  _DATADEFINITION_SNAP._serialized_start=1138
-  _DATADEFINITION_SNAP._serialized_end=1219
-  _DATADEFINITION_PART._serialized_start=1221
-  _DATADEFINITION_PART._serialized_end=1324
-  _DATADEFINITION_PARTSENTRY._serialized_start=1326
-  _DATADEFINITION_PARTSENTRY._serialized_end=1409
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
+  _globals['_DATADEFINITION_PARTSENTRY']._options = None
+  _globals['_DATADEFINITION_PARTSENTRY']._serialized_options = b'8\001'
+  _globals['_SCHEMATYPE']._serialized_start=1430
+  _globals['_SCHEMATYPE']._serialized_end=1478
+  _globals['_PARTTYPE']._serialized_start=1480
+  _globals['_PARTTYPE']._serialized_end=1543
+  _globals['_FIELDSCHEMA']._serialized_start=113
+  _globals['_FIELDSCHEMA']._serialized_end=344
+  _globals['_TABLESCHEMA']._serialized_start=346
+  _globals['_TABLESCHEMA']._serialized_end=406
+  _globals['_SCHEMADEFINITION']._serialized_start=409
+  _globals['_SCHEMADEFINITION']._serialized_end=595
+  _globals['_PARTKEY']._serialized_start=598
+  _globals['_PARTKEY']._serialized_end=855
+  _globals['_DATADEFINITION']._serialized_start=858
+  _globals['_DATADEFINITION']._serialized_end=1428
+  _globals['_DATADEFINITION_DELTA']._serialized_start=1091
+  _globals['_DATADEFINITION_DELTA']._serialized_end=1136
+  _globals['_DATADEFINITION_SNAP']._serialized_start=1138
+  _globals['_DATADEFINITION_SNAP']._serialized_end=1219
+  _globals['_DATADEFINITION_PART']._serialized_start=1221
+  _globals['_DATADEFINITION_PART']._serialized_end=1324
+  _globals['_DATADEFINITION_PARTSENTRY']._serialized_start=1326
+  _globals['_DATADEFINITION_PARTSENTRY']._serialized_end=1409
 # @@protoc_insertion_point(module_scope)

tracdap/rt/metadata/file_pb2.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/metadata/file.proto
+# Protobuf Python Version: 4.25.3
 """Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -16,12 +17,12 @@ from tracdap.metadata import object_id_pb2 as tracdap_dot_metadata_dot_object__i
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1btracdap/metadata/file.proto\x12\x10tracdap.metadata\x1a tracdap/metadata/object_id.proto\"\x95\x01\n\x0e\x46ileDefinition\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\textension\x18\x02 \x01(\t\x12\x10\n\x08mimeType\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x04\x12\x30\n\tstorageId\x18\x05 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x12\x10\n\x08\x64\x61taItem\x18\x06 \x01(\tB\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.file_pb2', globals())
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.file_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-
-  DESCRIPTOR._options = None
-  DESCRIPTOR._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
-  _FILEDEFINITION._serialized_start=84
-  _FILEDEFINITION._serialized_end=233
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
+  _globals['_FILEDEFINITION']._serialized_start=84
+  _globals['_FILEDEFINITION']._serialized_end=233
 # @@protoc_insertion_point(module_scope)

tracdap/rt/metadata/flow_pb2.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/metadata/flow.proto
+# Protobuf Python Version: 4.25.3
 """Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -19,36 +20,36 @@ from tracdap.metadata import tag_update_pb2 as tracdap_dot_metadata_dot_tag__upd
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1btracdap/metadata/flow.proto\x12\x10tracdap.metadata\x1a\x1btracdap/metadata/type.proto\x1a\x1ctracdap/metadata/model.proto\x1a\x1dtracdap/metadata/search.proto\x1a!tracdap/metadata/tag_update.proto\"\xe1\x01\n\x08\x46lowNode\x12\x30\n\x08nodeType\x18\x01 \x01(\x0e\x32\x1e.tracdap.metadata.FlowNodeType\x12\x0e\n\x06inputs\x18\x02 \x03(\t\x12\x0f\n\x07outputs\x18\x03 \x03(\t\x12\x36\n\nnodeSearch\x18\x04 \x01(\x0b\x32\".tracdap.metadata.SearchExpression\x12.\n\tnodeAttrs\x18\x05 \x03(\x0b\x32\x1b.tracdap.metadata.TagUpdate\x12\r\n\x05label\x18\x06 \x01(\tR\x0bparameters;\"*\n\nFlowSocket\x12\x0c\n\x04node\x18\x01 \x01(\t\x12\x0e\n\x06socket\x18\x02 \x01(\t\"f\n\x08\x46lowEdge\x12,\n\x06source\x18\x01 \x01(\x0b\x32\x1c.tracdap.metadata.FlowSocket\x12,\n\x06target\x18\x02 \x01(\x0b\x32\x1c.tracdap.metadata.FlowSocket\"\x82\x05\n\x0e\x46lowDefinition\x12:\n\x05nodes\x18\x01 \x03(\x0b\x32+.tracdap.metadata.FlowDefinition.NodesEntry\x12)\n\x05\x65\x64ges\x18\x02 \x03(\x0b\x32\x1a.tracdap.metadata.FlowEdge\x12\x44\n\nparameters\x18\x03 \x03(\x0b\x32\x30.tracdap.metadata.FlowDefinition.ParametersEntry\x12<\n\x06inputs\x18\x04 \x03(\x0b\x32,.tracdap.metadata.FlowDefinition.InputsEntry\x12>\n\x07outputs\x18\x05 \x03(\x0b\x32-.tracdap.metadata.FlowDefinition.OutputsEntry\x1aH\n\nNodesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.tracdap.metadata.FlowNode:\x02\x38\x01\x1aS\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .tracdap.metadata.ModelParameter:\x02\x38\x01\x1aQ\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".tracdap.metadata.ModelInputSchema:\x02\x38\x01\x1aS\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x32\n\x05value\x18\x02 \x01(\x0b\x32#.tracdap.metadata.ModelOutputSchema:\x02\x38\x01*V\n\x0c\x46lowNodeType\x12\x15\n\x11NODE_TYPE_NOT_SET\x10\x00\x12\x0e\n\nINPUT_NODE\x10\x01\x12\x0f\n\x0bOUTPUT_NODE\x10\x02\x12\x0e\n\nMODEL_NODE\x10\x03\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.flow_pb2', globals())
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.flow_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-
-  DESCRIPTOR._options = None
-  DESCRIPTOR._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
-  _FLOWDEFINITION_NODESENTRY._options = None
-  _FLOWDEFINITION_NODESENTRY._serialized_options = b'8\001'
-  _FLOWDEFINITION_PARAMETERSENTRY._options = None
-  _FLOWDEFINITION_PARAMETERSENTRY._serialized_options = b'8\001'
-  _FLOWDEFINITION_INPUTSENTRY._options = None
-  _FLOWDEFINITION_INPUTSENTRY._serialized_options = b'8\001'
-  _FLOWDEFINITION_OUTPUTSENTRY._options = None
-  _FLOWDEFINITION_OUTPUTSENTRY._serialized_options = b'8\001'
-  _FLOWNODETYPE._serialized_start=1195
-  _FLOWNODETYPE._serialized_end=1281
-  _FLOWNODE._serialized_start=175
-  _FLOWNODE._serialized_end=400
-  _FLOWSOCKET._serialized_start=402
-  _FLOWSOCKET._serialized_end=444
-  _FLOWEDGE._serialized_start=446
-  _FLOWEDGE._serialized_end=548
-  _FLOWDEFINITION._serialized_start=551
-  _FLOWDEFINITION._serialized_end=1193
-  _FLOWDEFINITION_NODESENTRY._serialized_start=868
-  _FLOWDEFINITION_NODESENTRY._serialized_end=940
-  _FLOWDEFINITION_PARAMETERSENTRY._serialized_start=942
-  _FLOWDEFINITION_PARAMETERSENTRY._serialized_end=1025
-  _FLOWDEFINITION_INPUTSENTRY._serialized_start=1027
-  _FLOWDEFINITION_INPUTSENTRY._serialized_end=1108
-  _FLOWDEFINITION_OUTPUTSENTRY._serialized_start=1110
-  _FLOWDEFINITION_OUTPUTSENTRY._serialized_end=1193
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
+  _globals['_FLOWDEFINITION_NODESENTRY']._options = None
+  _globals['_FLOWDEFINITION_NODESENTRY']._serialized_options = b'8\001'
+  _globals['_FLOWDEFINITION_PARAMETERSENTRY']._options = None
+  _globals['_FLOWDEFINITION_PARAMETERSENTRY']._serialized_options = b'8\001'
+  _globals['_FLOWDEFINITION_INPUTSENTRY']._options = None
+  _globals['_FLOWDEFINITION_INPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_FLOWDEFINITION_OUTPUTSENTRY']._options = None
+  _globals['_FLOWDEFINITION_OUTPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_FLOWNODETYPE']._serialized_start=1195
+  _globals['_FLOWNODETYPE']._serialized_end=1281
+  _globals['_FLOWNODE']._serialized_start=175
+  _globals['_FLOWNODE']._serialized_end=400
+  _globals['_FLOWSOCKET']._serialized_start=402
+  _globals['_FLOWSOCKET']._serialized_end=444
+  _globals['_FLOWEDGE']._serialized_start=446
+  _globals['_FLOWEDGE']._serialized_end=548
+  _globals['_FLOWDEFINITION']._serialized_start=551
+  _globals['_FLOWDEFINITION']._serialized_end=1193
+  _globals['_FLOWDEFINITION_NODESENTRY']._serialized_start=868
+  _globals['_FLOWDEFINITION_NODESENTRY']._serialized_end=940
+  _globals['_FLOWDEFINITION_PARAMETERSENTRY']._serialized_start=942
+  _globals['_FLOWDEFINITION_PARAMETERSENTRY']._serialized_end=1025
+  _globals['_FLOWDEFINITION_INPUTSENTRY']._serialized_start=1027
+  _globals['_FLOWDEFINITION_INPUTSENTRY']._serialized_end=1108
+  _globals['_FLOWDEFINITION_OUTPUTSENTRY']._serialized_start=1110
+  _globals['_FLOWDEFINITION_OUTPUTSENTRY']._serialized_end=1193
 # @@protoc_insertion_point(module_scope)

tracdap/rt/metadata/job_pb2.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/metadata/job.proto
+# Protobuf Python Version: 4.25.3
 """Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -18,58 +19,58 @@ from tracdap.metadata import tag_update_pb2 as tracdap_dot_metadata_dot_tag__upd
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1atracdap/metadata/job.proto\x12\x10tracdap.metadata\x1a\x1btracdap/metadata/type.proto\x1a tracdap/metadata/object_id.proto\x1a!tracdap/metadata/tag_update.proto\"\xe6\x01\n\rJobDefinition\x12*\n\x07jobType\x18\x01 \x01(\x0e\x32\x19.tracdap.metadata.JobType\x12\x31\n\x08runModel\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.RunModelJobH\x00\x12/\n\x07runFlow\x18\x03 \x01(\x0b\x32\x1c.tracdap.metadata.RunFlowJobH\x00\x12\x37\n\x0bimportModel\x18\x04 \x01(\x0b\x32 .tracdap.metadata.ImportModelJobH\x00\x42\x0c\n\njobDetails\"\xac\x05\n\x0bRunModelJob\x12,\n\x05model\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x12\x41\n\nparameters\x18\x02 \x03(\x0b\x32-.tracdap.metadata.RunModelJob.ParametersEntry\x12\x39\n\x06inputs\x18\x03 \x03(\x0b\x32).tracdap.metadata.RunModelJob.InputsEntry\x12;\n\x07outputs\x18\x04 \x03(\x0b\x32*.tracdap.metadata.RunModelJob.OutputsEntry\x12\x45\n\x0cpriorOutputs\x18\x05 \x03(\x0b\x32/.tracdap.metadata.RunModelJob.PriorOutputsEntry\x12\x30\n\x0boutputAttrs\x18\x06 \x03(\x0b\x32\x1b.tracdap.metadata.TagUpdate\x1aJ\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x1aL\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector:\x02\x38\x01\x1aM\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector:\x02\x38\x01\x1aR\n\x11PriorOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector:\x02\x38\x01\"\xae\x06\n\nRunFlowJob\x12+\n\x04\x66low\x18\x01 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector\x12@\n\nparameters\x18\x02 \x03(\x0b\x32,.tracdap.metadata.RunFlowJob.ParametersEntry\x12\x38\n\x06inputs\x18\x03 \x03(\x0b\x32(.tracdap.metadata.RunFlowJob.InputsEntry\x12:\n\x07outputs\x18\x04 \x03(\x0b\x32).tracdap.metadata.RunFlowJob.OutputsEntry\x12\x44\n\x0cpriorOutputs\x18\x05 \x03(\x0b\x32..tracdap.metadata.RunFlowJob.PriorOutputsEntry\x12\x38\n\x06models\x18\x06 \x03(\x0b\x32(.tracdap.metadata.RunFlowJob.ModelsEntry\x12\x30\n\x0boutputAttrs\x18\x07 \x03(\x0b\x32\x1b.tracdap.metadata.TagUpdate\x1aJ\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x1aL\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector:\x02\x38\x01\x1aM\n\x0cOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector:\x02\x38\x01\x1aR\n\x11PriorOutputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector:\x02\x38\x01\x1aL\n\x0bModelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.TagSelector:\x02\x38\x01\"\xd7\x01\n\x0eImportModelJob\x12\x10\n\x08language\x18\x01 \x01(\t\x12\x12\n\nrepository\x18\x02 \x01(\t\x12\x19\n\x0cpackageGroup\x18\x07 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07package\x18\x08 \x01(\t\x12\x0f\n\x07version\x18\x05 \x01(\t\x12\x12\n\nentryPoint\x18\x04 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12/\n\nmodelAttrs\x18\x06 \x03(\x0b\x32\x1b.tracdap.metadata.TagUpdateB\x0f\n\r_packageGroup*_\n\x07JobType\x12\x14\n\x10JOB_TYPE_NOT_SET\x10\x00\x12\r\n\tRUN_MODEL\x10\x01\x12\x0c\n\x08RUN_FLOW\x10\x02\x12\x10\n\x0cIMPORT_MODEL\x10\x03\x12\x0f\n\x0bIMPORT_DATA\x10\x04*\xb8\x01\n\rJobStatusCode\x12\x1b\n\x17JOB_STATUS_CODE_NOT_SET\x10\x00\x12\r\n\tPREPARING\x10\x01\x12\r\n\tVALIDATED\x10\x02\x12\x0b\n\x07PENDING\x10\x03\x12\n\n\x06QUEUED\x10\x04\x12\r\n\tSUBMITTED\x10\x05\x12\x0b\n\x07RUNNING\x10\x06\x12\r\n\tFINISHING\x10\x07\x12\r\n\tSUCCEEDED\x10\x08\x12\n\n\x06\x46\x41ILED\x10\t\x12\r\n\tCANCELLED\x10\nB\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.job_pb2', globals())
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.metadata.job_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-
-  DESCRIPTOR._options = None
-  DESCRIPTOR._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
-  _RUNMODELJOB_PARAMETERSENTRY._options = None
-  _RUNMODELJOB_PARAMETERSENTRY._serialized_options = b'8\001'
-  _RUNMODELJOB_INPUTSENTRY._options = None
-  _RUNMODELJOB_INPUTSENTRY._serialized_options = b'8\001'
-  _RUNMODELJOB_OUTPUTSENTRY._options = None
-  _RUNMODELJOB_OUTPUTSENTRY._serialized_options = b'8\001'
-  _RUNMODELJOB_PRIOROUTPUTSENTRY._options = None
-  _RUNMODELJOB_PRIOROUTPUTSENTRY._serialized_options = b'8\001'
-  _RUNFLOWJOB_PARAMETERSENTRY._options = None
-  _RUNFLOWJOB_PARAMETERSENTRY._serialized_options = b'8\001'
-  _RUNFLOWJOB_INPUTSENTRY._options = None
-  _RUNFLOWJOB_INPUTSENTRY._serialized_options = b'8\001'
-  _RUNFLOWJOB_OUTPUTSENTRY._options = None
-  _RUNFLOWJOB_OUTPUTSENTRY._serialized_options = b'8\001'
-  _RUNFLOWJOB_PRIOROUTPUTSENTRY._options = None
-  _RUNFLOWJOB_PRIOROUTPUTSENTRY._serialized_options = b'8\001'
-  _RUNFLOWJOB_MODELSENTRY._options = None
-  _RUNFLOWJOB_MODELSENTRY._serialized_options = b'8\001'
-  _JOBTYPE._serialized_start=2101
-  _JOBTYPE._serialized_end=2196
-  _JOBSTATUSCODE._serialized_start=2199
-  _JOBSTATUSCODE._serialized_end=2383
-  _JOBDEFINITION._serialized_start=147
-  _JOBDEFINITION._serialized_end=377
-  _RUNMODELJOB._serialized_start=380
-  _RUNMODELJOB._serialized_end=1064
-  _RUNMODELJOB_PARAMETERSENTRY._serialized_start=749
-  _RUNMODELJOB_PARAMETERSENTRY._serialized_end=823
-  _RUNMODELJOB_INPUTSENTRY._serialized_start=825
-  _RUNMODELJOB_INPUTSENTRY._serialized_end=901
-  _RUNMODELJOB_OUTPUTSENTRY._serialized_start=903
-  _RUNMODELJOB_OUTPUTSENTRY._serialized_end=980
-  _RUNMODELJOB_PRIOROUTPUTSENTRY._serialized_start=982
-  _RUNMODELJOB_PRIOROUTPUTSENTRY._serialized_end=1064
-  _RUNFLOWJOB._serialized_start=1067
-  _RUNFLOWJOB._serialized_end=1881
-  _RUNFLOWJOB_PARAMETERSENTRY._serialized_start=749
-  _RUNFLOWJOB_PARAMETERSENTRY._serialized_end=823
-  _RUNFLOWJOB_INPUTSENTRY._serialized_start=825
-  _RUNFLOWJOB_INPUTSENTRY._serialized_end=901
-  _RUNFLOWJOB_OUTPUTSENTRY._serialized_start=903
-  _RUNFLOWJOB_OUTPUTSENTRY._serialized_end=980
-  _RUNFLOWJOB_PRIOROUTPUTSENTRY._serialized_start=982
-  _RUNFLOWJOB_PRIOROUTPUTSENTRY._serialized_end=1064
-  _RUNFLOWJOB_MODELSENTRY._serialized_start=1805
-  _RUNFLOWJOB_MODELSENTRY._serialized_end=1881
-  _IMPORTMODELJOB._serialized_start=1884
-  _IMPORTMODELJOB._serialized_end=2099
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
+  _globals['_RUNMODELJOB_PARAMETERSENTRY']._options = None
+  _globals['_RUNMODELJOB_PARAMETERSENTRY']._serialized_options = b'8\001'
+  _globals['_RUNMODELJOB_INPUTSENTRY']._options = None
+  _globals['_RUNMODELJOB_INPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_RUNMODELJOB_OUTPUTSENTRY']._options = None
+  _globals['_RUNMODELJOB_OUTPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_RUNMODELJOB_PRIOROUTPUTSENTRY']._options = None
+  _globals['_RUNMODELJOB_PRIOROUTPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_RUNFLOWJOB_PARAMETERSENTRY']._options = None
+  _globals['_RUNFLOWJOB_PARAMETERSENTRY']._serialized_options = b'8\001'
+  _globals['_RUNFLOWJOB_INPUTSENTRY']._options = None
+  _globals['_RUNFLOWJOB_INPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_RUNFLOWJOB_OUTPUTSENTRY']._options = None
+  _globals['_RUNFLOWJOB_OUTPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_RUNFLOWJOB_PRIOROUTPUTSENTRY']._options = None
+  _globals['_RUNFLOWJOB_PRIOROUTPUTSENTRY']._serialized_options = b'8\001'
+  _globals['_RUNFLOWJOB_MODELSENTRY']._options = None
+  _globals['_RUNFLOWJOB_MODELSENTRY']._serialized_options = b'8\001'
+  _globals['_JOBTYPE']._serialized_start=2101
+  _globals['_JOBTYPE']._serialized_end=2196
+  _globals['_JOBSTATUSCODE']._serialized_start=2199
+  _globals['_JOBSTATUSCODE']._serialized_end=2383
+  _globals['_JOBDEFINITION']._serialized_start=147
+  _globals['_JOBDEFINITION']._serialized_end=377
+  _globals['_RUNMODELJOB']._serialized_start=380
+  _globals['_RUNMODELJOB']._serialized_end=1064
+  _globals['_RUNMODELJOB_PARAMETERSENTRY']._serialized_start=749
+  _globals['_RUNMODELJOB_PARAMETERSENTRY']._serialized_end=823
+  _globals['_RUNMODELJOB_INPUTSENTRY']._serialized_start=825
+  _globals['_RUNMODELJOB_INPUTSENTRY']._serialized_end=901
+  _globals['_RUNMODELJOB_OUTPUTSENTRY']._serialized_start=903
+  _globals['_RUNMODELJOB_OUTPUTSENTRY']._serialized_end=980
+  _globals['_RUNMODELJOB_PRIOROUTPUTSENTRY']._serialized_start=982
+  _globals['_RUNMODELJOB_PRIOROUTPUTSENTRY']._serialized_end=1064
+  _globals['_RUNFLOWJOB']._serialized_start=1067
+  _globals['_RUNFLOWJOB']._serialized_end=1881
+  _globals['_RUNFLOWJOB_PARAMETERSENTRY']._serialized_start=749
+  _globals['_RUNFLOWJOB_PARAMETERSENTRY']._serialized_end=823
+  _globals['_RUNFLOWJOB_INPUTSENTRY']._serialized_start=825
+  _globals['_RUNFLOWJOB_INPUTSENTRY']._serialized_end=901
+  _globals['_RUNFLOWJOB_OUTPUTSENTRY']._serialized_start=903
+  _globals['_RUNFLOWJOB_OUTPUTSENTRY']._serialized_end=980
+  _globals['_RUNFLOWJOB_PRIOROUTPUTSENTRY']._serialized_start=982
+  _globals['_RUNFLOWJOB_PRIOROUTPUTSENTRY']._serialized_end=1064
+  _globals['_RUNFLOWJOB_MODELSENTRY']._serialized_start=1805
+  _globals['_RUNFLOWJOB_MODELSENTRY']._serialized_end=1881
+  _globals['_IMPORTMODELJOB']._serialized_start=1884
+  _globals['_IMPORTMODELJOB']._serialized_end=2099
 # @@protoc_insertion_point(module_scope)