tracdap-runtime 0.8.0rc1__py3-none-any.whl → 0.9.0b1__py3-none-any.whl

tracdap/rt/_impl/grpc/tracdap/metadata/object_id_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -15,7 +15,7 @@ _sym_db = _symbol_database.Default()
 from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_type__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\"\x98\x02\n\tTagHeader\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x10\n\x08objectId\x18\x02 \x01(\t\x12\x15\n\robjectVersion\x18\x03 \x01(\x05\x12\x38\n\x0fobjectTimestamp\x18\x04 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12\x12\n\ntagVersion\x18\x05 \x01(\x05\x12\x35\n\x0ctagTimestamp\x18\x06 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12\x16\n\x0eisLatestObject\x18\x07 \x01(\x08\x12\x13\n\x0bisLatestTag\x18\x08 \x01(\x08\"\xb9\x02\n\x0bTagSelector\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x10\n\x08objectId\x18\x02 \x01(\t\x12\x16\n\x0clatestObject\x18\x03 \x01(\x08H\x00\x12\x17\n\robjectVersion\x18\x04 \x01(\x05H\x00\x12\x35\n\nobjectAsOf\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValueH\x00\x12\x13\n\tlatestTag\x18\x06 \x01(\x08H\x01\x12\x14\n\ntagVersion\x18\x07 \x01(\x05H\x01\x12\x32\n\x07tagAsOf\x18\x08 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValueH\x01\x42\x10\n\x0eobjectCriteriaB\r\n\x0btagCriteria*\x88\x01\n\nObjectType\x12\x17\n\x13OBJECT_TYPE_NOT_SET\x10\x00\x12\x08\n\x04\x44\x41TA\x10\x01\x12\t\n\x05MODEL\x10\x02\x12\x08\n\x04\x46LOW\x10\x03\x12\x07\n\x03JOB\x10\x04\x12\x08\n\x04\x46ILE\x10\x05\x12\n\n\x06\x43USTOM\x10\x06\x12\x0b\n\x07STORAGE\x10\x07\x12\n\n\x06SCHEMA\x10\x08\x12\n\n\x06RESULT\x10\tB\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\"\x98\x02\n\tTagHeader\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x10\n\x08objectId\x18\x02 \x01(\t\x12\x15\n\robjectVersion\x18\x03 \x01(\x05\x12\x38\n\x0fobjectTimestamp\x18\x04 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12\x12\n\ntagVersion\x18\x05 \x01(\x05\x12\x35\n\x0ctagTimestamp\x18\x06 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12\x16\n\x0eisLatestObject\x18\x07 \x01(\x08\x12\x13\n\x0bisLatestTag\x18\x08 \x01(\x08\"\xb9\x02\n\x0bTagSelector\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x10\n\x08objectId\x18\x02 \x01(\t\x12\x16\n\x0clatestObject\x18\x03 \x01(\x08H\x00\x12\x17\n\robjectVersion\x18\x04 \x01(\x05H\x00\x12\x35\n\nobjectAsOf\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValueH\x00\x12\x13\n\tlatestTag\x18\x06 \x01(\x08H\x01\x12\x14\n\ntagVersion\x18\x07 \x01(\x05H\x01\x12\x32\n\x07tagAsOf\x18\x08 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValueH\x01\x42\x10\n\x0eobjectCriteriaB\r\n\x0btagCriteria*\xa2\x01\n\nObjectType\x12\x17\n\x13OBJECT_TYPE_NOT_SET\x10\x00\x12\x08\n\x04\x44\x41TA\x10\x01\x12\t\n\x05MODEL\x10\x02\x12\x08\n\x04\x46LOW\x10\x03\x12\x07\n\x03JOB\x10\x04\x12\x08\n\x04\x46ILE\x10\x05\x12\n\n\x06\x43USTOM\x10\x06\x12\x0b\n\x07STORAGE\x10\x07\x12\n\n\x06SCHEMA\x10\x08\x12\n\n\x06RESULT\x10\t\x12\n\n\x06\x43ONFIG\x10\n\x12\x0c\n\x08RESOURCE\x10\x0b\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -24,7 +24,7 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._options = None
   _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
   _globals['_OBJECTTYPE']._serialized_start=727
-  _globals['_OBJECTTYPE']._serialized_end=863
+  _globals['_OBJECTTYPE']._serialized_end=889
   _globals['_TAGHEADER']._serialized_start=128
   _globals['_TAGHEADER']._serialized_end=408
   _globals['_TAGSELECTOR']._serialized_start=411

tracdap/rt/_impl/grpc/tracdap/metadata/object_id_pb2.pyi

@@ -18,6 +18,8 @@ class ObjectType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     STORAGE: _ClassVar[ObjectType]
     SCHEMA: _ClassVar[ObjectType]
     RESULT: _ClassVar[ObjectType]
+    CONFIG: _ClassVar[ObjectType]
+    RESOURCE: _ClassVar[ObjectType]
 OBJECT_TYPE_NOT_SET: ObjectType
 DATA: ObjectType
 MODEL: ObjectType
@@ -28,6 +30,8 @@ CUSTOM: ObjectType
 STORAGE: ObjectType
 SCHEMA: ObjectType
 RESULT: ObjectType
+CONFIG: ObjectType
+RESOURCE: ObjectType
 
 class TagHeader(_message.Message):
     __slots__ = ("objectType", "objectId", "objectVersion", "objectTimestamp", "tagVersion", "tagTimestamp", "isLatestObject", "isLatestTag")

tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/object.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -21,9 +21,11 @@ from tracdap.rt._impl.grpc.tracdap.metadata import job_pb2 as tracdap_dot_rt_dot
 from tracdap.rt._impl.grpc.tracdap.metadata import file_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_file__pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import custom_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_custom__pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import storage_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_storage__pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import config_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_config__pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import resource_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_resource__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x32tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto\x1a\x34tracdap/rt/_impl/grpc/tracdap/metadata/storage.proto\"\xbd\x05\n\x10ObjectDefinition\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .tracdap.metadata.DataDefinitionH\x00\x12\x32\n\x05model\x18\x03 \x01(\x0b\x32!.tracdap.metadata.ModelDefinitionH\x00\x12\x30\n\x04\x66low\x18\x04 \x01(\x0b\x32 .tracdap.metadata.FlowDefinitionH\x00\x12.\n\x03job\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinitionH\x00\x12\x30\n\x04\x66ile\x18\x06 \x01(\x0b\x32 .tracdap.metadata.FileDefinitionH\x00\x12\x34\n\x06\x63ustom\x18\x07 \x01(\x0b\x32\".tracdap.metadata.CustomDefinitionH\x00\x12\x36\n\x07storage\x18\x08 \x01(\x0b\x32#.tracdap.metadata.StorageDefinitionH\x00\x12\x34\n\x06schema\x18\t \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12\x34\n\x06result\x18\n \x01(\x0b\x32\".tracdap.metadata.ResultDefinitionH\x00\x12H\n\x0bobjectProps\x18\x64 \x03(\x0b\x32\x33.tracdap.metadata.ObjectDefinition.ObjectPropsEntry\x1aK\n\x10ObjectPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0c\n\ndefinitionB2\n\x1aorg.finos.tracdap.metadataB\x12ObjectProtoWrapperP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x32tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto\x1a\x34tracdap/rt/_impl/grpc/tracdap/metadata/storage.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/config.proto\x1a\x35tracdap/rt/_impl/grpc/tracdap/metadata/resource.proto\"\xad\x06\n\x10ObjectDefinition\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .tracdap.metadata.DataDefinitionH\x00\x12\x32\n\x05model\x18\x03 \x01(\x0b\x32!.tracdap.metadata.ModelDefinitionH\x00\x12\x30\n\x04\x66low\x18\x04 \x01(\x0b\x32 .tracdap.metadata.FlowDefinitionH\x00\x12.\n\x03job\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinitionH\x00\x12\x30\n\x04\x66ile\x18\x06 \x01(\x0b\x32 .tracdap.metadata.FileDefinitionH\x00\x12\x34\n\x06\x63ustom\x18\x07 \x01(\x0b\x32\".tracdap.metadata.CustomDefinitionH\x00\x12\x36\n\x07storage\x18\x08 \x01(\x0b\x32#.tracdap.metadata.StorageDefinitionH\x00\x12\x34\n\x06schema\x18\t \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12\x34\n\x06result\x18\n \x01(\x0b\x32\".tracdap.metadata.ResultDefinitionH\x00\x12\x34\n\x06\x63onfig\x18\x0b \x01(\x0b\x32\".tracdap.metadata.ConfigDefinitionH\x00\x12\x38\n\x08resource\x18\x0c \x01(\x0b\x32$.tracdap.metadata.ResourceDefinitionH\x00\x12H\n\x0bobjectProps\x18\x64 \x03(\x0b\x32\x33.tracdap.metadata.ObjectDefinition.ObjectPropsEntry\x1aK\n\x10ObjectPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0c\n\ndefinitionB2\n\x1aorg.finos.tracdap.metadataB\x12ObjectProtoWrapperP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -33,8 +35,8 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataB\022ObjectProtoWrapperP\001'
   _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._options = None
   _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_options = b'8\001'
-  _globals['_OBJECTDEFINITION']._serialized_start=543
-  _globals['_OBJECTDEFINITION']._serialized_end=1244
-  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_start=1155
-  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_end=1230
+  _globals['_OBJECTDEFINITION']._serialized_start=651
+  _globals['_OBJECTDEFINITION']._serialized_end=1464
+  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_start=1375
+  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_end=1450
 # @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.pyi

@@ -7,6 +7,8 @@ from tracdap.rt._impl.grpc.tracdap.metadata import job_pb2 as _job_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import file_pb2 as _file_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import custom_pb2 as _custom_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import storage_pb2 as _storage_pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import config_pb2 as _config_pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import resource_pb2 as _resource_pb2
 from google.protobuf.internal import containers as _containers
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
@@ -15,7 +17,7 @@ from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Opti
 DESCRIPTOR: _descriptor.FileDescriptor
 
 class ObjectDefinition(_message.Message):
-    __slots__ = ("objectType", "data", "model", "flow", "job", "file", "custom", "storage", "schema", "result", "objectProps")
+    __slots__ = ("objectType", "data", "model", "flow", "job", "file", "custom", "storage", "schema", "result", "config", "resource", "objectProps")
     class ObjectPropsEntry(_message.Message):
         __slots__ = ("key", "value")
         KEY_FIELD_NUMBER: _ClassVar[int]
@@ -33,6 +35,8 @@ class ObjectDefinition(_message.Message):
     STORAGE_FIELD_NUMBER: _ClassVar[int]
     SCHEMA_FIELD_NUMBER: _ClassVar[int]
     RESULT_FIELD_NUMBER: _ClassVar[int]
+    CONFIG_FIELD_NUMBER: _ClassVar[int]
+    RESOURCE_FIELD_NUMBER: _ClassVar[int]
     OBJECTPROPS_FIELD_NUMBER: _ClassVar[int]
     objectType: _object_id_pb2.ObjectType
     data: _data_pb2.DataDefinition
@@ -44,5 +48,7 @@ class ObjectDefinition(_message.Message):
     storage: _storage_pb2.StorageDefinition
     schema: _data_pb2.SchemaDefinition
     result: _job_pb2.ResultDefinition
+    config: _config_pb2.ConfigDefinition
+    resource: _resource_pb2.ResourceDefinition
     objectProps: _containers.MessageMap[str, _type_pb2.Value]
-    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., data: _Optional[_Union[_data_pb2.DataDefinition, _Mapping]] = ..., model: _Optional[_Union[_model_pb2.ModelDefinition, _Mapping]] = ..., flow: _Optional[_Union[_flow_pb2.FlowDefinition, _Mapping]] = ..., job: _Optional[_Union[_job_pb2.JobDefinition, _Mapping]] = ..., file: _Optional[_Union[_file_pb2.FileDefinition, _Mapping]] = ..., custom: _Optional[_Union[_custom_pb2.CustomDefinition, _Mapping]] = ..., storage: _Optional[_Union[_storage_pb2.StorageDefinition, _Mapping]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., result: _Optional[_Union[_job_pb2.ResultDefinition, _Mapping]] = ..., objectProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
+    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., data: _Optional[_Union[_data_pb2.DataDefinition, _Mapping]] = ..., model: _Optional[_Union[_model_pb2.ModelDefinition, _Mapping]] = ..., flow: _Optional[_Union[_flow_pb2.FlowDefinition, _Mapping]] = ..., job: _Optional[_Union[_job_pb2.JobDefinition, _Mapping]] = ..., file: _Optional[_Union[_file_pb2.FileDefinition, _Mapping]] = ..., custom: _Optional[_Union[_custom_pb2.CustomDefinition, _Mapping]] = ..., storage: _Optional[_Union[_storage_pb2.StorageDefinition, _Mapping]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., result: _Optional[_Union[_job_pb2.ResultDefinition, _Mapping]] = ..., config: _Optional[_Union[_config_pb2.ConfigDefinition, _Mapping]] = ..., resource: _Optional[_Union[_resource_pb2.ResourceDefinition, _Mapping]] = ..., objectProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...

tracdap/rt/_impl/grpc/tracdap/metadata/resource_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/resource.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -13,10 +13,9 @@ _sym_db = _symbol_database.Default()
 
 
 from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__id__pb2
-from tracdap.rt._impl.grpc.tracdap.metadata import object_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n5tracdap/rt/_impl/grpc/tracdap/metadata/resource.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/object.proto*m\n\x0cResourceType\x12\x19\n\x15RESOURCE_TYPE_NOT_SET\x10\x00\x12\x14\n\x10MODEL_REPOSITORY\x10\x01\x12\x14\n\x10INTERNAL_STORAGE\x10\x02\"\x04\x08\x03\x10\x03*\x10\x45XTERNAL_STORAGEB\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n5tracdap/rt/_impl/grpc/tracdap/metadata/resource.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\"\x86\x04\n\x12ResourceDefinition\x12\x34\n\x0cresourceType\x18\x01 \x01(\x0e\x32\x1e.tracdap.metadata.ResourceType\x12\x10\n\x08protocol\x18\x02 \x01(\t\x12\x18\n\x0bsubProtocol\x18\x03 \x01(\tH\x00\x88\x01\x01\x12T\n\x10publicProperties\x18\x04 \x03(\x0b\x32:.tracdap.metadata.ResourceDefinition.PublicPropertiesEntry\x12H\n\nproperties\x18\x05 \x03(\x0b\x32\x34.tracdap.metadata.ResourceDefinition.PropertiesEntry\x12\x42\n\x07secrets\x18\x06 \x03(\x0b\x32\x31.tracdap.metadata.ResourceDefinition.SecretsEntry\x1a\x37\n\x15PublicPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a.\n\x0cSecretsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_subProtocol*k\n\x0cResourceType\x12\x19\n\x15RESOURCE_TYPE_NOT_SET\x10\x00\x12\x14\n\x10MODEL_REPOSITORY\x10\x01\x12\x14\n\x10INTERNAL_STORAGE\x10\x02\x12\x14\n\x10\x45XTERNAL_STORAGE\x10\x03\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -24,6 +23,20 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracd
 if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._options = None
   _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
-  _globals['_RESOURCETYPE']._serialized_start=184
-  _globals['_RESOURCETYPE']._serialized_end=293
+  _globals['_RESOURCEDEFINITION_PUBLICPROPERTIESENTRY']._options = None
+  _globals['_RESOURCEDEFINITION_PUBLICPROPERTIESENTRY']._serialized_options = b'8\001'
+  _globals['_RESOURCEDEFINITION_PROPERTIESENTRY']._options = None
+  _globals['_RESOURCEDEFINITION_PROPERTIESENTRY']._serialized_options = b'8\001'
+  _globals['_RESOURCEDEFINITION_SECRETSENTRY']._options = None
+  _globals['_RESOURCEDEFINITION_SECRETSENTRY']._serialized_options = b'8\001'
+  _globals['_RESOURCETYPE']._serialized_start=652
+  _globals['_RESOURCETYPE']._serialized_end=759
+  _globals['_RESOURCEDEFINITION']._serialized_start=132
+  _globals['_RESOURCEDEFINITION']._serialized_end=650
+  _globals['_RESOURCEDEFINITION_PUBLICPROPERTIESENTRY']._serialized_start=480
+  _globals['_RESOURCEDEFINITION_PUBLICPROPERTIESENTRY']._serialized_end=535
+  _globals['_RESOURCEDEFINITION_PROPERTIESENTRY']._serialized_start=537
+  _globals['_RESOURCEDEFINITION_PROPERTIESENTRY']._serialized_end=586
+  _globals['_RESOURCEDEFINITION_SECRETSENTRY']._serialized_start=588
+  _globals['_RESOURCEDEFINITION_SECRETSENTRY']._serialized_end=634
 # @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/resource_pb2.pyi

@@ -1,8 +1,9 @@
 from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as _object_id_pb2
-from tracdap.rt._impl.grpc.tracdap.metadata import object_pb2 as _object_pb2
+from google.protobuf.internal import containers as _containers
 from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
 from google.protobuf import descriptor as _descriptor
-from typing import ClassVar as _ClassVar
+from google.protobuf import message as _message
+from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union
 
 DESCRIPTOR: _descriptor.FileDescriptor
 
@@ -11,6 +12,45 @@ class ResourceType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     RESOURCE_TYPE_NOT_SET: _ClassVar[ResourceType]
     MODEL_REPOSITORY: _ClassVar[ResourceType]
     INTERNAL_STORAGE: _ClassVar[ResourceType]
+    EXTERNAL_STORAGE: _ClassVar[ResourceType]
 RESOURCE_TYPE_NOT_SET: ResourceType
 MODEL_REPOSITORY: ResourceType
 INTERNAL_STORAGE: ResourceType
+EXTERNAL_STORAGE: ResourceType
+
+class ResourceDefinition(_message.Message):
+    __slots__ = ("resourceType", "protocol", "subProtocol", "publicProperties", "properties", "secrets")
+    class PublicPropertiesEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    class PropertiesEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    class SecretsEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    RESOURCETYPE_FIELD_NUMBER: _ClassVar[int]
+    PROTOCOL_FIELD_NUMBER: _ClassVar[int]
+    SUBPROTOCOL_FIELD_NUMBER: _ClassVar[int]
+    PUBLICPROPERTIES_FIELD_NUMBER: _ClassVar[int]
+    PROPERTIES_FIELD_NUMBER: _ClassVar[int]
+    SECRETS_FIELD_NUMBER: _ClassVar[int]
+    resourceType: ResourceType
+    protocol: str
+    subProtocol: str
+    publicProperties: _containers.ScalarMap[str, str]
+    properties: _containers.ScalarMap[str, str]
+    secrets: _containers.ScalarMap[str, str]
+    def __init__(self, resourceType: _Optional[_Union[ResourceType, str]] = ..., protocol: _Optional[str] = ..., subProtocol: _Optional[str] = ..., publicProperties: _Optional[_Mapping[str, str]] = ..., properties: _Optional[_Mapping[str, str]] = ..., secrets: _Optional[_Mapping[str, str]] = ...) -> None: ...

tracdap/rt/_impl/grpc/tracdap/metadata/search_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/search.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

tracdap/rt/_impl/grpc/tracdap/metadata/storage_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/storage.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -15,7 +15,7 @@ _sym_db = _symbol_database.Default()
 from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_type__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n4tracdap/rt/_impl/grpc/tracdap/metadata/storage.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\"\xd2\x02\n\x0bStorageCopy\x12\x12\n\nstorageKey\x18\x01 \x01(\t\x12\x13\n\x0bstoragePath\x18\x02 \x01(\t\x12\x15\n\rstorageFormat\x18\x03 \x01(\t\x12\x30\n\ncopyStatus\x18\x04 \x01(\x0e\x32\x1c.tracdap.metadata.CopyStatus\x12\x36\n\rcopyTimestamp\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12I\n\x0estorageOptions\x18\x06 \x03(\x0b\x32\x31.tracdap.metadata.StorageCopy.StorageOptionsEntry\x1aN\n\x13StorageOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\"\xdc\x01\n\x12StorageIncarnation\x12-\n\x06\x63opies\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.StorageCopy\x12\x18\n\x10incarnationIndex\x18\x02 \x01(\x05\x12=\n\x14incarnationTimestamp\x18\x03 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12>\n\x11incarnationStatus\x18\x04 \x01(\x0e\x32#.tracdap.metadata.IncarnationStatus\"I\n\x0bStorageItem\x12:\n\x0cincarnations\x18\x01 \x03(\x0b\x32$.tracdap.metadata.StorageIncarnation\"\xe1\x02\n\x11StorageDefinition\x12\x45\n\tdataItems\x18\x01 \x03(\x0b\x32\x32.tracdap.metadata.StorageDefinition.DataItemsEntry\x12O\n\x0estorageOptions\x18\x03 \x03(\x0b\x32\x37.tracdap.metadata.StorageDefinition.StorageOptionsEntry\x1aO\n\x0e\x44\x61taItemsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.StorageItem:\x02\x38\x01\x1aN\n\x13StorageOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01J\x04\x08\x02\x10\x03R\rstorageFormat*L\n\nCopyStatus\x12\x17\n\x13\x43OPY_STATUS_NOT_SET\x10\x00\x12\x12\n\x0e\x43OPY_AVAILABLE\x10\x01\x12\x11\n\rCOPY_EXPUNGED\x10\x02*h\n\x11IncarnationStatus\x12\x1e\n\x1aINCARNATION_STATUS_NOT_SET\x10\x00\x12\x19\n\x15INCARNATION_AVAILABLE\x10\x01\x12\x18\n\x14INCARNATION_EXPUNGED\x10\x02\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n4tracdap/rt/_impl/grpc/tracdap/metadata/storage.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\"\xd2\x02\n\x0bStorageCopy\x12\x12\n\nstorageKey\x18\x01 \x01(\t\x12\x13\n\x0bstoragePath\x18\x02 \x01(\t\x12\x15\n\rstorageFormat\x18\x03 \x01(\t\x12\x30\n\ncopyStatus\x18\x04 \x01(\x0e\x32\x1c.tracdap.metadata.CopyStatus\x12\x36\n\rcopyTimestamp\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12I\n\x0estorageOptions\x18\x06 \x03(\x0b\x32\x31.tracdap.metadata.StorageCopy.StorageOptionsEntry\x1aN\n\x13StorageOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\"\xdc\x01\n\x12StorageIncarnation\x12-\n\x06\x63opies\x18\x01 \x03(\x0b\x32\x1d.tracdap.metadata.StorageCopy\x12\x18\n\x10incarnationIndex\x18\x02 \x01(\x05\x12=\n\x14incarnationTimestamp\x18\x03 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12>\n\x11incarnationStatus\x18\x04 \x01(\x0e\x32#.tracdap.metadata.IncarnationStatus\"I\n\x0bStorageItem\x12:\n\x0cincarnations\x18\x01 \x03(\x0b\x32$.tracdap.metadata.StorageIncarnation\"\x92\x03\n\x11StorageDefinition\x12\x45\n\tdataItems\x18\x01 \x03(\x0b\x32\x32.tracdap.metadata.StorageDefinition.DataItemsEntry\x12O\n\x0estorageOptions\x18\x03 \x03(\x0b\x32\x37.tracdap.metadata.StorageDefinition.StorageOptionsEntry\x12/\n\x06layout\x18\x04 \x01(\x0e\x32\x1f.tracdap.metadata.StorageLayout\x1aO\n\x0e\x44\x61taItemsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.tracdap.metadata.StorageItem:\x02\x38\x01\x1aN\n\x13StorageOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01J\x04\x08\x02\x10\x03R\rstorageFormat*L\n\nCopyStatus\x12\x17\n\x13\x43OPY_STATUS_NOT_SET\x10\x00\x12\x12\n\x0e\x43OPY_AVAILABLE\x10\x01\x12\x11\n\rCOPY_EXPUNGED\x10\x02*h\n\x11IncarnationStatus\x12\x1e\n\x1aINCARNATION_STATUS_NOT_SET\x10\x00\x12\x19\n\x15INCARNATION_AVAILABLE\x10\x01\x12\x18\n\x14INCARNATION_EXPUNGED\x10\x02*;\n\rStorageLayout\x12\x14\n\x10OBJECT_ID_LAYOUT\x10\x00\x12\x14\n\x10\x44\x45VELOPER_LAYOUT\x10\x01\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -29,10 +29,12 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_STORAGEDEFINITION_DATAITEMSENTRY']._serialized_options = b'8\001'
   _globals['_STORAGEDEFINITION_STORAGEOPTIONSENTRY']._options = None
   _globals['_STORAGEDEFINITION_STORAGEOPTIONSENTRY']._serialized_options = b'8\001'
-  _globals['_COPYSTATUS']._serialized_start=1120
-  _globals['_COPYSTATUS']._serialized_end=1196
-  _globals['_INCARNATIONSTATUS']._serialized_start=1198
-  _globals['_INCARNATIONSTATUS']._serialized_end=1302
+  _globals['_COPYSTATUS']._serialized_start=1169
+  _globals['_COPYSTATUS']._serialized_end=1245
+  _globals['_INCARNATIONSTATUS']._serialized_start=1247
+  _globals['_INCARNATIONSTATUS']._serialized_end=1351
+  _globals['_STORAGELAYOUT']._serialized_start=1353
+  _globals['_STORAGELAYOUT']._serialized_end=1412
   _globals['_STORAGECOPY']._serialized_start=126
   _globals['_STORAGECOPY']._serialized_end=464
   _globals['_STORAGECOPY_STORAGEOPTIONSENTRY']._serialized_start=386
@@ -42,9 +44,9 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_STORAGEITEM']._serialized_start=689
   _globals['_STORAGEITEM']._serialized_end=762
   _globals['_STORAGEDEFINITION']._serialized_start=765
-  _globals['_STORAGEDEFINITION']._serialized_end=1118
-  _globals['_STORAGEDEFINITION_DATAITEMSENTRY']._serialized_start=938
-  _globals['_STORAGEDEFINITION_DATAITEMSENTRY']._serialized_end=1017
+  _globals['_STORAGEDEFINITION']._serialized_end=1167
+  _globals['_STORAGEDEFINITION_DATAITEMSENTRY']._serialized_start=987
+  _globals['_STORAGEDEFINITION_DATAITEMSENTRY']._serialized_end=1066
   _globals['_STORAGEDEFINITION_STORAGEOPTIONSENTRY']._serialized_start=386
   _globals['_STORAGEDEFINITION_STORAGEOPTIONSENTRY']._serialized_end=464
 # @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/storage_pb2.pyi

@@ -18,12 +18,19 @@ class IncarnationStatus(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     INCARNATION_STATUS_NOT_SET: _ClassVar[IncarnationStatus]
     INCARNATION_AVAILABLE: _ClassVar[IncarnationStatus]
     INCARNATION_EXPUNGED: _ClassVar[IncarnationStatus]
+
+class StorageLayout(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    OBJECT_ID_LAYOUT: _ClassVar[StorageLayout]
+    DEVELOPER_LAYOUT: _ClassVar[StorageLayout]
 COPY_STATUS_NOT_SET: CopyStatus
 COPY_AVAILABLE: CopyStatus
 COPY_EXPUNGED: CopyStatus
 INCARNATION_STATUS_NOT_SET: IncarnationStatus
 INCARNATION_AVAILABLE: IncarnationStatus
 INCARNATION_EXPUNGED: IncarnationStatus
+OBJECT_ID_LAYOUT: StorageLayout
+DEVELOPER_LAYOUT: StorageLayout
 
 class StorageCopy(_message.Message):
     __slots__ = ("storageKey", "storagePath", "storageFormat", "copyStatus", "copyTimestamp", "storageOptions")
@@ -67,7 +74,7 @@ class StorageItem(_message.Message):
     def __init__(self, incarnations: _Optional[_Iterable[_Union[StorageIncarnation, _Mapping]]] = ...) -> None: ...
 
 class StorageDefinition(_message.Message):
-    __slots__ = ("dataItems", "storageOptions")
+    __slots__ = ("dataItems", "storageOptions", "layout")
     class DataItemsEntry(_message.Message):
         __slots__ = ("key", "value")
         KEY_FIELD_NUMBER: _ClassVar[int]
@@ -84,6 +91,8 @@ class StorageDefinition(_message.Message):
         def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[_type_pb2.Value, _Mapping]] = ...) -> None: ...
     DATAITEMS_FIELD_NUMBER: _ClassVar[int]
     STORAGEOPTIONS_FIELD_NUMBER: _ClassVar[int]
+    LAYOUT_FIELD_NUMBER: _ClassVar[int]
     dataItems: _containers.MessageMap[str, StorageItem]
     storageOptions: _containers.MessageMap[str, _type_pb2.Value]
-    def __init__(self, dataItems: _Optional[_Mapping[str, StorageItem]] = ..., storageOptions: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
+    layout: StorageLayout
+    def __init__(self, dataItems: _Optional[_Mapping[str, StorageItem]] = ..., storageOptions: _Optional[_Mapping[str, _type_pb2.Value]] = ..., layout: _Optional[_Union[StorageLayout, str]] = ...) -> None: ...

tracdap/rt/_impl/grpc/tracdap/metadata/tag_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/tag.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

tracdap/rt/_impl/grpc/tracdap/metadata/tag_update_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/tag_update.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/type.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

tracdap/rt/_impl/runtime.py

@@ -30,6 +30,7 @@ import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as _plugins
 import tracdap.rt._impl.core.config_parser as _cparse
+import tracdap.rt._impl.core.data as _data
 import tracdap.rt._impl.core.guard_rails as _guard
 import tracdap.rt._impl.core.logging as _logging
 import tracdap.rt._impl.core.models as _models
@@ -167,6 +168,13 @@ class TracRuntime:
             else:
                 self._log.info("Using embedded system config")
 
+            # Check whether to enable categorical processing in the data layer
+
+            if _data.DataMapping.CATEGORICAL_CONFIG_KEY in self._sys_config.properties:
+                categorical_prop = self._sys_config.properties[_data.DataMapping.CATEGORICAL_CONFIG_KEY]
+                categorical_flag = categorical_prop and categorical_prop.lower() == "true"
+                _data.DataMapping.enable_categorical(categorical_flag)
+
             # Dev mode translation is controlled by the dev mode flag
             # I.e. it can be applied to embedded configs
 

tracdap/rt/_plugins/repo_git.py

@@ -23,6 +23,7 @@ import time
 import dulwich.repo as git_repo
 import dulwich.client as git_client
 import dulwich.index as git_index
+import urllib3.exceptions  # noqa
 
 import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
@@ -75,20 +76,45 @@ class GitRepository(IModelRepository):
 
     def do_checkout(self, model_def: meta.ModelDefinition, checkout_dir: pathlib.Path) -> pathlib.Path:
 
-        self._log.info(
-            f"Git checkout: repo = [{model_def.repository}], " +
-            f"group = [{model_def.packageGroup}], package = [{model_def.package}], version = [{model_def.version}]")
+        try:
 
-        self._log.info(f"Checkout location: [{checkout_dir}]")
+            self._log.info(
+                f"Git checkout: repo = [{model_def.repository}], " +
+                f"group = [{model_def.packageGroup}], package = [{model_def.package}], version = [{model_def.version}]")
 
-        if self._native_git:
-            package_path = self._do_native_checkout(model_def, checkout_dir)
-        else:
-            package_path = self._do_python_checkout(model_def, checkout_dir)
+            self._log.info(f"Checkout location: [{checkout_dir}]")
+
+            if self._native_git:
+                package_path = self._do_native_checkout(model_def, checkout_dir)
+            else:
+                package_path = self._do_python_checkout(model_def, checkout_dir)
+
+            self._log.info(f"Git checkout succeeded for {model_def.package} {model_def.version}")
+
+            return package_path
+
+        except Exception as e:
+
+            error = e
+
+            # For retry failures, try to find the original cause
+            while e.__cause__ is not None:
+                if isinstance(e, urllib3.exceptions.MaxRetryError):
+                    error = e.__cause__
+                    break
+                else:
+                    e = e.__cause__
+
+            # Try to sanitize error messages from urllib3
+            if isinstance(error, urllib3.exceptions.HTTPError):
+                detail = self._clean_urllib3_error(error)
+            else:
+                detail = str(error)
 
-        self._log.info(f"Git checkout succeeded for {model_def.package} {model_def.version}")
+            message = f"Failed to check out [{model_def.repository}]: {detail}"
 
-        return package_path
+            self._log.error(message)
+            raise ex.EModelRepo(message) from error
 
     def _do_native_checkout(self, model_def: meta.ModelDefinition, checkout_dir: pathlib.Path) -> pathlib.Path:
 
@@ -150,10 +176,15 @@ class GitRepository(IModelRepository):
                 for line in cmd_err:
                     self._log.info(line)
 
-            else:
+            elif cmd_err:
+
                 for line in cmd_err:
                     self._log.error(line)
 
+                raise ex.EModelRepo(cmd_err[-1])
+
+            else:
+
                 error_msg = f"Git checkout failed for {model_def.package} {model_def.version}"
                 self._log.error(error_msg)
                 raise ex.EModelRepo(error_msg)
@@ -265,6 +296,20 @@ class GitRepository(IModelRepository):
     def _ref_key(key):
         return bytes(key, "ascii")
 
+    @classmethod
+    def _clean_urllib3_error(cls, error: urllib3.exceptions.HTTPError):
+
+        match = cls._URLLIB3_ERROR_PATTERN.match(str(error))
+
+        # Best efforts to clean up the message, fall back on str(error)
+        if match:
+            return match.group(1)
+        else:
+            return str(error)
+
+    # Error message format is like this:
+    # <pkg.ClassName object at 0xXXXXXXX>: Message
+    _URLLIB3_ERROR_PATTERN = re.compile(r"<[^>]*>: (.*)")
 
 # Register plugin
 plugins.PluginManager.register_plugin(IModelRepository, GitRepository, ["git"])

tracdap/rt/_version.py

@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.8.0rc1"
+__version__ = "0.9.0b1"

tracdap/rt/config/__init__.py

@@ -1,6 +1,7 @@
 # Code generated by TRAC
 
-from .job import JobConfig
+from .result import JobResultAttrs
+from .result import JobResult
 
 from .common import _ConfigFile
 from .common import PluginConfig
@@ -8,9 +9,6 @@ from .common import PlatformInfo
 from .common import StorageConfig
 from .common import ServiceConfig
 
-from .result import TagUpdateList
-from .result import JobResult
-
 from .platform import RoutingProtocol
 from .platform import DeploymentLayout
 from .platform import PlatformConfig
@@ -26,3 +24,7 @@ from .platform import ClientConfig
 
 from .runtime import RuntimeConfig
 from .runtime import SparkSettings
+
+from .job import JobConfig
+
+from .dynamic import DynamicConfig

tracdap/rt/config/common.py

@@ -4,6 +4,9 @@ import typing as _tp  # noqa
 import dataclasses as _dc  # noqa
 import enum as _enum  # noqa
 
+import tracdap.rt.metadata as metadata
+
+
 
 @_dc.dataclass
 class _ConfigFile:
@@ -46,6 +49,8 @@ class StorageConfig:
 
     defaultFormat: "str" = ""
 
+    defaultLayout: "metadata.StorageLayout" = metadata.StorageLayout.OBJECT_ID_LAYOUT
+
 
 @_dc.dataclass
 class ServiceConfig:

tracdap/rt/config/dynamic.py

@@ -0,0 +1,28 @@
+# Code generated by TRAC
+
+import typing as _tp  # noqa
+import dataclasses as _dc  # noqa
+import enum as _enum  # noqa
+
+import tracdap.rt.metadata as metadata
+
+
+
+@_dc.dataclass
+class DynamicConfig:
+
+    """
+    Represent config that is normally held in the metadata store
+    
+    This can be used to supply config to isolated processes, including
+    the runtime when operating in batch mode. It can also be used to
+    to supply config for testing or to bootstrap the metadata store.
+    """
+
+    configClass: "str" = ""
+
+    resourceClass: "str" = ""
+
+    config: "_tp.Dict[str, metadata.ConfigDefinition]" = _dc.field(default_factory=dict)
+
+    resources: "_tp.Dict[str, metadata.ResourceDefinition]" = _dc.field(default_factory=dict)

tracdap/rt/config/job.py

@@ -13,10 +13,20 @@ class JobConfig:
 
     jobId: "metadata.TagHeader" = _dc.field(default_factory=lambda: metadata.TagHeader())
 
+    """ID and definition of the job being submitted"""
+
     job: "metadata.JobDefinition" = _dc.field(default_factory=lambda: metadata.JobDefinition())
 
-    resources: "_tp.Dict[str, metadata.ObjectDefinition]" = _dc.field(default_factory=dict)
+    objectMapping: "_tp.Dict[str, metadata.TagHeader]" = _dc.field(default_factory=dict)
+
+    """Metadata needed to execute the job (objects referred to in the job definition)"""
+
+    objects: "_tp.Dict[str, metadata.ObjectDefinition]" = _dc.field(default_factory=dict)
+
+    tags: "_tp.Dict[str, metadata.Tag]" = _dc.field(default_factory=dict)
+
+    resultId: "metadata.TagHeader" = _dc.field(default_factory=lambda: metadata.TagHeader())
 
-    resourceMapping: "_tp.Dict[str, metadata.TagHeader]" = _dc.field(default_factory=dict)
+    """Preallocated IDs for job outputs"""
 
-    resultMapping: "_tp.Dict[str, metadata.TagHeader]" = _dc.field(default_factory=dict)
+    preallocatedIds: "_tp.List[metadata.TagHeader]" = _dc.field(default_factory=list)

tracdap/rt/config/result.py

@@ -9,7 +9,7 @@ import tracdap.rt.metadata as metadata
 
 
 @_dc.dataclass
-class TagUpdateList:
+class JobResultAttrs:
 
     attrs: "_tp.List[metadata.TagUpdate]" = _dc.field(default_factory=list)
 
@@ -19,8 +19,12 @@ class JobResult:
 
     jobId: "metadata.TagHeader" = _dc.field(default_factory=lambda: metadata.TagHeader())
 
-    statusCode: "metadata.JobStatusCode" = metadata.JobStatusCode.JOB_STATUS_CODE_NOT_SET
+    resultId: "metadata.TagHeader" = _dc.field(default_factory=lambda: metadata.TagHeader())
 
-    statusMessage: "str" = ""
+    result: "metadata.ResultDefinition" = _dc.field(default_factory=lambda: metadata.ResultDefinition())
 
-    results: "_tp.Dict[str, metadata.ObjectDefinition]" = _dc.field(default_factory=dict)
+    objectIds: "_tp.List[metadata.TagHeader]" = _dc.field(default_factory=list)
+
+    objects: "_tp.Dict[str, metadata.ObjectDefinition]" = _dc.field(default_factory=dict)
+
+    attrs: "_tp.Dict[str, JobResultAttrs]" = _dc.field(default_factory=dict)

tracdap/rt/config/runtime.py

@@ -21,6 +21,8 @@ class RuntimeConfig:
 
     runtimeApi: "ServiceConfig" = _dc.field(default_factory=lambda: ServiceConfig())
 
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
 
 @_dc.dataclass
 class SparkSettings: