tracdap-runtime 0.6.0rc1__py3-none-any.whl → 0.6.0rc3__py3-none-any.whl

tracdap/rt/_plugins/storage_gcp.py

@@ -21,16 +21,25 @@ import tracdap.rt.exceptions as ex
 import tracdap.rt.ext.plugins as plugins
 from tracdap.rt.ext.storage import *
 
-from pyarrow import fs as afs
+from pyarrow import fs as pa_fs
 
 # Set of common helpers across the core plugins (do not reference rt._impl)
 from . import _helpers
 
 
-class GcpStorageProvider(IStorageProvider):
+try:
+    # These dependencies are provided by the optional [gcp] feature
+    # For local development, pip install -r requirements_plugins.txt
+    import google.cloud.storage as gcs  # noqa
+    import gcsfs  # noqa
+    __gcp_available = True
+except ImportError:
+    gcs = None
+    gcsfs = None
+    __gcp_available = False
+
 
-    ARROW_NATIVE_FS_PROPERTY = "arrowNativeFs"
-    ARROW_NATIVE_FS_DEFAULT = False
+class GcpStorageProvider(IStorageProvider):
 
     BUCKET_PROPERTY = "bucket"
     PREFIX_PROPERTY = "prefix"
@@ -45,35 +54,78 @@ class GcpStorageProvider(IStorageProvider):
     ACCESS_TOKEN_EXPIRY = "accessTokenExpiry"
     ACCESS_TOKEN_EXPIRY_DEFAULT = 3600
 
+    RUNTIME_FS_PROPERTY = "runtimeFs"
+    RUNTIME_FS_AUTO = "auto"
+    RUNTIME_FS_ARROW = "arrow"
+    RUNTIME_FS_FSSPEC = "fsspec"
+    RUNTIME_FS_DEFAULT = RUNTIME_FS_AUTO
+
+    ARROW_CLIENT_ARGS = {
+        REGION_PROPERTY: "default_bucket_location",
+        ENDPOINT_PROPERTY: "endpoint_override"
+    }
+
+    FSSPEC_CLIENT_ARGS = {
+        REGION_PROPERTY: "default_location",
+        ENDPOINT_PROPERTY: "endpoint_url"
+    }
+
+    try:
+        __arrow_available = pa_fs.GcsFileSystem is not None
+    except ImportError:
+        __arrow_available = False
+
     def __init__(self, properties: tp.Dict[str, str]):
 
         self._log = _helpers.logger_for_object(self)
         self._properties = properties
 
-        self._arrow_native = _helpers.get_plugin_property_boolean(
-            properties, self.ARROW_NATIVE_FS_PROPERTY, self.ARROW_NATIVE_FS_DEFAULT)
+        self._runtime_fs = _helpers.get_plugin_property(
+            properties, self.RUNTIME_FS_PROPERTY) \
+            or self.RUNTIME_FS_DEFAULT
 
     def has_arrow_native(self) -> bool:
         return True
 
-    def get_arrow_native(self) -> afs.SubTreeFileSystem:
+    def get_arrow_native(self) -> pa_fs.SubTreeFileSystem:
 
-        gcs_args = self.setup_client_args()
-        gcs_fs = afs.GcsFileSystem(**gcs_args)
+        if self._runtime_fs == self.RUNTIME_FS_AUTO:
+            gcs_fs = self.create_arrow() if self.__arrow_available else self.create_fsspec()
+        elif self._runtime_fs == self.RUNTIME_FS_ARROW:
+            gcs_fs = self.create_arrow()
+        elif self._runtime_fs == self.RUNTIME_FS_FSSPEC:
+            gcs_fs = self.create_fsspec()
+        else:
+            message = f"Requested runtime FS [{self._runtime_fs}] is not available for GCP storage"
+            self._log.error(message)
+            raise ex.EStartup(message)
 
         bucket = _helpers.get_plugin_property(self._properties, self.BUCKET_PROPERTY)
+        prefix = _helpers.get_plugin_property(self._properties, self.PREFIX_PROPERTY)
 
         if bucket is None or len(bucket.strip()) == 0:
             message = f"Missing required config property [{self.BUCKET_PROPERTY}] for GCP storage"
             self._log.error(message)
             raise ex.EConfigParse(message)
 
-        prefix = _helpers.get_plugin_property(self._properties, self.PREFIX_PROPERTY)
         root_path = f"{bucket}/{prefix}" if prefix else bucket
 
-        return afs.SubTreeFileSystem(root_path, gcs_fs)
+        return pa_fs.SubTreeFileSystem(root_path, gcs_fs)
+
+    def create_arrow(self) -> pa_fs.FileSystem:
+
+        gcs_arrow_args = self.setup_client_args(self.ARROW_CLIENT_ARGS)
+
+        return pa_fs.GcsFileSystem(**gcs_arrow_args)
 
-    def setup_client_args(self) -> tp.Dict[str, tp.Any]:
+    def create_fsspec(self) -> pa_fs.FileSystem:
+
+        gcs_fsspec_args = self.setup_client_args(self.FSSPEC_CLIENT_ARGS)
+        gcs_fsspec = gcsfs.GCSFileSystem(**gcs_fsspec_args)
+
+        return pa_fs.PyFileSystem(pa_fs.FSSpecHandler(gcs_fsspec))
+
+    def setup_client_args(self, arg_mapping: tp.Dict[str, str]) -> tp.Dict[str, tp.Any]:
 
         client_args = dict()
 
@@ -81,10 +133,12 @@ class GcpStorageProvider(IStorageProvider):
         endpoint = _helpers.get_plugin_property(self._properties, self.ENDPOINT_PROPERTY)
 
         if region is not None:
-            client_args["default_bucket_location"] = region
+            region_key = arg_mapping[self.REGION_PROPERTY]
+            client_args[region_key] = region
 
         if endpoint is not None:
-            client_args["endpoint_override"] = endpoint
+            endpoint_key = arg_mapping[self.ENDPOINT_PROPERTY]
+            client_args[endpoint_key] = endpoint
 
         credentials = self.setup_credentials()
         client_args.update(credentials)
@@ -104,6 +158,8 @@ class GcpStorageProvider(IStorageProvider):
 
         if mechanism == self.CREDENTIALS_ACCESS_TOKEN:
 
+            self._log.info(f"Using [{self.CREDENTIALS_ACCESS_TOKEN}] credentials mechanism")
+
             access_token = _helpers.get_plugin_property(self._properties, self.ACCESS_TOKEN)
             access_token_expiry = _helpers.get_plugin_property(self._properties, self.ACCESS_TOKEN_EXPIRY)
 
@@ -123,4 +179,5 @@ class GcpStorageProvider(IStorageProvider):
         raise ex.EStartup(message)
 
 
-plugins.PluginManager.register_plugin(IStorageProvider, GcpStorageProvider, ["GCS"])
+if __gcp_available:
+    plugins.PluginManager.register_plugin(IStorageProvider, GcpStorageProvider, ["GCS"])

tracdap/rt/_plugins/storage_local.py

@@ -35,8 +35,12 @@ from . import _helpers
 class LocalStorageProvider(IStorageProvider):
 
     ROOT_PATH_PROPERTY = "rootPath"
-    ARROW_NATIVE_FS_PROPERTY = "arrowNativeFs"
-    ARROW_NATIVE_FS_DEFAULT = False
+
+    RUNTIME_FS_PROPERTY = "runtimeFs"
+    RUNTIME_FS_AUTO = "auto"
+    RUNTIME_FS_ARROW = "arrow"
+    RUNTIME_FS_PYTHON = "python"
+    RUNTIME_FS_DEFAULT = RUNTIME_FS_AUTO
 
     def __init__(self, properties: tp.Dict[str, str]):
 
@@ -45,14 +49,15 @@ class LocalStorageProvider(IStorageProvider):
 
         self._root_path = self.check_root_path(self._properties, self._log)
 
-        self._arrow_native = _helpers.get_plugin_property_boolean(
-            properties, self.ARROW_NATIVE_FS_PROPERTY, self.ARROW_NATIVE_FS_DEFAULT)
+        self._runtime_fs = _helpers.get_plugin_property(
+            properties, self.RUNTIME_FS_PROPERTY) \
+            or self.RUNTIME_FS_DEFAULT
 
     def has_arrow_native(self) -> bool:
-        return True if self._arrow_native else False
+        return self._runtime_fs in [self.RUNTIME_FS_ARROW, self.RUNTIME_FS_AUTO]
 
     def has_file_storage(self) -> bool:
-        return False if self._arrow_native else True
+        return self._runtime_fs == self.RUNTIME_FS_PYTHON
 
     def get_arrow_native(self) -> afs.SubTreeFileSystem:
         root_fs = afs.LocalFileSystem()

tracdap/rt/_version.py

@@ -12,4 +12,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.6.0rc1"
+__version__ = "0.6.0rc3"

tracdap/rt/config/__init__.py

@@ -1,35 +1,30 @@
 # Code generated by TRAC
 
-from .job import JobConfig
-
-from .result import TagUpdateList
-from .result import JobResult
-
 from .common import _ConfigFile
 from .common import PluginConfig
 from .common import PlatformInfo
 from .common import AuthenticationConfig
+from .common import StorageConfig
 
+from .platform import RoutingProtocol
+from .platform import DeploymentLayout
 from .platform import PlatformConfig
 from .platform import MetadataConfig
-from .platform import StorageConfig
 from .platform import TenantConfig
 from .platform import WebServerConfig
 from .platform import WebServerRewriteRule
 from .platform import WebServerRedirect
-from .platform import InstantMap
-from .platform import InstanceConfig
-from .platform import ServiceMap
+from .platform import GatewayConfig
+from .platform import RouteConfig
+from .platform import RoutingMatch
+from .platform import RoutingTarget
 from .platform import ServiceConfig
+from .platform import DeploymentConfig
+
+from .result import TagUpdateList
+from .result import JobResult
 
 from .runtime import RuntimeConfig
 from .runtime import SparkSettings
 
-from .gateway import GwProtocol
-from .gateway import GwRestMapping
-from .gateway import GatewayConfig
-from .gateway import GwRoute
-from .gateway import GwMatch
-from .gateway import GwTarget
-from .gateway import GwServiceMap
-from .gateway import GwService
+from .job import JobConfig

tracdap/rt/config/common.py

@@ -56,3 +56,13 @@ class AuthenticationConfig:
     systemTicketDuration: int = None
 
     systemTicketRefresh: int = None
+
+
+@_dc.dataclass
+class StorageConfig:
+
+    buckets: _tp.Dict[str, PluginConfig] = _dc.field(default_factory=dict)
+
+    defaultBucket: str = None
+
+    defaultFormat: str = None

tracdap/rt/config/common_pb2.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/config/common.proto
+# Protobuf Python Version: 4.25.3
 """Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -13,36 +14,42 @@ _sym_db = _symbol_database.Default()
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1btracdap/config/common.proto\x12\x0etracdap.config\"u\n\x0b_ConfigFile\x12\x37\n\x06\x63onfig\x18\x01 \x03(\x0b\x32\'.tracdap.config._ConfigFile.ConfigEntry\x1a-\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x81\x02\n\x0cPluginConfig\x12\x10\n\x08protocol\x18\x01 \x01(\t\x12@\n\nproperties\x18\x02 \x03(\x0b\x32,.tracdap.config.PluginConfig.PropertiesEntry\x12:\n\x07secrets\x18\x03 \x03(\x0b\x32).tracdap.config.PluginConfig.SecretsEntry\x1a\x31\n\x0fPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a.\n\x0cSecretsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb8\x01\n\x0cPlatformInfo\x12\x13\n\x0b\x65nvironment\x18\x01 \x01(\t\x12\x12\n\nproduction\x18\x02 \x01(\x08\x12H\n\x0e\x64\x65ploymentInfo\x18\x03 \x03(\x0b\x32\x30.tracdap.config.PlatformInfo.DeploymentInfoEntry\x1a\x35\n\x13\x44\x65ploymentInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xba\x02\n\x14\x41uthenticationConfig\x12\x11\n\tjwtIssuer\x18\x01 \x01(\t\x12\x11\n\tjwtExpiry\x18\x02 \x01(\x11\x12\x10\n\x08jwtLimit\x18\x06 \x01(\x11\x12\x12\n\njwtRefresh\x18\x07 \x01(\x11\x12\x33\n\x08provider\x18\x03 \x01(\x0b\x32\x1c.tracdap.config.PluginConfigH\x00\x88\x01\x01\x12\x13\n\x0b\x64isableAuth\x18\x04 \x01(\x08\x12\x16\n\x0e\x64isableSigning\x18\x05 \x01(\x08\x12\x14\n\x0csystemUserId\x18\x08 \x01(\t\x12\x16\n\x0esystemUserName\x18\t \x01(\t\x12\x1c\n\x14systemTicketDuration\x18\n \x01(\x11\x12\x1b\n\x13systemTicketRefresh\x18\x0b \x01(\x11\x42\x0b\n\t_providerB\x1c\n\x18org.finos.tracdap.configP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1btracdap/config/common.proto\x12\x0etracdap.config\"u\n\x0b_ConfigFile\x12\x37\n\x06\x63onfig\x18\x01 \x03(\x0b\x32\'.tracdap.config._ConfigFile.ConfigEntry\x1a-\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x81\x02\n\x0cPluginConfig\x12\x10\n\x08protocol\x18\x01 \x01(\t\x12@\n\nproperties\x18\x02 \x03(\x0b\x32,.tracdap.config.PluginConfig.PropertiesEntry\x12:\n\x07secrets\x18\x03 \x03(\x0b\x32).tracdap.config.PluginConfig.SecretsEntry\x1a\x31\n\x0fPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a.\n\x0cSecretsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb8\x01\n\x0cPlatformInfo\x12\x13\n\x0b\x65nvironment\x18\x01 \x01(\t\x12\x12\n\nproduction\x18\x02 \x01(\x08\x12H\n\x0e\x64\x65ploymentInfo\x18\x03 \x03(\x0b\x32\x30.tracdap.config.PlatformInfo.DeploymentInfoEntry\x1a\x35\n\x13\x44\x65ploymentInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xba\x02\n\x14\x41uthenticationConfig\x12\x11\n\tjwtIssuer\x18\x01 \x01(\t\x12\x11\n\tjwtExpiry\x18\x02 \x01(\x11\x12\x10\n\x08jwtLimit\x18\x06 \x01(\x11\x12\x12\n\njwtRefresh\x18\x07 \x01(\x11\x12\x33\n\x08provider\x18\x03 \x01(\x0b\x32\x1c.tracdap.config.PluginConfigH\x00\x88\x01\x01\x12\x13\n\x0b\x64isableAuth\x18\x04 \x01(\x08\x12\x16\n\x0e\x64isableSigning\x18\x05 \x01(\x08\x12\x14\n\x0csystemUserId\x18\x08 \x01(\t\x12\x16\n\x0esystemUserName\x18\t \x01(\t\x12\x1c\n\x14systemTicketDuration\x18\n \x01(\x11\x12\x1b\n\x13systemTicketRefresh\x18\x0b \x01(\x11\x42\x0b\n\t_provider\"\xc8\x01\n\rStorageConfig\x12;\n\x07\x62uckets\x18\x01 \x03(\x0b\x32*.tracdap.config.StorageConfig.BucketsEntry\x12\x15\n\rdefaultBucket\x18\x02 \x01(\t\x12\x15\n\rdefaultFormat\x18\x03 \x01(\t\x1aL\n\x0c\x42ucketsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.tracdap.config.PluginConfig:\x02\x38\x01\x42\x1c\n\x18org.finos.tracdap.configP\x01\x62\x06proto3')
 
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.config.common_pb2', globals())
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.config.common_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-
-  DESCRIPTOR._options = None
-  DESCRIPTOR._serialized_options = b'\n\030org.finos.tracdap.configP\001'
-  __CONFIGFILE_CONFIGENTRY._options = None
-  __CONFIGFILE_CONFIGENTRY._serialized_options = b'8\001'
-  _PLUGINCONFIG_PROPERTIESENTRY._options = None
-  _PLUGINCONFIG_PROPERTIESENTRY._serialized_options = b'8\001'
-  _PLUGINCONFIG_SECRETSENTRY._options = None
-  _PLUGINCONFIG_SECRETSENTRY._serialized_options = b'8\001'
-  _PLATFORMINFO_DEPLOYMENTINFOENTRY._options = None
-  _PLATFORMINFO_DEPLOYMENTINFOENTRY._serialized_options = b'8\001'
-  __CONFIGFILE._serialized_start=47
-  __CONFIGFILE._serialized_end=164
-  __CONFIGFILE_CONFIGENTRY._serialized_start=119
-  __CONFIGFILE_CONFIGENTRY._serialized_end=164
-  _PLUGINCONFIG._serialized_start=167
-  _PLUGINCONFIG._serialized_end=424
-  _PLUGINCONFIG_PROPERTIESENTRY._serialized_start=327
-  _PLUGINCONFIG_PROPERTIESENTRY._serialized_end=376
-  _PLUGINCONFIG_SECRETSENTRY._serialized_start=378
-  _PLUGINCONFIG_SECRETSENTRY._serialized_end=424
-  _PLATFORMINFO._serialized_start=427
-  _PLATFORMINFO._serialized_end=611
-  _PLATFORMINFO_DEPLOYMENTINFOENTRY._serialized_start=558
-  _PLATFORMINFO_DEPLOYMENTINFOENTRY._serialized_end=611
-  _AUTHENTICATIONCONFIG._serialized_start=614
-  _AUTHENTICATIONCONFIG._serialized_end=928
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\030org.finos.tracdap.configP\001'
+  _globals['__CONFIGFILE_CONFIGENTRY']._options = None
+  _globals['__CONFIGFILE_CONFIGENTRY']._serialized_options = b'8\001'
+  _globals['_PLUGINCONFIG_PROPERTIESENTRY']._options = None
+  _globals['_PLUGINCONFIG_PROPERTIESENTRY']._serialized_options = b'8\001'
+  _globals['_PLUGINCONFIG_SECRETSENTRY']._options = None
+  _globals['_PLUGINCONFIG_SECRETSENTRY']._serialized_options = b'8\001'
+  _globals['_PLATFORMINFO_DEPLOYMENTINFOENTRY']._options = None
+  _globals['_PLATFORMINFO_DEPLOYMENTINFOENTRY']._serialized_options = b'8\001'
+  _globals['_STORAGECONFIG_BUCKETSENTRY']._options = None
+  _globals['_STORAGECONFIG_BUCKETSENTRY']._serialized_options = b'8\001'
+  _globals['__CONFIGFILE']._serialized_start=47
+  _globals['__CONFIGFILE']._serialized_end=164
+  _globals['__CONFIGFILE_CONFIGENTRY']._serialized_start=119
+  _globals['__CONFIGFILE_CONFIGENTRY']._serialized_end=164
+  _globals['_PLUGINCONFIG']._serialized_start=167
+  _globals['_PLUGINCONFIG']._serialized_end=424
+  _globals['_PLUGINCONFIG_PROPERTIESENTRY']._serialized_start=327
+  _globals['_PLUGINCONFIG_PROPERTIESENTRY']._serialized_end=376
+  _globals['_PLUGINCONFIG_SECRETSENTRY']._serialized_start=378
+  _globals['_PLUGINCONFIG_SECRETSENTRY']._serialized_end=424
+  _globals['_PLATFORMINFO']._serialized_start=427
+  _globals['_PLATFORMINFO']._serialized_end=611
+  _globals['_PLATFORMINFO_DEPLOYMENTINFOENTRY']._serialized_start=558
+  _globals['_PLATFORMINFO_DEPLOYMENTINFOENTRY']._serialized_end=611
+  _globals['_AUTHENTICATIONCONFIG']._serialized_start=614
+  _globals['_AUTHENTICATIONCONFIG']._serialized_end=928
+  _globals['_STORAGECONFIG']._serialized_start=931
+  _globals['_STORAGECONFIG']._serialized_end=1131
+  _globals['_STORAGECONFIG_BUCKETSENTRY']._serialized_start=1055
+  _globals['_STORAGECONFIG_BUCKETSENTRY']._serialized_end=1131
 # @@protoc_insertion_point(module_scope)

tracdap/rt/config/job_pb2.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/config/job.proto
+# Protobuf Python Version: 4.25.3
 """Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -18,24 +19,24 @@ from tracdap.metadata import job_pb2 as tracdap_dot_metadata_dot_job__pb2
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18tracdap/config/job.proto\x12\x0etracdap.config\x1a tracdap/metadata/object_id.proto\x1a\x1dtracdap/metadata/object.proto\x1a\x1atracdap/metadata/job.proto\"\xae\x04\n\tJobConfig\x12*\n\x05jobId\x18\x01 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader\x12,\n\x03job\x18\x02 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinition\x12;\n\tresources\x18\x03 \x03(\x0b\x32(.tracdap.config.JobConfig.ResourcesEntry\x12G\n\x0fresourceMapping\x18\x04 \x03(\x0b\x32..tracdap.config.JobConfig.ResourceMappingEntry\x12\x43\n\rresultMapping\x18\x05 \x03(\x0b\x32,.tracdap.config.JobConfig.ResultMappingEntry\x1aT\n\x0eResourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".tracdap.metadata.ObjectDefinition:\x02\x38\x01\x1aS\n\x14ResourceMappingEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader:\x02\x38\x01\x1aQ\n\x12ResultMappingEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.tracdap.metadata.TagHeader:\x02\x38\x01\x42\x1c\n\x18org.finos.tracdap.configP\x01\x62\x06proto3')
 
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.config.job_pb2', globals())
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.config.job_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-
-  DESCRIPTOR._options = None
-  DESCRIPTOR._serialized_options = b'\n\030org.finos.tracdap.configP\001'
-  _JOBCONFIG_RESOURCESENTRY._options = None
-  _JOBCONFIG_RESOURCESENTRY._serialized_options = b'8\001'
-  _JOBCONFIG_RESOURCEMAPPINGENTRY._options = None
-  _JOBCONFIG_RESOURCEMAPPINGENTRY._serialized_options = b'8\001'
-  _JOBCONFIG_RESULTMAPPINGENTRY._options = None
-  _JOBCONFIG_RESULTMAPPINGENTRY._serialized_options = b'8\001'
-  _JOBCONFIG._serialized_start=138
-  _JOBCONFIG._serialized_end=696
-  _JOBCONFIG_RESOURCESENTRY._serialized_start=444
-  _JOBCONFIG_RESOURCESENTRY._serialized_end=528
-  _JOBCONFIG_RESOURCEMAPPINGENTRY._serialized_start=530
-  _JOBCONFIG_RESOURCEMAPPINGENTRY._serialized_end=613
-  _JOBCONFIG_RESULTMAPPINGENTRY._serialized_start=615
-  _JOBCONFIG_RESULTMAPPINGENTRY._serialized_end=696
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\030org.finos.tracdap.configP\001'
+  _globals['_JOBCONFIG_RESOURCESENTRY']._options = None
+  _globals['_JOBCONFIG_RESOURCESENTRY']._serialized_options = b'8\001'
+  _globals['_JOBCONFIG_RESOURCEMAPPINGENTRY']._options = None
+  _globals['_JOBCONFIG_RESOURCEMAPPINGENTRY']._serialized_options = b'8\001'
+  _globals['_JOBCONFIG_RESULTMAPPINGENTRY']._options = None
+  _globals['_JOBCONFIG_RESULTMAPPINGENTRY']._serialized_options = b'8\001'
+  _globals['_JOBCONFIG']._serialized_start=138
+  _globals['_JOBCONFIG']._serialized_end=696
+  _globals['_JOBCONFIG_RESOURCESENTRY']._serialized_start=444
+  _globals['_JOBCONFIG_RESOURCESENTRY']._serialized_end=528
+  _globals['_JOBCONFIG_RESOURCEMAPPINGENTRY']._serialized_start=530
+  _globals['_JOBCONFIG_RESOURCEMAPPINGENTRY']._serialized_end=613
+  _globals['_JOBCONFIG_RESULTMAPPINGENTRY']._serialized_start=615
+  _globals['_JOBCONFIG_RESULTMAPPINGENTRY']._serialized_end=696
 # @@protoc_insertion_point(module_scope)

tracdap/rt/config/platform.py

@@ -9,6 +9,29 @@ import tracdap.rt.metadata as metadata
 from .common import *  # noqa
 
 
+class RoutingProtocol(_enum.Enum):
+
+    PROTOCOL_NOT_SET = 0, 
+
+    HTTP = 1, 
+
+    GRPC = 2, 
+
+    GRPC_WEB = 3, 
+
+    REST = 4, 
+
+
+class DeploymentLayout(_enum.Enum):
+
+    LAYOUT_NOT_SET = 0, 
+
+    SANDBOX = 1, 
+
+    HOSTED = 2, 
+
+    CUSTOM = 3, 
+
 
 @_dc.dataclass
 class PlatformConfig:
@@ -33,9 +56,11 @@ class PlatformConfig:
 
     webServer: _tp.Optional[WebServerConfig] = None
 
-    instances: InstantMap = None
+    gateway: _tp.Optional[GatewayConfig] = None
+
+    services: _tp.Dict[str, ServiceConfig] = _dc.field(default_factory=dict)
 
-    services: ServiceMap = None
+    deployment: DeploymentConfig = None
 
 
 @_dc.dataclass
@@ -46,16 +71,6 @@ class MetadataConfig:
     format: metadata.MetadataFormat = metadata.MetadataFormat.METADATA_FORMAT_NOT_SET
 
 
-@_dc.dataclass
-class StorageConfig:
-
-    buckets: _tp.Dict[str, PluginConfig] = _dc.field(default_factory=dict)
-
-    defaultBucket: str = None
-
-    defaultFormat: str = None
-
-
 @_dc.dataclass
 class TenantConfig:
 
@@ -69,8 +84,6 @@ class WebServerConfig:
 
     enabled: bool = None
 
-    port: int = None
-
     contentRoot: PluginConfig = None
 
     rewriteRules: _tp.List[WebServerRewriteRule] = _dc.field(default_factory=list)
@@ -97,17 +110,37 @@ class WebServerRedirect:
 
 
 @_dc.dataclass
-class InstantMap:
+class GatewayConfig:
+
+    idleTimeout: int = None
+
+    routes: _tp.List[RouteConfig] = _dc.field(default_factory=list)
+
+
+@_dc.dataclass
+class RouteConfig:
+
+    routeName: str = None
+
+    routeType: RoutingProtocol = RoutingProtocol.PROTOCOL_NOT_SET
 
-    meta: _tp.List[InstanceConfig] = _dc.field(default_factory=list)
+    protocols: _tp.List[RoutingProtocol] = _dc.field(default_factory=list)
 
-    data: _tp.List[InstanceConfig] = _dc.field(default_factory=list)
+    match: RoutingMatch = None
 
-    orch: _tp.List[InstanceConfig] = _dc.field(default_factory=list)
+    target: RoutingTarget = None
 
 
 @_dc.dataclass
-class InstanceConfig:
+class RoutingMatch:
+
+    host: str = None
+
+    path: str = None
+
+
+@_dc.dataclass
+class RoutingTarget:
 
     scheme: str = None
 
@@ -115,18 +148,20 @@ class InstanceConfig:
 
     port: int = None
 
+    path: str = None
+
 
 @_dc.dataclass
-class ServiceMap:
+class ServiceConfig:
 
-    meta: ServiceConfig = None
+    enabled: _tp.Optional[bool] = None
 
-    data: ServiceConfig = None
+    alias: str = None
 
-    orch: ServiceConfig = None
+    port: int = None
 
 
 @_dc.dataclass
-class ServiceConfig:
+class DeploymentConfig:
 
-    port: int = None
+    layout: DeploymentLayout = DeploymentLayout.LAYOUT_NOT_SET