tracdap-runtime 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl

tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.py

@@ -13,6 +13,7 @@ _sym_db = _symbol_database.Default()
 
 
 from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__id__pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_type__pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import data_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_data__pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import model_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_model__pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import flow_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_flow__pb2
@@ -22,7 +23,7 @@ from tracdap.rt._impl.grpc.tracdap.metadata import custom_pb2 as tracdap_dot_rt_
 from tracdap.rt._impl.grpc.tracdap.metadata import stoarge_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_stoarge__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x32tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto\x1a\x34tracdap/rt/_impl/grpc/tracdap/metadata/stoarge.proto\"\xf0\x03\n\x10ObjectDefinition\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .tracdap.metadata.DataDefinitionH\x00\x12\x32\n\x05model\x18\x03 \x01(\x0b\x32!.tracdap.metadata.ModelDefinitionH\x00\x12\x30\n\x04\x66low\x18\x04 \x01(\x0b\x32 .tracdap.metadata.FlowDefinitionH\x00\x12.\n\x03job\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinitionH\x00\x12\x30\n\x04\x66ile\x18\x06 \x01(\x0b\x32 .tracdap.metadata.FileDefinitionH\x00\x12\x34\n\x06\x63ustom\x18\x07 \x01(\x0b\x32\".tracdap.metadata.CustomDefinitionH\x00\x12\x36\n\x07storage\x18\x08 \x01(\x0b\x32#.tracdap.metadata.StorageDefinitionH\x00\x12\x34\n\x06schema\x18\t \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x42\x0c\n\ndefinitionB2\n\x1aorg.finos.tracdap.metadataB\x12ObjectProtoWrapperP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x32tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto\x1a\x34tracdap/rt/_impl/grpc/tracdap/metadata/stoarge.proto\"\x87\x05\n\x10ObjectDefinition\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .tracdap.metadata.DataDefinitionH\x00\x12\x32\n\x05model\x18\x03 \x01(\x0b\x32!.tracdap.metadata.ModelDefinitionH\x00\x12\x30\n\x04\x66low\x18\x04 \x01(\x0b\x32 .tracdap.metadata.FlowDefinitionH\x00\x12.\n\x03job\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinitionH\x00\x12\x30\n\x04\x66ile\x18\x06 \x01(\x0b\x32 .tracdap.metadata.FileDefinitionH\x00\x12\x34\n\x06\x63ustom\x18\x07 \x01(\x0b\x32\".tracdap.metadata.CustomDefinitionH\x00\x12\x36\n\x07storage\x18\x08 \x01(\x0b\x32#.tracdap.metadata.StorageDefinitionH\x00\x12\x34\n\x06schema\x18\t \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12H\n\x0bobjectProps\x18\x64 \x03(\x0b\x32\x33.tracdap.metadata.ObjectDefinition.ObjectPropsEntry\x1aK\n\x10ObjectPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0c\n\ndefinitionB2\n\x1aorg.finos.tracdap.metadataB\x12ObjectProtoWrapperP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -30,6 +31,10 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracd
 if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._options = None
   _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataB\022ObjectProtoWrapperP\001'
-  _globals['_OBJECTDEFINITION']._serialized_start=492
-  _globals['_OBJECTDEFINITION']._serialized_end=988
+  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._options = None
+  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_options = b'8\001'
+  _globals['_OBJECTDEFINITION']._serialized_start=543
+  _globals['_OBJECTDEFINITION']._serialized_end=1190
+  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_start=1101
+  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_end=1176
 # @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.pyi

@@ -1,4 +1,5 @@
 from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as _object_id_pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as _type_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import data_pb2 as _data_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import model_pb2 as _model_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import flow_pb2 as _flow_pb2
@@ -6,6 +7,7 @@ from tracdap.rt._impl.grpc.tracdap.metadata import job_pb2 as _job_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import file_pb2 as _file_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import custom_pb2 as _custom_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import stoarge_pb2 as _stoarge_pb2
+from google.protobuf.internal import containers as _containers
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
 from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union
@@ -13,7 +15,14 @@ from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Opti
 DESCRIPTOR: _descriptor.FileDescriptor
 
 class ObjectDefinition(_message.Message):
-    __slots__ = ("objectType", "data", "model", "flow", "job", "file", "custom", "storage", "schema")
+    __slots__ = ("objectType", "data", "model", "flow", "job", "file", "custom", "storage", "schema", "objectProps")
+    class ObjectPropsEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: _type_pb2.Value
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[_type_pb2.Value, _Mapping]] = ...) -> None: ...
     OBJECTTYPE_FIELD_NUMBER: _ClassVar[int]
     DATA_FIELD_NUMBER: _ClassVar[int]
     MODEL_FIELD_NUMBER: _ClassVar[int]
@@ -23,6 +32,7 @@ class ObjectDefinition(_message.Message):
     CUSTOM_FIELD_NUMBER: _ClassVar[int]
     STORAGE_FIELD_NUMBER: _ClassVar[int]
     SCHEMA_FIELD_NUMBER: _ClassVar[int]
+    OBJECTPROPS_FIELD_NUMBER: _ClassVar[int]
     objectType: _object_id_pb2.ObjectType
     data: _data_pb2.DataDefinition
     model: _model_pb2.ModelDefinition
@@ -32,4 +42,5 @@ class ObjectDefinition(_message.Message):
     custom: _custom_pb2.CustomDefinition
     storage: _stoarge_pb2.StorageDefinition
     schema: _data_pb2.SchemaDefinition
-    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., data: _Optional[_Union[_data_pb2.DataDefinition, _Mapping]] = ..., model: _Optional[_Union[_model_pb2.ModelDefinition, _Mapping]] = ..., flow: _Optional[_Union[_flow_pb2.FlowDefinition, _Mapping]] = ..., job: _Optional[_Union[_job_pb2.JobDefinition, _Mapping]] = ..., file: _Optional[_Union[_file_pb2.FileDefinition, _Mapping]] = ..., custom: _Optional[_Union[_custom_pb2.CustomDefinition, _Mapping]] = ..., storage: _Optional[_Union[_stoarge_pb2.StorageDefinition, _Mapping]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ...) -> None: ...
+    objectProps: _containers.MessageMap[str, _type_pb2.Value]
+    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., data: _Optional[_Union[_data_pb2.DataDefinition, _Mapping]] = ..., model: _Optional[_Union[_model_pb2.ModelDefinition, _Mapping]] = ..., flow: _Optional[_Union[_flow_pb2.FlowDefinition, _Mapping]] = ..., job: _Optional[_Union[_job_pb2.JobDefinition, _Mapping]] = ..., file: _Optional[_Union[_file_pb2.FileDefinition, _Mapping]] = ..., custom: _Optional[_Union[_custom_pb2.CustomDefinition, _Mapping]] = ..., storage: _Optional[_Union[_stoarge_pb2.StorageDefinition, _Mapping]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., objectProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...

tracdap/rt/_impl/guard_rails.py

@@ -46,6 +46,27 @@ def _get_package_path(module_name):
     return module_path.parents[depth]
 
 
+def run_model_guard(operation: str = None):
+
+    # A simple guard method to block model code from accessing parts of the TRAC runtime framework
+    # To blocks calls to the Python stdlib or 3rd party libs, use PythonGuardRails instead
+
+    stack = inspect.stack()
+    frame = stack[-1]
+
+    if operation is None:
+        operation = f"Calling {frame.function}()"
+
+    for frame_index in range(len(stack) - 2, 0, -1):
+
+        parent_frame = frame
+        frame = stack[frame_index]
+
+        if frame.function == "run_model" and parent_frame.function == "_execute":
+            err = f"{operation} is not allowed inside run_model()"
+            raise ex.ERuntimeValidation(err)
+
+
 class PythonGuardRails:
 
     DANGEROUS_BUILTIN_FUNCTIONS = ["exec", "eval", "compile", "open", "input", "memoryview"]

tracdap/rt/_impl/models.py

@@ -215,12 +215,15 @@ class ModelLoader:
 
             for name, param in model_def.parameters.items():
                 self.__log.info(f"Parameter [{name}] - {param.paramType.basicType.name}")
+                param.paramProps = self._encoded_props(param.paramProps, "parameter", name)
 
             for name, schema in model_def.inputs.items():
                 self.__log.info(f"Input [{name}] - {schema.schema.schemaType.name}")
+                schema.inputProps = self._encoded_props(schema.inputProps, "input", name)
 
             for name, schema in model_def.outputs.items():
                 self.__log.info(f"Output [{name}] - {schema.schema.schemaType.name}")
+                schema.outputProps = self._encoded_props(schema.outputProps, "input", name)
 
             return model_def
 
@@ -231,3 +234,25 @@ class ModelLoader:
 
             self.__log.error(msg, exc_info=True)
             raise _ex.EModelValidation(msg) from e
+
+    @staticmethod
+    def _encoded_props(
+            raw_props: tp.Dict[str, tp.Any],
+            item_type: str, item_name: str) \
+            -> tp.Dict[str, _meta.Value]:
+
+        if raw_props is None:
+            return dict()
+
+        encoded_props = dict()
+
+        for key, raw_value in raw_props.items():
+
+            if raw_value is None:
+                raise _ex.EModelValidation(f"Invalid null property [{key}] for {item_type} [{item_name}]")
+            elif isinstance(raw_value, _meta.Value):
+                encoded_props[key] = raw_value
+            else:
+                encoded_props[key] = _types.MetadataCodec.encode_value(raw_value)
+
+        return encoded_props

tracdap/rt/_impl/static_api.py

@@ -71,10 +71,14 @@ class StaticApiImpl(_StaticApiHook):
 
     def define_parameter(
             self, param_name: str, param_type: _tp.Union[_meta.TypeDescriptor, _meta.BasicType],
-            label: str, default_value: _tp.Optional[_tp.Any] = None) \
+            label: str, default_value: _tp.Optional[_tp.Any] = None,
+            *, param_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
             -> _Named[_meta.ModelParameter]:
 
-        _val.validate_signature(self.define_parameter, param_name, param_type, label, default_value)
+        _val.validate_signature(
+            self.define_parameter,
+            param_name, param_type, label, default_value,
+            param_props=param_props)
 
         if isinstance(param_type, _meta.TypeDescriptor):
             param_type_descriptor = param_type
@@ -88,7 +92,9 @@ class StaticApiImpl(_StaticApiHook):
                 msg = f"Default value for parameter [{param_name}] does not match the declared type"
                 raise _ex.EModelValidation(msg) from e
 
-        return _Named(param_name, _meta.ModelParameter(param_type_descriptor, label, default_value))
+        return _Named(param_name, _meta.ModelParameter(
+            param_type_descriptor, label, default_value,
+            paramProps=param_props))
 
     def define_parameters(
             self, *params: _tp.Union[_Named[_meta.ModelParameter], _tp.List[_Named[_meta.ModelParameter]]]) \
@@ -148,25 +154,49 @@ class StaticApiImpl(_StaticApiHook):
 
     def define_input_table(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None,
-            optional: bool = False) \
+            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
+            input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
             -> _meta.ModelInputSchema:
 
-        _val.validate_signature(self.define_input_table, *fields, label=label, optional=optional)
+        _val.validate_signature(
+            self.define_input_table, *fields,
+            label=label, optional=optional, dynamic=dynamic,
+            input_props=input_props)
+
+        # Do not define details for dynamic schemas
+
+        if dynamic:
+            schema_def = _meta.SchemaDefinition(_meta.SchemaType.TABLE)
+        else:
+            schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
 
-        schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
-        return _meta.ModelInputSchema(schema=schema_def, label=label, optional=optional)
+        return _meta.ModelInputSchema(
+            schema=schema_def, label=label,
+            optional=optional, dynamic=dynamic,
+            inputProps=input_props)
 
     def define_output_table(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None,
-            optional: bool = False) \
+            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
+            output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
             -> _meta.ModelOutputSchema:
 
-        _val.validate_signature(self.define_output_table, *fields, label=label, optional=optional)
+        _val.validate_signature(
+            self.define_output_table, *fields,
+            label=label, optional=optional, dynamic=dynamic,
+            output_props=output_props)
+
+        # Do not define details for dynamic schemas
+
+        if dynamic:
+            schema_def = _meta.SchemaDefinition(_meta.SchemaType.TABLE)
+        else:
+            schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
 
-        schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
-        return _meta.ModelOutputSchema(schema=schema_def, label=label, optional=optional)
+        return _meta.ModelOutputSchema(
+            schema=schema_def, label=label,
+            optional=optional, dynamic=dynamic,
+            outputProps=output_props)
 
     @staticmethod
     def _build_named_dict(

tracdap/rt/_impl/type_system.py

@@ -171,6 +171,23 @@ class MetadataCodec:
             type_desc = _meta.TypeDescriptor(_meta.BasicType.DATE)
             return _meta.Value(type_desc, dateValue=_meta.DateValue(value.isoformat()))
 
+        if isinstance(value, list):
+
+            if len(value) == 0:
+                raise _ex.ETracInternal("Cannot encode an empty list")
+
+            array_raw_type = type(value[0])
+            array_trac_type = TypeMapping.python_to_trac(array_raw_type)
+
+            if any(map(lambda x: type(x) != array_raw_type, value)):
+                raise _ex.ETracInternal("Cannot encode a list with values of different types")
+
+            encoded_items = list(map(lambda x: cls.convert_value(x, array_trac_type), value))
+
+            return _meta.Value(
+                _meta.TypeDescriptor(_meta.BasicType.ARRAY, arrayType=array_trac_type),
+                arrayValue=_meta.ArrayValue(encoded_items))
+
         raise _ex.ETracInternal(f"Value type [{type(value)}] is not supported yet")
 
     @classmethod

tracdap/rt/_impl/validation.py

@@ -38,7 +38,7 @@ def check_type(expected_type: tp.Type, value: tp.Any) -> bool:
 
 
 def quick_validate_model_def(model_def: meta.ModelDefinition):
-    _StaticValidator.quick_validate_model_def(model_def)
+    StaticValidator.quick_validate_model_def(model_def)
 
 
 class _TypeValidator:
@@ -233,7 +233,7 @@ class _TypeValidator:
         return type_var.__name__
 
 
-class _StaticValidator:
+class StaticValidator:
 
     __identifier_pattern = re.compile("\\A[a-zA-Z_]\\w*\\Z", re.ASCII)
     __reserved_identifier_pattern = re.compile("\\A(_|trac_)", re.ASCII)
@@ -301,6 +301,28 @@ class _StaticValidator:
         cls._check_inputs_or_outputs(model_def.inputs)
         cls._check_inputs_or_outputs(model_def.outputs)
 
+    @classmethod
+    def quick_validate_schema(cls, schema: meta.SchemaDefinition):
+
+        if schema.schemaType != meta.SchemaType.TABLE:
+            cls._fail(f"Unsupported schema type [{schema.schemaType}]")
+
+        if schema.partType != meta.PartType.PART_ROOT:
+            cls._fail(f"Unsupported partition type [{schema.partType}]")
+
+        if schema.table is None or schema.table.fields is None or len(schema.table.fields) == 0:
+            cls._fail(f"Table schema does not define any fields")
+
+        fields = schema.table.fields
+        field_names = list(map(lambda f: f.fieldName, fields))
+        property_type = f"field"
+
+        cls._valid_identifiers(field_names, property_type)
+        cls._case_insensitive_duplicates(field_names, property_type)
+
+        for field in fields:
+            cls._check_single_field(field, property_type)
+
     @classmethod
     def _check_label(cls, label, param_name):
         if label is not None:
@@ -320,6 +342,9 @@ class _StaticValidator:
             else:
                 cls._check_label(param.label, param_name)
 
+            if param.paramProps is not None:
+                cls._valid_identifiers(param.paramProps.keys(), "entry in param props")
+
     @classmethod
     def _check_inputs_or_outputs(cls, inputs_or_outputs):
 
@@ -327,10 +352,20 @@ class _StaticValidator:
 
             cls._log.info(f"Checking {input_name}")
 
+            if input_schema.dynamic:
+                if input_schema.schema and input_schema.schema.table:
+                    error = "Dynamic schemas must have schema.table = None"
+                    cls._fail(f"Invalid schema for [{input_name}]: {error}")
+                else:
+                    continue
+
             fields = input_schema.schema.table.fields
             field_names = list(map(lambda f: f.fieldName, fields))
             property_type = f"field in [{input_name}]"
 
+            if len(fields) == 0:
+                cls._fail(f"Invalid schema for [{input_name}]: No fields defined")
+
             cls._valid_identifiers(field_names, property_type)
             cls._case_insensitive_duplicates(field_names, property_type)
 
@@ -340,6 +375,13 @@ class _StaticValidator:
             label = input_schema.label
             cls._check_label(label, input_name)
 
+            if isinstance(input_schema, meta.ModelInputSchema):
+                if input_schema.inputProps is not None:
+                    cls._valid_identifiers(input_schema.inputProps.keys(), "entry in input props")
+            else:
+                if input_schema.outputProps is not None:
+                    cls._valid_identifiers(input_schema.outputProps.keys(), "entry in output props")
+
     @classmethod
     def _check_single_field(cls, field: meta.FieldSchema, property_type):
 
@@ -365,8 +407,9 @@ class _StaticValidator:
         if field.categorical and field.fieldType != meta.BasicType.STRING:
             cls._fail(f"Invalid {property_type}: [{field.fieldName}] fieldType {field.fieldType} used as categorical")
 
-        if field.businessKey and not field.notNull:
-            cls._fail(f"Invalid {property_type}: [{field.fieldName}] is a business key but not_null = False")
+        # Do not require notNull = True for business keys here
+        # Instead setting businessKey = True will cause notNull = True to be set during normalization
+        # This agrees with the semantics in platform API and CSV schema loader
 
     @classmethod
     def _valid_identifiers(cls, keys, property_type):

tracdap/rt/_plugins/config_local.py

@@ -0,0 +1,49 @@
+#  Copyright 2024 Accenture Global Solutions Limited
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import pathlib as pathlib
+import typing as tp
+
+import tracdap.rt.ext.plugins as plugins
+import tracdap.rt.exceptions as ex
+
+from tracdap.rt.ext.config import *
+
+
+class LocalConfigLoader(IConfigLoader):
+
+    # Properties dict will be empty for config plugins
+    def __init__(self, properties: tp.Dict[str, str]):  # noqa
+        pass
+
+    def has_config_file(self, config_url: str) -> bool:
+        if config_url.startswith("file:"):
+            config_url = config_url[5:]
+        config_path = pathlib.Path(config_url).resolve()
+        return config_path.exists() and config_path.is_file()
+
+    def load_config_file(self, config_url: str) -> bytes:
+        if config_url.startswith("file:"):
+            config_url = config_url[5:]
+        config_path = pathlib.Path(config_url).resolve()
+        return config_path.read_bytes()
+
+    def has_config_dict(self, config_url: str) -> bool:
+        return False
+
+    def load_config_dict(self, config_url: str) -> dict:
+        raise ex.ETracInternal("Local config loader does not support loading objects")
+
+
+plugins.PluginManager.register_plugin(IConfigLoader, LocalConfigLoader, ["LOCAL", "file"])

tracdap/rt/_version.py

@@ -12,4 +12,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.6.2"
+__version__ = "0.6.4"

tracdap/rt/api/hook.py

@@ -80,7 +80,8 @@ class _StaticApiHook:
     @_abc.abstractmethod
     def define_parameter(
             self, param_name: str, param_type: _tp.Union[_meta.TypeDescriptor, _meta.BasicType],
-            label: str, default_value: _tp.Optional[_tp.Any] = None) \
+            label: str, default_value: _tp.Optional[_tp.Any] = None,
+            *, param_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
             -> _Named[_meta.ModelParameter]:
 
         pass
@@ -120,8 +121,8 @@ class _StaticApiHook:
     @_abc.abstractmethod
     def define_input_table(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None,
-            optional: bool = False) \
+            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
+            input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
             -> _meta.ModelInputSchema:
 
         pass
@@ -129,8 +130,8 @@ class _StaticApiHook:
     @_abc.abstractmethod
     def define_output_table(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None,
-            optional: bool = False) \
+            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
+            output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
             -> _meta.ModelOutputSchema:
 
         pass

tracdap/rt/api/model_api.py

@@ -100,13 +100,26 @@ class TracContext:
         """
         Get the schema of a model input or output
 
-        The schema of an input or output can be retrieved and examined at runtime using this method.
-        Inputs must be defined in :py:meth:`TracModel.define_inputs`
-        and outputs in :py:meth:`TracModel.define_outputs`.
-        Input and output names are case-sensitive.
-
-        In the current version of the runtime all model inputs and outputs are defined statically,
-        :py:meth:`get_schema` will return the schema as it was defined.
+        Use this method to get the :py:class:`SchemaDefinition <tracdap.rt.metadata.SchemaDefinition>`
+        for any input or output of the current model.
+        For datasets with static schemas, this will be the same schema that was defined in the
+        :py:class:`TracModel <tracdap.rt.api.TracModel>` methods
+        :py:meth:`define_inputs() <tracdap.rt.api.TracModel.define_inputs>` and
+        :py:meth:`define_outputs() <tracdap.rt.api.TracModel.define_outputs>`.
+
+        For inputs with dynamic schemas, the schema of the provided input dataset will be returned.
+        For outputs with dynamic schemas the schema must be set by calling
+        :py:meth:`put_schema() <tracdap.rt.api.TracContext.put_schema>`, after which this method
+        will return that schema. Calling :py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>`
+        for a dynamic output before the schema is set will result in a runtime validation error.
+
+        For optional inputs, use :py:meth:`has_dataset() <tracdap.rt.api.TracContext.has_dataset>`
+        to check whether the input was provided. Calling :py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>`
+        for an optional input that was not provided will always result in a validation error,
+        regardless of whether the input using a static or dynamic schema. For optional outputs
+        :py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>` can be called, with the
+        normal proviso that dynamic schemas must first be set by calling
+        :py:meth:`put_schema() <tracdap.rt.api.TracContext.put_schema>`.
 
         Attempting to retrieve the schema for a dataset that is not defined as a model input or output
         will result in a runtime validation error, even if that dataset exists in the job config and
@@ -152,6 +165,36 @@ class TracContext:
         """
         pass
 
+    @_abc.abstractmethod
+    def put_schema(self, dataset_name: str, schema: SchemaDefinition):
+
+        """
+        Set the schema of a dynamic model output
+
+        For outputs marked as dynamic, a :py:class:`SchemaDefinition <tracdap.rt.metadata.SchemaDefinition>`
+        must be supplied before attempting to save the data. TRAC API functions are available to help with
+        building schemas, such as :py:func:`trac.F() <tracdap.rt.api.F>` to define fields or
+        :py:func:`load_schema() <tracdap.rt.api.load_schema>` to load predefined schemas.
+        Once a schema has been set, it can be retrieved by calling
+        :py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>` as normal.
+        If :py:meth:`put_schema() <tracdap.rt.api.TracContext.put_schema>` is called for an optional
+        output the model must also supply data for that output, otherwise TRAC will report a runtime
+        validation error after the model completes.
+
+        Each schema can only be set once and the schema will be validated using the normal
+        validation rules. If validation fails this method will raise
+        :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`.
+        Attempting to set the schema for a dataset that is not defined as a dynamic model output
+        for the current model will result in a runtime validation error.
+
+        :param dataset_name: The name of the output to set the schema for
+        :param schema: A TRAC schema definition to use for the named output
+        :type schema: :py:class:`SchemaDefinition <tracdap.rt.metadata.SchemaDefinition>`
+        :raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
+        """
+
+        pass
+
     @_abc.abstractmethod
     def put_pandas_table(self, dataset_name: str, dataset: pandas.DataFrame):