tracdap-runtime 0.8.0rc2__py3-none-any.whl → 0.9.0b2__py3-none-any.whl

tracdap/rt/_impl/core/repos.py

@@ -17,6 +17,7 @@ import typing as _tp
 
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt.config as cfg
+import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
 import tracdap.rt._impl.core.logging as _logging
 
@@ -34,19 +35,27 @@ class RepositoryManager:
         # Initialize all repos in the system config
         # Any errors for missing repo types (plugins) will be raised during startup
 
-        for repo_name, repo_config in sys_config.repositories.items():
+        for resource_key, resource in sys_config.resources.items():
+            if resource.resourceType == meta.ResourceType.MODEL_REPOSITORY:
 
-            try:
+                try:
 
-                self._repos[repo_name] = plugins.PluginManager.load_plugin(IModelRepository, repo_config)
+                    # Add global properties related to the repo protocol
+                    related_props = {
+                        k: v for (k, v) in sys_config.properties.items()
+                        if k.startswith(f"{resource.protocol}.")}
 
-            except ex.EPluginNotAvailable as e:
+                    resource.properties.update(related_props)
 
-                msg = f"Model repository type [{repo_config.protocol}] is not recognised" \
-                      + " (this could indicate a missing model repository plugin)"
+                    self._repos[resource_key] = plugins.PluginManager.load_plugin(IModelRepository, resource)
 
-                self._log.error(msg)
-                raise ex.EStartup(msg) from e
+                except ex.EPluginNotAvailable as e:
+
+                    msg = f"Model repository type [{resource.protocol}] is not recognised" \
+                          + " (this could indicate a missing model repository plugin)"
+
+                    self._log.error(msg)
+                    raise ex.EStartup(msg) from e
 
     def get_repository(self, repo_name: str) -> IModelRepository:
 

tracdap/rt/_impl/core/storage.py

@@ -29,6 +29,7 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as plugins
+import tracdap.rt._impl.core.config_parser as _cfg_p
 import tracdap.rt._impl.core.data as _data
 import tracdap.rt._impl.core.logging as _logging
 import tracdap.rt._impl.core.util as _util
@@ -81,25 +82,35 @@ class StorageManager:
         self.__file_storage: tp.Dict[str, IFileStorage] = dict()
         self.__data_storage: tp.Dict[str, IDataStorage] = dict()
         self.__external: tp.List[str] = list()
-        self.__settings = sys_config.storage
+        self.__sys_config = sys_config
 
-        for storage_key, storage_config in sys_config.storage.buckets.items():
-            self.create_storage(storage_key, storage_config)
+        self.__default_location = _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
+        self.__default_format = _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_FORMAT, "CSV")
 
-        for storage_key, storage_config in sys_config.storage.external.items():
-            if storage_key in self.__file_storage or storage_key in self.__data_storage:
-                raise _ex.EConfig(f"Storage key [{storage_key}] is defined as both internal and external storage")
-            self.__external.append(storage_key)
-            self.create_storage(storage_key, storage_config)
+        for storage_key, storage_config in sys_config.resources.items():
+
+            if storage_config.resourceType == _meta.ResourceType.INTERNAL_STORAGE:
+                self.create_storage(storage_key, storage_config)
+
+            elif storage_config.resourceType == _meta.ResourceType.EXTERNAL_STORAGE:
+                self.__external.append(storage_key)
+                self.create_storage(storage_key, storage_config)
 
     def default_storage_key(self):
-        return self.__settings.defaultBucket
+        return self.__default_location
 
     def default_storage_format(self):
-        return self.__settings.defaultFormat
+        return self.__default_format
 
     def create_storage(self, storage_key: str, storage_config: _cfg.PluginConfig):
 
+        # Add global properties related to the storage protocol
+        related_props = {
+            k: v for (k, v) in self.__sys_config.properties.items()
+            if k.startswith(f"{storage_config.protocol}.")}
+
+        storage_config.properties.update(related_props)
+
         if plugins.PluginManager.is_plugin_available(IStorageProvider, storage_config.protocol):
             self._create_storage_from_provider(storage_key, storage_config)
         else:
@@ -109,11 +120,12 @@ class StorageManager:
 
         provider = plugins.PluginManager.load_plugin(IStorageProvider, storage_config)
 
-        if provider.has_file_storage():
-            file_storage = provider.get_file_storage()
-        elif provider.has_arrow_native():
+        # Prefer arrow native FS if it is available
+        if provider.has_arrow_native():
             fs = provider.get_arrow_native()
             file_storage = CommonFileStorage(storage_key, storage_config, fs)
+        elif provider.has_file_storage():
+            file_storage = provider.get_file_storage()
         else:
             file_storage = None
 

tracdap/rt/_impl/core/struct.py

@@ -37,6 +37,12 @@ import tracdap.rt._impl.core.type_system as _meta_types
 
 class StructProcessor:
 
+    JSON_FORMAT = "text/json"
+    JSON_ALT_FORMATS = ["json", ".json"]
+
+    YAML_FORMAT = "text/yaml"
+    YAML_ALT_FORMATS = ["yaml", ".yaml", "yml"]
+
     __primitive_types: dict[type, _meta.BasicType] = {
         bool: _meta.BasicType.BOOLEAN,
         int: _meta.BasicType.INTEGER,
@@ -68,13 +74,16 @@ class StructProcessor:
 
 
     @classmethod
-    def define_struct(cls, python_type: type) -> _meta.StructSchema:
+    def define_struct(cls, python_type: type) -> _meta.SchemaDefinition:
+
+        named_types = dict()
+        named_enums = dict()
 
         if _dc.is_dataclass(python_type):
-            return cls._define_struct_for_dataclass(python_type)
+            return cls._define_struct_for_dataclass(python_type, named_types, named_enums, type_stack=[])
 
         if _pyd and issubclass(python_type, _pyd.BaseModel):
-            return cls._define_struct_for_pydantic(python_type)
+            return cls._define_struct_for_pydantic(python_type, named_types, named_enums, type_stack=[])
 
         raise _ex.EUnexpected()
 
@@ -83,10 +92,10 @@ class StructProcessor:
 
         try:
 
-            if src_format == "yaml" or src_format == "yml":
+            if src_format == cls.YAML_FORMAT or src_format.lower() in cls.YAML_ALT_FORMATS:
                 config_dict = _yaml.safe_load(src)
 
-            elif src_format == "json":
+            elif src_format == cls.JSON_FORMAT or src_format.lower() in cls.JSON_ALT_FORMATS:
                 config_dict = _json.load(src)
 
             else:
@@ -113,111 +122,297 @@ class StructProcessor:
         StructQuoter.quote(struct, dst, dst_format)
 
     @classmethod
-    def parse_struct(cls, data: dict, schema: _meta.StructSchema = None, python_type: type = None) -> object:
+    def parse_struct(cls, data: dict, schema: _meta.SchemaDefinition = None, python_type: type = None) -> object:
 
         parser = StructParser()
         return parser.parse(python_type, data)
 
     @classmethod
-    def _define_struct_for_dataclass(cls, python_type: _dc.dataclass) -> _meta.StructSchema:
+    def _define_struct_for_dataclass(
+            cls, python_type: _dc.dataclass,
+            named_types: _tp.Dict[str, _meta.SchemaDefinition],
+            named_enums: _tp.Dict[str, _meta.EnumValues],
+            type_stack: _tp.List[str]) \
+            -> _meta.SchemaDefinition:
 
-        type_hints = _tp.get_type_hints(python_type)
-        trac_fields = dict()
+        try:
 
-        for dc_field in _dc.fields(python_type):
+            type_stack.append(cls._qualified_type_name(python_type))
 
-            field_name = dc_field.name
-            python_type = type_hints[field_name]
+            type_hints = _tp.get_type_hints(python_type)
+            trac_fields = list()
+
+            for field_index, dc_field in enumerate(_dc.fields(python_type)):
 
-            trac_field = cls._define_field(python_type, dc_field=dc_field)
-            trac_fields[field_name] = trac_field
+                field_name = dc_field.name
+                python_type = type_hints[field_name]
 
-        return _meta.StructSchema(fields=trac_fields)
+                trac_field = cls._define_field(
+                    field_name, field_index, python_type, dc_field=dc_field,
+                    named_types=named_types, named_enums=named_enums, type_stack=type_stack)
+
+                trac_fields.append(trac_field)
+
+            if len(type_stack) == 1:
+                return _meta.SchemaDefinition(
+                    schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields,
+                    namedTypes=named_types, namedEnums=named_enums)
+            else:
+                return _meta.SchemaDefinition(
+                    schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields)
+
+        finally:
+
+            type_stack.pop()
 
     @classmethod
-    def _define_struct_for_pydantic(cls, python_type: "type[_pyd.BaseModel]") -> _meta.StructSchema:
+    def _define_struct_for_pydantic(
+            cls, python_type: "type[_pyd.BaseModel]",
+            named_types: _tp.Dict[str, _meta.SchemaDefinition],
+            named_enums: _tp.Dict[str, _meta.EnumValues],
+            type_stack: _tp.List[str]) \
+            -> _meta.SchemaDefinition:
 
-        type_hints = _tp.get_type_hints(python_type)
-        trac_fields = dict()
+        try:
 
-        for field_name, pyd_field in python_type.model_fields.items():
+            type_stack.append(cls._qualified_type_name(python_type))
+
+            type_hints = _tp.get_type_hints(python_type)
+            trac_fields = list()
+
+            field_index = 0
+
+            for field_name, pyd_field in python_type.model_fields.items():
 
-            python_type = type_hints[field_name]
+                python_type = type_hints[field_name]
 
-            trac_field = cls._define_field(python_type, pyd_field=pyd_field)
+                trac_field = cls._define_field(
+                    field_name, field_index, python_type, pyd_field=pyd_field,
+                    named_types=named_types, named_enums=named_enums, type_stack=type_stack)
+
+                if trac_field is not None:
+                    trac_fields.append(trac_field)
+                    field_index += 1
+
+            if len(type_stack) == 1:
+                return _meta.SchemaDefinition(
+                    schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields,
+                    namedTypes=named_types, namedEnums=named_enums)
+            else:
+                return _meta.SchemaDefinition(
+                    schemaType=_meta.SchemaType.STRUCT_SCHEMA, fields=trac_fields)
 
-            if trac_field is not None:
-                trac_fields[field_name] = trac_field
+        finally:
 
-        return _meta.StructSchema(fields=trac_fields)
+            type_stack.pop()
 
     @classmethod
     def _define_field(
-            cls, python_type: type, *,
-            dc_field: _dc.Field = None,
-            pyd_field: "_pyd.fields.FieldInfo" = None) \
-            -> _meta.StructField:
+            cls, name, index, python_type: type, optional=False, *,
+            named_types: _tp.Dict[str, _meta.SchemaDefinition],
+            named_enums: _tp.Dict[str, _meta.EnumValues],
+            type_stack: _tp.List[str],
+            dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
+            -> _meta.FieldSchema:
 
         if python_type in cls.__primitive_types:
-            return cls._define_primitive_field(python_type, dc_field=dc_field, pyd_field=pyd_field)
+
+            return cls._define_primitive_field(
+                name, index, python_type, optional,
+                dc_field=dc_field, pyd_field=pyd_field)
 
         elif any(map(lambda _t: isinstance(python_type, _t), cls.__generic_types)):
-            return cls._define_generic_field(python_type, pyd_field=pyd_field)
 
-        elif dc_field is not None and _dc.is_dataclass(python_type):
-            pass
+            return cls._define_generic_field(
+                name, index, python_type,
+                dc_field=dc_field, pyd_field=pyd_field,
+                named_types=named_types, named_enums=named_enums,
+                type_stack=type_stack)
+
+        elif isinstance(python_type, _enum.EnumMeta):
+
+            type_name = cls._qualified_type_name(python_type)
+
+            if type_name not in named_enums:
+                enum_values = cls._define_enum_values(python_type)
+                named_enums[type_name] = enum_values
+
+            return cls._define_enum_field(
+                name, index, python_type, optional,
+                dc_field=dc_field, pyd_field=pyd_field)
+
+        elif _dc.is_dataclass(python_type):
 
-        elif pyd_field is not None and issubclass(python_type, _pyd.BaseModel):
-            pass
+            type_name = cls._qualified_type_name(python_type)
+
+            if type_name in type_stack:
+                raise _ex.EValidation("Recursive types are not supported")
+
+            if type_name not in named_types:
+                struct_type = cls._define_struct_for_dataclass(python_type, named_types, named_enums, type_stack)
+                named_types[type_name] = struct_type
+
+            return _meta.FieldSchema(
+                fieldName=name,
+                fieldOrder=index,
+                fieldType=_meta.BasicType.STRUCT,
+                notNull=not optional,
+                namedType=type_name)
+
+        elif issubclass(python_type, _pyd.BaseModel):
+
+            type_name = cls._qualified_type_name(python_type)
+
+            if type_name in type_stack:
+                raise _ex.EValidation("Recursive types are not supported")
+
+            if type_name not in named_types:
+                struct_type = cls._define_struct_for_pydantic(python_type, named_types, named_enums, type_stack)
+                named_types[type_name] = struct_type
+
+            return _meta.FieldSchema(
+                fieldName=name,
+                fieldOrder=index,
+                fieldType=_meta.BasicType.STRUCT,
+                notNull=not optional,
+                namedType=type_name)
 
         else:
             raise _ex.ETracInternal("Cannot encode field type: " + str(python_type))
 
     @classmethod
     def _define_primitive_field(
-            cls, python_type: type, optional=False, *,
-            dc_field: _dc.Field = None,
-            pyd_field: "_pyd.fields.FieldInfo" = None) \
-            -> _meta.StructField:
+            cls, name: str, index: int, python_type: type, optional=False, *,
+            dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
+            -> _meta.FieldSchema:
+
+        default_value = None
+
+        if dc_field is not None:
+            if dc_field.default not in [_dc.MISSING, None]:
+                default_value = _meta_types.MetadataCodec.encode_value(dc_field.default)
+            elif dc_field.default_factory not in [_dc.MISSING, None]:
+                native_value = dc_field.default_factory()
+                default_value = _meta_types.MetadataCodec.encode_value(native_value)
+
+        elif pyd_field is not None:
+            if pyd_field.default not in [_pyd.fields.PydanticUndefined, None]:
+                default_value = _meta_types.MetadataCodec.encode_value(pyd_field.default)
+            elif pyd_field.default_factory not in [_pyd.fields.PydanticUndefined, None]:
+                native_value = pyd_field.default_factory()
+                default_value = _meta_types.MetadataCodec.encode_value(native_value)
+
+        return _meta.FieldSchema(
+            fieldName=name,
+            fieldOrder=index,
+            fieldType=cls.__primitive_types[python_type],
+            notNull=not optional,
+            defaultValue=default_value)
+
+    @classmethod
+    def _define_enum_field(
+            cls, name: str, index: int, enum_type: _enum.EnumMeta, optional=False, *,
+            dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
+            -> _meta.FieldSchema:
 
-        struct_field = _meta.StructField()
-        struct_field.fieldType = _meta.TypeDescriptor(basicType=cls.__primitive_types[python_type])
-        struct_field.notNull = not optional
+        default_value = None
 
-        if dc_field is not None and  dc_field.default is not _dc.MISSING:
-            struct_field.defaultValue = _meta_types.MetadataCodec.encode_value(dc_field.default)
+        if dc_field is not None and  dc_field.default not in [_dc.MISSING, None]:
+            default_value = _meta_types.MetadataCodec.encode_value(dc_field.default.name)
 
-        if pyd_field is not None and pyd_field.default is not _pyd.fields.PydanticUndefined:
-            struct_field.defaultValue = _meta_types.MetadataCodec.encode_value(pyd_field.default)
+        if pyd_field is not None and pyd_field.default not in [_pyd.fields.PydanticUndefined, None]:
+            default_value = _meta_types.MetadataCodec.encode_value(pyd_field.default.name)
 
-        return struct_field
+        return _meta.FieldSchema(
+            fieldName=name,
+            fieldOrder=index,
+            fieldType=_meta.BasicType.STRING,
+            categorical=True,
+            notNull=not optional,
+            namedEnum=cls._qualified_type_name(enum_type),
+            defaultValue=default_value)
+
+    @classmethod
+    def _define_enum_values(cls, enum_type: _enum.EnumMeta) -> _meta.EnumValues:
+
+        values = list(map(lambda value: value.name, enum_type))
+        return _meta.EnumValues(values=values)
 
     @classmethod
     def _define_generic_field(
-            cls, python_type: type, *,
-            dc_field: _dc.Field = None,
-            pyd_field: "_pyd.fields.FieldInfo" = None) -> _meta.StructField:
+            cls, name, index, python_type: type, *,
+            named_types: _tp.Dict[str, _meta.SchemaDefinition],
+            named_enums: _tp.Dict[str, _meta.EnumValues],
+            type_stack: _tp.List[str],
+            dc_field: _dc.Field = None, pyd_field: "_pyd.fields.FieldInfo" = None) \
+            -> _meta.FieldSchema:
 
         origin = _tp.get_origin(python_type)
         args = _tp.get_args(python_type)
 
         # types.NoneType not available in Python 3.9, so use type(None) instead
-        if origin in cls.__union_types and len(args) == 2 and args[1] is type(None):
-            optional_type = args[0]
-            return cls._define_primitive_field(optional_type, optional=True, dc_field=dc_field, pyd_field=pyd_field)
+        if origin in cls.__union_types and len(args) == 2 and type(None) in args:
+            optional_type = args[0] if args[1] is type(None) else args[1]
+            return cls._define_field(
+                name, index, optional_type, optional=True,
+                dc_field=dc_field, pyd_field=pyd_field,
+                named_types=named_types, named_enums=named_enums,
+                type_stack=type_stack)
 
         elif origin in [list, _tp.List]:
-            list_type = args[0]
-            pass
+
+            item_type = args[0]
+            item_field = cls._define_field(
+                "item", 0, item_type, optional=False,
+                named_types=named_types, named_enums=named_enums,
+                type_stack=type_stack)
+
+            return _meta.FieldSchema(
+                fieldName=name,
+                fieldOrder=index,
+                fieldType=_meta.BasicType.ARRAY,
+                notNull=True,
+                children=[item_field])
 
         elif origin in [dict, _tp.Dict]:
+
             key_type = args[0]
+            key_field = _meta.FieldSchema(
+                fieldName="key",
+                fieldOrder=0,
+                fieldType=_meta.BasicType.STRING,
+                notNull=True)
+
             value_type = args[1]
-            pass
+            value_field = cls._define_field(
+                "value", 1, value_type, optional=False,
+                named_types=named_types, named_enums=named_enums,
+                type_stack=type_stack)
+
+            return _meta.FieldSchema(
+                fieldName=name,
+                fieldOrder=index,
+                fieldType=_meta.BasicType.MAP,
+                notNull=True,
+                children=[key_field, value_field])
 
         else:
             raise _ex.ETracInternal("Cannot encode field type: " + str(python_type))
 
+    @classmethod
+    def _qualified_type_name(cls, python_type: type):
+
+        name = python_type.__name__
+        module = python_type.__module__
+
+        if module.startswith(cls.__SHIM_PREFIX):
+            shim_root_index = module.index(".", len(cls.__SHIM_PREFIX)) + 1
+            module = module[shim_root_index:]
+
+        return f"{module}.{name}"
+
+    __SHIM_PREFIX = "tracdap.shim."
+
 
 class StructParser:
 
@@ -476,19 +671,18 @@ class StructQuoter:
     # New implementation of STRUCT quoting, copied from config_parser
     # After a period of stabilization, config_parser will switch to this implementation
 
-    JSON_FORMAT = "json"
-    YAML_FORMAT = "yaml"
     INDENT = 3
 
     @classmethod
     def quote(cls, obj: _tp.Any, dst: _tp.TextIO, dst_format: str):
 
-        if dst_format.lower() == cls.JSON_FORMAT:
-            return cls.quote_json(obj, dst)
 
-        if dst_format.lower() == cls.YAML_FORMAT:
+        if dst_format == StructProcessor.YAML_FORMAT or dst_format.lower() in StructProcessor.YAML_ALT_FORMATS:
             return cls.quote_yaml(obj, dst)
 
+        if dst_format == StructProcessor.JSON_FORMAT or dst_format.lower() in StructProcessor.JSON_ALT_FORMATS:
+            return cls.quote_json(obj, dst)
+
         # TODO :This is probably an error in the user-supplied parameters
         raise _ex.ETracInternal(f"Unsupported output format [{dst_format}]")