tracdap-runtime 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

tracdap/rt/_impl/core/__init__.py

@@ -0,0 +1,14 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

tracdap/rt/_impl/{config_parser.py → core/config_parser.py}

@@ -13,8 +13,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from __future__ import annotations
-
 import dataclasses as _dc
 import decimal
 import enum
@@ -24,6 +22,7 @@ import json
 import os
 import pathlib
 import re
+import types as ts
 import typing as tp
 import urllib.parse as _urlp
 import uuid
@@ -32,18 +31,24 @@ import tracdap.rt.config as _config
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as _plugins
 import tracdap.rt.ext.config as _config_ext
-import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.util as _util
 
 import yaml
 import yaml.parser
 
+try:
+    import pydantic as _pyd  # noqa
+except ModuleNotFoundError:
+    _pyd = None
+
 _T = tp.TypeVar('_T')
 
 
 class ConfigManager:
 
     @classmethod
-    def for_root_config(cls, root_config_file: tp.Union[str, pathlib.Path, None]) -> ConfigManager:
+    def for_root_config(cls, root_config_file: tp.Union[str, pathlib.Path, None]) -> "ConfigManager":
 
         if isinstance(root_config_file, pathlib.Path):
             root_file_path = cls._resolve_scheme(root_config_file)
@@ -69,7 +74,7 @@ class ConfigManager:
             return ConfigManager(working_dir_url, None)
 
     @classmethod
-    def for_root_dir(cls, root_config_dir: tp.Union[str, pathlib.Path]) -> ConfigManager:
+    def for_root_dir(cls, root_config_dir: tp.Union[str, pathlib.Path]) -> "ConfigManager":
 
         if isinstance(root_config_dir, pathlib.Path):
             root_dir_path = cls._resolve_scheme(root_config_dir)
@@ -103,7 +108,7 @@ class ConfigManager:
             return raw_url
 
     def __init__(self, root_dir_url: _urlp.ParseResult, root_file_url: tp.Optional[_urlp.ParseResult]):
-        self._log = _util.logger_for_object(self)
+        self._log = _logging.logger_for_object(self)
         self._root_dir_url = root_dir_url
         self._root_file_url = root_file_url
 
@@ -279,9 +284,17 @@ class ConfigManager:
 
 class ConfigParser(tp.Generic[_T]):
 
-    # The metaclass for generic types varies between versions of the typing library
-    # To work around this, detect the correct metaclass by inspecting a generic type variable
-    __generic_metaclass = type(tp.List[object])
+    # Support both new and old styles for generic, union and optional types
+    # Old-style annotations are still valid, even when the new style is fully supported
+    __generic_types: list[type] = [
+        ts.GenericAlias,
+        type(tp.List[int]),
+        type(tp.Optional[int])
+    ]
+
+    # UnionType was added to the types module in Python 3.10, we support 3.9 (Jan 2025)
+    if hasattr(ts, "UnionType"):
+        __generic_types.append(ts.UnionType)
 
     __primitive_types: tp.Dict[type, callable] = {
         bool: bool,
@@ -294,7 +307,7 @@ class ConfigParser(tp.Generic[_T]):
     }
 
     def __init__(self, config_class: _T.__class__, dev_mode_locations: tp.List[str] = None):
-        self._log = _util.logger_for_object(self)
+        self._log = _logging.logger_for_object(self)
         self._config_class = config_class
         self._dev_mode_locations = dev_mode_locations or []
         self._errors = []
@@ -358,7 +371,11 @@ class ConfigParser(tp.Generic[_T]):
         if _dc.is_dataclass(annotation):
             return self._parse_simple_class(location, raw_value, annotation)
 
-        if isinstance(annotation, self.__generic_metaclass):
+        # Basic support for Pydantic, if it is installed
+        if _pyd and isinstance(annotation, type) and issubclass(annotation, _pyd.BaseModel):
+            return self._parse_simple_class(location, raw_value, annotation)
+
+        if any(map(lambda _t: isinstance(annotation, _t), self.__generic_types)):
             return self._parse_generic_class(location, raw_value, annotation)  # noqa
 
         return self._error(location, f"Cannot parse value of type {annotation.__name__}")
@@ -429,7 +446,7 @@ class ConfigParser(tp.Generic[_T]):
         init_signature = inspect.signature(metaclass.__init__)
         init_types = tp.get_type_hints(metaclass.__init__)
         init_params = iter(init_signature.parameters.items())
-        init_values: tp.List[tp.Any] = list()
+        init_values: tp.Dict[str, tp.Any] = dict()
 
         # Do not process 'self'
         next(init_params)
@@ -443,20 +460,20 @@ class ConfigParser(tp.Generic[_T]):
                 message = f"Class {metaclass.__name__} does not support config decoding: " + \
                           f"Missing type information for init parameter '{param_name}'"
                 self._error(location, message)
-                init_values.append(None)
+                init_values[param_name] = None
 
             elif param_name in raw_dict and raw_dict[param_name] is not None:
                 param_value = self._parse_value(param_location, raw_dict[param_name], param_type)
-                init_values.append(param_value)
+                init_values[param_name] = param_value
 
             elif param.default != inspect._empty:  # noqa
-                init_values.append(param.default)
+                init_values[param_name] = param.default
 
             else:
                 self._error(location, f"Missing required value '{param_name}'")
-                init_values.append(None)
+                init_values[param_name] = None
 
-        binding = init_signature.bind(obj, *init_values)
+        binding = init_signature.bind(obj, **init_values)
         metaclass.__init__(*binding.args, **binding.kwargs)
 
         # Now go back over the members and look for any that weren't declared in __init__
@@ -477,7 +494,7 @@ class ConfigParser(tp.Generic[_T]):
                 self._error(location, message)
 
             # Generic members must be declared in __init__ since that is the only way to get the full annotation
-            if isinstance(type(default_value), self.__generic_metaclass):
+            if any(map(lambda _t: isinstance(type(default_value), _t), self.__generic_types)):
                 message = f"Class {metaclass.__name__} does not support config decoding: " + \
                           f"Members with no default value must be declared in __init__: '{member_name}'"
                 self._error(location, message)
@@ -503,7 +520,7 @@ class ConfigParser(tp.Generic[_T]):
 
         return obj
 
-    def _parse_generic_class(self, location: str, raw_value: tp.Any, metaclass:  __generic_metaclass):
+    def _parse_generic_class(self, location: str, raw_value: tp.Any,  metaclass: type):
 
         origin = _util.get_origin(metaclass)
         args = _util.get_args(metaclass)

tracdap/rt/_impl/{data.py → core/data.py}

@@ -14,6 +14,7 @@
 #  limitations under the License.
 
 import abc
+import copy
 import dataclasses as dc
 import typing as tp
 import datetime as dt
@@ -36,17 +37,62 @@ except ModuleNotFoundError:
 import tracdap.rt.api.experimental as _api
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.core.logging as _log
 
 
 @dc.dataclass(frozen=True)
 class DataSpec:
 
+    object_type: _meta.ObjectType
+    schema_type: _meta.SchemaType
     data_item: str
+
     data_def: _meta.DataDefinition
+    file_def: _meta.FileDefinition
     storage_def: _meta.StorageDefinition
     schema_def: tp.Optional[_meta.SchemaDefinition]
 
+    @staticmethod
+    def create_data_spec(
+            data_item: str,
+            data_def: _meta.DataDefinition,
+            storage_def: _meta.StorageDefinition,
+            schema_def: tp.Optional[_meta.SchemaDefinition] = None) -> "DataSpec":
+
+        if schema_def:
+            schema_type = schema_def.schemaType
+        elif data_def.schema:
+            schema_type = data_def.schema.schemaType
+        else:
+            schema_type = _meta.SchemaType.SCHEMA_TYPE_NOT_SET
+
+        return DataSpec(
+            _meta.ObjectType.DATA, schema_type, data_item,
+            data_def,
+            storage_def=storage_def,
+            schema_def=schema_def,
+            file_def=None)
+
+    @staticmethod
+    def create_file_spec(
+            data_item: str,
+            file_def: _meta.FileDefinition,
+            storage_def: _meta.StorageDefinition) -> "DataSpec":
+
+        return DataSpec(
+            _meta.ObjectType.FILE, _meta.SchemaType.SCHEMA_TYPE_NOT_SET, data_item,
+            file_def=file_def,
+            storage_def=storage_def,
+            data_def=None,
+            schema_def=None)
+
+    @staticmethod
+    def create_empty_spec(object_type: _meta.ObjectType, schema_type: _meta.SchemaType):
+        return DataSpec(object_type, schema_type, None, None, None, None, None)
+
+    def is_empty(self):
+        return self.data_item is None or len(self.data_item) == 0
+
 
 @dc.dataclass(frozen=True)
 class DataPartKey:
@@ -61,44 +107,104 @@ class DataPartKey:
 @dc.dataclass(frozen=True)
 class DataItem:
 
-    schema: pa.Schema
-    table: tp.Optional[pa.Table] = None
-    batches: tp.Optional[tp.List[pa.RecordBatch]] = None
+    object_type: _meta.ObjectType
+    schema_type: _meta.SchemaType
+
+    content: tp.Any = None
+    content_type: tp.Type = None
+    content_func: tp.Callable[[], tp.Any] = None
 
-    pandas: "tp.Optional[pandas.DataFrame]" = None
-    pyspark: tp.Any = None
+    trac_schema: _meta.SchemaDefinition = None
+    native_schema: tp.Any = None
+
+    # TODO: Remove legacy API and use content / native_schema instead
+    schema: pa.Schema = None
+    table: tp.Optional[pa.Table] = None
 
     def is_empty(self) -> bool:
-        return self.table is None and (self.batches is None or len(self.batches) == 0)
+        return self.content is None
+
+    @staticmethod
+    def create_empty(
+            object_type: _meta.ObjectType = _meta.ObjectType.DATA,
+            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE) -> "DataItem":
+
+        if object_type == _meta.ObjectType.DATA and schema_type == _meta.SchemaType.TABLE:
+            return DataItem(_meta.ObjectType.DATA, _meta.SchemaType.TABLE, schema=pa.schema([]))
+        else:
+            return DataItem(object_type, schema_type)
 
     @staticmethod
-    def create_empty() -> "DataItem":
-        return DataItem(pa.schema([]))
+    def for_table(table: pa.Table, schema: pa.Schema, trac_schema: _meta.SchemaDefinition) -> "DataItem":
+
+        return DataItem(
+            _meta.ObjectType.DATA, _meta.SchemaType.TABLE,
+            content=table, content_type=pa.Table,
+            trac_schema=trac_schema, native_schema=schema,
+            table=table, schema=schema)
+
+    @staticmethod
+    def for_struct(content: tp.Any):
+
+        return DataItem(
+            _meta.ObjectType.DATA, _meta.SchemaType.STRUCT,
+            content=content, content_type=type(content))
+
+    @staticmethod
+    def for_file_content(content: bytes):
+
+        return DataItem(
+            _meta.ObjectType.FILE, _meta.SchemaType.SCHEMA_TYPE_NOT_SET,
+            content=content, content_type=bytes)
 
 
 @dc.dataclass(frozen=True)
 class DataView:
 
-    trac_schema: _meta.SchemaDefinition
-    arrow_schema: pa.Schema
+    object_type: _meta.ObjectType
 
-    parts: tp.Dict[DataPartKey, tp.List[DataItem]]
+    trac_schema: _meta.SchemaDefinition = None
+    arrow_schema: pa.Schema = None
+
+    parts: tp.Dict[DataPartKey, tp.List[DataItem]] = None
+    file_item: tp.Optional[DataItem] = None
 
     @staticmethod
-    def create_empty() -> "DataView":
-        return DataView(_meta.SchemaDefinition(), pa.schema([]), dict())
+    def create_empty(object_type: _meta.ObjectType = _meta.ObjectType.DATA) -> "DataView":
+        if object_type == _meta.ObjectType.DATA:
+            return DataView(object_type, _meta.SchemaDefinition(), pa.schema([]), dict())
+        else:
+            return DataView(object_type)
 
     @staticmethod
     def for_trac_schema(trac_schema: _meta.SchemaDefinition):
-        arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
-        return DataView(trac_schema, arrow_schema, dict())
+        if trac_schema.schemaType == _meta.SchemaType.TABLE:
+            arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
+            return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, dict())
+        else:
+            return DataView(_meta.ObjectType.DATA, trac_schema, parts = dict())
+
+    @staticmethod
+    def for_file_item(file_item: DataItem):
+        return DataView(file_item.object_type, file_item=file_item)
 
     def with_trac_schema(self, trac_schema: _meta.SchemaDefinition):
         arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
-        return DataView(trac_schema, arrow_schema, self.parts)
+        return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, self.parts)
+
+    def with_part(self, part_key: DataPartKey, part: DataItem):
+        new_parts = copy.copy(self.parts)
+        new_parts[part_key] = [part]
+        return DataView(self.object_type, self.trac_schema, self.arrow_schema, new_parts)
+
+    def with_file_item(self, file_item: DataItem):
+        return DataView(self.object_type, file_item=file_item)
 
     def is_empty(self) -> bool:
-        return self.parts is None or not any(self.parts.values())
+        if self.object_type == _meta.ObjectType.FILE:
+            return self.file_item is None
+        else:
+            return self.parts is None or not any(self.parts.values())
 
 
 class _DataInternal:
@@ -115,7 +221,7 @@ class DataMapping:
     :py:class:`TypeMapping <tracdap.rt.impl.type_system.MetadataCodec>`.
     """
 
-    __log = _util.logger_for_namespace(_DataInternal.__module__ + ".DataMapping")
+    __log = _log.logger_for_namespace(_DataInternal.__module__ + ".DataMapping")
 
     # Matches TRAC_ARROW_TYPE_MAPPING in ArrowSchema, tracdap-lib-data
 
@@ -293,7 +399,7 @@ class DataMapping:
         deltas = [*prior_deltas, item]
         parts = {**view.parts, part: deltas}
 
-        return DataView(view.trac_schema, view.arrow_schema, parts)
+        return DataView(view.object_type, view.trac_schema, view.arrow_schema, parts=parts)
 
     @classmethod
     def view_to_arrow(cls, view: DataView, part: DataPartKey) -> pa.Table:
@@ -308,29 +414,27 @@ class DataMapping:
         if not deltas:
             raise _ex.ETracInternal(f"Data view for part [{part.opaque_key}] does not contain any items")
 
+        # For a single delta, use the existing Arrow content
         if len(deltas) == 1:
             return cls.item_to_arrow(deltas[0])
 
-        batches = {
+        # For multiple deltas, construct a new table by assembling the record batches
+        # Atm no consideration is given to overwriting records based on business key
+        batches = iter(
             batch
             for delta in deltas
-            for batch in (
-                delta.batches
-                if delta.batches
-                else delta.table.to_batches())}
+            for batch in cls.item_to_arrow(delta).to_batches())
 
         return pa.Table.from_batches(batches) # noqa
 
     @classmethod
     def item_to_arrow(cls, item: DataItem) -> pa.Table:
 
-        if item.table is not None:
-            return item.table
-
-        if item.batches is not None:
-            return pa.Table.from_batches(item.batches, item.schema)  # noqa
+        if item.content_type != pa.Table:
+            detail = f"expected Arrow table, got [{item.content_type}]"
+            raise _ex.ETracInternal(f"Data item does not contain tabular data ({detail})")
 
-        raise _ex.ETracInternal(f"Data item does not contain any usable data")
+        return item.content
 
     @classmethod
     def arrow_to_pandas(
@@ -642,7 +746,7 @@ class DataConformance:
     Check and/or apply conformance between datasets and schemas.
     """
 
-    __log = _util.logger_for_namespace(_DataInternal.__module__ + ".DataConformance")
+    __log = _log.logger_for_namespace(_DataInternal.__module__ + ".DataConformance")
 
     __E_FIELD_MISSING = \
         "Field [{field_name}] is missing from the data"

tracdap/rt/_impl/core/logging.py

@@ -0,0 +1,195 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import io as _io
+import sys as _sys
+import typing as _tp
+
+from logging import *
+
+
+class PlainFormatter(Formatter):
+
+    FORMAT = f"%(asctime)s [%(threadName)s] %(levelname)s %(name)s" + \
+                     f" - %(message)s"
+
+    def __init__(self):
+        super().__init__(self.FORMAT)
+
+
+class ColorFormatter(Formatter):
+
+    _BLACK, _RED, _GREEN, _YELLOW, _BLUE, _MAGENTA, _CYAN, _WHITE, _DEFAULT_WHITE = range(9)
+    _DARK_BASE = 30
+    _LIGHT_BASE = 90
+
+    # DARK_BASE + WHITE = light grey
+    # DARK_BASE + DEFAULT_WHITE = regular console white
+    # LIGHT_BASE + WHITE = bright white (0xffffff), very bright!
+
+    def __init__(self, is_bright: bool):
+
+        super().__init__(self._base_fmt(is_bright))
+        self._level_colors = self._make_level_colors(is_bright)
+        self._default_color = self._make_default_color(is_bright)
+
+    def format(self, record):
+
+        level_name = record.levelname
+        level_color = self._level_colors.get(level_name)
+
+        if level_color:
+            record.levelname = level_color
+        else:
+            record.levelname = self._default_color + level_name
+
+        return Formatter.format(self, record)
+
+    def _base_fmt(self, is_bright: bool):
+
+        if is_bright:
+            base_color = self._make_ansi_code(self._DARK_BASE, self._DEFAULT_WHITE, is_bold=False)
+            message_color = self._make_ansi_code(self._LIGHT_BASE, self._CYAN, is_bold=False)
+        else:
+            base_color = self._make_ansi_code(self._DARK_BASE, self._WHITE, is_bold=False)
+            message_color = self._make_ansi_code(self._DARK_BASE, self._CYAN, is_bold=False)
+
+        return f"{base_color}%(asctime)s [%(threadName)s] %(levelname)s{base_color} %(name)s" + \
+            f" - {message_color}%(message)s"
+
+    def _make_level_colors(self, is_bright: bool):
+
+        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
+
+        green = self._make_ansi_code(base_code, self._GREEN, is_bold=is_bright)
+        yellow = self._make_ansi_code(base_code, self._YELLOW, is_bold=is_bright)
+        red = self._make_ansi_code(base_code, self._RED, is_bold=is_bright)
+
+        level_colors = {
+            'CRITICAL': f"{red}CRITICAL",
+            'ERROR': f"{red}ERROR",
+            'WARNING': f"{yellow}WARNING",
+            'INFO': f"{green}INFO"
+        }
+
+        return level_colors
+
+    def _make_default_color(self, is_bright: bool):
+
+        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
+        blue = self._make_ansi_code(base_code, self._BLUE, is_bold=is_bright)
+
+        return blue
+
+    @classmethod
+    def _make_ansi_code(cls, base_code: int, color_offset: int, is_bold: bool):
+        return f"\033[{1 if is_bold else 0};{base_code + color_offset}m"
+
+
+def configure_logging(enable_debug=False):
+
+    root_logger = getLogger()
+    log_level = DEBUG if enable_debug else INFO
+
+    if not root_logger.hasHandlers():
+
+        console_formatter = ColorFormatter(is_bright=True)
+        console_handler = StreamHandler(_sys.stdout)
+        console_handler.setFormatter(console_formatter)
+        console_handler.setLevel(INFO)
+        root_logger.addHandler(console_handler)
+        root_logger.setLevel(log_level)
+
+        # Use is_bright=False for logs from the TRAC runtime, so model logs stand out
+
+        trac_logger = getLogger("tracdap.rt")
+
+        console_formatter = ColorFormatter(is_bright=False)
+        console_handler = StreamHandler(_sys.stdout)
+        console_handler.setFormatter(console_formatter)
+        console_handler.setLevel(log_level)
+        trac_logger.addHandler(console_handler)
+        trac_logger.propagate = False
+
+
+def logger_for_object(obj: object) -> Logger:
+    return logger_for_class(obj.__class__)
+
+
+def logger_for_class(clazz: type) -> Logger:
+    qualified_class_name = f"{clazz.__module__}.{clazz.__name__}"
+    return getLogger(qualified_class_name)
+
+
+def logger_for_namespace(namespace: str) -> Logger:
+    return getLogger(namespace)
+
+
+class JobLogger(Logger):
+
+    def __init__(self, sys_log: Logger, *handlers: Handler):
+
+        super().__init__(sys_log.name, sys_log.level)
+        self._sys_log = sys_log._log
+        self._job_log = super()._log
+
+        for handler in handlers:
+            self.addHandler(handler)
+
+    def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, stacklevel=1):
+
+        self._sys_log(level, msg, args, exc_info, extra, stack_info, stacklevel)
+        self._job_log(level, msg, args, exc_info, extra, stack_info, stacklevel)
+
+
+class LogProvider:
+
+    def logger_for_object(self, obj: object) -> Logger:
+        return logger_for_object(obj)
+
+    def logger_for_class(self, clazz: type) -> Logger:
+        return logger_for_class(clazz)
+
+    def logger_for_namespace(self, namespace: str) -> Logger:
+        return logger_for_namespace(namespace)
+
+
+class JobLogProvider(LogProvider):
+
+    def __init__(self, *handlers: Handler):
+        self.__handlers = handlers
+
+    def logger_for_object(self, obj: object) -> Logger:
+        base_logger = logger_for_object(obj)
+        return JobLogger(base_logger, *self.__handlers)
+
+    def logger_for_class(self, clazz: type) -> Logger:
+        base_logger = logger_for_class(clazz)
+        return JobLogger(base_logger, *self.__handlers)
+
+    def logger_for_namespace(self, namespace: str) -> Logger:
+        base_logger = logger_for_namespace(namespace)
+        return JobLogger(base_logger, *self.__handlers)
+
+
+def job_log_provider(target: _tp.BinaryIO) -> JobLogProvider:
+
+    stream = _io.TextIOWrapper(target, newline="\r\n")
+    formatter = PlainFormatter()
+
+    handler = StreamHandler(stream)
+    handler.setFormatter(formatter)
+
+    return JobLogProvider(handler)

tracdap/rt/_impl/{models.py → core/models.py}

@@ -25,11 +25,12 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 
-import tracdap.rt._impl.type_system as _types
-import tracdap.rt._impl.repos as _repos
-import tracdap.rt._impl.shim as _shim
-import tracdap.rt._impl.util as _util
-import tracdap.rt._impl.validation as _val
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.repos as _repos
+import tracdap.rt._impl.core.shim as _shim
+import tracdap.rt._impl.core.type_system as _types
+import tracdap.rt._impl.core.util as _util
+import tracdap.rt._impl.core.validation as _val
 
 
 class ModelLoader:
@@ -43,7 +44,7 @@ class ModelLoader:
 
     def __init__(self, sys_config: _cfg.RuntimeConfig, scratch_dir: pathlib.Path):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
 
         self.__scratch_dir = scratch_dir.joinpath("models")
         self.__repos = _repos.RepositoryManager(sys_config)
@@ -226,13 +227,15 @@ class ModelLoader:
                 self.__log.info(f"Parameter [{name}] - {param.paramType.basicType.name}")
                 param.paramProps = self._encoded_props(param.paramProps, "parameter", name)
 
-            for name, schema in model_def.inputs.items():
-                self.__log.info(f"Input [{name}] - {schema.schema.schemaType.name}")
-                schema.inputProps = self._encoded_props(schema.inputProps, "input", name)
+            for name, input_def in model_def.inputs.items():
+                input_type = input_def.schema.schemaType.name if input_def.objectType == _meta.ObjectType.DATA else input_def.objectType.name
+                self.__log.info(f"Input [{name}] - {input_type}")
+                input_def.inputProps = self._encoded_props(input_def.inputProps, "input", name)
 
-            for name, schema in model_def.outputs.items():
-                self.__log.info(f"Output [{name}] - {schema.schema.schemaType.name}")
-                schema.outputProps = self._encoded_props(schema.outputProps, "input", name)
+            for name, output_def in model_def.outputs.items():
+                output_type = output_def.schema.schemaType.name if output_def.objectType == _meta.ObjectType.DATA else output_def.objectType.name
+                self.__log.info(f"Output [{name}] - {output_type}")
+                output_def.outputProps = self._encoded_props(output_def.outputProps, "input", name)
 
             return model_def