tracdap-runtime 0.7.0rc1__py3-none-any.whl → 0.8.0b2__py3-none-any.whl

tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.py

@@ -23,7 +23,7 @@ from tracdap.rt._impl.grpc.tracdap.metadata import custom_pb2 as tracdap_dot_rt_
 from tracdap.rt._impl.grpc.tracdap.metadata import stoarge_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_stoarge__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x32tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto\x1a\x34tracdap/rt/_impl/grpc/tracdap/metadata/stoarge.proto\"\x87\x05\n\x10ObjectDefinition\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .tracdap.metadata.DataDefinitionH\x00\x12\x32\n\x05model\x18\x03 \x01(\x0b\x32!.tracdap.metadata.ModelDefinitionH\x00\x12\x30\n\x04\x66low\x18\x04 \x01(\x0b\x32 .tracdap.metadata.FlowDefinitionH\x00\x12.\n\x03job\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinitionH\x00\x12\x30\n\x04\x66ile\x18\x06 \x01(\x0b\x32 .tracdap.metadata.FileDefinitionH\x00\x12\x34\n\x06\x63ustom\x18\x07 \x01(\x0b\x32\".tracdap.metadata.CustomDefinitionH\x00\x12\x36\n\x07storage\x18\x08 \x01(\x0b\x32#.tracdap.metadata.StorageDefinitionH\x00\x12\x34\n\x06schema\x18\t \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12H\n\x0bobjectProps\x18\x64 \x03(\x0b\x32\x33.tracdap.metadata.ObjectDefinition.ObjectPropsEntry\x1aK\n\x10ObjectPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0c\n\ndefinitionB2\n\x1aorg.finos.tracdap.metadataB\x12ObjectProtoWrapperP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x32tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto\x1a\x34tracdap/rt/_impl/grpc/tracdap/metadata/stoarge.proto\"\xbd\x05\n\x10ObjectDefinition\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .tracdap.metadata.DataDefinitionH\x00\x12\x32\n\x05model\x18\x03 \x01(\x0b\x32!.tracdap.metadata.ModelDefinitionH\x00\x12\x30\n\x04\x66low\x18\x04 \x01(\x0b\x32 .tracdap.metadata.FlowDefinitionH\x00\x12.\n\x03job\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinitionH\x00\x12\x30\n\x04\x66ile\x18\x06 \x01(\x0b\x32 .tracdap.metadata.FileDefinitionH\x00\x12\x34\n\x06\x63ustom\x18\x07 \x01(\x0b\x32\".tracdap.metadata.CustomDefinitionH\x00\x12\x36\n\x07storage\x18\x08 \x01(\x0b\x32#.tracdap.metadata.StorageDefinitionH\x00\x12\x34\n\x06schema\x18\t \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12\x34\n\x06result\x18\n \x01(\x0b\x32\".tracdap.metadata.ResultDefinitionH\x00\x12H\n\x0bobjectProps\x18\x64 \x03(\x0b\x32\x33.tracdap.metadata.ObjectDefinition.ObjectPropsEntry\x1aK\n\x10ObjectPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0c\n\ndefinitionB2\n\x1aorg.finos.tracdap.metadataB\x12ObjectProtoWrapperP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -34,7 +34,7 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._options = None
   _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_options = b'8\001'
   _globals['_OBJECTDEFINITION']._serialized_start=543
-  _globals['_OBJECTDEFINITION']._serialized_end=1190
-  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_start=1101
-  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_end=1176
+  _globals['_OBJECTDEFINITION']._serialized_end=1244
+  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_start=1155
+  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_end=1230
 # @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.pyi

@@ -15,7 +15,7 @@ from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Opti
 DESCRIPTOR: _descriptor.FileDescriptor
 
 class ObjectDefinition(_message.Message):
-    __slots__ = ("objectType", "data", "model", "flow", "job", "file", "custom", "storage", "schema", "objectProps")
+    __slots__ = ("objectType", "data", "model", "flow", "job", "file", "custom", "storage", "schema", "result", "objectProps")
     class ObjectPropsEntry(_message.Message):
         __slots__ = ("key", "value")
         KEY_FIELD_NUMBER: _ClassVar[int]
@@ -32,6 +32,7 @@ class ObjectDefinition(_message.Message):
     CUSTOM_FIELD_NUMBER: _ClassVar[int]
     STORAGE_FIELD_NUMBER: _ClassVar[int]
     SCHEMA_FIELD_NUMBER: _ClassVar[int]
+    RESULT_FIELD_NUMBER: _ClassVar[int]
     OBJECTPROPS_FIELD_NUMBER: _ClassVar[int]
     objectType: _object_id_pb2.ObjectType
     data: _data_pb2.DataDefinition
@@ -42,5 +43,6 @@ class ObjectDefinition(_message.Message):
     custom: _custom_pb2.CustomDefinition
     storage: _stoarge_pb2.StorageDefinition
     schema: _data_pb2.SchemaDefinition
+    result: _job_pb2.ResultDefinition
     objectProps: _containers.MessageMap[str, _type_pb2.Value]
-    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., data: _Optional[_Union[_data_pb2.DataDefinition, _Mapping]] = ..., model: _Optional[_Union[_model_pb2.ModelDefinition, _Mapping]] = ..., flow: _Optional[_Union[_flow_pb2.FlowDefinition, _Mapping]] = ..., job: _Optional[_Union[_job_pb2.JobDefinition, _Mapping]] = ..., file: _Optional[_Union[_file_pb2.FileDefinition, _Mapping]] = ..., custom: _Optional[_Union[_custom_pb2.CustomDefinition, _Mapping]] = ..., storage: _Optional[_Union[_stoarge_pb2.StorageDefinition, _Mapping]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., objectProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
+    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., data: _Optional[_Union[_data_pb2.DataDefinition, _Mapping]] = ..., model: _Optional[_Union[_model_pb2.ModelDefinition, _Mapping]] = ..., flow: _Optional[_Union[_flow_pb2.FlowDefinition, _Mapping]] = ..., job: _Optional[_Union[_job_pb2.JobDefinition, _Mapping]] = ..., file: _Optional[_Union[_file_pb2.FileDefinition, _Mapping]] = ..., custom: _Optional[_Union[_custom_pb2.CustomDefinition, _Mapping]] = ..., storage: _Optional[_Union[_stoarge_pb2.StorageDefinition, _Mapping]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., result: _Optional[_Union[_job_pb2.ResultDefinition, _Mapping]] = ..., objectProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...

tracdap/rt/_impl/logging.py

@@ -0,0 +1,195 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import io as _io
+import sys as _sys
+import typing as _tp
+
+from logging import *
+
+
+class PlainFormatter(Formatter):
+
+    FORMAT = f"%(asctime)s [%(threadName)s] %(levelname)s %(name)s" + \
+                     f" - %(message)s"
+
+    def __init__(self):
+        super().__init__(self.FORMAT)
+
+
+class ColorFormatter(Formatter):
+
+    _BLACK, _RED, _GREEN, _YELLOW, _BLUE, _MAGENTA, _CYAN, _WHITE, _DEFAULT_WHITE = range(9)
+    _DARK_BASE = 30
+    _LIGHT_BASE = 90
+
+    # DARK_BASE + WHITE = light grey
+    # DARK_BASE + DEFAULT_WHITE = regular console white
+    # LIGHT_BASE + WHITE = bright white (0xffffff), very bright!
+
+    def __init__(self, is_bright: bool):
+
+        super().__init__(self._base_fmt(is_bright))
+        self._level_colors = self._make_level_colors(is_bright)
+        self._default_color = self._make_default_color(is_bright)
+
+    def format(self, record):
+
+        level_name = record.levelname
+        level_color = self._level_colors.get(level_name)
+
+        if level_color:
+            record.levelname = level_color
+        else:
+            record.levelname = self._default_color + level_name
+
+        return Formatter.format(self, record)
+
+    def _base_fmt(self, is_bright: bool):
+
+        if is_bright:
+            base_color = self._make_ansi_code(self._DARK_BASE, self._DEFAULT_WHITE, is_bold=False)
+            message_color = self._make_ansi_code(self._LIGHT_BASE, self._CYAN, is_bold=False)
+        else:
+            base_color = self._make_ansi_code(self._DARK_BASE, self._WHITE, is_bold=False)
+            message_color = self._make_ansi_code(self._DARK_BASE, self._CYAN, is_bold=False)
+
+        return f"{base_color}%(asctime)s [%(threadName)s] %(levelname)s{base_color} %(name)s" + \
+            f" - {message_color}%(message)s"
+
+    def _make_level_colors(self, is_bright: bool):
+
+        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
+
+        green = self._make_ansi_code(base_code, self._GREEN, is_bold=is_bright)
+        yellow = self._make_ansi_code(base_code, self._YELLOW, is_bold=is_bright)
+        red = self._make_ansi_code(base_code, self._RED, is_bold=is_bright)
+
+        level_colors = {
+            'CRITICAL': f"{red}CRITICAL",
+            'ERROR': f"{red}ERROR",
+            'WARNING': f"{yellow}WARNING",
+            'INFO': f"{green}INFO"
+        }
+
+        return level_colors
+
+    def _make_default_color(self, is_bright: bool):
+
+        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
+        blue = self._make_ansi_code(base_code, self._BLUE, is_bold=is_bright)
+
+        return blue
+
+    @classmethod
+    def _make_ansi_code(cls, base_code: int, color_offset: int, is_bold: bool):
+        return f"\033[{1 if is_bold else 0};{base_code + color_offset}m"
+
+
+def configure_logging(enable_debug=False):
+
+    root_logger = getLogger()
+    log_level = DEBUG if enable_debug else INFO
+
+    if not root_logger.hasHandlers():
+
+        console_formatter = ColorFormatter(is_bright=True)
+        console_handler = StreamHandler(_sys.stdout)
+        console_handler.setFormatter(console_formatter)
+        console_handler.setLevel(INFO)
+        root_logger.addHandler(console_handler)
+        root_logger.setLevel(log_level)
+
+        # Use is_bright=False for logs from the TRAC runtime, so model logs stand out
+
+        trac_logger = getLogger("tracdap.rt")
+
+        console_formatter = ColorFormatter(is_bright=False)
+        console_handler = StreamHandler(_sys.stdout)
+        console_handler.setFormatter(console_formatter)
+        console_handler.setLevel(log_level)
+        trac_logger.addHandler(console_handler)
+        trac_logger.propagate = False
+
+
+def logger_for_object(obj: object) -> Logger:
+    return logger_for_class(obj.__class__)
+
+
+def logger_for_class(clazz: type) -> Logger:
+    qualified_class_name = f"{clazz.__module__}.{clazz.__name__}"
+    return getLogger(qualified_class_name)
+
+
+def logger_for_namespace(namespace: str) -> Logger:
+    return getLogger(namespace)
+
+
+class JobLogger(Logger):
+
+    def __init__(self, sys_log: Logger, *handlers: Handler):
+
+        super().__init__(sys_log.name, sys_log.level)
+        self._sys_log = sys_log._log
+        self._job_log = super()._log
+
+        for handler in handlers:
+            self.addHandler(handler)
+
+    def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, stacklevel=1):
+
+        self._sys_log(level, msg, args, exc_info, extra, stack_info, stacklevel)
+        self._job_log(level, msg, args, exc_info, extra, stack_info, stacklevel)
+
+
+class LogProvider:
+
+    def logger_for_object(self, obj: object) -> Logger:
+        return logger_for_object(obj)
+
+    def logger_for_class(self, clazz: type) -> Logger:
+        return logger_for_class(clazz)
+
+    def logger_for_namespace(self, namespace: str) -> Logger:
+        return logger_for_namespace(namespace)
+
+
+class JobLogProvider(LogProvider):
+
+    def __init__(self, *handlers: Handler):
+        self.__handlers = handlers
+
+    def logger_for_object(self, obj: object) -> Logger:
+        base_logger = logger_for_object(obj)
+        return JobLogger(base_logger, *self.__handlers)
+
+    def logger_for_class(self, clazz: type) -> Logger:
+        base_logger = logger_for_class(clazz)
+        return JobLogger(base_logger, *self.__handlers)
+
+    def logger_for_namespace(self, namespace: str) -> Logger:
+        base_logger = logger_for_namespace(namespace)
+        return JobLogger(base_logger, *self.__handlers)
+
+
+def job_log_provider(target: _tp.BinaryIO) -> JobLogProvider:
+
+    stream = _io.TextIOWrapper(target, newline="\r\n")
+    formatter = PlainFormatter()
+
+    handler = StreamHandler(stream)
+    handler.setFormatter(formatter)
+
+    return JobLogProvider(handler)

tracdap/rt/_impl/models.py

@@ -25,9 +25,10 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 
-import tracdap.rt._impl.type_system as _types
+import tracdap.rt._impl.logging as _logging
 import tracdap.rt._impl.repos as _repos
 import tracdap.rt._impl.shim as _shim
+import tracdap.rt._impl.type_system as _types
 import tracdap.rt._impl.util as _util
 import tracdap.rt._impl.validation as _val
 
@@ -43,7 +44,7 @@ class ModelLoader:
 
     def __init__(self, sys_config: _cfg.RuntimeConfig, scratch_dir: pathlib.Path):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
 
         self.__scratch_dir = scratch_dir.joinpath("models")
         self.__repos = _repos.RepositoryManager(sys_config)
@@ -226,13 +227,15 @@ class ModelLoader:
                 self.__log.info(f"Parameter [{name}] - {param.paramType.basicType.name}")
                 param.paramProps = self._encoded_props(param.paramProps, "parameter", name)
 
-            for name, schema in model_def.inputs.items():
-                self.__log.info(f"Input [{name}] - {schema.schema.schemaType.name}")
-                schema.inputProps = self._encoded_props(schema.inputProps, "input", name)
+            for name, input_def in model_def.inputs.items():
+                input_type = input_def.schema.schemaType.name if input_def.objectType == _meta.ObjectType.DATA else input_def.objectType.name
+                self.__log.info(f"Input [{name}] - {input_type}")
+                input_def.inputProps = self._encoded_props(input_def.inputProps, "input", name)
 
-            for name, schema in model_def.outputs.items():
-                self.__log.info(f"Output [{name}] - {schema.schema.schemaType.name}")
-                schema.outputProps = self._encoded_props(schema.outputProps, "input", name)
+            for name, output_def in model_def.outputs.items():
+                output_type = output_def.schema.schemaType.name if output_def.objectType == _meta.ObjectType.DATA else output_def.objectType.name
+                self.__log.info(f"Output [{name}] - {output_type}")
+                output_def.outputProps = self._encoded_props(output_def.outputProps, "input", name)
 
             return model_def
 

tracdap/rt/_impl/repos.py

@@ -13,10 +13,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import typing as _tp
+
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt.config as cfg
 import tracdap.rt.exceptions as ex
-import tracdap.rt._impl.util as util
+import tracdap.rt._impl.logging as _logging
 
 # Import repo interfaces
 from tracdap.rt.ext.repos import *
@@ -26,8 +28,8 @@ class RepositoryManager:
 
     def __init__(self, sys_config: cfg.RuntimeConfig):
 
-        self._log = util.logger_for_object(self)
-        self._repos: tp.Dict[str, IModelRepository] = dict()
+        self._log = _logging.logger_for_object(self)
+        self._repos: _tp.Dict[str, IModelRepository] = dict()
 
         # Initialize all repos in the system config
         # Any errors for missing repo types (plugins) will be raised during startup

tracdap/rt/_impl/schemas.py

@@ -22,9 +22,9 @@ import pyarrow as pa
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
 import tracdap.rt._impl.data as _data
+import tracdap.rt._impl.logging as _log
 import tracdap.rt._impl.storage as _storage
 import tracdap.rt._impl.shim as _shim
-import tracdap.rt._impl.util as _util
 
 
 class SchemaLoader:
@@ -169,4 +169,4 @@ class SchemaLoader:
         raise _ex.EDataConformance(err)
 
 
-SchemaLoader._log = _util.logger_for_class(SchemaLoader)
+SchemaLoader._log = _log.logger_for_class(SchemaLoader)

tracdap/rt/_impl/shim.py

@@ -31,6 +31,7 @@ import importlib.resources as _ilr
 
 import tracdap.rt.exceptions as _ex
 import tracdap.rt._impl.guard_rails as _guard
+import tracdap.rt._impl.logging as _log
 import tracdap.rt._impl.util as _util
 
 
@@ -138,7 +139,7 @@ class _NamespaceShimFinder(_ila.MetaPathFinder):
     def __init__(self, shim_map: tp.Dict[str, _Shim], active_shim: _ActiveShim):
         self.__shim_map = shim_map
         self.__active_shim = active_shim
-        self._log = _util.logger_for_class(ShimLoader)
+        self._log = _log.logger_for_class(ShimLoader)
 
     def find_spec(
             self, fullname: str,
@@ -561,5 +562,5 @@ class ShimLoader:
                 raise _ex.ERuntimeValidation(err)
 
 
-ShimLoader._log = _util.logger_for_class(ShimLoader)
+ShimLoader._log = _log.logger_for_class(ShimLoader)
 ShimLoader._init()  # noqa

tracdap/rt/_impl/static_api.py

@@ -152,14 +152,18 @@ class StaticApiImpl(_StaticApiHook):
 
     def define_schema(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE) \
+            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE, dynamic: bool = False) \
             -> _meta.SchemaDefinition:
 
-        _val.validate_signature(self.define_schema, *fields, schema_type=schema_type)
+        _val.validate_signature(self.define_schema, *fields, schema_type=schema_type, dynamic=dynamic)
 
         if schema_type == _meta.SchemaType.TABLE:
 
-            table_schema = self._build_table_schema(*fields)
+            if dynamic and not fields:
+                table_schema = None
+            else:
+                table_schema = self._build_table_schema(*fields)
+
             return _meta.SchemaDefinition(_meta.SchemaType.TABLE, table=table_schema)
 
         raise _ex.ERuntimeValidation(f"Invalid schema type [{schema_type.name}]")
@@ -182,51 +186,67 @@ class StaticApiImpl(_StaticApiHook):
 
         return converter.infer_schema(dataset)
 
-    def define_input_table(
-            self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
-            input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
-            -> _meta.ModelInputSchema:
+    def define_file_type(self, extension: str, mime_type: str) -> _meta.FileType:
+
+        _val.validate_signature(self.define_file_type, extension, mime_type)
+
+        return _meta.FileType(extension=extension, mimeType=mime_type)
+
+    def define_input(
+            self, requirement: _tp.Union[_meta.SchemaDefinition, _meta.FileType], *,
+            label: _tp.Optional[str] = None,
+            optional: bool = False, dynamic: bool = False,
+            input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None):
 
         _val.validate_signature(
-            self.define_input_table, *fields,
+            self.define_input, requirement,
             label=label, optional=optional, dynamic=dynamic,
             input_props=input_props)
 
-        # Do not define details for dynamic schemas
+        if isinstance(requirement, _meta.SchemaDefinition):
 
-        if dynamic:
-            schema_def = _meta.SchemaDefinition(_meta.SchemaType.TABLE)
-        else:
-            schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
+            return _meta.ModelInputSchema(
+                objectType=_meta.ObjectType.DATA, schema=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                inputProps=input_props)
 
-        return _meta.ModelInputSchema(
-            schema=schema_def, label=label,
-            optional=optional, dynamic=dynamic,
-            inputProps=input_props)
+        elif isinstance(requirement, _meta.FileType):
 
-    def define_output_table(
-            self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
-            output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
-            -> _meta.ModelOutputSchema:
+            return _meta.ModelInputSchema(
+                objectType=_meta.ObjectType.FILE, fileType=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                inputProps=input_props)
+
+        else:
+            raise _ex.EUnexpected()
+
+    def define_output(
+            self, requirement: _tp.Union[_meta.SchemaDefinition, _meta.FileType], *,
+            label: _tp.Optional[str] = None,
+            optional: bool = False, dynamic: bool = False,
+            output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None):
 
         _val.validate_signature(
-            self.define_output_table, *fields,
+            self.define_output, requirement,
             label=label, optional=optional, dynamic=dynamic,
             output_props=output_props)
 
-        # Do not define details for dynamic schemas
+        if isinstance(requirement, _meta.SchemaDefinition):
 
-        if dynamic:
-            schema_def = _meta.SchemaDefinition(_meta.SchemaType.TABLE)
-        else:
-            schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
+            return _meta.ModelOutputSchema(
+                objectType=_meta.ObjectType.DATA, schema=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                outputProps=output_props)
 
-        return _meta.ModelOutputSchema(
-            schema=schema_def, label=label,
-            optional=optional, dynamic=dynamic,
-            outputProps=output_props)
+        elif isinstance(requirement, _meta.FileType):
+
+            return _meta.ModelOutputSchema(
+                objectType=_meta.ObjectType.FILE, fileType=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                outputProps=output_props)
+
+        else:
+            raise _ex.EUnexpected()
 
     @staticmethod
     def _build_named_dict(

tracdap/rt/_impl/storage.py

@@ -30,6 +30,7 @@ import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt._impl.data as _data
+import tracdap.rt._impl.logging as _logging
 import tracdap.rt._impl.util as _util
 import tracdap.rt._impl.validation as _val
 
@@ -76,7 +77,7 @@ class StorageManager:
 
     def __init__(self, sys_config: _cfg.RuntimeConfig):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
         self.__file_storage: tp.Dict[str, IFileStorage] = dict()
         self.__data_storage: tp.Dict[str, IDataStorage] = dict()
         self.__external: tp.List[str] = list()
@@ -224,7 +225,7 @@ class CommonFileStorage(IFileStorage):
 
     def __init__(self, storage_key: str, storage_config: _cfg.PluginConfig, fs: pa_fs.SubTreeFileSystem):
 
-        self._log = _util.logger_for_object(self)
+        self._log = _logging.logger_for_object(self)
         self._key = storage_key
         self._config = storage_config
         self._fs = fs
@@ -731,7 +732,7 @@ class CommonDataStorage(IDataStorage):
             self, config: _cfg.PluginConfig, file_storage: IFileStorage,
             pushdown_pandas: bool = False, pushdown_spark: bool = False):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
 
         self.__config = config
         self.__file_storage = file_storage

tracdap/rt/_impl/util.py

@@ -14,11 +14,9 @@
 #  limitations under the License.
 
 import datetime as dt
-import logging
 import pathlib
 import platform
 
-import sys
 import typing as tp
 import uuid
 
@@ -38,114 +36,6 @@ def is_windows():
     return __IS_WINDOWS
 
 
-class ColorFormatter(logging.Formatter):
-
-    _BLACK, _RED, _GREEN, _YELLOW, _BLUE, _MAGENTA, _CYAN, _WHITE, _DEFAULT_WHITE = range(9)
-    _DARK_BASE = 30
-    _LIGHT_BASE = 90
-
-    # DARK_BASE + WHITE = light grey
-    # DARK_BASE + DEFAULT_WHITE = regular console white
-    # LIGHT_BASE + WHITE = bright white (0xffffff), very bright!
-
-    def __init__(self, is_bright: bool):
-
-        super().__init__(self._base_fmt(is_bright))
-        self._level_colors = self._make_level_colors(is_bright)
-        self._default_color = self._make_default_color(is_bright)
-
-    def format(self, record):
-
-        level_name = record.levelname
-        level_color = self._level_colors.get(level_name)
-
-        if level_color:
-            record.levelname = level_color
-        else:
-            record.levelname = self._default_color + level_name
-
-        return logging.Formatter.format(self, record)
-
-    def _base_fmt(self, is_bright: bool):
-
-        if is_bright:
-            base_color = self._make_ansi_code(self._DARK_BASE, self._DEFAULT_WHITE, is_bold=False)
-            message_color = self._make_ansi_code(self._LIGHT_BASE, self._CYAN, is_bold=False)
-        else:
-            base_color = self._make_ansi_code(self._DARK_BASE, self._WHITE, is_bold=False)
-            message_color = self._make_ansi_code(self._DARK_BASE, self._CYAN, is_bold=False)
-
-        return f"{base_color}%(asctime)s [%(threadName)s] %(levelname)s{base_color} %(name)s" + \
-               f" - {message_color}%(message)s"
-
-    def _make_level_colors(self, is_bright: bool):
-
-        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
-
-        green = self._make_ansi_code(base_code, self._GREEN, is_bold=is_bright)
-        yellow = self._make_ansi_code(base_code, self._YELLOW, is_bold=is_bright)
-        red = self._make_ansi_code(base_code, self._RED, is_bold=is_bright)
-
-        level_colors = {
-            'CRITICAL': f"{red}CRITICAL",
-            'ERROR': f"{red}ERROR",
-            'WARNING': f"{yellow}WARNING",
-            'INFO': f"{green}INFO"
-        }
-
-        return level_colors
-
-    def _make_default_color(self, is_bright: bool):
-
-        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
-        blue = self._make_ansi_code(base_code, self._BLUE, is_bold=is_bright)
-
-        return blue
-
-    @classmethod
-    def _make_ansi_code(cls, base_code: int, color_offset: int, is_bold: bool):
-        return f"\033[{1 if is_bold else 0};{base_code + color_offset}m"
-
-
-def configure_logging(enable_debug=False):
-
-    root_logger = logging.getLogger()
-    log_level = logging.DEBUG if enable_debug else logging.INFO
-
-    if not root_logger.hasHandlers():
-
-        console_formatter = ColorFormatter(is_bright=True)
-        console_handler = logging.StreamHandler(sys.stdout)
-        console_handler.setFormatter(console_formatter)
-        console_handler.setLevel(logging.INFO)
-        root_logger.addHandler(console_handler)
-        root_logger.setLevel(log_level)
-
-        # Use is_bright=False for logs from the TRAC runtime, so model logs stand out
-
-        trac_logger = logging.getLogger("tracdap.rt")
-
-        console_formatter = ColorFormatter(is_bright=False)
-        console_handler = logging.StreamHandler(sys.stdout)
-        console_handler.setFormatter(console_formatter)
-        console_handler.setLevel(log_level)
-        trac_logger.addHandler(console_handler)
-        trac_logger.propagate = False
-
-
-def logger_for_object(obj: object) -> logging.Logger:
-    return logger_for_class(obj.__class__)
-
-
-def logger_for_class(clazz: type) -> logging.Logger:
-    qualified_class_name = f"{clazz.__module__}.{clazz.__name__}"
-    return logging.getLogger(qualified_class_name)
-
-
-def logger_for_namespace(namespace: str) -> logging.Logger:
-    return logging.getLogger(namespace)
-
-
 def format_file_size(size: int) -> str:
 
     if size < 1024:
@@ -235,7 +125,7 @@ def get_job_resource(
     if optional:
         return None
 
-    err = f"Missing required {selector.objectType} resource [{object_key(selector)}]"
+    err = f"Missing required {selector.objectType.name} resource [{object_key(selector)}]"
     raise ex.ERuntimeValidation(err)