tracdap-runtime 0.8.0b1__py3-none-any.whl → 0.8.0b2__py3-none-any.whl

tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.pyi

@@ -61,7 +61,7 @@ SEQUENTIAL_JOB_GROUP: JobGroupType
 PARALLEL_JOB_GROUP: JobGroupType
 
 class JobDefinition(_message.Message):
-    __slots__ = ("jobType", "runModel", "runFlow", "importModel", "importData", "exportData", "jobGroup")
+    __slots__ = ("jobType", "runModel", "runFlow", "importModel", "importData", "exportData", "jobGroup", "resultId")
     JOBTYPE_FIELD_NUMBER: _ClassVar[int]
     RUNMODEL_FIELD_NUMBER: _ClassVar[int]
     RUNFLOW_FIELD_NUMBER: _ClassVar[int]
@@ -69,6 +69,7 @@ class JobDefinition(_message.Message):
     IMPORTDATA_FIELD_NUMBER: _ClassVar[int]
     EXPORTDATA_FIELD_NUMBER: _ClassVar[int]
     JOBGROUP_FIELD_NUMBER: _ClassVar[int]
+    RESULTID_FIELD_NUMBER: _ClassVar[int]
     jobType: JobType
     runModel: RunModelJob
     runFlow: RunFlowJob
@@ -76,7 +77,20 @@ class JobDefinition(_message.Message):
     importData: ImportDataJob
     exportData: ExportDataJob
     jobGroup: JobGroup
-    def __init__(self, jobType: _Optional[_Union[JobType, str]] = ..., runModel: _Optional[_Union[RunModelJob, _Mapping]] = ..., runFlow: _Optional[_Union[RunFlowJob, _Mapping]] = ..., importModel: _Optional[_Union[ImportModelJob, _Mapping]] = ..., importData: _Optional[_Union[ImportDataJob, _Mapping]] = ..., exportData: _Optional[_Union[ExportDataJob, _Mapping]] = ..., jobGroup: _Optional[_Union[JobGroup, _Mapping]] = ...) -> None: ...
+    resultId: _object_id_pb2.TagSelector
+    def __init__(self, jobType: _Optional[_Union[JobType, str]] = ..., runModel: _Optional[_Union[RunModelJob, _Mapping]] = ..., runFlow: _Optional[_Union[RunFlowJob, _Mapping]] = ..., importModel: _Optional[_Union[ImportModelJob, _Mapping]] = ..., importData: _Optional[_Union[ImportDataJob, _Mapping]] = ..., exportData: _Optional[_Union[ExportDataJob, _Mapping]] = ..., jobGroup: _Optional[_Union[JobGroup, _Mapping]] = ..., resultId: _Optional[_Union[_object_id_pb2.TagSelector, _Mapping]] = ...) -> None: ...
+
+class ResultDefinition(_message.Message):
+    __slots__ = ("jobId", "statusCode", "statusMessage", "logFileId")
+    JOBID_FIELD_NUMBER: _ClassVar[int]
+    STATUSCODE_FIELD_NUMBER: _ClassVar[int]
+    STATUSMESSAGE_FIELD_NUMBER: _ClassVar[int]
+    LOGFILEID_FIELD_NUMBER: _ClassVar[int]
+    jobId: _object_id_pb2.TagSelector
+    statusCode: JobStatusCode
+    statusMessage: str
+    logFileId: _object_id_pb2.TagSelector
+    def __init__(self, jobId: _Optional[_Union[_object_id_pb2.TagSelector, _Mapping]] = ..., statusCode: _Optional[_Union[JobStatusCode, str]] = ..., statusMessage: _Optional[str] = ..., logFileId: _Optional[_Union[_object_id_pb2.TagSelector, _Mapping]] = ...) -> None: ...
 
 class RunModelJob(_message.Message):
     __slots__ = ("model", "parameters", "inputs", "outputs", "priorOutputs", "outputAttrs")

tracdap/rt/_impl/grpc/tracdap/metadata/object_id_pb2.py

@@ -15,7 +15,7 @@ _sym_db = _symbol_database.Default()
 from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_type__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\"\x98\x02\n\tTagHeader\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x10\n\x08objectId\x18\x02 \x01(\t\x12\x15\n\robjectVersion\x18\x03 \x01(\x05\x12\x38\n\x0fobjectTimestamp\x18\x04 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12\x12\n\ntagVersion\x18\x05 \x01(\x05\x12\x35\n\x0ctagTimestamp\x18\x06 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12\x16\n\x0eisLatestObject\x18\x07 \x01(\x08\x12\x13\n\x0bisLatestTag\x18\x08 \x01(\x08\"\xb9\x02\n\x0bTagSelector\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x10\n\x08objectId\x18\x02 \x01(\t\x12\x16\n\x0clatestObject\x18\x03 \x01(\x08H\x00\x12\x17\n\robjectVersion\x18\x04 \x01(\x05H\x00\x12\x35\n\nobjectAsOf\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValueH\x00\x12\x13\n\tlatestTag\x18\x06 \x01(\x08H\x01\x12\x14\n\ntagVersion\x18\x07 \x01(\x05H\x01\x12\x32\n\x07tagAsOf\x18\x08 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValueH\x01\x42\x10\n\x0eobjectCriteriaB\r\n\x0btagCriteria*|\n\nObjectType\x12\x17\n\x13OBJECT_TYPE_NOT_SET\x10\x00\x12\x08\n\x04\x44\x41TA\x10\x01\x12\t\n\x05MODEL\x10\x02\x12\x08\n\x04\x46LOW\x10\x03\x12\x07\n\x03JOB\x10\x04\x12\x08\n\x04\x46ILE\x10\x05\x12\n\n\x06\x43USTOM\x10\x06\x12\x0b\n\x07STORAGE\x10\x07\x12\n\n\x06SCHEMA\x10\x08\x42\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x12\x10tracdap.metadata\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\"\x98\x02\n\tTagHeader\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x10\n\x08objectId\x18\x02 \x01(\t\x12\x15\n\robjectVersion\x18\x03 \x01(\x05\x12\x38\n\x0fobjectTimestamp\x18\x04 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12\x12\n\ntagVersion\x18\x05 \x01(\x05\x12\x35\n\x0ctagTimestamp\x18\x06 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValue\x12\x16\n\x0eisLatestObject\x18\x07 \x01(\x08\x12\x13\n\x0bisLatestTag\x18\x08 \x01(\x08\"\xb9\x02\n\x0bTagSelector\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x10\n\x08objectId\x18\x02 \x01(\t\x12\x16\n\x0clatestObject\x18\x03 \x01(\x08H\x00\x12\x17\n\robjectVersion\x18\x04 \x01(\x05H\x00\x12\x35\n\nobjectAsOf\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValueH\x00\x12\x13\n\tlatestTag\x18\x06 \x01(\x08H\x01\x12\x14\n\ntagVersion\x18\x07 \x01(\x05H\x01\x12\x32\n\x07tagAsOf\x18\x08 \x01(\x0b\x32\x1f.tracdap.metadata.DatetimeValueH\x01\x42\x10\n\x0eobjectCriteriaB\r\n\x0btagCriteria*\x88\x01\n\nObjectType\x12\x17\n\x13OBJECT_TYPE_NOT_SET\x10\x00\x12\x08\n\x04\x44\x41TA\x10\x01\x12\t\n\x05MODEL\x10\x02\x12\x08\n\x04\x46LOW\x10\x03\x12\x07\n\x03JOB\x10\x04\x12\x08\n\x04\x46ILE\x10\x05\x12\n\n\x06\x43USTOM\x10\x06\x12\x0b\n\x07STORAGE\x10\x07\x12\n\n\x06SCHEMA\x10\x08\x12\n\n\x06RESULT\x10\tB\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -23,8 +23,8 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracd
 if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['DESCRIPTOR']._options = None
   _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
-  _globals['_OBJECTTYPE']._serialized_start=726
-  _globals['_OBJECTTYPE']._serialized_end=850
+  _globals['_OBJECTTYPE']._serialized_start=727
+  _globals['_OBJECTTYPE']._serialized_end=863
   _globals['_TAGHEADER']._serialized_start=128
   _globals['_TAGHEADER']._serialized_end=408
   _globals['_TAGSELECTOR']._serialized_start=411

tracdap/rt/_impl/grpc/tracdap/metadata/object_id_pb2.pyi

@@ -17,6 +17,7 @@ class ObjectType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     CUSTOM: _ClassVar[ObjectType]
     STORAGE: _ClassVar[ObjectType]
     SCHEMA: _ClassVar[ObjectType]
+    RESULT: _ClassVar[ObjectType]
 OBJECT_TYPE_NOT_SET: ObjectType
 DATA: ObjectType
 MODEL: ObjectType
@@ -26,6 +27,7 @@ FILE: ObjectType
 CUSTOM: ObjectType
 STORAGE: ObjectType
 SCHEMA: ObjectType
+RESULT: ObjectType
 
 class TagHeader(_message.Message):
     __slots__ = ("objectType", "objectId", "objectVersion", "objectTimestamp", "tagVersion", "tagTimestamp", "isLatestObject", "isLatestTag")

tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.py

@@ -23,7 +23,7 @@ from tracdap.rt._impl.grpc.tracdap.metadata import custom_pb2 as tracdap_dot_rt_
 from tracdap.rt._impl.grpc.tracdap.metadata import stoarge_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_stoarge__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x32tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto\x1a\x34tracdap/rt/_impl/grpc/tracdap/metadata/stoarge.proto\"\x87\x05\n\x10ObjectDefinition\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .tracdap.metadata.DataDefinitionH\x00\x12\x32\n\x05model\x18\x03 \x01(\x0b\x32!.tracdap.metadata.ModelDefinitionH\x00\x12\x30\n\x04\x66low\x18\x04 \x01(\x0b\x32 .tracdap.metadata.FlowDefinitionH\x00\x12.\n\x03job\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinitionH\x00\x12\x30\n\x04\x66ile\x18\x06 \x01(\x0b\x32 .tracdap.metadata.FileDefinitionH\x00\x12\x34\n\x06\x63ustom\x18\x07 \x01(\x0b\x32\".tracdap.metadata.CustomDefinitionH\x00\x12\x36\n\x07storage\x18\x08 \x01(\x0b\x32#.tracdap.metadata.StorageDefinitionH\x00\x12\x34\n\x06schema\x18\t \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12H\n\x0bobjectProps\x18\x64 \x03(\x0b\x32\x33.tracdap.metadata.ObjectDefinition.ObjectPropsEntry\x1aK\n\x10ObjectPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0c\n\ndefinitionB2\n\x1aorg.finos.tracdap.metadataB\x12ObjectProtoWrapperP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3tracdap/rt/_impl/grpc/tracdap/metadata/object.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/type.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/data.proto\x1a\x32tracdap/rt/_impl/grpc/tracdap/metadata/model.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto\x1a\x30tracdap/rt/_impl/grpc/tracdap/metadata/job.proto\x1a\x31tracdap/rt/_impl/grpc/tracdap/metadata/file.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto\x1a\x34tracdap/rt/_impl/grpc/tracdap/metadata/stoarge.proto\"\xbd\x05\n\x10ObjectDefinition\x12\x30\n\nobjectType\x18\x01 \x01(\x0e\x32\x1c.tracdap.metadata.ObjectType\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .tracdap.metadata.DataDefinitionH\x00\x12\x32\n\x05model\x18\x03 \x01(\x0b\x32!.tracdap.metadata.ModelDefinitionH\x00\x12\x30\n\x04\x66low\x18\x04 \x01(\x0b\x32 .tracdap.metadata.FlowDefinitionH\x00\x12.\n\x03job\x18\x05 \x01(\x0b\x32\x1f.tracdap.metadata.JobDefinitionH\x00\x12\x30\n\x04\x66ile\x18\x06 \x01(\x0b\x32 .tracdap.metadata.FileDefinitionH\x00\x12\x34\n\x06\x63ustom\x18\x07 \x01(\x0b\x32\".tracdap.metadata.CustomDefinitionH\x00\x12\x36\n\x07storage\x18\x08 \x01(\x0b\x32#.tracdap.metadata.StorageDefinitionH\x00\x12\x34\n\x06schema\x18\t \x01(\x0b\x32\".tracdap.metadata.SchemaDefinitionH\x00\x12\x34\n\x06result\x18\n \x01(\x0b\x32\".tracdap.metadata.ResultDefinitionH\x00\x12H\n\x0bobjectProps\x18\x64 \x03(\x0b\x32\x33.tracdap.metadata.ObjectDefinition.ObjectPropsEntry\x1aK\n\x10ObjectPropsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.tracdap.metadata.Value:\x02\x38\x01\x42\x0c\n\ndefinitionB2\n\x1aorg.finos.tracdap.metadataB\x12ObjectProtoWrapperP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -34,7 +34,7 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._options = None
   _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_options = b'8\001'
   _globals['_OBJECTDEFINITION']._serialized_start=543
-  _globals['_OBJECTDEFINITION']._serialized_end=1190
-  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_start=1101
-  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_end=1176
+  _globals['_OBJECTDEFINITION']._serialized_end=1244
+  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_start=1155
+  _globals['_OBJECTDEFINITION_OBJECTPROPSENTRY']._serialized_end=1230
 # @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.pyi

@@ -15,7 +15,7 @@ from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Opti
 DESCRIPTOR: _descriptor.FileDescriptor
 
 class ObjectDefinition(_message.Message):
-    __slots__ = ("objectType", "data", "model", "flow", "job", "file", "custom", "storage", "schema", "objectProps")
+    __slots__ = ("objectType", "data", "model", "flow", "job", "file", "custom", "storage", "schema", "result", "objectProps")
     class ObjectPropsEntry(_message.Message):
         __slots__ = ("key", "value")
         KEY_FIELD_NUMBER: _ClassVar[int]
@@ -32,6 +32,7 @@ class ObjectDefinition(_message.Message):
     CUSTOM_FIELD_NUMBER: _ClassVar[int]
     STORAGE_FIELD_NUMBER: _ClassVar[int]
     SCHEMA_FIELD_NUMBER: _ClassVar[int]
+    RESULT_FIELD_NUMBER: _ClassVar[int]
     OBJECTPROPS_FIELD_NUMBER: _ClassVar[int]
     objectType: _object_id_pb2.ObjectType
     data: _data_pb2.DataDefinition
@@ -42,5 +43,6 @@ class ObjectDefinition(_message.Message):
     custom: _custom_pb2.CustomDefinition
     storage: _stoarge_pb2.StorageDefinition
     schema: _data_pb2.SchemaDefinition
+    result: _job_pb2.ResultDefinition
     objectProps: _containers.MessageMap[str, _type_pb2.Value]
-    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., data: _Optional[_Union[_data_pb2.DataDefinition, _Mapping]] = ..., model: _Optional[_Union[_model_pb2.ModelDefinition, _Mapping]] = ..., flow: _Optional[_Union[_flow_pb2.FlowDefinition, _Mapping]] = ..., job: _Optional[_Union[_job_pb2.JobDefinition, _Mapping]] = ..., file: _Optional[_Union[_file_pb2.FileDefinition, _Mapping]] = ..., custom: _Optional[_Union[_custom_pb2.CustomDefinition, _Mapping]] = ..., storage: _Optional[_Union[_stoarge_pb2.StorageDefinition, _Mapping]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., objectProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
+    def __init__(self, objectType: _Optional[_Union[_object_id_pb2.ObjectType, str]] = ..., data: _Optional[_Union[_data_pb2.DataDefinition, _Mapping]] = ..., model: _Optional[_Union[_model_pb2.ModelDefinition, _Mapping]] = ..., flow: _Optional[_Union[_flow_pb2.FlowDefinition, _Mapping]] = ..., job: _Optional[_Union[_job_pb2.JobDefinition, _Mapping]] = ..., file: _Optional[_Union[_file_pb2.FileDefinition, _Mapping]] = ..., custom: _Optional[_Union[_custom_pb2.CustomDefinition, _Mapping]] = ..., storage: _Optional[_Union[_stoarge_pb2.StorageDefinition, _Mapping]] = ..., schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., result: _Optional[_Union[_job_pb2.ResultDefinition, _Mapping]] = ..., objectProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...

tracdap/rt/_impl/logging.py

@@ -0,0 +1,195 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import io as _io
+import sys as _sys
+import typing as _tp
+
+from logging import *
+
+
+class PlainFormatter(Formatter):
+
+    FORMAT = f"%(asctime)s [%(threadName)s] %(levelname)s %(name)s" + \
+                     f" - %(message)s"
+
+    def __init__(self):
+        super().__init__(self.FORMAT)
+
+
+class ColorFormatter(Formatter):
+
+    _BLACK, _RED, _GREEN, _YELLOW, _BLUE, _MAGENTA, _CYAN, _WHITE, _DEFAULT_WHITE = range(9)
+    _DARK_BASE = 30
+    _LIGHT_BASE = 90
+
+    # DARK_BASE + WHITE = light grey
+    # DARK_BASE + DEFAULT_WHITE = regular console white
+    # LIGHT_BASE + WHITE = bright white (0xffffff), very bright!
+
+    def __init__(self, is_bright: bool):
+
+        super().__init__(self._base_fmt(is_bright))
+        self._level_colors = self._make_level_colors(is_bright)
+        self._default_color = self._make_default_color(is_bright)
+
+    def format(self, record):
+
+        level_name = record.levelname
+        level_color = self._level_colors.get(level_name)
+
+        if level_color:
+            record.levelname = level_color
+        else:
+            record.levelname = self._default_color + level_name
+
+        return Formatter.format(self, record)
+
+    def _base_fmt(self, is_bright: bool):
+
+        if is_bright:
+            base_color = self._make_ansi_code(self._DARK_BASE, self._DEFAULT_WHITE, is_bold=False)
+            message_color = self._make_ansi_code(self._LIGHT_BASE, self._CYAN, is_bold=False)
+        else:
+            base_color = self._make_ansi_code(self._DARK_BASE, self._WHITE, is_bold=False)
+            message_color = self._make_ansi_code(self._DARK_BASE, self._CYAN, is_bold=False)
+
+        return f"{base_color}%(asctime)s [%(threadName)s] %(levelname)s{base_color} %(name)s" + \
+            f" - {message_color}%(message)s"
+
+    def _make_level_colors(self, is_bright: bool):
+
+        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
+
+        green = self._make_ansi_code(base_code, self._GREEN, is_bold=is_bright)
+        yellow = self._make_ansi_code(base_code, self._YELLOW, is_bold=is_bright)
+        red = self._make_ansi_code(base_code, self._RED, is_bold=is_bright)
+
+        level_colors = {
+            'CRITICAL': f"{red}CRITICAL",
+            'ERROR': f"{red}ERROR",
+            'WARNING': f"{yellow}WARNING",
+            'INFO': f"{green}INFO"
+        }
+
+        return level_colors
+
+    def _make_default_color(self, is_bright: bool):
+
+        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
+        blue = self._make_ansi_code(base_code, self._BLUE, is_bold=is_bright)
+
+        return blue
+
+    @classmethod
+    def _make_ansi_code(cls, base_code: int, color_offset: int, is_bold: bool):
+        return f"\033[{1 if is_bold else 0};{base_code + color_offset}m"
+
+
+def configure_logging(enable_debug=False):
+
+    root_logger = getLogger()
+    log_level = DEBUG if enable_debug else INFO
+
+    if not root_logger.hasHandlers():
+
+        console_formatter = ColorFormatter(is_bright=True)
+        console_handler = StreamHandler(_sys.stdout)
+        console_handler.setFormatter(console_formatter)
+        console_handler.setLevel(INFO)
+        root_logger.addHandler(console_handler)
+        root_logger.setLevel(log_level)
+
+        # Use is_bright=False for logs from the TRAC runtime, so model logs stand out
+
+        trac_logger = getLogger("tracdap.rt")
+
+        console_formatter = ColorFormatter(is_bright=False)
+        console_handler = StreamHandler(_sys.stdout)
+        console_handler.setFormatter(console_formatter)
+        console_handler.setLevel(log_level)
+        trac_logger.addHandler(console_handler)
+        trac_logger.propagate = False
+
+
+def logger_for_object(obj: object) -> Logger:
+    return logger_for_class(obj.__class__)
+
+
+def logger_for_class(clazz: type) -> Logger:
+    qualified_class_name = f"{clazz.__module__}.{clazz.__name__}"
+    return getLogger(qualified_class_name)
+
+
+def logger_for_namespace(namespace: str) -> Logger:
+    return getLogger(namespace)
+
+
+class JobLogger(Logger):
+
+    def __init__(self, sys_log: Logger, *handlers: Handler):
+
+        super().__init__(sys_log.name, sys_log.level)
+        self._sys_log = sys_log._log
+        self._job_log = super()._log
+
+        for handler in handlers:
+            self.addHandler(handler)
+
+    def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, stacklevel=1):
+
+        self._sys_log(level, msg, args, exc_info, extra, stack_info, stacklevel)
+        self._job_log(level, msg, args, exc_info, extra, stack_info, stacklevel)
+
+
+class LogProvider:
+
+    def logger_for_object(self, obj: object) -> Logger:
+        return logger_for_object(obj)
+
+    def logger_for_class(self, clazz: type) -> Logger:
+        return logger_for_class(clazz)
+
+    def logger_for_namespace(self, namespace: str) -> Logger:
+        return logger_for_namespace(namespace)
+
+
+class JobLogProvider(LogProvider):
+
+    def __init__(self, *handlers: Handler):
+        self.__handlers = handlers
+
+    def logger_for_object(self, obj: object) -> Logger:
+        base_logger = logger_for_object(obj)
+        return JobLogger(base_logger, *self.__handlers)
+
+    def logger_for_class(self, clazz: type) -> Logger:
+        base_logger = logger_for_class(clazz)
+        return JobLogger(base_logger, *self.__handlers)
+
+    def logger_for_namespace(self, namespace: str) -> Logger:
+        base_logger = logger_for_namespace(namespace)
+        return JobLogger(base_logger, *self.__handlers)
+
+
+def job_log_provider(target: _tp.BinaryIO) -> JobLogProvider:
+
+    stream = _io.TextIOWrapper(target, newline="\r\n")
+    formatter = PlainFormatter()
+
+    handler = StreamHandler(stream)
+    handler.setFormatter(formatter)
+
+    return JobLogProvider(handler)

tracdap/rt/_impl/models.py

@@ -25,9 +25,10 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 
-import tracdap.rt._impl.type_system as _types
+import tracdap.rt._impl.logging as _logging
 import tracdap.rt._impl.repos as _repos
 import tracdap.rt._impl.shim as _shim
+import tracdap.rt._impl.type_system as _types
 import tracdap.rt._impl.util as _util
 import tracdap.rt._impl.validation as _val
 
@@ -43,7 +44,7 @@ class ModelLoader:
 
     def __init__(self, sys_config: _cfg.RuntimeConfig, scratch_dir: pathlib.Path):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
 
         self.__scratch_dir = scratch_dir.joinpath("models")
         self.__repos = _repos.RepositoryManager(sys_config)

tracdap/rt/_impl/repos.py

@@ -13,10 +13,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import typing as _tp
+
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt.config as cfg
 import tracdap.rt.exceptions as ex
-import tracdap.rt._impl.util as util
+import tracdap.rt._impl.logging as _logging
 
 # Import repo interfaces
 from tracdap.rt.ext.repos import *
@@ -26,8 +28,8 @@ class RepositoryManager:
 
     def __init__(self, sys_config: cfg.RuntimeConfig):
 
-        self._log = util.logger_for_object(self)
-        self._repos: tp.Dict[str, IModelRepository] = dict()
+        self._log = _logging.logger_for_object(self)
+        self._repos: _tp.Dict[str, IModelRepository] = dict()
 
         # Initialize all repos in the system config
         # Any errors for missing repo types (plugins) will be raised during startup

tracdap/rt/_impl/schemas.py

@@ -22,9 +22,9 @@ import pyarrow as pa
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
 import tracdap.rt._impl.data as _data
+import tracdap.rt._impl.logging as _log
 import tracdap.rt._impl.storage as _storage
 import tracdap.rt._impl.shim as _shim
-import tracdap.rt._impl.util as _util
 
 
 class SchemaLoader:
@@ -169,4 +169,4 @@ class SchemaLoader:
         raise _ex.EDataConformance(err)
 
 
-SchemaLoader._log = _util.logger_for_class(SchemaLoader)
+SchemaLoader._log = _log.logger_for_class(SchemaLoader)

tracdap/rt/_impl/shim.py

@@ -31,6 +31,7 @@ import importlib.resources as _ilr
 
 import tracdap.rt.exceptions as _ex
 import tracdap.rt._impl.guard_rails as _guard
+import tracdap.rt._impl.logging as _log
 import tracdap.rt._impl.util as _util
 
 
@@ -138,7 +139,7 @@ class _NamespaceShimFinder(_ila.MetaPathFinder):
     def __init__(self, shim_map: tp.Dict[str, _Shim], active_shim: _ActiveShim):
         self.__shim_map = shim_map
         self.__active_shim = active_shim
-        self._log = _util.logger_for_class(ShimLoader)
+        self._log = _log.logger_for_class(ShimLoader)
 
     def find_spec(
             self, fullname: str,
@@ -561,5 +562,5 @@ class ShimLoader:
                 raise _ex.ERuntimeValidation(err)
 
 
-ShimLoader._log = _util.logger_for_class(ShimLoader)
+ShimLoader._log = _log.logger_for_class(ShimLoader)
 ShimLoader._init()  # noqa

tracdap/rt/_impl/storage.py

@@ -30,6 +30,7 @@ import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt._impl.data as _data
+import tracdap.rt._impl.logging as _logging
 import tracdap.rt._impl.util as _util
 import tracdap.rt._impl.validation as _val
 
@@ -76,7 +77,7 @@ class StorageManager:
 
     def __init__(self, sys_config: _cfg.RuntimeConfig):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
         self.__file_storage: tp.Dict[str, IFileStorage] = dict()
         self.__data_storage: tp.Dict[str, IDataStorage] = dict()
         self.__external: tp.List[str] = list()
@@ -224,7 +225,7 @@ class CommonFileStorage(IFileStorage):
 
     def __init__(self, storage_key: str, storage_config: _cfg.PluginConfig, fs: pa_fs.SubTreeFileSystem):
 
-        self._log = _util.logger_for_object(self)
+        self._log = _logging.logger_for_object(self)
         self._key = storage_key
         self._config = storage_config
         self._fs = fs
@@ -731,7 +732,7 @@ class CommonDataStorage(IDataStorage):
             self, config: _cfg.PluginConfig, file_storage: IFileStorage,
             pushdown_pandas: bool = False, pushdown_spark: bool = False):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
 
         self.__config = config
         self.__file_storage = file_storage

tracdap/rt/_impl/util.py

@@ -14,11 +14,9 @@
 #  limitations under the License.
 
 import datetime as dt
-import logging
 import pathlib
 import platform
 
-import sys
 import typing as tp
 import uuid
 
@@ -38,114 +36,6 @@ def is_windows():
     return __IS_WINDOWS
 
 
-class ColorFormatter(logging.Formatter):
-
-    _BLACK, _RED, _GREEN, _YELLOW, _BLUE, _MAGENTA, _CYAN, _WHITE, _DEFAULT_WHITE = range(9)
-    _DARK_BASE = 30
-    _LIGHT_BASE = 90
-
-    # DARK_BASE + WHITE = light grey
-    # DARK_BASE + DEFAULT_WHITE = regular console white
-    # LIGHT_BASE + WHITE = bright white (0xffffff), very bright!
-
-    def __init__(self, is_bright: bool):
-
-        super().__init__(self._base_fmt(is_bright))
-        self._level_colors = self._make_level_colors(is_bright)
-        self._default_color = self._make_default_color(is_bright)
-
-    def format(self, record):
-
-        level_name = record.levelname
-        level_color = self._level_colors.get(level_name)
-
-        if level_color:
-            record.levelname = level_color
-        else:
-            record.levelname = self._default_color + level_name
-
-        return logging.Formatter.format(self, record)
-
-    def _base_fmt(self, is_bright: bool):
-
-        if is_bright:
-            base_color = self._make_ansi_code(self._DARK_BASE, self._DEFAULT_WHITE, is_bold=False)
-            message_color = self._make_ansi_code(self._LIGHT_BASE, self._CYAN, is_bold=False)
-        else:
-            base_color = self._make_ansi_code(self._DARK_BASE, self._WHITE, is_bold=False)
-            message_color = self._make_ansi_code(self._DARK_BASE, self._CYAN, is_bold=False)
-
-        return f"{base_color}%(asctime)s [%(threadName)s] %(levelname)s{base_color} %(name)s" + \
-               f" - {message_color}%(message)s"
-
-    def _make_level_colors(self, is_bright: bool):
-
-        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
-
-        green = self._make_ansi_code(base_code, self._GREEN, is_bold=is_bright)
-        yellow = self._make_ansi_code(base_code, self._YELLOW, is_bold=is_bright)
-        red = self._make_ansi_code(base_code, self._RED, is_bold=is_bright)
-
-        level_colors = {
-            'CRITICAL': f"{red}CRITICAL",
-            'ERROR': f"{red}ERROR",
-            'WARNING': f"{yellow}WARNING",
-            'INFO': f"{green}INFO"
-        }
-
-        return level_colors
-
-    def _make_default_color(self, is_bright: bool):
-
-        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
-        blue = self._make_ansi_code(base_code, self._BLUE, is_bold=is_bright)
-
-        return blue
-
-    @classmethod
-    def _make_ansi_code(cls, base_code: int, color_offset: int, is_bold: bool):
-        return f"\033[{1 if is_bold else 0};{base_code + color_offset}m"
-
-
-def configure_logging(enable_debug=False):
-
-    root_logger = logging.getLogger()
-    log_level = logging.DEBUG if enable_debug else logging.INFO
-
-    if not root_logger.hasHandlers():
-
-        console_formatter = ColorFormatter(is_bright=True)
-        console_handler = logging.StreamHandler(sys.stdout)
-        console_handler.setFormatter(console_formatter)
-        console_handler.setLevel(logging.INFO)
-        root_logger.addHandler(console_handler)
-        root_logger.setLevel(log_level)
-
-        # Use is_bright=False for logs from the TRAC runtime, so model logs stand out
-
-        trac_logger = logging.getLogger("tracdap.rt")
-
-        console_formatter = ColorFormatter(is_bright=False)
-        console_handler = logging.StreamHandler(sys.stdout)
-        console_handler.setFormatter(console_formatter)
-        console_handler.setLevel(log_level)
-        trac_logger.addHandler(console_handler)
-        trac_logger.propagate = False
-
-
-def logger_for_object(obj: object) -> logging.Logger:
-    return logger_for_class(obj.__class__)
-
-
-def logger_for_class(clazz: type) -> logging.Logger:
-    qualified_class_name = f"{clazz.__module__}.{clazz.__name__}"
-    return logging.getLogger(qualified_class_name)
-
-
-def logger_for_namespace(namespace: str) -> logging.Logger:
-    return logging.getLogger(namespace)
-
-
 def format_file_size(size: int) -> str:
 
     if size < 1024:

tracdap/rt/_impl/validation.py

@@ -22,6 +22,7 @@ import pathlib
 
 import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
+import tracdap.rt._impl.logging as log
 import tracdap.rt._impl.util as util
 
 # _Named placeholder type from API hook is needed for API type checking
@@ -74,7 +75,7 @@ class _TypeValidator:
     # Inspecting a function signature can take ~ half a second in Python 3.7
     __method_cache: tp.Dict[str, tp.Tuple[inspect.Signature, tp.Any]] = dict()
 
-    _log: logging.Logger = util.logger_for_namespace(__name__)
+    _log: logging.Logger = log.logger_for_namespace(__name__)
 
     @classmethod
     def validate_signature(cls, method: tp.Callable, *args, **kwargs):
@@ -324,7 +325,7 @@ class StaticValidator:
         meta.BasicType.INTEGER,
         meta.BasicType.DATE]
 
-    _log: logging.Logger = util.logger_for_namespace(__name__)
+    _log: logging.Logger = log.logger_for_namespace(__name__)
 
     @classmethod
     def is_primitive_type(cls, basic_type: meta.BasicType) -> bool:

tracdap/rt/_version.py

@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.8.0b1"
+__version__ = "0.8.0b2"

tracdap/rt/ext/plugins.py

@@ -14,13 +14,13 @@
 #  limitations under the License.
 
 import typing as _tp
-import logging as _log
 import pkgutil as _pkg
 import importlib as _il
 
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt._impl.guard_rails as _guard  # noqa
+import tracdap.rt._impl.logging as _logging  # noqa
 from tracdap.rt.exceptions import EStartup
 
 
@@ -28,7 +28,7 @@ class PluginManager:
 
     T_SERVICE = _tp.TypeVar("T_SERVICE")
 
-    __log = _log.getLogger(f"{__name__}.PluginManager")
+    __log = _logging.getLogger(f"{__name__}.PluginManager")
 
     __core_registered = False
     __3rd_party_registered = list()

tracdap/rt/metadata/__init__.py

@@ -50,6 +50,7 @@ from .job import JobType
 from .job import JobStatusCode
 from .job import JobGroupType
 from .job import JobDefinition
+from .job import ResultDefinition
 from .job import RunModelJob
 from .job import RunFlowJob
 from .job import ImportModelJob

tracdap/rt/metadata/job.py

@@ -117,6 +117,22 @@ class JobDefinition:
 
     jobGroup: "_tp.Optional[JobGroup]" = None
 
+    resultId: "TagSelector" = _dc.field(default_factory=lambda: TagSelector())
+
+
+@_dc.dataclass
+class ResultDefinition:
+
+    """Define the result of a job after it has completed"""
+
+    jobId: "TagSelector" = _dc.field(default_factory=lambda: TagSelector())
+
+    statusCode: "JobStatusCode" = JobStatusCode.JOB_STATUS_CODE_NOT_SET
+
+    statusMessage: "str" = ""
+
+    logFileId: "TagSelector" = _dc.field(default_factory=lambda: TagSelector())
+
 
 @_dc.dataclass
 class RunModelJob: