tracdap-runtime 0.7.0rc1__py3-none-any.whl → 0.8.0__py3-none-any.whl

tracdap/rt/_impl/core/logging.py

@@ -0,0 +1,195 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import io as _io
+import sys as _sys
+import typing as _tp
+
+from logging import *
+
+
+class PlainFormatter(Formatter):
+
+    FORMAT = f"%(asctime)s [%(threadName)s] %(levelname)s %(name)s" + \
+                     f" - %(message)s"
+
+    def __init__(self):
+        super().__init__(self.FORMAT)
+
+
+class ColorFormatter(Formatter):
+
+    _BLACK, _RED, _GREEN, _YELLOW, _BLUE, _MAGENTA, _CYAN, _WHITE, _DEFAULT_WHITE = range(9)
+    _DARK_BASE = 30
+    _LIGHT_BASE = 90
+
+    # DARK_BASE + WHITE = light grey
+    # DARK_BASE + DEFAULT_WHITE = regular console white
+    # LIGHT_BASE + WHITE = bright white (0xffffff), very bright!
+
+    def __init__(self, is_bright: bool):
+
+        super().__init__(self._base_fmt(is_bright))
+        self._level_colors = self._make_level_colors(is_bright)
+        self._default_color = self._make_default_color(is_bright)
+
+    def format(self, record):
+
+        level_name = record.levelname
+        level_color = self._level_colors.get(level_name)
+
+        if level_color:
+            record.levelname = level_color
+        else:
+            record.levelname = self._default_color + level_name
+
+        return Formatter.format(self, record)
+
+    def _base_fmt(self, is_bright: bool):
+
+        if is_bright:
+            base_color = self._make_ansi_code(self._DARK_BASE, self._DEFAULT_WHITE, is_bold=False)
+            message_color = self._make_ansi_code(self._LIGHT_BASE, self._CYAN, is_bold=False)
+        else:
+            base_color = self._make_ansi_code(self._DARK_BASE, self._WHITE, is_bold=False)
+            message_color = self._make_ansi_code(self._DARK_BASE, self._CYAN, is_bold=False)
+
+        return f"{base_color}%(asctime)s [%(threadName)s] %(levelname)s{base_color} %(name)s" + \
+            f" - {message_color}%(message)s"
+
+    def _make_level_colors(self, is_bright: bool):
+
+        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
+
+        green = self._make_ansi_code(base_code, self._GREEN, is_bold=is_bright)
+        yellow = self._make_ansi_code(base_code, self._YELLOW, is_bold=is_bright)
+        red = self._make_ansi_code(base_code, self._RED, is_bold=is_bright)
+
+        level_colors = {
+            'CRITICAL': f"{red}CRITICAL",
+            'ERROR': f"{red}ERROR",
+            'WARNING': f"{yellow}WARNING",
+            'INFO': f"{green}INFO"
+        }
+
+        return level_colors
+
+    def _make_default_color(self, is_bright: bool):
+
+        base_code = self._LIGHT_BASE if is_bright else self._DARK_BASE
+        blue = self._make_ansi_code(base_code, self._BLUE, is_bold=is_bright)
+
+        return blue
+
+    @classmethod
+    def _make_ansi_code(cls, base_code: int, color_offset: int, is_bold: bool):
+        return f"\033[{1 if is_bold else 0};{base_code + color_offset}m"
+
+
+def configure_logging(enable_debug=False):
+
+    root_logger = getLogger()
+    log_level = DEBUG if enable_debug else INFO
+
+    if not root_logger.hasHandlers():
+
+        console_formatter = ColorFormatter(is_bright=True)
+        console_handler = StreamHandler(_sys.stdout)
+        console_handler.setFormatter(console_formatter)
+        console_handler.setLevel(INFO)
+        root_logger.addHandler(console_handler)
+        root_logger.setLevel(log_level)
+
+        # Use is_bright=False for logs from the TRAC runtime, so model logs stand out
+
+        trac_logger = getLogger("tracdap.rt")
+
+        console_formatter = ColorFormatter(is_bright=False)
+        console_handler = StreamHandler(_sys.stdout)
+        console_handler.setFormatter(console_formatter)
+        console_handler.setLevel(log_level)
+        trac_logger.addHandler(console_handler)
+        trac_logger.propagate = False
+
+
+def logger_for_object(obj: object) -> Logger:
+    return logger_for_class(obj.__class__)
+
+
+def logger_for_class(clazz: type) -> Logger:
+    qualified_class_name = f"{clazz.__module__}.{clazz.__name__}"
+    return getLogger(qualified_class_name)
+
+
+def logger_for_namespace(namespace: str) -> Logger:
+    return getLogger(namespace)
+
+
+class JobLogger(Logger):
+
+    def __init__(self, sys_log: Logger, *handlers: Handler):
+
+        super().__init__(sys_log.name, sys_log.level)
+        self._sys_log = sys_log._log
+        self._job_log = super()._log
+
+        for handler in handlers:
+            self.addHandler(handler)
+
+    def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, stacklevel=1):
+
+        self._sys_log(level, msg, args, exc_info, extra, stack_info, stacklevel)
+        self._job_log(level, msg, args, exc_info, extra, stack_info, stacklevel)
+
+
+class LogProvider:
+
+    def logger_for_object(self, obj: object) -> Logger:
+        return logger_for_object(obj)
+
+    def logger_for_class(self, clazz: type) -> Logger:
+        return logger_for_class(clazz)
+
+    def logger_for_namespace(self, namespace: str) -> Logger:
+        return logger_for_namespace(namespace)
+
+
+class JobLogProvider(LogProvider):
+
+    def __init__(self, *handlers: Handler):
+        self.__handlers = handlers
+
+    def logger_for_object(self, obj: object) -> Logger:
+        base_logger = logger_for_object(obj)
+        return JobLogger(base_logger, *self.__handlers)
+
+    def logger_for_class(self, clazz: type) -> Logger:
+        base_logger = logger_for_class(clazz)
+        return JobLogger(base_logger, *self.__handlers)
+
+    def logger_for_namespace(self, namespace: str) -> Logger:
+        base_logger = logger_for_namespace(namespace)
+        return JobLogger(base_logger, *self.__handlers)
+
+
+def job_log_provider(target: _tp.BinaryIO) -> JobLogProvider:
+
+    stream = _io.TextIOWrapper(target, newline="\r\n")
+    formatter = PlainFormatter()
+
+    handler = StreamHandler(stream)
+    handler.setFormatter(formatter)
+
+    return JobLogProvider(handler)

tracdap/rt/_impl/{models.py → core/models.py}

@@ -25,11 +25,12 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 
-import tracdap.rt._impl.type_system as _types
-import tracdap.rt._impl.repos as _repos
-import tracdap.rt._impl.shim as _shim
-import tracdap.rt._impl.util as _util
-import tracdap.rt._impl.validation as _val
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.repos as _repos
+import tracdap.rt._impl.core.shim as _shim
+import tracdap.rt._impl.core.type_system as _types
+import tracdap.rt._impl.core.util as _util
+import tracdap.rt._impl.core.validation as _val
 
 
 class ModelLoader:
@@ -43,7 +44,7 @@ class ModelLoader:
 
     def __init__(self, sys_config: _cfg.RuntimeConfig, scratch_dir: pathlib.Path):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
 
         self.__scratch_dir = scratch_dir.joinpath("models")
         self.__repos = _repos.RepositoryManager(sys_config)
@@ -226,13 +227,15 @@ class ModelLoader:
                 self.__log.info(f"Parameter [{name}] - {param.paramType.basicType.name}")
                 param.paramProps = self._encoded_props(param.paramProps, "parameter", name)
 
-            for name, schema in model_def.inputs.items():
-                self.__log.info(f"Input [{name}] - {schema.schema.schemaType.name}")
-                schema.inputProps = self._encoded_props(schema.inputProps, "input", name)
+            for name, input_def in model_def.inputs.items():
+                input_type = input_def.schema.schemaType.name if input_def.objectType == _meta.ObjectType.DATA else input_def.objectType.name
+                self.__log.info(f"Input [{name}] - {input_type}")
+                input_def.inputProps = self._encoded_props(input_def.inputProps, "input", name)
 
-            for name, schema in model_def.outputs.items():
-                self.__log.info(f"Output [{name}] - {schema.schema.schemaType.name}")
-                schema.outputProps = self._encoded_props(schema.outputProps, "input", name)
+            for name, output_def in model_def.outputs.items():
+                output_type = output_def.schema.schemaType.name if output_def.objectType == _meta.ObjectType.DATA else output_def.objectType.name
+                self.__log.info(f"Output [{name}] - {output_type}")
+                output_def.outputProps = self._encoded_props(output_def.outputProps, "input", name)
 
             return model_def
 

tracdap/rt/_impl/{repos.py → core/repos.py}

@@ -13,10 +13,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import typing as _tp
+
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt.config as cfg
 import tracdap.rt.exceptions as ex
-import tracdap.rt._impl.util as util
+import tracdap.rt._impl.core.logging as _logging
 
 # Import repo interfaces
 from tracdap.rt.ext.repos import *
@@ -26,8 +28,8 @@ class RepositoryManager:
 
     def __init__(self, sys_config: cfg.RuntimeConfig):
 
-        self._log = util.logger_for_object(self)
-        self._repos: tp.Dict[str, IModelRepository] = dict()
+        self._log = _logging.logger_for_object(self)
+        self._repos: _tp.Dict[str, IModelRepository] = dict()
 
         # Initialize all repos in the system config
         # Any errors for missing repo types (plugins) will be raised during startup
@@ -36,6 +38,13 @@ class RepositoryManager:
 
             try:
 
+                # Add global properties related to the repo protocol
+                related_props = {
+                    k: v for (k, v) in sys_config.properties.items()
+                    if k.startswith(f"{repo_config.protocol}.")}
+
+                repo_config.properties.update(related_props)
+
                 self._repos[repo_name] = plugins.PluginManager.load_plugin(IModelRepository, repo_config)
 
             except ex.EPluginNotAvailable as e:

tracdap/rt/_impl/{schemas.py → core/schemas.py}

@@ -21,10 +21,10 @@ import pyarrow as pa
 
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.data as _data
-import tracdap.rt._impl.storage as _storage
-import tracdap.rt._impl.shim as _shim
-import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.logging as _log
+import tracdap.rt._impl.core.storage as _storage
+import tracdap.rt._impl.core.shim as _shim
 
 
 class SchemaLoader:
@@ -169,4 +169,4 @@ class SchemaLoader:
         raise _ex.EDataConformance(err)
 
 
-SchemaLoader._log = _util.logger_for_class(SchemaLoader)
+SchemaLoader._log = _log.logger_for_class(SchemaLoader)

tracdap/rt/_impl/{shim.py → core/shim.py}

@@ -30,8 +30,9 @@ import importlib.machinery as _ilm
 import importlib.resources as _ilr
 
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.guard_rails as _guard
-import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.core.guard_rails as _guard
+import tracdap.rt._impl.core.logging as _log
+import tracdap.rt._impl.core.util as _util
 
 
 class _Shim:
@@ -138,7 +139,7 @@ class _NamespaceShimFinder(_ila.MetaPathFinder):
     def __init__(self, shim_map: tp.Dict[str, _Shim], active_shim: _ActiveShim):
         self.__shim_map = shim_map
         self.__active_shim = active_shim
-        self._log = _util.logger_for_class(ShimLoader)
+        self._log = _log.logger_for_class(ShimLoader)
 
     def find_spec(
             self, fullname: str,
@@ -561,5 +562,5 @@ class ShimLoader:
                 raise _ex.ERuntimeValidation(err)
 
 
-ShimLoader._log = _util.logger_for_class(ShimLoader)
+ShimLoader._log = _log.logger_for_class(ShimLoader)
 ShimLoader._init()  # noqa

tracdap/rt/_impl/{storage.py → core/storage.py}

@@ -29,9 +29,10 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as plugins
-import tracdap.rt._impl.data as _data
-import tracdap.rt._impl.util as _util
-import tracdap.rt._impl.validation as _val
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.util as _util
+import tracdap.rt._impl.core.validation as _val
 
 # Import storage interfaces (using the internal version, it has extra bits that are not public)
 from tracdap.rt._impl.ext.storage import *
@@ -76,11 +77,11 @@ class StorageManager:
 
     def __init__(self, sys_config: _cfg.RuntimeConfig):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
         self.__file_storage: tp.Dict[str, IFileStorage] = dict()
         self.__data_storage: tp.Dict[str, IDataStorage] = dict()
         self.__external: tp.List[str] = list()
-        self.__settings = sys_config.storage
+        self.__sys_config = sys_config
 
         for storage_key, storage_config in sys_config.storage.buckets.items():
             self.create_storage(storage_key, storage_config)
@@ -92,13 +93,20 @@ class StorageManager:
             self.create_storage(storage_key, storage_config)
 
     def default_storage_key(self):
-        return self.__settings.defaultBucket
+        return self.__sys_config.storage.defaultBucket
 
     def default_storage_format(self):
-        return self.__settings.defaultFormat
+        return self.__sys_config.storage.defaultFormat
 
     def create_storage(self, storage_key: str, storage_config: _cfg.PluginConfig):
 
+        # Add global properties related to the storage protocol
+        related_props = {
+            k: v for (k, v) in self.__sys_config.properties.items()
+            if k.startswith(f"{storage_config.protocol}.")}
+
+        storage_config.properties.update(related_props)
+
         if plugins.PluginManager.is_plugin_available(IStorageProvider, storage_config.protocol):
             self._create_storage_from_provider(storage_key, storage_config)
         else:
@@ -224,7 +232,7 @@ class CommonFileStorage(IFileStorage):
 
     def __init__(self, storage_key: str, storage_config: _cfg.PluginConfig, fs: pa_fs.SubTreeFileSystem):
 
-        self._log = _util.logger_for_object(self)
+        self._log = _logging.logger_for_object(self)
         self._key = storage_key
         self._config = storage_config
         self._fs = fs
@@ -495,7 +503,10 @@ class CommonFileStorage(IFileStorage):
 
         # For successful write streams, log the total size written
         if is_write and not error:
-            file_size = _util.format_file_size(stream.tell())
+            if not stream.closed:
+                file_size = _util.format_file_size(stream.tell())
+            else:
+                file_size = self._fs.get_file_info(storage_path).size
             self._log.info(f"File size [{self._key}]: {file_size} [{storage_path}]")
 
         # Close the stream - this may take time for write streams that are not flushed
@@ -731,7 +742,7 @@ class CommonDataStorage(IDataStorage):
             self, config: _cfg.PluginConfig, file_storage: IFileStorage,
             pushdown_pandas: bool = False, pushdown_spark: bool = False):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
 
         self.__config = config
         self.__file_storage = file_storage