backend.ai-logging 24.9.0__tar.gz

backend_ai_logging-24.9.0/MANIFEST.in

@@ -0,0 +1 @@
+include *.py

backend_ai_logging-24.9.0/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.1
+Name: backend.ai-logging
+Version: 24.9.0
+Summary: Backend.AI Logging Subsystem
+Home-page: https://github.com/lablup/backend.ai
+Author: Lablup Inc. and contributors
+License: MIT
+Project-URL: Documentation, https://docs.backend.ai/
+Project-URL: Source, https://github.com/lablup/backend.ai
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Environment :: No Input/Output (Daemon)
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.12,<3.13
+Description-Content-Type: text/markdown
+Requires-Dist: coloredlogs~=15.0
+Requires-Dist: graypy==2.1.0
+Requires-Dist: msgpack~=1.1.0
+Requires-Dist: python-json-logger>=2.0.1
+Requires-Dist: pyzmq~=26.2
+Requires-Dist: trafaret~=2.1
+Requires-Dist: yarl~=1.13.1
+
+Backend.AI Logging Subsystem
+============================
+
+Package Structure
+-----------------
+
+* `ai.backend.logging`
+  - `abc`: Abstract base classes
+  - `logger`: The core logging facility
+    - `Logger`: The standard multiprocess-friendly logger using `RelayHandler` based on ZeroMQ
+    - `LocalLogger`: A minimalized console/file logger that does not require serialization via networks at all
+  - `handler`: Collection of vendor-specific handler implementations
+  - `formatter`: Collection of formatters
+  - `types`: Definition of enums/types like `LogLevel`
+  - `utils`: Brace-style message formatting adapters and other extras

backend_ai_logging-24.9.0/ai/backend/logging/VERSION

@@ -0,0 +1 @@
+24.09.0

backend_ai_logging-24.9.0/ai/backend/logging/__init__.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+
+from .abc import AbstractLogger
+from .exceptions import ConfigurationError
+from .logger import LocalLogger, Logger, NoopLogger, is_active
+from .types import LogFormat, LogLevel
+from .utils import BraceStyleAdapter
+
+__version__ = (Path(__file__).parent / "VERSION").read_text().strip()
+
+__all__ = (
+    "AbstractLogger",
+    "Logger",
+    "LocalLogger",
+    "NoopLogger",
+    "BraceStyleAdapter",
+    "is_active",
+    "ConfigurationError",
+    "LogFormat",
+    "LogLevel",
+)

backend_ai_logging-24.9.0/ai/backend/logging/abc.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from abc import ABCMeta, abstractmethod
+from typing import Any, MutableMapping, Self
+
+
+class AbstractLogger(metaclass=ABCMeta):
+    def __init__(
+        self,
+        logging_config: MutableMapping[str, Any],
+    ) -> None:
+        pass
+
+    @abstractmethod
+    def __enter__(self) -> Self:
+        raise NotImplementedError
+
+    @abstractmethod
+    def __exit__(self, *exc_info_args) -> bool | None:
+        raise NotImplementedError

backend_ai_logging-24.9.0/ai/backend/logging/config.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from collections.abc import MutableMapping
+from typing import Any
+
+import trafaret as t
+
+from .types import (
+    DirPathTrafaret,
+    LogFormat,
+    LogLevel,
+    SimpleBinarySizeTrafaret,
+)
+
+default_pkg_ns: dict[str, LogLevel] = {
+    "": LogLevel.WARNING,
+    "ai.backend": LogLevel.INFO,
+    "tests": LogLevel.DEBUG,
+}
+
+logging_config_iv = t.Dict({
+    t.Key("level", default=LogLevel.INFO): LogLevel.as_trafaret(),
+    t.Key("pkg-ns", default=default_pkg_ns): t.Mapping(
+        t.String(allow_blank=True), LogLevel.as_trafaret()
+    ),
+    t.Key("drivers", default=["console"]): t.List(
+        t.Enum(
+            "console",
+            "logstash",
+            "file",
+            "graylog",
+        )
+    ),
+    t.Key(
+        "console",
+        default={
+            "colored": None,
+            "format": LogFormat.VERBOSE,
+        },
+    ): t.Dict({
+        t.Key("colored", default=None): t.Null | t.ToBool,
+        t.Key("format", default=LogFormat.VERBOSE): LogFormat.as_trafaret(),
+    }).allow_extra("*"),
+    t.Key("file", default=None): t.Null
+    | t.Dict({
+        t.Key("path"): DirPathTrafaret(auto_create=True),
+        t.Key("filename"): t.String,
+        t.Key("backup-count", default=5): t.ToInt[1:100],
+        t.Key("rotation-size", default="10M"): SimpleBinarySizeTrafaret,
+        t.Key("format", default=LogFormat.VERBOSE): LogFormat.as_trafaret(),
+    }).allow_extra("*"),
+    t.Key("logstash", default=None): t.Null
+    | t.Dict({
+        t.Key("endpoint"): t.Tuple(t.String, t.ToInt[1:65535])
+        | t.Dict({
+            t.Key("host"): t.String,
+            t.Key("port"): t.ToInt[1:65535],
+        }),
+        t.Key("protocol", default="tcp"): t.Enum("zmq.push", "zmq.pub", "tcp", "udp"),
+        t.Key("ssl-enabled", default=True): t.ToBool,
+        t.Key("ssl-verify", default=True): t.ToBool,
+        # NOTE: logstash does not have format option.
+    }).allow_extra("*"),
+    t.Key("graylog", default=None): t.Null
+    | t.Dict({
+        t.Key("host"): t.String,
+        t.Key("port"): t.ToInt[1024:65535],
+        t.Key("level", default=LogLevel.INFO): LogLevel.as_trafaret(),
+        t.Key("ssl-verify", default=False): t.Bool,
+        t.Key("ca-certs", default=None): t.Null | t.String(allow_blank=True),
+        t.Key("keyfile", default=None): t.Null | t.String(allow_blank=True),
+        t.Key("certfile", default=None): t.Null | t.String(allow_blank=True),
+        t.Key("fqdn", default=True): t.ToBool,
+        t.Key("localname", default=None): t.Null | t.String(),
+    }).allow_extra("*"),
+}).allow_extra("*")
+
+
+def override_key(table: MutableMapping[str, Any], key_path: tuple[str, ...], value: Any):
+    for k in key_path[:-1]:
+        if k not in table:
+            table[k] = {}
+        table = table[k]
+    table[key_path[-1]] = value

backend_ai_logging-24.9.0/ai/backend/logging/exceptions.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+
+class ConfigurationError(Exception):
+    invalid_data: Mapping[str, Any]
+
+    def __init__(self, invalid_data: Mapping[str, Any]) -> None:
+        super().__init__(invalid_data)
+        self.invalid_data = invalid_data

backend_ai_logging-24.9.0/ai/backend/logging/formatter.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import logging
+import pprint
+import time
+from datetime import datetime
+from typing import Any
+
+import coloredlogs
+from pythonjsonlogger.jsonlogger import JsonFormatter
+
+
+def format_exception(self, ei) -> str:
+    s = "".join(ei)
+    if s[-1:] == "\n":
+        s = s[:-1]
+    return s
+
+
+class SerializedExceptionFormatter(logging.Formatter):
+    def formatException(self, ei) -> str:
+        return format_exception(self, ei)
+
+
+class ConsoleFormatter(logging.Formatter):
+    def formatException(self, ei) -> str:
+        return format_exception(self, ei)
+
+    def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str:
+        ct = self.converter(record.created)  # type: ignore
+        if datefmt:
+            datefmt = datefmt.replace("%f", f"{int(record.msecs):03d}")
+            return time.strftime(datefmt, ct)
+        else:
+            t = time.strftime("%Y-%m-%d %H:%M:%S", ct)
+            return f"{t}.{int(record.msecs):03d}"
+
+
+class CustomJsonFormatter(JsonFormatter):
+    def formatException(self, ei) -> str:
+        return format_exception(self, ei)
+
+    def add_fields(
+        self,
+        log_record: dict[str, Any],  # the manipulated entry object
+        record: logging.LogRecord,  # the source log record
+        message_dict: dict[str, Any],
+    ) -> None:
+        super().add_fields(log_record, record, message_dict)
+        if not log_record.get("timestamp"):
+            # this doesn't use record.created, so it is slightly off
+            now = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+            log_record["timestamp"] = now
+        if loglevel := log_record.get("level"):
+            log_record["level"] = loglevel.upper()
+        else:
+            log_record["level"] = record.levelname.upper()
+
+
+class ColorizedFormatter(coloredlogs.ColoredFormatter):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        coloredlogs.logging.Formatter.formatException = format_exception
+
+
+class pretty:
+    """A simple object wrapper to pretty-format it when formatting the log record."""
+
+    def __init__(self, obj: Any) -> None:
+        self.obj = obj
+
+    def __repr__(self) -> str:
+        return pprint.pformat(self.obj)

backend_ai_logging-24.9.0/ai/backend/logging/handler/graylog.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import logging
+import socket
+import ssl
+from typing import Any, Mapping, Optional
+
+import graypy
+
+
+class GELFTLSHandler(graypy.GELFTLSHandler):
+    ssl_ctx: ssl.SSLContext
+
+    def __init__(self, host, port=12204, validate=False, ca_certs=None, **kwargs) -> None:
+        """Initialize the GELFTLSHandler
+
+        :param host: GELF TLS input host.
+        :type host: str
+
+        :param port: GELF TLS input port.
+        :type port: int
+
+        :param validate: If :obj:`True`, validate the Graylog server's
+            certificate. In this case specifying ``ca_certs`` is also
+            required.
+        :type validate: bool
+
+        :param ca_certs: Path to CA bundle file.
+        :type ca_certs: str
+        """
+
+        super().__init__(host, port=port, validate=validate, **kwargs)
+        self.ssl_ctx = ssl.create_default_context(capath=ca_certs)
+        if not validate:
+            self.ssl_ctx.check_hostname = False
+            self.ssl_ctx.verify_mode = ssl.CERT_NONE
+
+    def makeSocket(self, timeout: float = 1):
+        """Create a TLS wrapped socket"""
+        plain_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+        if hasattr(plain_socket, "settimeout"):
+            plain_socket.settimeout(timeout)
+
+        wrapped_socket = self.ssl_ctx.wrap_socket(
+            plain_socket,
+            server_hostname=self.host,
+        )
+        wrapped_socket.connect((self.host, self.port))
+
+        return wrapped_socket
+
+
+def setup_graylog_handler(config: Mapping[str, Any]) -> Optional[logging.Handler]:
+    drv_config = config["graylog"]
+    graylog_params = {
+        "host": drv_config["host"],
+        "port": drv_config["port"],
+        "validate": drv_config["ssl-verify"],
+        "ca_certs": drv_config["ca-certs"],
+        "keyfile": drv_config["keyfile"],
+        "certfile": drv_config["certfile"],
+    }
+    if drv_config["localname"]:
+        graylog_params["localname"] = drv_config["localname"]
+    else:
+        graylog_params["fqdn"] = drv_config["fqdn"]
+
+    graylog_handler = GELFTLSHandler(**graylog_params)
+    graylog_handler.setLevel(config["level"])
+    return graylog_handler

backend_ai_logging-24.9.0/ai/backend/logging/handler/intrinsic.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import logging
+import sys
+import traceback
+from typing import TYPE_CHECKING, Optional, override
+
+import msgpack
+import zmq
+
+if TYPE_CHECKING:
+    from ..logger import MsgpackOptions
+
+
+class RelayHandler(logging.Handler):
+    _sock: zmq.Socket | None
+
+    def __init__(self, *, endpoint: str, msgpack_options: MsgpackOptions) -> None:
+        super().__init__()
+        self.endpoint = endpoint
+        self.msgpack_options = msgpack_options
+        self._zctx = zmq.Context()
+        # We should use PUSH-PULL socket pairs to avoid
+        # lost of synchronization sentinel messages.
+        if endpoint:
+            self._sock = self._zctx.socket(zmq.PUSH)
+            assert self._sock is not None
+            self._sock.setsockopt(zmq.LINGER, 100)
+            self._sock.connect(self.endpoint)
+        else:
+            self._sock = None
+
+    def close(self) -> None:
+        if self._sock is not None:
+            self._sock.close()
+        self._zctx.term()
+
+    def _fallback(self, record: Optional[logging.LogRecord]) -> None:
+        if record is None:
+            return
+        print(record.getMessage(), file=sys.stderr)
+
+    @override
+    def emit(self, record: Optional[logging.LogRecord]) -> None:
+        if self._sock is None:
+            self._fallback(record)
+            return
+        # record may be None to signal shutdown.
+        if record:
+            log_body = {
+                "name": record.name,
+                "pathname": record.pathname,
+                "lineno": record.lineno,
+                "msg": record.getMessage(),
+                "levelno": record.levelno,
+                "levelname": record.levelname,
+            }
+            if record.exc_info:
+                log_body["exc_info"] = traceback.format_exception(*record.exc_info)
+        else:
+            log_body = None
+        try:
+            serialized_record = msgpack.packb(log_body, **self.msgpack_options["pack_opts"])
+            self._sock.send(serialized_record)
+        except zmq.ZMQError:
+            self._fallback(record)

backend_ai_logging-24.9.0/ai/backend/logging/handler/logstash.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import json
+import logging
+import socket
+import ssl
+from collections import OrderedDict
+from datetime import datetime
+from typing import override
+
+import zmq
+
+from ..exceptions import ConfigurationError
+
+
+class LogstashHandler(logging.Handler):
+    _sock: socket.socket | zmq.Socket | None
+    _sslctx: ssl.SSLContext | None
+    _zmqctx: zmq.Context | None
+
+    def __init__(
+        self,
+        endpoint: tuple[str, int],
+        protocol: str,
+        *,
+        ssl_enabled: bool = True,
+        ssl_verify: bool = True,
+        myhost: str | None = None,
+    ) -> None:
+        super().__init__()
+        self._endpoint = endpoint
+        self._protocol = protocol
+        self._ssl_enabled = ssl_enabled
+        self._ssl_verify = ssl_verify
+        self._myhost = myhost
+        self._sock = None
+        self._sslctx = None
+        self._zmqctx = None
+
+    def _setup_transport(self) -> None:
+        if self._sock is not None:
+            return
+        if self._protocol == "zmq.push":
+            self._zmqctx = zmq.Context()
+            zsock = self._zmqctx.socket(zmq.PUSH)
+            zsock.setsockopt(zmq.LINGER, 50)
+            zsock.setsockopt(zmq.SNDHWM, 20)
+            zsock.connect(f"tcp://{self._endpoint[0]}:{self._endpoint[1]}")
+            self._sock = zsock
+        elif self._protocol == "zmq.pub":
+            self._zmqctx = zmq.Context()
+            zsock = self._zmqctx.socket(zmq.PUB)
+            zsock.setsockopt(zmq.LINGER, 50)
+            zsock.setsockopt(zmq.SNDHWM, 20)
+            zsock.connect(f"tcp://{self._endpoint[0]}:{self._endpoint[1]}")
+            self._sock = zsock
+        elif self._protocol == "tcp":
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            if self._ssl_enabled:
+                self._sslctx = ssl.create_default_context()
+                self._sslctx.minimum_version = ssl.TLSVersion.TLSv1_2
+                if not self._ssl_verify:
+                    self._sslctx.check_hostname = False
+                    self._sslctx.verify_mode = ssl.CERT_NONE
+                sock = self._sslctx.wrap_socket(sock, server_hostname=self._endpoint[0])
+            sock.connect((self._endpoint[0], self._endpoint[1]))
+            self._sock = sock
+        elif self._protocol == "udp":
+            sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            sock.connect((self._endpoint[0], self._endpoint[1]))
+            self._sock = sock
+        else:
+            raise ConfigurationError({
+                "logging.LogstashHandler": f"unsupported protocol: {self._protocol}"
+            })
+
+    def cleanup(self) -> None:
+        if self._sock:
+            self._sock.close()
+        self._sslctx = None
+        if self._zmqctx:
+            self._zmqctx.term()
+
+    @override
+    def emit(self, record: logging.LogRecord) -> None:
+        self._setup_transport()
+        tags: set[str] = set()
+
+        # This log format follows logstash's event format.
+        log = OrderedDict([
+            ("@timestamp", datetime.now().isoformat()),
+            ("@version", 1),
+            ("host", self._myhost),
+            ("logger", record.name),
+            ("path", record.pathname),
+            ("func", record.funcName),
+            ("lineno", record.lineno),
+            ("message", record.getMessage()),
+            ("level", record.levelname),
+            ("tags", list(tags)),
+        ])
+        if self._protocol.startswith("zmq"):
+            assert isinstance(self._sock, zmq.Socket)
+            self._sock.send_json(log)
+        else:
+            # TODO: reconnect if disconnected
+            assert isinstance(self._sock, socket.socket)
+            self._sock.sendall(json.dumps(log).encode("utf-8"))

backend_ai_logging-24.9.0/ai/backend/logging/logger.py

@@ -0,0 +1,348 @@
+from __future__ import annotations
+
+import logging
+import logging.config
+import logging.handlers
+import os
+import sys
+import threading
+from collections.abc import Mapping, MutableMapping
+from contextvars import ContextVar
+from pathlib import Path
+from typing import Any, Self, TypedDict, override
+
+import msgpack
+import yarl
+import zmq
+
+from .abc import AbstractLogger
+from .config import logging_config_iv, override_key
+from .exceptions import ConfigurationError
+from .formatter import (
+    ColorizedFormatter,
+    ConsoleFormatter,
+    CustomJsonFormatter,
+    SerializedExceptionFormatter,
+)
+from .handler.intrinsic import RelayHandler
+
+is_active: ContextVar[bool] = ContextVar("is_active", default=False)
+
+
+def _check_driver_config_exists_if_activated(cfg, driver):
+    if driver in cfg["drivers"] and cfg[driver] is None:
+        raise ConfigurationError({"logging": f"{driver} driver is activated but no config given."})
+
+
+class MsgpackOptions(TypedDict):
+    pack_opts: Mapping[str, Any]
+    unpack_opts: Mapping[str, Any]
+
+
+class NoopLogger(AbstractLogger):
+    def __init__(
+        self,
+        logging_config: MutableMapping[str, Any],
+    ) -> None:
+        pass
+
+    @override
+    def __enter__(self) -> Self:
+        return self
+
+    @override
+    def __exit__(self, *exc_info_args) -> bool | None:
+        pass
+
+
+class LocalLogger(AbstractLogger):
+    def __init__(
+        self,
+        logging_config: MutableMapping[str, Any],
+    ) -> None:
+        cfg = logging_config_iv.check(logging_config)
+        _check_driver_config_exists_if_activated(cfg, "console")
+        self.logging_config = cfg
+        log_handlers = []
+        if "console" in self.logging_config["drivers"]:
+            console_handler = setup_console_log_handler(self.logging_config)
+            log_handlers.append(console_handler)
+        if "file" in self.logging_config["drivers"]:
+            file_handler = setup_file_log_handler(self.logging_config)
+            log_handlers.append(file_handler)
+        self.log_config = {
+            "version": 1,
+            "disable_existing_loggers": False,
+            "handlers": {
+                "null": {"class": "logging.NullHandler"},
+            },
+            "loggers": {
+                "": {
+                    "handlers": [],
+                    "level": cfg["level"],
+                },
+                **{
+                    k: {
+                        "handlers": [],
+                        "level": v,
+                        "propagate": False,
+                    }
+                    for k, v in cfg["pkg-ns"].items()
+                },
+            },
+        }
+        logging.config.dictConfig(self.log_config)
+        root_logger = logging.getLogger(None)
+        for h in log_handlers:
+            root_logger.addHandler(h)
+        for pkg_ns in cfg["pkg-ns"].keys():
+            ns_logger = logging.getLogger(pkg_ns)
+            for h in log_handlers:
+                ns_logger.addHandler(h)
+
+    @override
+    def __enter__(self) -> Self:
+        return self
+
+    @override
+    def __exit__(self, *exc_info_args) -> bool | None:
+        pass
+
+
+class Logger(AbstractLogger):
+    is_master: bool
+    log_endpoint: str
+    logging_config: Mapping[str, Any]
+    log_config: dict[str, Any]
+    log_worker: threading.Thread
+
+    def __init__(
+        self,
+        logging_config: MutableMapping[str, Any],
+        *,
+        is_master: bool,
+        log_endpoint: str,
+        msgpack_options: MsgpackOptions,
+    ) -> None:
+        if (env_legacy_logfile_path := os.environ.get("BACKEND_LOG_FILE", None)) is not None:
+            p = Path(env_legacy_logfile_path)
+            override_key(logging_config, ("file", "path"), p.parent)
+            override_key(logging_config, ("file", "filename"), p.name)
+        if (env_legacy_backup_count := os.environ.get("BACKEND_LOG_FILE_COUNT", None)) is not None:
+            override_key(logging_config, ("file", "backup-count"), env_legacy_backup_count)
+        if (env_legacy_logfile_size := os.environ.get("BACKEND_LOG_FILE_SIZE", None)) is not None:
+            legacy_logfile_size = f"{env_legacy_logfile_size}M"
+            override_key(logging_config, ("file", "rotation-size"), legacy_logfile_size)
+
+        cfg = logging_config_iv.check(logging_config)
+
+        _check_driver_config_exists_if_activated(cfg, "console")
+        _check_driver_config_exists_if_activated(cfg, "file")
+        _check_driver_config_exists_if_activated(cfg, "logstash")
+        _check_driver_config_exists_if_activated(cfg, "graylog")
+
+        self.is_master = is_master
+        self.msgpack_options = msgpack_options
+        self.log_endpoint = log_endpoint
+        self.logging_config = cfg
+        self.log_config = {
+            "version": 1,
+            "disable_existing_loggers": False,
+            "handlers": {
+                "null": {"class": "logging.NullHandler"},
+            },
+            "loggers": {
+                "": {"handlers": [], "level": cfg["level"]},
+                **{
+                    k: {"handlers": [], "level": v, "propagate": False}
+                    for k, v in cfg["pkg-ns"].items()
+                },
+            },
+        }
+
+    @override
+    def __enter__(self) -> Self:
+        self.log_config["handlers"]["relay"] = {
+            "class": "ai.backend.logging.handler.intrinsic.RelayHandler",
+            "level": self.logging_config["level"],
+            "endpoint": self.log_endpoint,
+            "msgpack_options": self.msgpack_options,
+        }
+        for _logger in self.log_config["loggers"].values():
+            _logger["handlers"].append("relay")
+        logging.config.dictConfig(self.log_config)
+        self._is_active_token = is_active.set(True)
+        if self.is_master and self.log_endpoint:
+            self.relay_handler = logging.getLogger("").handlers[0]
+            self.ready_event = threading.Event()
+            assert isinstance(self.relay_handler, RelayHandler)
+            self.log_worker = threading.Thread(
+                target=log_worker,
+                name="Logger",
+                args=(
+                    self.logging_config,
+                    os.getpid(),
+                    self.log_endpoint,
+                    self.ready_event,
+                    self.msgpack_options,
+                ),
+            )
+            self.log_worker.start()
+            self.ready_event.wait()
+        return self
+
+    @override
+    def __exit__(self, *exc_info_args) -> bool | None:
+        # Resetting generates "different context" errors.
+        # Since practically we only need to check activeness in alembic scripts
+        # and it should be active until the program terminates,
+        # just leave it as-is.
+        is_active.reset(self._is_active_token)
+        if self.is_master and self.log_endpoint:
+            assert isinstance(self.relay_handler, RelayHandler)
+            self.relay_handler.emit(None)
+            self.log_worker.join()
+            self.relay_handler.close()
+            ep_url = yarl.URL(self.log_endpoint)
+            if ep_url.scheme.lower() == "ipc" and (ep_sock := Path(ep_url.path)).exists():
+                ep_sock.unlink()
+        return None
+
+
+def setup_console_log_handler(config: Mapping[str, Any]) -> logging.Handler:
+    log_formats = {
+        "simple": "%(levelname)s %(message)s",
+        "verbose": "%(asctime)s %(levelname)s %(name)s [%(process)d] %(message)s",
+    }
+    drv_config = config["console"]
+    console_formatter: logging.Formatter
+    colored = drv_config["colored"]
+    if colored is None:
+        colored = sys.stderr.isatty()
+    if colored:
+        console_formatter = ColorizedFormatter(
+            log_formats[drv_config["format"]],
+            datefmt="%Y-%m-%d %H:%M:%S.%f",  # coloredlogs has intrinsic support for msec
+            field_styles={
+                "levelname": {"color": 248, "bold": True},
+                "name": {"color": 246, "bold": False},
+                "process": {"color": "cyan"},
+                "asctime": {"color": 240},
+            },
+            level_styles={
+                "debug": {"color": "green"},
+                "verbose": {"color": "green", "bright": True},
+                "info": {"color": "cyan", "bright": True},
+                "notice": {"color": "cyan", "bold": True},
+                "warning": {"color": "yellow"},
+                "error": {"color": "red", "bright": True},
+                "success": {"color": 77},
+                "critical": {"background": "red", "color": 255, "bold": True},
+            },
+        )
+    else:
+        console_formatter = ConsoleFormatter(
+            log_formats[drv_config["format"]],
+            datefmt="%Y-%m-%d %H:%M:%S.%f",
+        )
+    console_handler = logging.StreamHandler(
+        stream=sys.stderr,
+    )
+    console_handler.setLevel(config["level"])
+    console_handler.setFormatter(console_formatter)
+    return console_handler
+
+
+def setup_file_log_handler(config: Mapping[str, Any]) -> logging.Handler:
+    drv_config = config["file"]
+    fmt = "%(timestamp) %(level) %(name) %(processName) %(message)"
+    file_handler = logging.handlers.RotatingFileHandler(
+        filename=drv_config["path"] / drv_config["filename"],
+        backupCount=drv_config["backup-count"],
+        maxBytes=drv_config["rotation-size"],
+        encoding="utf-8",
+    )
+    file_handler.setLevel(config["level"])
+    file_handler.setFormatter(CustomJsonFormatter(fmt))
+    return file_handler
+
+
+def log_worker(
+    logging_config: Mapping[str, Any],
+    parent_pid: int,
+    log_endpoint: str,
+    ready_event: threading.Event,
+    msgpack_options: MsgpackOptions,
+) -> None:
+    console_handler = None
+    file_handler = None
+    logstash_handler = None
+    graylog_handler = None
+
+    # For future references: when implementing new kind of logging adapters,
+    # make sure to adapt our custom `Formatter.formatException()` approach;
+    # Otherwise it won't print out EXCEPTION level log (along with the traceback).
+    if "console" in logging_config["drivers"]:
+        console_handler = setup_console_log_handler(logging_config)
+
+    if "file" in logging_config["drivers"]:
+        file_handler = setup_file_log_handler(logging_config)
+
+    if "logstash" in logging_config["drivers"]:
+        from .handler.logstash import LogstashHandler
+
+        drv_config = logging_config["logstash"]
+        logstash_handler = LogstashHandler(
+            endpoint=drv_config["endpoint"],
+            protocol=drv_config["protocol"],
+            ssl_enabled=drv_config["ssl-enabled"],
+            ssl_verify=drv_config["ssl-verify"],
+            myhost="hostname",  # TODO: implement
+        )
+        logstash_handler.setLevel(logging_config["level"])
+        logstash_handler.setFormatter(SerializedExceptionFormatter())
+
+    if "graylog" in logging_config["drivers"]:
+        from .handler.graylog import setup_graylog_handler
+
+        graylog_handler = setup_graylog_handler(logging_config)
+        assert graylog_handler is not None
+        graylog_handler.setFormatter(SerializedExceptionFormatter())
+
+    zctx = zmq.Context()
+    agg_sock = zctx.socket(zmq.PULL)
+    agg_sock.bind(log_endpoint)
+    ep_url = yarl.URL(log_endpoint)
+    if ep_url.scheme.lower() == "ipc":
+        os.chmod(ep_url.path, 0o777)
+    try:
+        ready_event.set()
+        while True:
+            data = agg_sock.recv()
+            if not data:
+                return
+            unpacked_data = msgpack.unpackb(data, **msgpack_options["unpack_opts"])
+            if not unpacked_data:
+                break
+            rec = logging.makeLogRecord(unpacked_data)
+            if rec is None:
+                break
+            if console_handler:
+                console_handler.emit(rec)
+            try:
+                if file_handler:
+                    file_handler.emit(rec)
+                if logstash_handler:
+                    logstash_handler.emit(rec)
+                if graylog_handler:
+                    graylog_handler.emit(rec)
+            except OSError:
+                # don't terminate the log worker.
+                continue
+    finally:
+        if logstash_handler:
+            logstash_handler.cleanup()
+        if graylog_handler:
+            graylog_handler.close()
+        agg_sock.close()
+        zctx.term()

backend_ai_logging-24.9.0/ai/backend/logging/py.typed

@@ -0,0 +1 @@
+placeholder

backend_ai_logging-24.9.0/ai/backend/logging/types.py

@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+import enum
+from decimal import Decimal
+from pathlib import Path
+from typing import Any, Generic, Self, TypeVar, override
+
+import trafaret as t
+
+
+class CIStrEnum(enum.StrEnum):
+    """
+    An StrEnum variant to allow case-insenstive matching of the members while the values are
+    lowercased.
+    """
+
+    @override
+    @classmethod
+    def _missing_(cls, value: Any) -> Self | None:
+        assert isinstance(value, str)  # since this is an StrEnum
+        value = value.lower()
+        # To prevent infinite recursion, we don't rely on "cls(value)" but manually search the
+        # members as the official stdlib example suggests.
+        for member in cls:
+            if member.value == value:
+                return member
+        return None
+
+    # The defualt behavior of `enum.auto()` is to set the value to the lowercased member name.
+
+    @classmethod
+    def as_trafaret(cls) -> t.Trafaret:
+        return CIStrEnumTrafaret(cls)
+
+
+class CIUpperStrEnum(CIStrEnum):
+    """
+    An StrEnum variant to allow case-insenstive matching of the members while the values are
+    UPPERCASED.
+    """
+
+    @override
+    @classmethod
+    def _missing_(cls, value: Any) -> Self | None:
+        assert isinstance(value, str)  # since this is an StrEnum
+        value = value.upper()
+        for member in cls:
+            if member.value == value:
+                return member
+        return None
+
+    @override
+    @staticmethod
+    def _generate_next_value_(name, start, count, last_values) -> str:
+        return name.upper()
+
+    @classmethod
+    def as_trafaret(cls) -> t.Trafaret:
+        return CIUpperStrEnumTrafaret(cls)
+
+
+T_enum = TypeVar("T_enum", bound=enum.Enum)
+
+
+class CIStrEnumTrafaret(t.Trafaret, Generic[T_enum]):
+    """
+    A case-insensitive version of trafaret to parse StrEnum values.
+    """
+
+    def __init__(self, enum_cls: type[T_enum]) -> None:
+        self.enum_cls = enum_cls
+
+    def check_and_return(self, value: str) -> T_enum:
+        try:
+            # Assume that the enum values are lowercases.
+            return self.enum_cls(value.lower())
+        except (KeyError, ValueError):
+            self._failure(f"value is not a valid member of {self.enum_cls.__name__}", value=value)
+
+
+class CIUpperStrEnumTrafaret(t.Trafaret, Generic[T_enum]):
+    """
+    A case-insensitive version of trafaret to parse StrEnum values.
+    """
+
+    def __init__(self, enum_cls: type[T_enum]) -> None:
+        self.enum_cls = enum_cls
+
+    def check_and_return(self, value: str) -> T_enum:
+        try:
+            # Assume that the enum values are lowercases.
+            return self.enum_cls(value.upper())
+        except (KeyError, ValueError):
+            self._failure(f"value is not a valid member of {self.enum_cls.__name__}", value=value)
+
+
+class LogLevel(CIUpperStrEnum):
+    # The logging stdlib only accepts uppercased loglevel names,
+    # so we subclass `CIUpperStrEnum` here.
+    CRITICAL = enum.auto()
+    ERROR = enum.auto()
+    WARNING = enum.auto()
+    INFO = enum.auto()
+    DEBUG = enum.auto()
+    NOTSET = enum.auto()
+
+
+class LogFormat(CIStrEnum):
+    SIMPLE = enum.auto()
+    VERBOSE = enum.auto()
+
+
+class SimpleBinarySizeTrafaret(t.Trafaret):
+    suffix_map = {
+        "y": 2**80,  # yotta
+        "z": 2**70,  # zetta
+        "e": 2**60,  # exa
+        "p": 2**50,  # peta
+        "t": 2**40,  # tera
+        "g": 2**30,  # giga
+        "m": 2**20,  # mega
+        "k": 2**10,  # kilo
+        " ": 1,
+    }
+    endings = ("ibytes", "ibyte", "ib", "bytes", "byte", "b")
+
+    def check_and_return(self, value: str) -> int:
+        orig_value = value
+        value = value.strip().replace("_", "")
+        try:
+            return int(value)
+        except ValueError:
+            value = value.lower()
+            dec_expr: Decimal
+            try:
+                for ending in self.endings:
+                    if (stem := value.removesuffix(ending)) != value:
+                        suffix = stem[-1]
+                        dec_expr = Decimal(stem[:-1])
+                        break
+                else:
+                    # when there is suffix without scale (e.g., "2K")
+                    if not str.isnumeric(value[-1]):
+                        suffix = value[-1]
+                        dec_expr = Decimal(value[:-1])
+                    else:
+                        # has no suffix and is not an integer
+                        # -> fractional bytes (e.g., 1.5 byte)
+                        raise ValueError("Fractional bytes are not allowed")
+            except ArithmeticError:
+                raise ValueError("Unconvertible value", orig_value)
+            try:
+                multiplier = self.suffix_map[suffix]
+            except KeyError:
+                raise ValueError("Unconvertible value", orig_value)
+            return int(dec_expr * multiplier)
+
+
+class DirPathTrafaret(t.Trafaret):
+    def __init__(
+        self,
+        *,
+        auto_create: bool = False,
+    ) -> None:
+        super().__init__()
+        self._auto_create = auto_create
+
+    def check_and_return(self, value: Any) -> Path:
+        try:
+            p = Path(value).resolve()
+        except (TypeError, ValueError):
+            self._failure("cannot parse value as a path", value=value)
+        else:
+            if self._auto_create:
+                p.mkdir(parents=True, exist_ok=True)
+            if not p.is_dir():
+                self._failure("value is not a directory", value=value)
+            return p

backend_ai_logging-24.9.0/ai/backend/logging/utils.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterable
+from typing import Any, LiteralString
+
+__all__ = (
+    "BraceMessage",
+    "BraceStyleAdapter",
+    "enforce_debug_logging",
+)
+
+
+class BraceMessage:
+    __slots__ = ("fmt", "args")
+
+    def __init__(self, fmt: LiteralString, args: tuple[Any, ...]):
+        self.fmt = fmt
+        self.args = args
+
+    def __str__(self):
+        return self.fmt.format(*self.args)
+
+
+class BraceStyleAdapter(logging.LoggerAdapter):
+    def __init__(self, logger, extra=None):
+        super().__init__(logger, extra)
+
+    def log(self, level, msg, *args, **kwargs):
+        if self.isEnabledFor(level):
+            msg, kwargs = self.process(msg, kwargs)
+            kwargs["stacklevel"] = kwargs.get("stacklevel", 1) + 1
+            self.logger._log(level, BraceMessage(msg, args), (), **kwargs)
+
+
+def enforce_debug_logging(loggers: Iterable[str]) -> None:
+    # Backend.AI's daemon logging:
+    # - All handlers are added to the root logger only.
+    #   -> Need to override the log level of the root logger's handlers.
+    # - Each logger has separate logging level.
+    #   -> Need to override the log level of the individual loggers.
+    root_logger = logging.getLogger()
+    for handler in root_logger.handlers:
+        handler.setLevel(logging.DEBUG)
+    for name in loggers:
+        instance = logging.getLogger(name)
+        instance.setLevel(logging.DEBUG)

backend_ai_logging-24.9.0/backend.ai_logging.egg-info/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.1
+Name: backend.ai-logging
+Version: 24.9.0
+Summary: Backend.AI Logging Subsystem
+Home-page: https://github.com/lablup/backend.ai
+Author: Lablup Inc. and contributors
+License: MIT
+Project-URL: Documentation, https://docs.backend.ai/
+Project-URL: Source, https://github.com/lablup/backend.ai
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Environment :: No Input/Output (Daemon)
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.12,<3.13
+Description-Content-Type: text/markdown
+Requires-Dist: coloredlogs~=15.0
+Requires-Dist: graypy==2.1.0
+Requires-Dist: msgpack~=1.1.0
+Requires-Dist: python-json-logger>=2.0.1
+Requires-Dist: pyzmq~=26.2
+Requires-Dist: trafaret~=2.1
+Requires-Dist: yarl~=1.13.1
+
+Backend.AI Logging Subsystem
+============================
+
+Package Structure
+-----------------
+
+* `ai.backend.logging`
+  - `abc`: Abstract base classes
+  - `logger`: The core logging facility
+    - `Logger`: The standard multiprocess-friendly logger using `RelayHandler` based on ZeroMQ
+    - `LocalLogger`: A minimalized console/file logger that does not require serialization via networks at all
+  - `handler`: Collection of vendor-specific handler implementations
+  - `formatter`: Collection of formatters
+  - `types`: Definition of enums/types like `LogLevel`
+  - `utils`: Brace-style message formatting adapters and other extras

backend_ai_logging-24.9.0/backend.ai_logging.egg-info/SOURCES.txt

@@ -0,0 +1,24 @@
+MANIFEST.in
+backend_shim.py
+setup.py
+ai/backend/logging/VERSION
+ai/backend/logging/__init__.py
+ai/backend/logging/abc.py
+ai/backend/logging/config.py
+ai/backend/logging/exceptions.py
+ai/backend/logging/formatter.py
+ai/backend/logging/logger.py
+ai/backend/logging/py.typed
+ai/backend/logging/types.py
+ai/backend/logging/utils.py
+ai/backend/logging/handler/__init__.py
+ai/backend/logging/handler/graylog.py
+ai/backend/logging/handler/intrinsic.py
+ai/backend/logging/handler/logstash.py
+backend.ai_logging.egg-info/PKG-INFO
+backend.ai_logging.egg-info/SOURCES.txt
+backend.ai_logging.egg-info/dependency_links.txt
+backend.ai_logging.egg-info/namespace_packages.txt
+backend.ai_logging.egg-info/not-zip-safe
+backend.ai_logging.egg-info/requires.txt
+backend.ai_logging.egg-info/top_level.txt

backend_ai_logging-24.9.0/backend.ai_logging.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

backend_ai_logging-24.9.0/backend.ai_logging.egg-info/namespace_packages.txt

@@ -0,0 +1 @@
+

backend_ai_logging-24.9.0/backend.ai_logging.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

backend_ai_logging-24.9.0/backend.ai_logging.egg-info/requires.txt

@@ -0,0 +1,7 @@
+coloredlogs~=15.0
+graypy==2.1.0
+msgpack~=1.1.0
+python-json-logger>=2.0.1
+pyzmq~=26.2
+trafaret~=2.1
+yarl~=1.13.1

backend_ai_logging-24.9.0/backend.ai_logging.egg-info/top_level.txt

@@ -0,0 +1 @@
+ai

backend_ai_logging-24.9.0/backend_shim.py

@@ -0,0 +1,31 @@
+
+# DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS
+
+import errno
+import os
+import setuptools.build_meta
+
+backend = setuptools.build_meta.__legacy__
+
+dist_dir = "dist/"
+build_wheel = True
+build_sdist = True
+wheel_config_settings = {
+}
+sdist_config_settings = {
+}
+
+# Python 2.7 doesn't have the exist_ok arg on os.makedirs().
+try:
+    os.makedirs(dist_dir)
+except OSError as e:
+    if e.errno != errno.EEXIST:
+        raise
+
+wheel_path = backend.build_wheel(dist_dir, wheel_config_settings) if build_wheel else None
+sdist_path = backend.build_sdist(dist_dir, sdist_config_settings) if build_sdist else None
+
+if wheel_path:
+    print("wheel: {wheel_path}".format(wheel_path=wheel_path))
+if sdist_path:
+    print("sdist: {sdist_path}".format(sdist_path=sdist_path))

backend_ai_logging-24.9.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

backend_ai_logging-24.9.0/setup.py

@@ -0,0 +1,72 @@
+
+# DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS
+# Target: src/ai/backend/logging:dist
+
+from setuptools import setup
+
+setup(**{
+    'author': 'Lablup Inc. and contributors',
+    'classifiers': [
+        'Intended Audience :: Developers',
+        'Operating System :: MacOS :: MacOS X',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+        'Environment :: No Input/Output (Daemon)',
+        'Topic :: Scientific/Engineering',
+        'Topic :: Software Development',
+        'Development Status :: 5 - Production/Stable',
+        'Programming Language :: Python :: 3.12',
+        'License :: OSI Approved :: MIT License',
+    ],
+    'description': 'Backend.AI Logging Subsystem',
+    'install_requires': (
+        'coloredlogs~=15.0',
+        'graypy==2.1.0',
+        'msgpack~=1.1.0',
+        'python-json-logger>=2.0.1',
+        'pyzmq~=26.2',
+        'trafaret~=2.1',
+        'yarl~=1.13.1',
+    ),
+    'license': 'MIT',
+    'long_description': """Backend.AI Logging Subsystem
+============================
+
+Package Structure
+-----------------
+
+* `ai.backend.logging`
+  - `abc`: Abstract base classes
+  - `logger`: The core logging facility
+    - `Logger`: The standard multiprocess-friendly logger using `RelayHandler` based on ZeroMQ
+    - `LocalLogger`: A minimalized console/file logger that does not require serialization via networks at all
+  - `handler`: Collection of vendor-specific handler implementations
+  - `formatter`: Collection of formatters
+  - `types`: Definition of enums/types like `LogLevel`
+  - `utils`: Brace-style message formatting adapters and other extras
+""",
+    'long_description_content_type': 'text/markdown',
+    'name': 'backend.ai-logging',
+    'namespace_packages': (
+    ),
+    'package_data': {
+        'ai.backend.logging': (
+            'VERSION',
+            'py.typed',
+        ),
+    },
+    'packages': (
+        'ai.backend.logging',
+        'ai.backend.logging.handler',
+    ),
+    'project_urls': {
+        'Documentation': 'https://docs.backend.ai/',
+        'Source': 'https://github.com/lablup/backend.ai',
+    },
+    'python_requires': '>=3.12,<3.13',
+    'url': 'https://github.com/lablup/backend.ai',
+    'version': """24.09.0
+""",
+    'zip_safe': False,
+})