xakki-log-python 0.4.0__py3-none-any.whl

log_python/formatters.py

@@ -0,0 +1,217 @@
+"""Wire and developer-facing formatters for stdlib :mod:`logging`."""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime
+from typing import Final
+
+from log_python.config import PylogConfig
+from log_python.context import get_bound_context
+from log_python.manager import LogManager
+from log_python.redactor import Redactor
+
+DEFAULT_MAX_BYTES: Final[int] = 1400
+_ELLIPSIS: Final[str] = "…"
+_REQUIRED_EXTRA_KEYS: Final[tuple[str, ...]] = ("app_name", "app_env", "app_ver", "log_ver", "tier")
+_LEVELS: Final[tuple[tuple[int, str, int], ...]] = (
+    (60, "emergency", 600),
+    (55, "alert", 550),
+    (logging.CRITICAL, "critical", 500),
+    (logging.ERROR, "error", 400),
+    (logging.WARNING, "warning", 300),
+    (25, "notice", 250),
+    (logging.INFO, "info", 200),
+    (logging.DEBUG, "debug", 100),
+)
+
+
+def _level(levelno: int) -> tuple[str, int]:
+    for threshold, name, number in _LEVELS:
+        if levelno >= threshold:
+            return name, number
+    return "debug", 100
+
+
+def _safe_json_value(value: object, redactor: Redactor) -> object:
+    try:
+        redacted = redactor.redact(value)
+        json.dumps(redacted, ensure_ascii=False, allow_nan=False)
+    except Exception:
+        return "<unserializable>"
+    return redacted
+
+
+def _json_bytes(data: Mapping[str, object]) -> bytes:
+    return json.dumps(data, ensure_ascii=False, separators=(",", ":"), allow_nan=False, default=str).encode()
+
+
+def _truncate_utf8(value: str, byte_limit: int) -> str:
+    marker_bytes = len(_ELLIPSIS.encode())
+    if byte_limit <= marker_bytes:
+        return _ELLIPSIS
+    raw = value.encode()
+    if len(raw) <= byte_limit:
+        return value
+    return raw[: byte_limit - marker_bytes].decode(errors="ignore") + _ELLIPSIS
+
+
+def _string_slots(data: dict[str, object]) -> list[tuple[dict[str, object], str]]:
+    slots: list[tuple[dict[str, object], str]] = []
+    for key in ("message", "channel"):
+        if isinstance(data.get(key), str):
+            slots.append((data, key))
+    for container_name in ("context", "extra"):
+        container = data.get(container_name)
+        if isinstance(container, dict):
+            slots.extend(
+                (container, key)
+                for key, value in container.items()
+                if isinstance(key, str) and isinstance(value, str) and not (container_name == "extra" and key in _REQUIRED_EXTRA_KEYS)
+            )
+    return slots
+
+
+def trim_log(data: dict[str, object], max_bytes: int = DEFAULT_MAX_BYTES) -> str:
+    """Serialize ``data`` as valid JSON no larger than ``max_bytes`` when possible."""
+    max_bytes = max(64, max_bytes)
+    encoded = _json_bytes(data)
+    previous = len(encoded) + 1
+    while len(encoded) > max_bytes and len(encoded) < previous:
+        previous = len(encoded)
+        slots = _string_slots(data)
+        if not slots:
+            break
+        excess = len(encoded) - max_bytes
+        reduction = max(1, excess // len(slots) + 1)
+        changed = False
+        for container, key in slots:
+            value = container[key]
+            if not isinstance(value, str):
+                continue
+            current = len(value.encode())
+            shortened = _truncate_utf8(value, max(len(_ELLIPSIS.encode()), current - reduction))
+            if shortened != value:
+                container[key] = shortened
+                changed = True
+        if not changed:
+            break
+        encoded = _json_bytes(data)
+    if len(encoded) > max_bytes:
+        context = data.get("context")
+        preserved: dict[str, object] = {}
+        if isinstance(context, dict):
+            for key in ("log_type", "request_id", "file", "exception", "exception_code"):
+                if key in context:
+                    preserved[key] = context[key]
+        preserved["_truncated"] = True
+        data["context"] = preserved
+        extra = data.get("extra")
+        data["extra"] = {key: extra[key] for key in _REQUIRED_EXTRA_KEYS if key in extra} if isinstance(extra, dict) else {}
+        encoded = _json_bytes(data)
+        while len(encoded) > max_bytes:
+            slots = _string_slots(data)
+            if not slots:
+                break
+            longest_container, longest_key = max(slots, key=lambda slot: len(str(slot[0][slot[1]]).encode()))
+            value = longest_container[longest_key]
+            if not isinstance(value, str) or value == _ELLIPSIS:
+                break
+            longest_container[longest_key] = _truncate_utf8(value, max(len(_ELLIPSIS.encode()), len(value.encode()) // 2))
+            encoded = _json_bytes(data)
+    return encoded.decode()
+
+
+class ExtraFilter(logging.Filter):
+    """Attach stable process metadata to every record."""
+
+    def __init__(self, config: PylogConfig, argv: Sequence[str] | None = None) -> None:
+        super().__init__()
+        values = {
+            "app_name": config.extra.app_name or "",
+            "app_env": config.extra.app_env or "",
+            "app_ver": config.extra.app_ver or "",
+            "log_ver": config.extra.log_ver,
+            "tier": config.extra.tier or "",
+            "release_tag": config.extra.release_tag,
+            "release_time": config.extra.release_time,
+        }
+        self._extra = {key: value for key, value in values.items() if value is not None}
+        resolved_argv = tuple(sys.argv if argv is None else argv)
+        if resolved_argv:
+            self._extra["console_argv"] = " ".join(resolved_argv)
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.pylog_extra = dict(self._extra)
+        return True
+
+
+class JsonFormatter(logging.Formatter):
+    """Emit one newline-terminated JSON record conforming to the shared wire contract."""
+
+    def __init__(self, config: PylogConfig | None = None, *, max_bytes: int = DEFAULT_MAX_BYTES) -> None:
+        super().__init__()
+        self._config = config or PylogConfig.from_env()
+        self._manager = LogManager(self._config)
+        self._redactor = Redactor.from_config(self._config)
+        self._max_bytes = max_bytes
+
+    def format(self, record: logging.LogRecord) -> str:
+        try:
+            raw_context = getattr(record, "context", None)
+            context = get_bound_context()
+            if isinstance(raw_context, Mapping):
+                context.update(raw_context)
+            if record.exc_info and isinstance(record.exc_info[1], BaseException) and "exception" not in context:
+                context["exception"] = record.exc_info[1]
+            message, enriched = self._manager.append_context(record.levelno, record.getMessage(), context)
+            message = self._redactor.redact_value(message)
+            level_name, level = _level(record.levelno)
+            raw_extra = getattr(record, "pylog_extra", {})
+            extra = raw_extra if isinstance(raw_extra, Mapping) else {}
+            data: dict[str, object] = {
+                "datetime": datetime.fromtimestamp(record.created, tz=UTC).isoformat(timespec="microseconds"),
+                "level": level,
+                "level_name": level_name,
+                "channel": record.name,
+                "message": message,
+                "context": {key: _safe_json_value(value, self._redactor) for key, value in enriched.items() if value is not None},
+                "extra": {str(key): _safe_json_value(value, self._redactor) for key, value in extra.items() if value is not None},
+            }
+            return trim_log(data, self._max_bytes) + "\n"
+        except Exception:
+            return '{"level":400,"level_name":"error","channel":"pylog","message":"logging format error","context":{},"extra":{}}\n'
+
+
+class LineFormatter(JsonFormatter):
+    """Readable single-line formatter with escaped user-controlled line breaks."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        data = json.loads(super().format(record))
+        message = str(data["message"]).replace("\r", "\\r").replace("\n", "\\n")
+        context = json.dumps(data["context"], ensure_ascii=False, separators=(",", ":"))
+        extra = json.dumps(data["extra"], ensure_ascii=False, separators=(",", ":"))
+        return f"[{data['datetime']}] {data['channel']}.{data['level_name']}: {message} {context} {extra}\n"
+
+
+class ConsoleFormatter(LineFormatter):
+    """ANSI-coloured formatter for local interactive development."""
+
+    _COLORS: Final[Mapping[str, str]] = {
+        "debug": "1;36",
+        "info": "0;32",
+        "notice": "0;34",
+        "warning": "1;33",
+        "error": "0;31",
+        "critical": "0;31",
+        "alert": "0;35",
+        "emergency": "0;35",
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        level_name, _ = _level(record.levelno)
+        line = super().format(record).rstrip("\n")
+        return f"\033[{self._COLORS[level_name]}m{line}\033[0m\n"

log_python/handlers/__init__.py

@@ -0,0 +1 @@
+"""Optional logging handlers."""

log_python/handlers/redis.py

@@ -0,0 +1,42 @@
+"""Redis list handler; importing this module does not import a Redis client."""
+
+from __future__ import annotations
+
+import logging
+from typing import Protocol
+
+
+class Pipeline(Protocol):
+    def lpush(self, key: str, value: str) -> Pipeline: ...
+    def ltrim(self, key: str, start: int, end: int) -> Pipeline: ...
+    def execute(self) -> object: ...
+
+
+class RedisClient(Protocol):
+    def lpush(self, key: str, value: str) -> object: ...
+    def pipeline(self, transaction: bool = True) -> Pipeline: ...
+
+
+class RedisHandler(logging.Handler):
+    """Best-effort ``LPUSH`` handler with an optional ``LTRIM`` cap."""
+
+    def __init__(self, client: RedisClient, *, key: str = "logs", cap_size: int = 10_000, level: int = logging.INFO) -> None:
+        super().__init__(level)
+        if cap_size < 0:
+            raise ValueError("cap_size must be non-negative")
+        self._client = client
+        self._key = key
+        self._cap_size = cap_size
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            payload = self.format(record)
+            if self._cap_size > 0:
+                pipeline = self._client.pipeline(transaction=True)
+                pipeline.lpush(self._key, payload)
+                pipeline.ltrim(self._key, 0, self._cap_size - 1)
+                pipeline.execute()
+            else:
+                self._client.lpush(self._key, payload)
+        except Exception:
+            self.handleError(record)

log_python/heartbeat.py

@@ -0,0 +1,52 @@
+"""Framework-neutral queue-worker heartbeat."""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from collections.abc import Callable
+
+from log_python.const import LOGGER_COUNT, LOGGER_MONITORING, LOGGER_STATUS
+
+
+class Heartbeat:
+    def __init__(self, logger: logging.Logger, *, name: str = "default", interval: float = 120, clock: Callable[[], float] = time.monotonic) -> None:
+        self._logger = logger
+        self._name = name
+        self._interval = interval
+        self._clock = clock
+        self._last = 0.0
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def tick(self, *, force: bool = False) -> bool:
+        now = self._clock()
+        if not force and (self._interval <= 0 or now - self._last <= self._interval):
+            return False
+        self._logger.info("heartbeat: %s", self._name, extra={"context": {LOGGER_MONITORING: "queue", LOGGER_COUNT: 0}})
+        self._last = now
+        return True
+
+    def start(self) -> None:
+        if self._interval <= 0:
+            return
+        if self._thread is not None and self._thread.is_alive():
+            return
+        self._stop.clear()
+        self._thread = threading.Thread(target=self._run, name=f"pylog-heartbeat-{self._name}", daemon=True)
+        self._thread.start()
+
+    @property
+    def running(self) -> bool:
+        return self._thread is not None and self._thread.is_alive()
+
+    def _run(self) -> None:
+        while not self._stop.wait(self._interval):
+            self.tick(force=True)
+
+    def stop(self, *, status: int = 0) -> None:
+        self._stop.set()
+        if self._thread is not None and self._thread is not threading.current_thread():
+            self._thread.join(timeout=max(1.0, self._interval))
+        self._logger.info("Queue stop signal", extra={"context": {LOGGER_MONITORING: "queue-stop", LOGGER_STATUS: status}})

log_python/integrations/__init__.py

@@ -0,0 +1 @@
+"""Optional framework integrations."""

log_python/integrations/sqlalchemy.py

@@ -0,0 +1,87 @@
+"""Optional SQLAlchemy slow-query instrumentation (dependency injected)."""
+
+from __future__ import annotations
+
+import importlib
+import json
+import logging
+import re
+import time
+from collections.abc import Callable
+from typing import Any, Protocol, cast
+
+from log_python.redactor import Redactor
+
+_SQL_RE = re.compile(r"(UPDATE|DELETE\s+FROM|INSERT\s+INTO|FROM)\s+[`\"\[]?([\w.]+)[`\"\]]?", re.IGNORECASE)
+
+
+class EventApi(Protocol):
+    def listen(self, target: object, name: str, callback: Callable[..., None]) -> None: ...
+
+
+def classify_sql(statement: str) -> tuple[str, str]:
+    match = _SQL_RE.search(statement)
+    if not match:
+        return "", ""
+    sql_type = re.sub(r"\s+", " ", match.group(1).upper())
+    sql_type = {"FROM": "SELECT", "DELETE FROM": "DELETE", "INSERT INTO": "INSERT"}.get(sql_type, sql_type)
+    return sql_type, match.group(2)
+
+
+def _safe_bindings(parameters: object, redactor: Redactor) -> str:
+    try:
+        if isinstance(parameters, (list, tuple)) and len(parameters) >= 20:
+            return ""
+        values = (
+            [f"`string({len(value)})`" if isinstance(value, str) and len(value) > 512 else value for value in parameters]
+            if isinstance(parameters, (list, tuple))
+            else parameters
+        )
+        return json.dumps(redactor.redact(values), default=str, ensure_ascii=False)
+    except Exception:
+        return ""
+
+
+def install_sqlalchemy_logging(
+    engine: object,
+    *,
+    logger: logging.Logger | None = None,
+    select_threshold_ms: int = 200,
+    all_threshold_ms: int = 500,
+    include_bindings: bool = False,
+    event_api: EventApi | None = None,
+    clock: Callable[[], float] = time.perf_counter,
+) -> None:
+    if event_api is None:
+        event_api = cast(EventApi, importlib.import_module("sqlalchemy.event"))
+    target_logger = logger or logging.getLogger("sql")
+    redactor = Redactor()
+
+    def before(_conn: object, _cursor: object, _statement: str, _parameters: object, context: Any, _executemany: bool) -> None:
+        started = clock()
+        if isinstance(context, dict):
+            context["_pylog_started"] = started
+        else:
+            context._pylog_started = started
+
+    def after(_conn: object, _cursor: object, statement: str, parameters: object, context: Any, _executemany: bool) -> None:
+        if "telescope" in statement.lower():
+            return
+        started = context.get("_pylog_started") if isinstance(context, dict) else getattr(context, "_pylog_started", clock())
+        started_value = float(started) if isinstance(started, (int, float, str)) else clock()
+        milliseconds = round((clock() - started_value) * 1000)
+        sql_type, table = classify_sql(statement)
+        threshold = select_threshold_ms if sql_type == "SELECT" and select_threshold_ms else all_threshold_ms
+        if threshold and milliseconds < threshold:
+            return
+        fields = {
+            "db_table": table,
+            "db_time_ms": milliseconds,
+            "db_bindings": _safe_bindings(parameters, redactor) if include_bindings else "",
+            "tag": "sql",
+            "sql_type": sql_type,
+        }
+        target_logger.info(statement, extra={"context": fields})
+
+    event_api.listen(engine, "before_cursor_execute", before)
+    event_api.listen(engine, "after_cursor_execute", after)

log_python/logtype.py

@@ -0,0 +1,171 @@
+"""Log origin classification, ported from the reference `src/LogType.php` (spec
+`docs/LoggingRules.md` §4.3.1, field `log_type`).
+
+Classifies WHO initiated a log record:
+
+- ``logger``    — an explicit call in code (the default).
+- ``trigger``   — reserved for a runtime warning/error integration.
+- ``exception`` — an uncaught-exception handler, or an explicit log call carrying
+  ``context['exception']``.
+- ``fatal``     — reserved for a supervisor/native-fatal integration.
+
+`install_handlers()` provides opt-in, idempotent and chained `sys.excepthook`,
+`threading.excepthook` and `warnings.showwarning` integration. Python has no PHP-style `error_get_last()` at
+normal `atexit`, so this module does not fabricate a reliable `fatal` event from a
+clean shutdown; native fatal signals remain the responsibility of the process
+supervisor/faulthandler.
+
+Python-idiom divergence from the reference (documented, not an oversight): PHP's
+`LogType::$origin` is a single process-global static (correct for PHP-FPM's one-request-
+per-process model). A bare global would leak across concurrent requests/tasks in a
+threaded or async Python process, so this port uses a `contextvars.ContextVar` instead —
+each thread/async-task context gets its own origin, isolated the way the spec's own
+`request_id` stability period ("stable within one request") already implies concurrent
+call-chains must not step on each other. The installed thread hook sets origin in the
+failing thread before emitting and chaining the record.
+
+Method names avoid `set`/`reset` as bare functions (`flake8-builtins`, enabled in this
+project's `ruff.toml`, flags shadowing the `set` builtin); the reference's `LogType::set()`
+/ `::reset()` become `LogType.set_origin()` / `LogType.reset_origin()` here.
+"""
+
+from __future__ import annotations
+
+import contextvars
+import logging
+import sys
+import threading
+import warnings
+from collections.abc import Callable
+from types import TracebackType
+from typing import ClassVar, Final, TextIO
+
+LOGGER: Final[str] = "logger"
+TRIGGER: Final[str] = "trigger"
+EXCEPTION: Final[str] = "exception"
+FATAL: Final[str] = "fatal"
+
+_ORIGIN: Final[contextvars.ContextVar[str | None]] = contextvars.ContextVar("pylog_log_type_origin", default=None)
+_HANDLER_LOCK: Final[threading.Lock] = threading.Lock()
+_INSTALLED = False
+_PREVIOUS_SYS_HOOK = sys.excepthook
+_PREVIOUS_THREAD_HOOK = threading.excepthook
+_PREVIOUS_WARNING_HOOK = warnings.showwarning
+_INSTALLED_SYS_HOOK: Callable[[type[BaseException], BaseException, TracebackType | None], None] | None = None
+_INSTALLED_THREAD_HOOK: Callable[[threading.ExceptHookArgs], None] | None = None
+_INSTALLED_WARNING_HOOK: Callable[[Warning | str, type[Warning], str, int, TextIO | None, str | None], None] | None = None
+
+
+class LogType:
+    """Namespace mirroring the reference `LogType` class (its four constants + the
+    process/context-scoped origin flag). Not instantiated — every member is a classmethod
+    or class-level constant, matching the PHP class's all-static shape.
+    """
+
+    LOGGER: Final[str] = LOGGER
+    TRIGGER: Final[str] = TRIGGER
+    EXCEPTION: Final[str] = EXCEPTION
+    FATAL: Final[str] = FATAL
+
+    _origin: ClassVar[contextvars.ContextVar[str | None]] = _ORIGIN
+
+    @classmethod
+    def current(cls) -> str:
+        """The current origin, or `LOGGER` when nothing more specific has been set —
+        reference `LogType::current()`.
+        """
+        return cls._origin.get() or cls.LOGGER
+
+    @classmethod
+    def set_origin(cls, value: str | None) -> None:
+        """Set the current context's origin. Reference `LogType::set()`."""
+        cls._origin.set(value)
+
+    @classmethod
+    def reset_origin(cls) -> None:
+        """Clear the current context's origin back to the `LOGGER` default. Reference
+        `LogType::reset()`.
+        """
+        cls._origin.set(None)
+
+
+def handlers_installed() -> bool:
+    """Whether this module currently owns the process exception hooks."""
+    return _INSTALLED
+
+
+def install_handlers() -> None:
+    """Install idempotent, chained uncaught-exception hooks."""
+    global _INSTALLED, _PREVIOUS_SYS_HOOK, _PREVIOUS_THREAD_HOOK, _PREVIOUS_WARNING_HOOK
+    global _INSTALLED_SYS_HOOK, _INSTALLED_THREAD_HOOK, _INSTALLED_WARNING_HOOK
+    with _HANDLER_LOCK:
+        if _INSTALLED:
+            return
+        _PREVIOUS_SYS_HOOK = sys.excepthook
+        _PREVIOUS_THREAD_HOOK = threading.excepthook
+        _PREVIOUS_WARNING_HOOK = warnings.showwarning
+
+        def sys_hook(exc_type: type[BaseException], exc: BaseException, tb: TracebackType | None) -> None:
+            LogType.set_origin(LogType.EXCEPTION)
+            try:
+                logging.getLogger("log_python.uncaught").critical("uncaught exception", exc_info=(exc_type, exc, tb))
+                _PREVIOUS_SYS_HOOK(exc_type, exc, tb)
+            finally:
+                LogType.reset_origin()
+
+        def thread_hook(args: threading.ExceptHookArgs) -> None:
+            LogType.set_origin(LogType.EXCEPTION)
+            try:
+                if args.exc_value is not None:
+                    logging.getLogger("log_python.uncaught").critical(
+                        "uncaught thread exception",
+                        exc_info=(args.exc_type, args.exc_value, args.exc_traceback),
+                    )
+                _PREVIOUS_THREAD_HOOK(args)
+            finally:
+                LogType.reset_origin()
+
+        def warning_hook(
+            message: Warning | str,
+            category: type[Warning],
+            filename: str,
+            lineno: int,
+            file: TextIO | None = None,
+            line: str | None = None,
+        ) -> None:
+            LogType.set_origin(LogType.TRIGGER)
+            try:
+                logging.getLogger("log_python.warning").warning(
+                    str(message),
+                    extra={"context": {"warning_category": category.__name__, "warning_file": filename, "warning_line": lineno}},
+                )
+                _PREVIOUS_WARNING_HOOK(message, category, filename, lineno, file, line)
+            finally:
+                LogType.reset_origin()
+
+        sys.excepthook = sys_hook
+        threading.excepthook = thread_hook
+        warnings.showwarning = warning_hook
+        _INSTALLED_SYS_HOOK = sys_hook
+        _INSTALLED_THREAD_HOOK = thread_hook
+        _INSTALLED_WARNING_HOOK = warning_hook
+        _INSTALLED = True
+
+
+def uninstall_handlers() -> None:
+    """Restore hooks captured by :func:`install_handlers`; primarily useful in tests."""
+    global _INSTALLED, _INSTALLED_SYS_HOOK, _INSTALLED_THREAD_HOOK, _INSTALLED_WARNING_HOOK
+    with _HANDLER_LOCK:
+        if not _INSTALLED:
+            return
+        if sys.excepthook is _INSTALLED_SYS_HOOK:
+            sys.excepthook = _PREVIOUS_SYS_HOOK
+        if threading.excepthook is _INSTALLED_THREAD_HOOK:
+            threading.excepthook = _PREVIOUS_THREAD_HOOK
+        if warnings.showwarning is _INSTALLED_WARNING_HOOK:
+            warnings.showwarning = _PREVIOUS_WARNING_HOOK
+        _INSTALLED_SYS_HOOK = None
+        _INSTALLED_THREAD_HOOK = None
+        _INSTALLED_WARNING_HOOK = None
+        _INSTALLED = False
+        LogType.reset_origin()