kitty-logger 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kitty_logger/__init__.py +19 -0
- kitty_logger/_client.py +83 -0
- kitty_logger/_env.py +8 -0
- kitty_logger/_formatters.py +60 -0
- kitty_logger/_server.py +171 -0
- kitty_logger/_setup.py +282 -0
- kitty_logger-0.1.0.dist-info/LICENSE +21 -0
- kitty_logger-0.1.0.dist-info/METADATA +86 -0
- kitty_logger-0.1.0.dist-info/RECORD +11 -0
- kitty_logger-0.1.0.dist-info/WHEEL +5 -0
- kitty_logger-0.1.0.dist-info/top_level.txt +1 -0
kitty_logger/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""kitty_logger:通过专用日志服务子进程实现的跨进程日志记录。
|
|
2
|
+
|
|
3
|
+
典型用法::
|
|
4
|
+
|
|
5
|
+
# 主进程
|
|
6
|
+
import kitty_logger
|
|
7
|
+
kitty_logger.setup_logging(log_file="app.log")
|
|
8
|
+
|
|
9
|
+
# 之后用 spawn 启动子进程(本库不支持 fork),在子进程中:
|
|
10
|
+
import kitty_logger
|
|
11
|
+
log = kitty_logger.getLogger(__name__)
|
|
12
|
+
log.info("hello from child")
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from ._client import getLogger
|
|
16
|
+
from ._setup import setup_logging, shutdown_logging
|
|
17
|
+
|
|
18
|
+
__all__ = ["setup_logging", "shutdown_logging", "getLogger"]
|
|
19
|
+
__version__ = "0.1.0"
|
kitty_logger/_client.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""子进程侧 API:``getLogger`` 返回一个连接到日志服务进程的 logger。"""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import logging.handlers
|
|
5
|
+
import os
|
|
6
|
+
import threading
|
|
7
|
+
|
|
8
|
+
from ._env import ENV_HOST, ENV_LEVEL, ENV_PORT
|
|
9
|
+
|
|
10
|
+
_lock = threading.Lock()
|
|
11
|
+
_configured_loggers: set[str] = set()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _read_endpoint() -> tuple[str, int]:
|
|
15
|
+
host = os.environ.get(ENV_HOST)
|
|
16
|
+
port = os.environ.get(ENV_PORT)
|
|
17
|
+
if not host or not port:
|
|
18
|
+
raise RuntimeError(
|
|
19
|
+
"kitty_logger 尚未在当前进程树中初始化。请先在主进程调用 kitty_logger.setup_logging(),再创建子进程。"
|
|
20
|
+
)
|
|
21
|
+
return host, int(port)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _read_level() -> int:
|
|
25
|
+
raw = os.environ.get(ENV_LEVEL)
|
|
26
|
+
if raw is None:
|
|
27
|
+
return logging.INFO
|
|
28
|
+
# 优先按整数解析;解析失败再当作 ``"INFO"`` 这类名字处理。
|
|
29
|
+
try:
|
|
30
|
+
return int(raw)
|
|
31
|
+
except ValueError:
|
|
32
|
+
pass
|
|
33
|
+
name_to_level = logging.getLevelNamesMapping()
|
|
34
|
+
return name_to_level.get(raw.upper(), logging.INFO)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _attach_socket_handler(
|
|
38
|
+
logger: logging.Logger, host: str, port: int, level: int
|
|
39
|
+
) -> None:
|
|
40
|
+
# 避免在同一个 logger 上重复挂 SocketHandler(例如模块被重新 import 时)。
|
|
41
|
+
for h in logger.handlers:
|
|
42
|
+
if isinstance(h, logging.handlers.SocketHandler) and (h.host, h.port) == (
|
|
43
|
+
host,
|
|
44
|
+
port,
|
|
45
|
+
):
|
|
46
|
+
return
|
|
47
|
+
handler = logging.handlers.SocketHandler(host, port)
|
|
48
|
+
# SocketHandler 直接 pickle LogRecord,由接收端负责格式化。
|
|
49
|
+
# 在这里设置 formatter 没有效果,会被忽略。
|
|
50
|
+
logger.addHandler(handler)
|
|
51
|
+
# 仅当 logger 还是默认的 NOTSET 时才设置 level,不要悄悄覆盖用户已经
|
|
52
|
+
# 显式设置过的级别(例如用户在 setup_logging 之前已经
|
|
53
|
+
# ``logging.getLogger().setLevel(DEBUG)``)。
|
|
54
|
+
if logger.level == logging.NOTSET:
|
|
55
|
+
logger.setLevel(level)
|
|
56
|
+
# 关闭向 root 的传播,避免子进程恰好已经有默认 StreamHandler
|
|
57
|
+
# (比如调用过 basicConfig)时出现重复输出。root logger 没有父级,
|
|
58
|
+
# 设置 propagate 没有意义,跳过。
|
|
59
|
+
if logger is not logging.getLogger():
|
|
60
|
+
logger.propagate = False
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def getLogger(name: str | None = None) -> logging.Logger:
|
|
64
|
+
"""返回一个会把日志记录发送到 kitty_logger 服务进程的 logger。
|
|
65
|
+
|
|
66
|
+
必须在祖先进程已经调用过 :func:`setup_logging` 之后才能使用
|
|
67
|
+
(子进程通过继承的环境变量 ``KITTY_LOGGER_HOST`` / ``KITTY_LOGGER_PORT``
|
|
68
|
+
定位服务地址)。
|
|
69
|
+
|
|
70
|
+
本库只支持 ``spawn`` 启动方式,因此模块级状态在子进程里天然是空的,
|
|
71
|
+
无需处理"继承自父进程的 SocketHandler"问题。
|
|
72
|
+
"""
|
|
73
|
+
host, port = _read_endpoint()
|
|
74
|
+
level = _read_level()
|
|
75
|
+
logger = logging.getLogger(name)
|
|
76
|
+
with _lock:
|
|
77
|
+
# ``logging.getLogger("")`` 与 ``logging.getLogger(None)`` 都是 root,
|
|
78
|
+
# 在本集合里共用 "";与 root 真正的别名一致,不需要单独区分。
|
|
79
|
+
key = name or ""
|
|
80
|
+
if key not in _configured_loggers:
|
|
81
|
+
_attach_socket_handler(logger, host, port, level)
|
|
82
|
+
_configured_loggers.add(key)
|
|
83
|
+
return logger
|
kitty_logger/_env.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""日志格式化器:彩色控制台 + 毫秒时间戳文件。
|
|
2
|
+
|
|
3
|
+
移植自 llm_engine.log;保留原本的 ANSI 配色与 ``formatTime`` 的毫秒精度。
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
import time
|
|
10
|
+
from typing import override
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class _MillisecondTimeFormatter(logging.Formatter):
|
|
14
|
+
"""``formatTime`` 输出 ``YYYY-MM-DD HH:MM:SS.mmm``(毫秒精度)。"""
|
|
15
|
+
|
|
16
|
+
@override
|
|
17
|
+
def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str:
|
|
18
|
+
ct = self.converter(record.created)
|
|
19
|
+
s = time.strftime(datefmt or "%Y-%m-%d %H:%M:%S", ct)
|
|
20
|
+
return f"{s}.{int(record.msecs):03d}"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ColorFormatter(_MillisecondTimeFormatter):
|
|
24
|
+
"""为控制台输出添加 ANSI 彩色的日志格式化器。
|
|
25
|
+
|
|
26
|
+
在格式串里可以使用以下额外字段:``%(color)s``、``%(name_color)s``、
|
|
27
|
+
``%(file_color)s``、``%(func_color)s``,以及对应的 ``%(reset)s``。
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
colors: dict[str, str] = {
|
|
31
|
+
"DEBUG": "\033[38;5;32m",
|
|
32
|
+
"INFO": "\033[38;5;70m",
|
|
33
|
+
"WARNING": "\033[38;5;186m",
|
|
34
|
+
"ERROR": "\033[38;5;206m",
|
|
35
|
+
"CRITICAL": "\033[1;38;5;196m",
|
|
36
|
+
}
|
|
37
|
+
name_color: str = "\033[38;5;36m"
|
|
38
|
+
file_color: str = "\033[38;5;45m"
|
|
39
|
+
func_color: str = "\033[38;5;208m"
|
|
40
|
+
reset: str = "\033[0m"
|
|
41
|
+
|
|
42
|
+
@override
|
|
43
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
44
|
+
record.color = self.colors.get(record.levelname, "") # type: ignore[attr-defined]
|
|
45
|
+
record.name_color = self.name_color # type: ignore[attr-defined]
|
|
46
|
+
record.file_color = self.file_color # type: ignore[attr-defined]
|
|
47
|
+
record.func_color = self.func_color # type: ignore[attr-defined]
|
|
48
|
+
record.reset = self.reset # type: ignore[attr-defined]
|
|
49
|
+
return super().format(record)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class FileFormatter(_MillisecondTimeFormatter):
|
|
53
|
+
"""用于日志文件的格式化器,无颜色,含毫秒时间戳。"""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# 默认格式:在 llm_engine 原版基础上加入 ``processName[process]``,
|
|
57
|
+
# 以便跨进程日志能区分来源进程。
|
|
58
|
+
DEFAULT_CONSOLE_FMT = "%(asctime)s | %(name_color)s%(name)s%(reset)s | %(processName)s[%(process)d] | %(file_color)s%(filename)s:%(lineno)d%(reset)s | %(func_color)s%(funcName)s%(reset)s | %(color)s%(levelname)s%(reset)s | %(message)s"
|
|
59
|
+
|
|
60
|
+
DEFAULT_FILE_FMT = "%(asctime)s | %(name)s | %(processName)s[%(process)d] | %(filename)s:%(lineno)d | %(funcName)s | %(levelname)s | %(message)s"
|
kitty_logger/_server.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""日志记录接收进程。
|
|
2
|
+
|
|
3
|
+
运行在一个独立的子进程中。接收父进程预先 bind 好的监听 socket,
|
|
4
|
+
反序列化客户端进程通过 ``logging.handlers.SocketHandler`` 发来的
|
|
5
|
+
``logging.LogRecord``,然后交给本进程内配置的日志 handler(文件、
|
|
6
|
+
stderr 等)输出。
|
|
7
|
+
|
|
8
|
+
实现参考自 Python ``logging`` cookbook。
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
import pickle
|
|
13
|
+
import socket
|
|
14
|
+
import socketserver
|
|
15
|
+
import struct
|
|
16
|
+
import sys
|
|
17
|
+
import threading
|
|
18
|
+
from typing import cast, final, override
|
|
19
|
+
|
|
20
|
+
from multiprocessing.synchronize import Event as MpEvent
|
|
21
|
+
|
|
22
|
+
from ._formatters import ColorFormatter, FileFormatter
|
|
23
|
+
|
|
24
|
+
# 单条 LogRecord 序列化后的最大字节数。超过即视为协议错乱或恶意流量,
|
|
25
|
+
# 直接断开连接,避免 ``recv`` 一口气分配几个 GB 的 buffer 把进程打爆。
|
|
26
|
+
# 16 MiB 已经远大于任何正常的 ``LogRecord``(含 traceback / extra)。
|
|
27
|
+
_MAX_RECORD_BYTES = 16 * 1024 * 1024
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _recv_exact(conn: socket.socket, n: int) -> bytes | None:
|
|
31
|
+
"""从 ``conn`` 上**读满** ``n`` 字节,对端关闭则返回 ``None``。
|
|
32
|
+
|
|
33
|
+
单次 ``recv`` 不保证读满指定长度(TCP 半包),而当对端关闭后 ``recv``
|
|
34
|
+
会立刻返回 ``b""`` —— 任何"按长度循环 recv"的实现都必须显式处理 EOF,
|
|
35
|
+
否则会在客户端中途断连时陷入死循环。
|
|
36
|
+
"""
|
|
37
|
+
buf = bytearray()
|
|
38
|
+
while len(buf) < n:
|
|
39
|
+
try:
|
|
40
|
+
chunk = conn.recv(n - len(buf))
|
|
41
|
+
except (ConnectionError, OSError):
|
|
42
|
+
return None
|
|
43
|
+
if not chunk:
|
|
44
|
+
return None
|
|
45
|
+
buf.extend(chunk)
|
|
46
|
+
return bytes(buf)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@final
|
|
50
|
+
class _LogRecordStreamHandler(socketserver.StreamRequestHandler):
|
|
51
|
+
"""处理一个客户端连接上的流式日志请求。"""
|
|
52
|
+
|
|
53
|
+
@override
|
|
54
|
+
def handle(self) -> None:
|
|
55
|
+
conn = cast(socket.socket, self.connection)
|
|
56
|
+
while True:
|
|
57
|
+
header = _recv_exact(conn, 4)
|
|
58
|
+
if header is None:
|
|
59
|
+
break
|
|
60
|
+
slen = cast(int, struct.unpack(">L", header)[0])
|
|
61
|
+
if slen == 0 or slen > _MAX_RECORD_BYTES:
|
|
62
|
+
# 长度异常的请求多半是协议不匹配或恶意流量,直接断连而不是
|
|
63
|
+
# 试图继续读,避免被诱导分配巨大 buffer。
|
|
64
|
+
break
|
|
65
|
+
body = _recv_exact(conn, slen)
|
|
66
|
+
if body is None:
|
|
67
|
+
break
|
|
68
|
+
try:
|
|
69
|
+
obj = cast(object, pickle.loads(body))
|
|
70
|
+
except Exception:
|
|
71
|
+
continue
|
|
72
|
+
if not isinstance(obj, dict):
|
|
73
|
+
continue
|
|
74
|
+
record = logging.makeLogRecord(cast(dict[str, object], obj))
|
|
75
|
+
logger = logging.getLogger(record.name)
|
|
76
|
+
logger.handle(record)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@final
|
|
80
|
+
class _LogRecordSocketReceiver(socketserver.ThreadingTCPServer):
|
|
81
|
+
allow_reuse_address: bool = True
|
|
82
|
+
daemon_threads: bool = True
|
|
83
|
+
|
|
84
|
+
@override
|
|
85
|
+
def __init__(self, listening_sock: socket.socket) -> None:
|
|
86
|
+
# 父类正常初始化(含 _BaseServer 的 shutdown 标志位等内部状态),
|
|
87
|
+
# 但跳过 bind/listen——监听 socket 是父进程已经绑好的。
|
|
88
|
+
addr = cast(tuple[str, int], listening_sock.getsockname()[:2])
|
|
89
|
+
super().__init__(addr, _LogRecordStreamHandler, bind_and_activate=False)
|
|
90
|
+
# 关闭 ``TCPServer.__init__`` 默认创建的那个空 socket,再换成外部传入的。
|
|
91
|
+
try:
|
|
92
|
+
self.socket.close()
|
|
93
|
+
except Exception:
|
|
94
|
+
pass
|
|
95
|
+
self.socket = listening_sock
|
|
96
|
+
self.server_address = addr
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _build_handlers(
|
|
100
|
+
log_file: str | None,
|
|
101
|
+
stream: bool,
|
|
102
|
+
console_fmt: str,
|
|
103
|
+
file_fmt: str,
|
|
104
|
+
datefmt: str | None,
|
|
105
|
+
) -> list[logging.Handler]:
|
|
106
|
+
handlers: list[logging.Handler] = []
|
|
107
|
+
if log_file:
|
|
108
|
+
fh = logging.FileHandler(log_file, encoding="utf-8")
|
|
109
|
+
fh.setFormatter(FileFormatter(fmt=file_fmt, datefmt=datefmt))
|
|
110
|
+
handlers.append(fh)
|
|
111
|
+
if stream:
|
|
112
|
+
sh = logging.StreamHandler(stream=sys.stderr)
|
|
113
|
+
sh.setFormatter(ColorFormatter(fmt=console_fmt, datefmt=datefmt))
|
|
114
|
+
handlers.append(sh)
|
|
115
|
+
return handlers
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def run_server(
|
|
119
|
+
listening_sock: socket.socket,
|
|
120
|
+
shutdown_event: MpEvent,
|
|
121
|
+
parent_alive_recv: socket.socket,
|
|
122
|
+
level: int,
|
|
123
|
+
log_file: str | None,
|
|
124
|
+
stream: bool,
|
|
125
|
+
console_fmt: str,
|
|
126
|
+
file_fmt: str,
|
|
127
|
+
datefmt: str | None,
|
|
128
|
+
) -> None:
|
|
129
|
+
"""日志服务子进程的入口函数。"""
|
|
130
|
+
root = logging.getLogger()
|
|
131
|
+
for h in list(root.handlers):
|
|
132
|
+
root.removeHandler(h)
|
|
133
|
+
for h in _build_handlers(log_file, stream, console_fmt, file_fmt, datefmt):
|
|
134
|
+
root.addHandler(h)
|
|
135
|
+
root.setLevel(level)
|
|
136
|
+
|
|
137
|
+
server = _LogRecordSocketReceiver(listening_sock)
|
|
138
|
+
|
|
139
|
+
def _wait_shutdown() -> None:
|
|
140
|
+
_ = shutdown_event.wait()
|
|
141
|
+
server.shutdown()
|
|
142
|
+
|
|
143
|
+
def _watch_parent_alive() -> None:
|
|
144
|
+
# 父进程持有 socketpair 的 send 端;任何原因导致父进程消失(正常退出、
|
|
145
|
+
# SIGKILL、段错误等),内核都会关闭它的 fd,本端 ``recv`` 会立刻得到
|
|
146
|
+
# 空字节(EOF)。借此触发服务自我退出,避免变成孤儿进程。
|
|
147
|
+
try:
|
|
148
|
+
while True:
|
|
149
|
+
data = parent_alive_recv.recv(64)
|
|
150
|
+
if not data:
|
|
151
|
+
break
|
|
152
|
+
except OSError:
|
|
153
|
+
pass
|
|
154
|
+
finally:
|
|
155
|
+
try:
|
|
156
|
+
parent_alive_recv.close()
|
|
157
|
+
except Exception:
|
|
158
|
+
pass
|
|
159
|
+
shutdown_event.set()
|
|
160
|
+
|
|
161
|
+
threading.Thread(target=_wait_shutdown, daemon=True).start()
|
|
162
|
+
threading.Thread(target=_watch_parent_alive, daemon=True).start()
|
|
163
|
+
|
|
164
|
+
try:
|
|
165
|
+
server.serve_forever()
|
|
166
|
+
finally:
|
|
167
|
+
try:
|
|
168
|
+
server.server_close()
|
|
169
|
+
except Exception:
|
|
170
|
+
pass
|
|
171
|
+
logging.shutdown()
|
kitty_logger/_setup.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
import atexit
|
|
2
|
+
import ipaddress
|
|
3
|
+
import logging
|
|
4
|
+
import multiprocessing as mp
|
|
5
|
+
import os
|
|
6
|
+
import socket
|
|
7
|
+
import warnings
|
|
8
|
+
|
|
9
|
+
from multiprocessing.process import BaseProcess
|
|
10
|
+
from multiprocessing.synchronize import Event as MpEvent
|
|
11
|
+
from typing import Literal, TypedDict, TypeGuard, cast
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
from ._env import ENV_HOST, ENV_LEVEL, ENV_PORT
|
|
15
|
+
from ._formatters import DEFAULT_CONSOLE_FMT, DEFAULT_FILE_FMT
|
|
16
|
+
from ._server import run_server
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class _NotRunningState(TypedDict):
|
|
20
|
+
running: Literal[False]
|
|
21
|
+
host: None
|
|
22
|
+
port: None
|
|
23
|
+
proc: None
|
|
24
|
+
shutdown_event: None
|
|
25
|
+
parent_alive_send: None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class _RunningState(TypedDict):
|
|
29
|
+
running: Literal[True]
|
|
30
|
+
host: str
|
|
31
|
+
port: int
|
|
32
|
+
proc: BaseProcess
|
|
33
|
+
shutdown_event: MpEvent
|
|
34
|
+
# 父进程持有的 socketpair 一端;只要父进程活着这个 fd 就开着,
|
|
35
|
+
# 父进程一旦消失(包括 SIGKILL/段错误),内核会自动关闭它,
|
|
36
|
+
# 子进程的另一端会读到 EOF 从而触发自我退出。
|
|
37
|
+
parent_alive_send: socket.socket
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
type _ServerState = _NotRunningState | _RunningState
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _make_not_running_state() -> _NotRunningState:
|
|
44
|
+
return {
|
|
45
|
+
"running": False,
|
|
46
|
+
"host": None,
|
|
47
|
+
"port": None,
|
|
48
|
+
"proc": None,
|
|
49
|
+
"shutdown_event": None,
|
|
50
|
+
"parent_alive_send": None,
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
_state: _ServerState = _make_not_running_state()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _is_running(state: _ServerState) -> TypeGuard[_RunningState]:
|
|
58
|
+
return state["running"] is True
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _warn_if_non_loopback(host: str) -> None:
|
|
62
|
+
"""非 loopback 地址会把 ``pickle.loads`` 暴露给可达端口的任何人,等于 RCE。"""
|
|
63
|
+
try:
|
|
64
|
+
addr = ipaddress.ip_address(host)
|
|
65
|
+
addrs: list[ipaddress.IPv4Address | ipaddress.IPv6Address] = [addr]
|
|
66
|
+
except ValueError:
|
|
67
|
+
# 主机名(例如 "localhost" / "my-host.example.com"):尽力解析后判断。
|
|
68
|
+
try:
|
|
69
|
+
infos = socket.getaddrinfo(host, None)
|
|
70
|
+
except OSError:
|
|
71
|
+
warnings.warn(
|
|
72
|
+
f"kitty_logger 无法解析 host={host!r},无法判断是否为 loopback;服务端使用 pickle 反序列化,请确认绑定地址不可被外部访问。",
|
|
73
|
+
stacklevel=3,
|
|
74
|
+
)
|
|
75
|
+
return
|
|
76
|
+
addrs = []
|
|
77
|
+
for info in infos:
|
|
78
|
+
sockaddr = info[4]
|
|
79
|
+
ip_str = sockaddr[0] if isinstance(sockaddr, tuple) else None
|
|
80
|
+
if not ip_str:
|
|
81
|
+
continue
|
|
82
|
+
try:
|
|
83
|
+
addrs.append(ipaddress.ip_address(ip_str))
|
|
84
|
+
except ValueError:
|
|
85
|
+
continue
|
|
86
|
+
if not addrs:
|
|
87
|
+
return
|
|
88
|
+
if any(not a.is_loopback for a in addrs):
|
|
89
|
+
warnings.warn(
|
|
90
|
+
f"kitty_logger 绑定到非 loopback 地址 {host!r}:服务端使用 pickle 反序列化,等同于把远程代码执行能力暴露给可达该端口的任何主机。请仅在 127.0.0.1 / ::1 上使用。",
|
|
91
|
+
stacklevel=3,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _bind_ipv4_listener(
|
|
96
|
+
host: str, port: int, backlog: int = 128
|
|
97
|
+
) -> tuple[socket.socket, str, int]:
|
|
98
|
+
"""创建一个 AF_INET TCP 监听 socket,返回 ``(sock, bound_host, bound_port)``。
|
|
99
|
+
|
|
100
|
+
把 "AF_INET 的 ``getsockname()`` 一定是 ``tuple[str, int]``" 这一事实
|
|
101
|
+
集中收敛到这里,使其它地方不必再写 ``cast``。
|
|
102
|
+
"""
|
|
103
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
104
|
+
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
105
|
+
sock.bind((host, port))
|
|
106
|
+
sock.listen(backlog)
|
|
107
|
+
bound_host, bound_port = cast(tuple[str, int], sock.getsockname())
|
|
108
|
+
return sock, bound_host, bound_port
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def setup_logging(
|
|
112
|
+
log_file: str | None = None,
|
|
113
|
+
level: int = logging.INFO,
|
|
114
|
+
host: str = "127.0.0.1",
|
|
115
|
+
port: int = 0,
|
|
116
|
+
stream: bool = True,
|
|
117
|
+
console_fmt: str = DEFAULT_CONSOLE_FMT,
|
|
118
|
+
file_fmt: str = DEFAULT_FILE_FMT,
|
|
119
|
+
datefmt: str | None = None,
|
|
120
|
+
attach_main_logger: bool = True,
|
|
121
|
+
) -> tuple[str, int]:
|
|
122
|
+
"""初始化跨进程日志服务。
|
|
123
|
+
|
|
124
|
+
启动一个专用子进程(始终使用 ``spawn`` 启动方式),通过 TCP 接收
|
|
125
|
+
``LogRecord`` 并写入 ``log_file`` 和/或 stderr。同时设置环境变量
|
|
126
|
+
``KITTY_LOGGER_HOST`` 和 ``KITTY_LOGGER_PORT``,让本次调用之后
|
|
127
|
+
spawn 出来的子进程可以通过 :func:`kitty_logger.getLogger` 连接到
|
|
128
|
+
该服务。
|
|
129
|
+
|
|
130
|
+
控制台输出使用 ``ColorFormatter``(带 ANSI 颜色),文件输出使用
|
|
131
|
+
``FileFormatter``(无颜色);两者都使用毫秒级时间戳。
|
|
132
|
+
|
|
133
|
+
返回服务实际绑定到的 ``(host, port)``(当 ``port=0`` 由 OS 自动
|
|
134
|
+
分配端口时,调用方可以从这里拿到真实端口)。
|
|
135
|
+
|
|
136
|
+
幂等:重复调用直接返回已经记录的 ``(host, port)``。
|
|
137
|
+
|
|
138
|
+
.. note::
|
|
139
|
+
本库**不支持** ``fork`` 启动方式。原因:
|
|
140
|
+
(1) ``fork`` 会让子进程继承父进程已建立的 ``SocketHandler`` 和
|
|
141
|
+
底层 TCP 连接,父子两个进程往同一个 socket 上交叉写 pickle 字节流,
|
|
142
|
+
服务端反序列化必然失败;
|
|
143
|
+
(2) 多线程程序里 ``fork`` 本身就不安全(其他线程持有的锁会原样
|
|
144
|
+
留在子进程里);
|
|
145
|
+
(3) 跨平台一致性:macOS / Windows 都不能用 ``fork``。
|
|
146
|
+
请始终使用 ``multiprocessing.get_context("spawn")`` 或顶层守卫
|
|
147
|
+
``if __name__ == "__main__":`` 配合默认 ``spawn``。
|
|
148
|
+
"""
|
|
149
|
+
global _state
|
|
150
|
+
if _is_running(state=_state):
|
|
151
|
+
return _state["host"], _state["port"]
|
|
152
|
+
|
|
153
|
+
_warn_if_non_loopback(host)
|
|
154
|
+
|
|
155
|
+
# 在父进程里直接 bind,使 ``port=0`` 时也能同步拿到真实端口;
|
|
156
|
+
# 然后把已绑定的 socket 交给服务子进程使用。
|
|
157
|
+
listening_sock, bound_host, bound_port = _bind_ipv4_listener(host, port)
|
|
158
|
+
|
|
159
|
+
# 父进程存活监测:父进程持有 send 端,子进程持有 recv 端。
|
|
160
|
+
# 父进程哪怕被 ``kill -9`` 也会让内核 close 它的 fd,子进程会读到 EOF。
|
|
161
|
+
parent_alive_send, parent_alive_recv = socket.socketpair()
|
|
162
|
+
|
|
163
|
+
# 始终使用 spawn:见上面 docstring 的解释。
|
|
164
|
+
ctx = mp.get_context("spawn")
|
|
165
|
+
shutdown_event = ctx.Event()
|
|
166
|
+
proc = ctx.Process(
|
|
167
|
+
target=run_server,
|
|
168
|
+
kwargs=dict(
|
|
169
|
+
listening_sock=listening_sock,
|
|
170
|
+
shutdown_event=shutdown_event,
|
|
171
|
+
parent_alive_recv=parent_alive_recv,
|
|
172
|
+
level=level,
|
|
173
|
+
log_file=log_file,
|
|
174
|
+
stream=stream,
|
|
175
|
+
console_fmt=console_fmt,
|
|
176
|
+
file_fmt=file_fmt,
|
|
177
|
+
datefmt=datefmt,
|
|
178
|
+
),
|
|
179
|
+
name="kitty-logger-server",
|
|
180
|
+
daemon=False,
|
|
181
|
+
)
|
|
182
|
+
proc.start()
|
|
183
|
+
# 父进程不再需要这两个 socket 副本:listening_sock 完全交给子进程;
|
|
184
|
+
# parent_alive_recv 只能由子进程持有,否则 EOF 永远不会到来。
|
|
185
|
+
listening_sock.close()
|
|
186
|
+
parent_alive_recv.close()
|
|
187
|
+
|
|
188
|
+
# 这些 ENV 是给"日后由本进程 spawn 出来的工作子进程"用的(它们靠 ENV
|
|
189
|
+
# 找到服务地址);服务子进程本身已经通过 kwargs 拿到全部参数,所以
|
|
190
|
+
# 在 ``proc.start()`` 之后再写 ENV 没有竞态问题。
|
|
191
|
+
os.environ[ENV_HOST] = bound_host
|
|
192
|
+
os.environ[ENV_PORT] = str(bound_port)
|
|
193
|
+
os.environ[ENV_LEVEL] = str(level)
|
|
194
|
+
|
|
195
|
+
new_state: _RunningState = {
|
|
196
|
+
"running": True,
|
|
197
|
+
"host": bound_host,
|
|
198
|
+
"port": bound_port,
|
|
199
|
+
"proc": proc,
|
|
200
|
+
"shutdown_event": shutdown_event,
|
|
201
|
+
"parent_alive_send": parent_alive_send,
|
|
202
|
+
}
|
|
203
|
+
_state = new_state
|
|
204
|
+
|
|
205
|
+
if attach_main_logger:
|
|
206
|
+
# 让主进程自己产生的日志也走同一个服务,避免输出渠道分裂。
|
|
207
|
+
from ._client import _attach_socket_handler
|
|
208
|
+
|
|
209
|
+
_attach_socket_handler(logging.getLogger(), bound_host, bound_port, level)
|
|
210
|
+
|
|
211
|
+
_ = atexit.register(shutdown_logging)
|
|
212
|
+
return bound_host, bound_port
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _detach_all_socket_handlers(host: str, port: int) -> None:
|
|
216
|
+
"""卸载本进程内所有指向 ``(host, port)`` 的 ``SocketHandler``。
|
|
217
|
+
|
|
218
|
+
``shutdown_logging`` 之后服务子进程已死,留在 logger 上的 handler 会让
|
|
219
|
+
后续 ``logging.xxx`` 调用打到一个不存在的端口(``logging`` 会静默吞掉
|
|
220
|
+
socket 错误,外部表现为日志凭空消失)。在 shutdown 时主动卸载它们,
|
|
221
|
+
保证状态对称:``setup_logging`` 装上去什么,``shutdown_logging`` 全部
|
|
222
|
+
取下来。
|
|
223
|
+
"""
|
|
224
|
+
from logging.handlers import SocketHandler
|
|
225
|
+
|
|
226
|
+
loggers: list[logging.Logger] = [logging.getLogger()]
|
|
227
|
+
for obj in logging.Logger.manager.loggerDict.values():
|
|
228
|
+
if isinstance(obj, logging.Logger):
|
|
229
|
+
loggers.append(obj)
|
|
230
|
+
for lg in loggers:
|
|
231
|
+
for h in list(lg.handlers):
|
|
232
|
+
if isinstance(h, SocketHandler) and (h.host, h.port) == (host, port):
|
|
233
|
+
lg.removeHandler(h)
|
|
234
|
+
try:
|
|
235
|
+
h.close()
|
|
236
|
+
except Exception:
|
|
237
|
+
pass
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def shutdown_logging(timeout: float = 5.0) -> None:
|
|
241
|
+
"""停止日志服务子进程并清理本进程内残留的 ``SocketHandler``。可被重复调用。"""
|
|
242
|
+
global _state
|
|
243
|
+
if not _is_running(state=_state):
|
|
244
|
+
return
|
|
245
|
+
event: MpEvent = _state["shutdown_event"]
|
|
246
|
+
proc: BaseProcess = _state["proc"]
|
|
247
|
+
parent_alive_send: socket.socket = _state["parent_alive_send"]
|
|
248
|
+
host: str = _state["host"]
|
|
249
|
+
port: int = _state["port"]
|
|
250
|
+
_state = _make_not_running_state()
|
|
251
|
+
try:
|
|
252
|
+
event.set()
|
|
253
|
+
except Exception:
|
|
254
|
+
pass
|
|
255
|
+
# 主动关闭父端 socket,作为 mp.Event 的兜底关闭信号。
|
|
256
|
+
try:
|
|
257
|
+
parent_alive_send.close()
|
|
258
|
+
except Exception:
|
|
259
|
+
pass
|
|
260
|
+
proc.join(timeout)
|
|
261
|
+
if proc.is_alive():
|
|
262
|
+
proc.terminate()
|
|
263
|
+
proc.join(1.0)
|
|
264
|
+
if proc.is_alive():
|
|
265
|
+
# ``terminate`` 没杀掉(被磁盘 IO 等阻塞在不可中断状态),最后兜底 kill。
|
|
266
|
+
try:
|
|
267
|
+
proc.kill()
|
|
268
|
+
except Exception:
|
|
269
|
+
pass
|
|
270
|
+
proc.join(1.0)
|
|
271
|
+
|
|
272
|
+
# 服务进程已经死了,本进程里指向它的 SocketHandler 必须全部取下,
|
|
273
|
+
# 否则后续 logging 调用会向一个失效端口发送,触发静默丢日志。
|
|
274
|
+
_detach_all_socket_handlers(host, port)
|
|
275
|
+
|
|
276
|
+
# 同步清理 _client 的 dedup 集合与 ENV,让"二次 setup"能真正回到干净状态。
|
|
277
|
+
from . import _client
|
|
278
|
+
|
|
279
|
+
with _client._lock:
|
|
280
|
+
_client._configured_loggers.clear()
|
|
281
|
+
for k in (ENV_HOST, ENV_PORT, ENV_LEVEL):
|
|
282
|
+
os.environ.pop(k, None)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kitty
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: kitty_logger
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Cross-process logging via a dedicated log server process and SocketHandler.
|
|
5
|
+
Author: Kitty
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Provides-Extra: test
|
|
11
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
12
|
+
|
|
13
|
+
# kitty_logger
|
|
14
|
+
|
|
15
|
+
Cross-process Python logger. The main process starts a dedicated log-server
|
|
16
|
+
subprocess; every other process (forked or spawned) ships `LogRecord`s to it
|
|
17
|
+
via `logging.handlers.SocketHandler`, so all log lines end up in one place
|
|
18
|
+
with no interleaving or file-locking issues.
|
|
19
|
+
|
|
20
|
+
## Install
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install .
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Usage
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
# main.py
|
|
30
|
+
import multiprocessing as mp
|
|
31
|
+
import kitty_logger
|
|
32
|
+
|
|
33
|
+
def worker(i):
|
|
34
|
+
log = kitty_logger.getLogger(f"worker.{i}")
|
|
35
|
+
log.info("hello from worker %d", i)
|
|
36
|
+
|
|
37
|
+
if __name__ == "__main__":
|
|
38
|
+
kitty_logger.setup_logging(log_file="app.log")
|
|
39
|
+
log = kitty_logger.getLogger("main")
|
|
40
|
+
log.info("starting")
|
|
41
|
+
|
|
42
|
+
with mp.get_context("spawn").Pool(4) as pool:
|
|
43
|
+
pool.map(worker, range(4))
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
`setup_logging` sets the env vars `KITTY_LOGGER_HOST` / `KITTY_LOGGER_PORT`,
|
|
47
|
+
which child processes inherit and use to connect.
|
|
48
|
+
|
|
49
|
+
## API
|
|
50
|
+
|
|
51
|
+
- `setup_logging(log_file=None, level=logging.INFO, host="127.0.0.1", port=0, stream=True, console_fmt=..., file_fmt=..., datefmt=None, attach_main_logger=True) -> (host, port)`
|
|
52
|
+
Starts the log-server subprocess (always uses the `spawn` start method).
|
|
53
|
+
Idempotent. Registered with `atexit`. `port=0` lets the OS pick a free port;
|
|
54
|
+
the actual `(host, port)` is returned.
|
|
55
|
+
- `getLogger(name=None) -> logging.Logger`
|
|
56
|
+
Returns a logger with a `SocketHandler` pointing at the server.
|
|
57
|
+
- `shutdown_logging()` — stop the server subprocess explicitly.
|
|
58
|
+
|
|
59
|
+
## Why spawn-only
|
|
60
|
+
|
|
61
|
+
Child processes started with `fork` inherit the parent's already-connected
|
|
62
|
+
`SocketHandler` (and its TCP fd). Parent and child would then interleave
|
|
63
|
+
pickled-record bytes on the same connection, which corrupts every record on
|
|
64
|
+
the wire. `fork` is also unsafe in multi-threaded parents and unavailable on
|
|
65
|
+
Windows. KittyLogger therefore only supports `spawn`. Use
|
|
66
|
+
`multiprocessing.get_context("spawn")` (or just rely on Python's defaults
|
|
67
|
+
under `if __name__ == "__main__":`).
|
|
68
|
+
|
|
69
|
+
## Security
|
|
70
|
+
|
|
71
|
+
The server uses `pickle` to deserialize incoming `LogRecord`s. **Only ever bind
|
|
72
|
+
to a loopback address** (the default `127.0.0.1`). Binding to `0.0.0.0` or any
|
|
73
|
+
externally reachable interface exposes a remote-code-execution surface to anyone
|
|
74
|
+
who can reach the port.
|
|
75
|
+
|
|
76
|
+
## Caveats
|
|
77
|
+
|
|
78
|
+
- Do **not** call `logging.basicConfig()` (or otherwise attach a `StreamHandler`
|
|
79
|
+
to the root logger) before `setup_logging(attach_main_logger=True)` — the main
|
|
80
|
+
process would emit each record twice: once via the local root handler and
|
|
81
|
+
once via the log-server. Either let kitty_logger be the only configurer, or
|
|
82
|
+
pass `attach_main_logger=False` and manage main-process handlers yourself.
|
|
83
|
+
- After `shutdown_logging()`, kitty_logger removes its own `SocketHandler`s
|
|
84
|
+
from this process's loggers and clears `KITTY_LOGGER_*` env vars, so the
|
|
85
|
+
process state is symmetric with `setup_logging`. If you attach extra handlers
|
|
86
|
+
yourself, you remain responsible for cleaning them up.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
kitty_logger/__init__.py,sha256=kafWaI39l1w8KMekxRYu1CV6AuAGFTBmV2V25ELQFRY,551
|
|
2
|
+
kitty_logger/_client.py,sha256=g57-Ly6Sqw8xVq6z87laF4usELhZ9hBGf5avNHaauCU,3198
|
|
3
|
+
kitty_logger/_env.py,sha256=CLKK0Ln0GOyWDFKBC2LqJE3YPuc2MJ_Zo9lg7UOe4Jk,221
|
|
4
|
+
kitty_logger/_formatters.py,sha256=C17X-7vIWpLyBBZrFTF5Tju2HB46Aa76fW-A9Mb4ulc,2433
|
|
5
|
+
kitty_logger/_server.py,sha256=YzGVbIhp0OB4iHaxcyFo7TjAc0lVGN1bFqQdecyp-cM,5770
|
|
6
|
+
kitty_logger/_setup.py,sha256=bVW6HT_d4WXH22WQJV9DJxuOCqixnxrE2o_Cs_kmdYI,10358
|
|
7
|
+
kitty_logger-0.1.0.dist-info/LICENSE,sha256=_xhnQ19LbBdV6FieZ9OsZxOUBw_fQW4QrUPLDx031b4,1062
|
|
8
|
+
kitty_logger-0.1.0.dist-info/METADATA,sha256=45f9gsBRuGf3qHsIPT3JPzq1HIdfGg9delBSDJgynS8,3214
|
|
9
|
+
kitty_logger-0.1.0.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
|
|
10
|
+
kitty_logger-0.1.0.dist-info/top_level.txt,sha256=OmkC-N3Zg_5NPlFWU9EkTd5eOM2A8qFGzY9ECVi-1Y8,13
|
|
11
|
+
kitty_logger-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
kitty_logger
|