stellarmesh-logging 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stellarmesh_logging-0.1.2/PKG-INFO +51 -0
- stellarmesh_logging-0.1.2/README.md +32 -0
- stellarmesh_logging-0.1.2/pyproject.toml +46 -0
- stellarmesh_logging-0.1.2/setup.cfg +4 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging/__init__.py +59 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging/client.py +495 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging/codec.py +63 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging/contracts.py +235 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging/handler.py +102 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging/logger.py +153 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging/py.typed +1 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging/sanitizer.py +111 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging/transport.py +73 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging.egg-info/PKG-INFO +51 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging.egg-info/SOURCES.txt +19 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging.egg-info/dependency_links.txt +1 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging.egg-info/requires.txt +13 -0
- stellarmesh_logging-0.1.2/src/stellarmesh_logging.egg-info/top_level.txt +1 -0
- stellarmesh_logging-0.1.2/tests/test_client.py +454 -0
- stellarmesh_logging-0.1.2/tests/test_contracts.py +191 -0
- stellarmesh_logging-0.1.2/tests/test_handler.py +244 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stellarmesh-logging
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Stellarmesh logging contract and asynchronous HTTP client
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: httpx<1,>=0.27
|
|
8
|
+
Requires-Dist: pydantic<3,>=2.7
|
|
9
|
+
Provides-Extra: dev
|
|
10
|
+
Requires-Dist: jsonschema<5,>=4; extra == "dev"
|
|
11
|
+
Requires-Dist: mypy<2,>=1.10; extra == "dev"
|
|
12
|
+
Requires-Dist: openapi-spec-validator<1,>=0.7; extra == "dev"
|
|
13
|
+
Requires-Dist: PyYAML<7,>=6; extra == "dev"
|
|
14
|
+
Requires-Dist: pytest<9,>=8; extra == "dev"
|
|
15
|
+
Requires-Dist: pytest-asyncio<2,>=0.23; extra == "dev"
|
|
16
|
+
Requires-Dist: ruff<1,>=0.6; extra == "dev"
|
|
17
|
+
Requires-Dist: types-jsonschema<5,>=4; extra == "dev"
|
|
18
|
+
Requires-Dist: types-PyYAML<7,>=6; extra == "dev"
|
|
19
|
+
|
|
20
|
+
# stellarmesh-logging
|
|
21
|
+
|
|
22
|
+
`stellarmesh-logging` 为 Python 3.11 及以上项目提供 Logging v1 严格模型、标准库 `logging.Handler`、结构化日志门面和有界异步批量 HTTP 客户端。
|
|
23
|
+
|
|
24
|
+
```sh
|
|
25
|
+
python -m pip install stellarmesh-logging==0.1.2
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
import logging
|
|
30
|
+
|
|
31
|
+
from stellarmesh_logging import Client, ClientConfig, StellarmeshHandler
|
|
32
|
+
|
|
33
|
+
client = Client(
|
|
34
|
+
ClientConfig(
|
|
35
|
+
base_url="http://logging-service:8091",
|
|
36
|
+
token="由业务配置层注入",
|
|
37
|
+
service="example-worker",
|
|
38
|
+
)
|
|
39
|
+
)
|
|
40
|
+
logger = logging.getLogger("example")
|
|
41
|
+
logger.addHandler(StellarmeshHandler(client))
|
|
42
|
+
logger.info("job started", extra={"job_id": "job-123"})
|
|
43
|
+
|
|
44
|
+
client.close(timeout=10.0)
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
客户端只在内存中排队日志,不提供本地持久 spool。日志调用成功只表示事件已经进入本地队列;收到 `logging-service` 的合法 `202` 后,才表示 Kafka 或服务端持久 spool 已经确认。网络结果不确定时可能产生重复事件,链路按 at-least-once 边界设计。
|
|
48
|
+
|
|
49
|
+
`service` 必须非空且没有首尾空白。token 只发送给 `logging-service`,metadata 会限制深度、数量和字符串长度,并对规范化后的敏感 key 脱敏。应用退出前应显式调用 `close()` 或 `aclose()`,并通过 `drop_handler` 观测队列满、校验失败、发送失败和关闭超时。
|
|
50
|
+
|
|
51
|
+
完整配置、标准 Handler、trace 传播、重试和关闭语义见项目中的 `docs/sdk/python/README.md`。
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# stellarmesh-logging
|
|
2
|
+
|
|
3
|
+
`stellarmesh-logging` 为 Python 3.11 及以上项目提供 Logging v1 严格模型、标准库 `logging.Handler`、结构化日志门面和有界异步批量 HTTP 客户端。
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
python -m pip install stellarmesh-logging==0.1.2
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
import logging
|
|
11
|
+
|
|
12
|
+
from stellarmesh_logging import Client, ClientConfig, StellarmeshHandler
|
|
13
|
+
|
|
14
|
+
client = Client(
|
|
15
|
+
ClientConfig(
|
|
16
|
+
base_url="http://logging-service:8091",
|
|
17
|
+
token="由业务配置层注入",
|
|
18
|
+
service="example-worker",
|
|
19
|
+
)
|
|
20
|
+
)
|
|
21
|
+
logger = logging.getLogger("example")
|
|
22
|
+
logger.addHandler(StellarmeshHandler(client))
|
|
23
|
+
logger.info("job started", extra={"job_id": "job-123"})
|
|
24
|
+
|
|
25
|
+
client.close(timeout=10.0)
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
客户端只在内存中排队日志,不提供本地持久 spool。日志调用成功只表示事件已经进入本地队列;收到 `logging-service` 的合法 `202` 后,才表示 Kafka 或服务端持久 spool 已经确认。网络结果不确定时可能产生重复事件,链路按 at-least-once 边界设计。
|
|
29
|
+
|
|
30
|
+
`service` 必须非空且没有首尾空白。token 只发送给 `logging-service`,metadata 会限制深度、数量和字符串长度,并对规范化后的敏感 key 脱敏。应用退出前应显式调用 `close()` 或 `aclose()`,并通过 `drop_handler` 观测队列满、校验失败、发送失败和关闭超时。
|
|
31
|
+
|
|
32
|
+
完整配置、标准 Handler、trace 传播、重试和关闭语义见项目中的 `docs/sdk/python/README.md`。
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "stellarmesh-logging"
|
|
7
|
+
version = "0.1.2"
|
|
8
|
+
description = "Stellarmesh logging contract and asynchronous HTTP client"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"httpx>=0.27,<1",
|
|
13
|
+
"pydantic>=2.7,<3",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[project.optional-dependencies]
|
|
17
|
+
dev = [
|
|
18
|
+
"jsonschema>=4,<5",
|
|
19
|
+
"mypy>=1.10,<2",
|
|
20
|
+
"openapi-spec-validator>=0.7,<1",
|
|
21
|
+
"PyYAML>=6,<7",
|
|
22
|
+
"pytest>=8,<9",
|
|
23
|
+
"pytest-asyncio>=0.23,<2",
|
|
24
|
+
"ruff>=0.6,<1",
|
|
25
|
+
"types-jsonschema>=4,<5",
|
|
26
|
+
"types-PyYAML>=6,<7",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[tool.setuptools.package-data]
|
|
30
|
+
stellarmesh_logging = ["py.typed"]
|
|
31
|
+
|
|
32
|
+
[tool.ruff]
|
|
33
|
+
line-length = 88
|
|
34
|
+
target-version = "py311"
|
|
35
|
+
|
|
36
|
+
[tool.ruff.lint]
|
|
37
|
+
select = ["E", "F", "I", "UP", "B", "SIM"]
|
|
38
|
+
|
|
39
|
+
[tool.mypy]
|
|
40
|
+
python_version = "3.11"
|
|
41
|
+
strict = true
|
|
42
|
+
packages = ["stellarmesh_logging"]
|
|
43
|
+
|
|
44
|
+
[tool.pytest.ini_options]
|
|
45
|
+
addopts = "-q"
|
|
46
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Stellarmesh 日志 SDK 的公共 API。"""
|
|
2
|
+
|
|
3
|
+
from .client import Client, ClientConfig, DropHandler, TraceIDProvider
|
|
4
|
+
from .codec import decode_event, encode_event
|
|
5
|
+
from .contracts import (
|
|
6
|
+
LOG_DEAD_LETTER_TOPIC,
|
|
7
|
+
LOG_EVENT_TOPIC,
|
|
8
|
+
MAX_EVENT_JSON_BYTES,
|
|
9
|
+
MAX_HTTP_BODY_BYTES,
|
|
10
|
+
MAX_KAFKA_KEY_VALUE_BYTES,
|
|
11
|
+
MAX_KAFKA_MESSAGE_BYTES,
|
|
12
|
+
BatchIngestRequest,
|
|
13
|
+
DeadLetter,
|
|
14
|
+
IngestRequest,
|
|
15
|
+
IngestResult,
|
|
16
|
+
Level,
|
|
17
|
+
LogEvent,
|
|
18
|
+
OversizeDeadLetter,
|
|
19
|
+
should_emit_level,
|
|
20
|
+
)
|
|
21
|
+
from .handler import StellarmeshHandler
|
|
22
|
+
from .logger import (
|
|
23
|
+
Logger,
|
|
24
|
+
get_logger,
|
|
25
|
+
set_default_client,
|
|
26
|
+
shutdown_logging,
|
|
27
|
+
shutdown_logging_sync,
|
|
28
|
+
)
|
|
29
|
+
from .sanitizer import sanitize_metadata
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"LOG_EVENT_TOPIC",
|
|
33
|
+
"LOG_DEAD_LETTER_TOPIC",
|
|
34
|
+
"MAX_EVENT_JSON_BYTES",
|
|
35
|
+
"MAX_HTTP_BODY_BYTES",
|
|
36
|
+
"MAX_KAFKA_KEY_VALUE_BYTES",
|
|
37
|
+
"MAX_KAFKA_MESSAGE_BYTES",
|
|
38
|
+
"BatchIngestRequest",
|
|
39
|
+
"Client",
|
|
40
|
+
"ClientConfig",
|
|
41
|
+
"DropHandler",
|
|
42
|
+
"DeadLetter",
|
|
43
|
+
"IngestRequest",
|
|
44
|
+
"IngestResult",
|
|
45
|
+
"Level",
|
|
46
|
+
"LogEvent",
|
|
47
|
+
"OversizeDeadLetter",
|
|
48
|
+
"Logger",
|
|
49
|
+
"StellarmeshHandler",
|
|
50
|
+
"TraceIDProvider",
|
|
51
|
+
"decode_event",
|
|
52
|
+
"encode_event",
|
|
53
|
+
"get_logger",
|
|
54
|
+
"sanitize_metadata",
|
|
55
|
+
"set_default_client",
|
|
56
|
+
"should_emit_level",
|
|
57
|
+
"shutdown_logging",
|
|
58
|
+
"shutdown_logging_sync",
|
|
59
|
+
]
|
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
"""面向 Stellarmesh 日志接收服务的非阻塞批量客户端。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import queue
|
|
7
|
+
import random
|
|
8
|
+
import sys
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from contextlib import suppress
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from datetime import UTC, datetime
|
|
15
|
+
from email.utils import parsedate_to_datetime
|
|
16
|
+
from enum import StrEnum
|
|
17
|
+
from typing import Any, TypeAlias, cast
|
|
18
|
+
|
|
19
|
+
import httpx
|
|
20
|
+
|
|
21
|
+
from .codec import encode_event
|
|
22
|
+
from .contracts import (
|
|
23
|
+
MAX_EVENT_JSON_BYTES,
|
|
24
|
+
MAX_HTTP_BODY_BYTES,
|
|
25
|
+
Level,
|
|
26
|
+
LogEvent,
|
|
27
|
+
normalize_level,
|
|
28
|
+
should_emit_level,
|
|
29
|
+
)
|
|
30
|
+
from .transport import BatchTransport
|
|
31
|
+
|
|
32
|
+
TraceIDProvider: TypeAlias = Callable[[], str]
|
|
33
|
+
DropHandler: TypeAlias = Callable[[LogEvent | None, Exception], None]
|
|
34
|
+
_STOP_WORKER = object()
|
|
35
|
+
_MAX_QUEUE_EVENTS = 1_000_000
|
|
36
|
+
_MAX_QUEUE_BYTES = 1 << 30
|
|
37
|
+
_MAX_BATCH_EVENTS = 10_000
|
|
38
|
+
_MAX_ATTEMPTS = 10
|
|
39
|
+
_MAX_DURATION_SECONDS = 3600.0
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _ClientState(StrEnum):
|
|
43
|
+
NEW = "new"
|
|
44
|
+
RUNNING = "running"
|
|
45
|
+
CLOSING = "closing"
|
|
46
|
+
CLOSED = "closed"
|
|
47
|
+
FAILED = "failed"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True, slots=True)
|
|
51
|
+
class ClientConfig:
|
|
52
|
+
"""异步日志客户端配置。"""
|
|
53
|
+
|
|
54
|
+
base_url: str
|
|
55
|
+
token: str
|
|
56
|
+
service: str
|
|
57
|
+
enabled: bool = True
|
|
58
|
+
minimum_level: Level | str = Level.INFO
|
|
59
|
+
timeout_seconds: float = 7.0
|
|
60
|
+
queue_size: int = 4096
|
|
61
|
+
queue_bytes: int = 16 << 20
|
|
62
|
+
batch_size: int = 128
|
|
63
|
+
flush_interval_ms: int = 100
|
|
64
|
+
max_body_bytes: int = MAX_HTTP_BODY_BYTES
|
|
65
|
+
max_attempts: int = 3
|
|
66
|
+
initial_backoff_seconds: float = 0.1
|
|
67
|
+
max_backoff_seconds: float = 1.0
|
|
68
|
+
max_retry_after_seconds: float = 30.0
|
|
69
|
+
trace_id_provider: TraceIDProvider | None = None
|
|
70
|
+
drop_handler: DropHandler | None = None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True, slots=True)
|
|
74
|
+
class _QueuedEvent:
|
|
75
|
+
event: LogEvent
|
|
76
|
+
bytes: int
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class Client:
|
|
80
|
+
"""在本地排队事件,并由工作线程通过有界 HTTP 批次发送。"""
|
|
81
|
+
|
|
82
|
+
def __init__(
|
|
83
|
+
self,
|
|
84
|
+
config: ClientConfig,
|
|
85
|
+
*,
|
|
86
|
+
transport: httpx.BaseTransport | None = None,
|
|
87
|
+
) -> None:
|
|
88
|
+
_validate_config(config)
|
|
89
|
+
self.config = config
|
|
90
|
+
self._minimum_level = normalize_level(config.minimum_level)
|
|
91
|
+
self._batch_size = config.batch_size
|
|
92
|
+
self._flush_interval = config.flush_interval_ms / 1000
|
|
93
|
+
self._max_body_bytes = config.max_body_bytes
|
|
94
|
+
self._max_attempts = config.max_attempts
|
|
95
|
+
self._initial_backoff = config.initial_backoff_seconds
|
|
96
|
+
self._max_backoff = config.max_backoff_seconds
|
|
97
|
+
self._max_retry_after = config.max_retry_after_seconds
|
|
98
|
+
self._transport = BatchTransport(
|
|
99
|
+
base_url=config.base_url,
|
|
100
|
+
token=config.token,
|
|
101
|
+
timeout_seconds=config.timeout_seconds,
|
|
102
|
+
transport=transport,
|
|
103
|
+
)
|
|
104
|
+
self._queue: queue.Queue[_QueuedEvent | object] = queue.Queue(
|
|
105
|
+
maxsize=config.queue_size
|
|
106
|
+
)
|
|
107
|
+
self._state_lock = threading.Lock()
|
|
108
|
+
self._fallback_lock = threading.Lock()
|
|
109
|
+
self._stop_requested = threading.Event()
|
|
110
|
+
self._abort_requested = threading.Event()
|
|
111
|
+
self._worker_done = threading.Event()
|
|
112
|
+
self._worker: threading.Thread | None = None
|
|
113
|
+
self._state = _ClientState.NEW
|
|
114
|
+
self._failure: Exception | None = None
|
|
115
|
+
self._last_fallback_warning = 0.0
|
|
116
|
+
self._queued_events = 0
|
|
117
|
+
self._queued_bytes = 0
|
|
118
|
+
|
|
119
|
+
def emit_event(
|
|
120
|
+
self,
|
|
121
|
+
level: Level | str,
|
|
122
|
+
*,
|
|
123
|
+
message: str,
|
|
124
|
+
trace_id: str | None = None,
|
|
125
|
+
metadata: dict[str, Any] | None = None,
|
|
126
|
+
timestamp: datetime | None = None,
|
|
127
|
+
service: str | None = None,
|
|
128
|
+
) -> bool:
|
|
129
|
+
"""构造并入队一个事件,不等待远端投递。"""
|
|
130
|
+
try:
|
|
131
|
+
normalized_level = normalize_level(level)
|
|
132
|
+
resolved_trace_id = trace_id
|
|
133
|
+
if resolved_trace_id is None:
|
|
134
|
+
provider = self.config.trace_id_provider
|
|
135
|
+
resolved_trace_id = provider() if provider is not None else ""
|
|
136
|
+
event = LogEvent(
|
|
137
|
+
timestamp=timestamp or datetime.now(UTC),
|
|
138
|
+
level=normalized_level,
|
|
139
|
+
service=service or self.config.service,
|
|
140
|
+
message=message,
|
|
141
|
+
trace_id=resolved_trace_id,
|
|
142
|
+
metadata=metadata or {},
|
|
143
|
+
)
|
|
144
|
+
except Exception as exc: # noqa: BLE001 - provider 异常需要隔离。
|
|
145
|
+
self._drop(None, exc)
|
|
146
|
+
return False
|
|
147
|
+
return self.enqueue(event)
|
|
148
|
+
|
|
149
|
+
def enqueue(self, event: LogEvent) -> bool:
|
|
150
|
+
"""将已校验的事件加入队列。"""
|
|
151
|
+
if not self.config.enabled or not should_emit_level(
|
|
152
|
+
event.level, self._minimum_level
|
|
153
|
+
):
|
|
154
|
+
return False
|
|
155
|
+
try:
|
|
156
|
+
snapshot = event.model_copy(deep=True)
|
|
157
|
+
event_bytes = len(encode_event(snapshot))
|
|
158
|
+
except Exception as exc: # noqa: BLE001 - 序列化异常需要隔离。
|
|
159
|
+
self._drop(event, exc)
|
|
160
|
+
return False
|
|
161
|
+
if event_bytes > MAX_EVENT_JSON_BYTES:
|
|
162
|
+
self._drop(event, ValueError("logging event exceeds the contract limit"))
|
|
163
|
+
return False
|
|
164
|
+
|
|
165
|
+
failure: Exception | None = None
|
|
166
|
+
with self._state_lock:
|
|
167
|
+
if self._state in {
|
|
168
|
+
_ClientState.CLOSING,
|
|
169
|
+
_ClientState.CLOSED,
|
|
170
|
+
_ClientState.FAILED,
|
|
171
|
+
}:
|
|
172
|
+
failure = self._failure or RuntimeError(
|
|
173
|
+
f"logging client is {self._state.value}"
|
|
174
|
+
)
|
|
175
|
+
elif (
|
|
176
|
+
self._queued_events >= self.config.queue_size
|
|
177
|
+
or event_bytes > self.config.queue_bytes - self._queued_bytes
|
|
178
|
+
):
|
|
179
|
+
failure = RuntimeError("logging client queue is full")
|
|
180
|
+
else:
|
|
181
|
+
try:
|
|
182
|
+
self._queue.put_nowait(_QueuedEvent(snapshot, event_bytes))
|
|
183
|
+
except queue.Full:
|
|
184
|
+
failure = RuntimeError("logging client queue is full")
|
|
185
|
+
else:
|
|
186
|
+
self._queued_events += 1
|
|
187
|
+
self._queued_bytes += event_bytes
|
|
188
|
+
self._ensure_worker_locked()
|
|
189
|
+
if failure is not None:
|
|
190
|
+
self._drop(event, failure)
|
|
191
|
+
return False
|
|
192
|
+
return True
|
|
193
|
+
|
|
194
|
+
def close(self, *, timeout: float = 2.0) -> bool:
|
|
195
|
+
"""停止接收事件,并等待队列中的投递完成。"""
|
|
196
|
+
worker = self._request_close()
|
|
197
|
+
if worker is None or worker is threading.current_thread():
|
|
198
|
+
return self._state_snapshot() is not _ClientState.FAILED
|
|
199
|
+
worker.join(timeout=max(timeout, 0.0))
|
|
200
|
+
if worker.is_alive():
|
|
201
|
+
self._abort_requested.set()
|
|
202
|
+
self._fallback_warning(
|
|
203
|
+
f"logging client drain timed out; remaining={self._pending_count()}"
|
|
204
|
+
)
|
|
205
|
+
return False
|
|
206
|
+
return self._state_snapshot() is _ClientState.CLOSED
|
|
207
|
+
|
|
208
|
+
async def aclose(self, *, timeout: float = 2.0) -> bool:
|
|
209
|
+
"""在不阻塞 asyncio 事件循环的情况下排空队列。"""
|
|
210
|
+
worker = self._request_close()
|
|
211
|
+
if worker is None or worker is threading.current_thread():
|
|
212
|
+
return self._state_snapshot() is not _ClientState.FAILED
|
|
213
|
+
loop = asyncio.get_running_loop()
|
|
214
|
+
deadline = loop.time() + max(timeout, 0.0)
|
|
215
|
+
while worker.is_alive():
|
|
216
|
+
remaining = deadline - loop.time()
|
|
217
|
+
if remaining <= 0:
|
|
218
|
+
self._abort_requested.set()
|
|
219
|
+
self._fallback_warning(
|
|
220
|
+
f"logging client drain timed out; remaining={self._pending_count()}"
|
|
221
|
+
)
|
|
222
|
+
return False
|
|
223
|
+
await asyncio.sleep(min(0.05, remaining))
|
|
224
|
+
return self._state_snapshot() is _ClientState.CLOSED
|
|
225
|
+
|
|
226
|
+
def _ensure_worker_locked(self) -> None:
|
|
227
|
+
if self._worker is not None:
|
|
228
|
+
return
|
|
229
|
+
self._state = _ClientState.RUNNING
|
|
230
|
+
self._worker = threading.Thread(
|
|
231
|
+
target=self._worker_loop,
|
|
232
|
+
name="stellarmesh-logging-client",
|
|
233
|
+
daemon=True,
|
|
234
|
+
)
|
|
235
|
+
self._worker.start()
|
|
236
|
+
|
|
237
|
+
def _request_close(self) -> threading.Thread | None:
|
|
238
|
+
with self._state_lock:
|
|
239
|
+
if self._state is _ClientState.NEW:
|
|
240
|
+
self._state = _ClientState.CLOSED
|
|
241
|
+
self._transport.close()
|
|
242
|
+
self._worker_done.set()
|
|
243
|
+
return None
|
|
244
|
+
if self._state is _ClientState.RUNNING:
|
|
245
|
+
self._state = _ClientState.CLOSING
|
|
246
|
+
self._stop_requested.set()
|
|
247
|
+
with suppress(queue.Full):
|
|
248
|
+
self._queue.put_nowait(_STOP_WORKER)
|
|
249
|
+
return self._worker
|
|
250
|
+
|
|
251
|
+
def _worker_loop(self) -> None:
|
|
252
|
+
failure: Exception | None = None
|
|
253
|
+
try:
|
|
254
|
+
self._run_worker()
|
|
255
|
+
except Exception as exc: # noqa: BLE001 - 隔离日志工作线程故障。
|
|
256
|
+
failure = exc
|
|
257
|
+
self._fallback_warning(f"logging worker failed: {exc}")
|
|
258
|
+
finally:
|
|
259
|
+
self._transport.close()
|
|
260
|
+
if failure is not None:
|
|
261
|
+
self._drain_failed_queue(failure)
|
|
262
|
+
with self._state_lock:
|
|
263
|
+
if failure is None:
|
|
264
|
+
self._state = _ClientState.CLOSED
|
|
265
|
+
else:
|
|
266
|
+
self._state = _ClientState.FAILED
|
|
267
|
+
self._failure = failure
|
|
268
|
+
self._worker_done.set()
|
|
269
|
+
|
|
270
|
+
def _run_worker(self) -> None:
|
|
271
|
+
while True:
|
|
272
|
+
if self._stop_requested.is_set() and self._queue.empty():
|
|
273
|
+
return
|
|
274
|
+
try:
|
|
275
|
+
first = self._queue.get(timeout=0.1)
|
|
276
|
+
except queue.Empty:
|
|
277
|
+
continue
|
|
278
|
+
if first is _STOP_WORKER:
|
|
279
|
+
self._queue.task_done()
|
|
280
|
+
continue
|
|
281
|
+
|
|
282
|
+
batch = [cast(_QueuedEvent, first)]
|
|
283
|
+
deadline = time.monotonic() + self._flush_interval
|
|
284
|
+
while len(batch) < self._batch_size:
|
|
285
|
+
try:
|
|
286
|
+
if self._stop_requested.is_set():
|
|
287
|
+
queued = self._queue.get_nowait()
|
|
288
|
+
else:
|
|
289
|
+
remaining = deadline - time.monotonic()
|
|
290
|
+
if remaining <= 0:
|
|
291
|
+
break
|
|
292
|
+
queued = self._queue.get(timeout=remaining)
|
|
293
|
+
except queue.Empty:
|
|
294
|
+
break
|
|
295
|
+
if queued is _STOP_WORKER:
|
|
296
|
+
self._queue.task_done()
|
|
297
|
+
break
|
|
298
|
+
batch.append(cast(_QueuedEvent, queued))
|
|
299
|
+
|
|
300
|
+
try:
|
|
301
|
+
self._send_batch([item.event for item in batch])
|
|
302
|
+
finally:
|
|
303
|
+
for item in batch:
|
|
304
|
+
self._queue.task_done()
|
|
305
|
+
self._release(item)
|
|
306
|
+
|
|
307
|
+
def _send_batch(self, events: list[LogEvent]) -> bool:
|
|
308
|
+
try:
|
|
309
|
+
payload = self._transport.encode(events)
|
|
310
|
+
except Exception as exc: # noqa: BLE001 - 隔离序列化故障。
|
|
311
|
+
for event in events:
|
|
312
|
+
self._drop(event, exc)
|
|
313
|
+
return False
|
|
314
|
+
if len(payload) > self._max_body_bytes:
|
|
315
|
+
if len(events) == 1:
|
|
316
|
+
self._drop(
|
|
317
|
+
events[0],
|
|
318
|
+
ValueError("logging event exceeds the client body limit"),
|
|
319
|
+
)
|
|
320
|
+
return False
|
|
321
|
+
midpoint = len(events) // 2
|
|
322
|
+
left_sent = self._send_batch(events[:midpoint])
|
|
323
|
+
right_sent = self._send_batch(events[midpoint:])
|
|
324
|
+
return left_sent and right_sent
|
|
325
|
+
|
|
326
|
+
last_error: Exception | None = None
|
|
327
|
+
for attempt in range(1, self._max_attempts + 1):
|
|
328
|
+
try:
|
|
329
|
+
self._transport.send(events, payload)
|
|
330
|
+
except Exception as exc: # noqa: BLE001 - 日志不能中断调用方。
|
|
331
|
+
last_error = exc
|
|
332
|
+
if not _retryable_error(exc) or attempt == self._max_attempts:
|
|
333
|
+
break
|
|
334
|
+
retry_after = _retry_after_delay(exc, self._max_retry_after)
|
|
335
|
+
if self._abort_requested.wait(
|
|
336
|
+
max(self._retry_delay(attempt), retry_after)
|
|
337
|
+
):
|
|
338
|
+
break
|
|
339
|
+
else:
|
|
340
|
+
return True
|
|
341
|
+
assert last_error is not None
|
|
342
|
+
for event in events:
|
|
343
|
+
self._drop(event, last_error)
|
|
344
|
+
return False
|
|
345
|
+
|
|
346
|
+
def _drain_failed_queue(self, failure: Exception) -> None:
|
|
347
|
+
while True:
|
|
348
|
+
try:
|
|
349
|
+
queued = self._queue.get_nowait()
|
|
350
|
+
except queue.Empty:
|
|
351
|
+
return
|
|
352
|
+
try:
|
|
353
|
+
if queued is not _STOP_WORKER:
|
|
354
|
+
item = cast(_QueuedEvent, queued)
|
|
355
|
+
self._drop(item.event, failure)
|
|
356
|
+
self._release(item)
|
|
357
|
+
finally:
|
|
358
|
+
self._queue.task_done()
|
|
359
|
+
|
|
360
|
+
def _drop(self, event: LogEvent | None, exc: Exception) -> None:
|
|
361
|
+
handler = self.config.drop_handler
|
|
362
|
+
if handler is None:
|
|
363
|
+
self._fallback_warning(str(exc))
|
|
364
|
+
return
|
|
365
|
+
try:
|
|
366
|
+
handler(event, exc)
|
|
367
|
+
except Exception as callback_error: # noqa: BLE001 - callback 异常需要隔离。
|
|
368
|
+
self._fallback_warning(f"logging drop handler failed: {callback_error}")
|
|
369
|
+
|
|
370
|
+
def _fallback_warning(self, message: str) -> None:
|
|
371
|
+
with self._fallback_lock:
|
|
372
|
+
now = time.monotonic()
|
|
373
|
+
if (
|
|
374
|
+
self._last_fallback_warning > 0
|
|
375
|
+
and now - self._last_fallback_warning < 30
|
|
376
|
+
):
|
|
377
|
+
return
|
|
378
|
+
self._last_fallback_warning = now
|
|
379
|
+
print(f"[stellarmesh-logging-fallback] {message}", file=sys.stderr)
|
|
380
|
+
|
|
381
|
+
def _state_snapshot(self) -> _ClientState:
|
|
382
|
+
with self._state_lock:
|
|
383
|
+
return self._state
|
|
384
|
+
|
|
385
|
+
def _pending_count(self) -> int:
|
|
386
|
+
with self._state_lock:
|
|
387
|
+
return self._queued_events
|
|
388
|
+
|
|
389
|
+
def _release(self, item: _QueuedEvent) -> None:
|
|
390
|
+
with self._state_lock:
|
|
391
|
+
self._queued_events -= 1
|
|
392
|
+
self._queued_bytes -= item.bytes
|
|
393
|
+
|
|
394
|
+
def _retry_delay(self, failed_attempt: int) -> float:
|
|
395
|
+
delay = min(
|
|
396
|
+
self._initial_backoff * (2 ** (failed_attempt - 1)),
|
|
397
|
+
self._max_backoff,
|
|
398
|
+
)
|
|
399
|
+
return random.uniform(0.0, delay)
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _validate_config(config: ClientConfig) -> None:
|
|
403
|
+
try:
|
|
404
|
+
url = httpx.URL(config.base_url)
|
|
405
|
+
except Exception as exc: # noqa: BLE001 - 统一配置错误。
|
|
406
|
+
raise ValueError("logging base URL is invalid") from exc
|
|
407
|
+
if url.scheme not in {"http", "https"} or not url.host:
|
|
408
|
+
raise ValueError("logging base URL must be an absolute HTTP or HTTPS URL")
|
|
409
|
+
if not config.token.strip():
|
|
410
|
+
raise ValueError("logging service token is required")
|
|
411
|
+
if not config.service.strip() or config.service != config.service.strip():
|
|
412
|
+
raise ValueError("logging service name must be non-empty and trimmed")
|
|
413
|
+
if config.timeout_seconds <= 0:
|
|
414
|
+
raise ValueError("logging timeout_seconds must be positive")
|
|
415
|
+
if config.queue_size <= 0:
|
|
416
|
+
raise ValueError("logging queue_size must be positive")
|
|
417
|
+
if config.queue_bytes <= 0:
|
|
418
|
+
raise ValueError("logging queue_bytes must be positive")
|
|
419
|
+
if config.batch_size <= 0:
|
|
420
|
+
raise ValueError("logging batch_size must be positive")
|
|
421
|
+
if config.flush_interval_ms <= 0:
|
|
422
|
+
raise ValueError("logging flush_interval_ms must be positive")
|
|
423
|
+
if config.max_body_bytes <= 0:
|
|
424
|
+
raise ValueError("logging max_body_bytes must be positive")
|
|
425
|
+
if config.max_body_bytes > MAX_HTTP_BODY_BYTES:
|
|
426
|
+
raise ValueError(
|
|
427
|
+
f"logging max_body_bytes must not exceed {MAX_HTTP_BODY_BYTES}"
|
|
428
|
+
)
|
|
429
|
+
if config.max_attempts <= 0:
|
|
430
|
+
raise ValueError("logging max_attempts must be positive")
|
|
431
|
+
if config.initial_backoff_seconds <= 0:
|
|
432
|
+
raise ValueError("logging initial_backoff_seconds must be positive")
|
|
433
|
+
if config.max_backoff_seconds <= 0:
|
|
434
|
+
raise ValueError("logging max_backoff_seconds must be positive")
|
|
435
|
+
if config.max_retry_after_seconds <= 0:
|
|
436
|
+
raise ValueError("logging max_retry_after_seconds must be positive")
|
|
437
|
+
if config.initial_backoff_seconds > config.max_backoff_seconds:
|
|
438
|
+
raise ValueError(
|
|
439
|
+
"logging initial_backoff_seconds must not exceed max_backoff_seconds"
|
|
440
|
+
)
|
|
441
|
+
if config.max_retry_after_seconds < config.max_backoff_seconds:
|
|
442
|
+
raise ValueError(
|
|
443
|
+
"logging max_retry_after_seconds must not be less than max_backoff_seconds"
|
|
444
|
+
)
|
|
445
|
+
if config.queue_size > _MAX_QUEUE_EVENTS:
|
|
446
|
+
raise ValueError("logging queue_size is outside supported bounds")
|
|
447
|
+
if config.queue_bytes > _MAX_QUEUE_BYTES:
|
|
448
|
+
raise ValueError("logging queue_bytes is outside supported bounds")
|
|
449
|
+
if config.batch_size > _MAX_BATCH_EVENTS:
|
|
450
|
+
raise ValueError("logging batch_size is outside supported bounds")
|
|
451
|
+
if config.max_attempts > _MAX_ATTEMPTS:
|
|
452
|
+
raise ValueError("logging max_attempts is outside supported bounds")
|
|
453
|
+
if any(
|
|
454
|
+
duration > _MAX_DURATION_SECONDS
|
|
455
|
+
for duration in (
|
|
456
|
+
config.timeout_seconds,
|
|
457
|
+
config.flush_interval_ms / 1000,
|
|
458
|
+
config.initial_backoff_seconds,
|
|
459
|
+
config.max_backoff_seconds,
|
|
460
|
+
config.max_retry_after_seconds,
|
|
461
|
+
)
|
|
462
|
+
):
|
|
463
|
+
raise ValueError("logging client duration is outside supported bounds")
|
|
464
|
+
normalize_level(config.minimum_level)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _retryable_error(exc: Exception) -> bool:
|
|
468
|
+
if isinstance(exc, httpx.TransportError):
|
|
469
|
+
return True
|
|
470
|
+
if isinstance(exc, httpx.HTTPStatusError):
|
|
471
|
+
return exc.response.status_code in {408, 425, 429, 500, 502, 503, 504}
|
|
472
|
+
return False
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def _retry_after_delay(exc: Exception, maximum: float) -> float:
|
|
476
|
+
if not isinstance(exc, httpx.HTTPStatusError):
|
|
477
|
+
return 0.0
|
|
478
|
+
value = exc.response.headers.get("Retry-After", "").strip()
|
|
479
|
+
if not value:
|
|
480
|
+
return 0.0
|
|
481
|
+
try:
|
|
482
|
+
seconds = int(value)
|
|
483
|
+
except ValueError:
|
|
484
|
+
try:
|
|
485
|
+
when = parsedate_to_datetime(value)
|
|
486
|
+
except (TypeError, ValueError, OverflowError):
|
|
487
|
+
return 0.0
|
|
488
|
+
if when.tzinfo is None:
|
|
489
|
+
return 0.0
|
|
490
|
+
seconds_value = (when - datetime.now(UTC)).total_seconds()
|
|
491
|
+
else:
|
|
492
|
+
seconds_value = float(seconds)
|
|
493
|
+
if seconds_value <= 0:
|
|
494
|
+
return 0.0
|
|
495
|
+
return min(seconds_value, maximum)
|