logs-sdk 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- logs_sdk/__init__.py +15 -0
- logs_sdk/buffer.py +42 -0
- logs_sdk/client.py +92 -0
- logs_sdk/middleware.py +142 -0
- logs_sdk/offline.py +57 -0
- logs_sdk/types.py +120 -0
- logs_sdk-0.3.0.dist-info/METADATA +151 -0
- logs_sdk-0.3.0.dist-info/RECORD +9 -0
- logs_sdk-0.3.0.dist-info/WHEEL +4 -0
logs_sdk/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""日志管理平台 Python SDK — FastAPI/Starlette/Flask 中间件,一行代码接入日志采集。
|
|
2
|
+
|
|
3
|
+
使用方法:
|
|
4
|
+
from logs_sdk import LogSDK
|
|
5
|
+
logger = LogSDK(endpoint="https://api.logs.codexs.cn/api/v1/ingest/logs",
|
|
6
|
+
api_key="clog_pk_xxx", api_secret="clog_sk_xxx",
|
|
7
|
+
project_slug="my-project")
|
|
8
|
+
app.add_middleware(logger.fastapi_middleware) # FastAPI
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from .client import LogSDK
|
|
12
|
+
from .types import LogEntry, LogConfig
|
|
13
|
+
|
|
14
|
+
__version__ = "0.3.0"
|
|
15
|
+
__all__ = ["LogSDK", "LogEntry", "LogConfig"]
|
logs_sdk/buffer.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""环形缓冲区 — 80% 自动触发 flush"""
|
|
2
|
+
import threading
|
|
3
|
+
from typing import List, Callable
|
|
4
|
+
from .types import LogEntry
|
|
5
|
+
|
|
6
|
+
class RingBuffer:
|
|
7
|
+
def __init__(self, capacity: int, flush_fn: Callable):
|
|
8
|
+
self.capacity = max(capacity, 100)
|
|
9
|
+
self.buf = [None] * self.capacity
|
|
10
|
+
self.head = 0
|
|
11
|
+
self.tail = 0
|
|
12
|
+
self.count = 0
|
|
13
|
+
self.lock = threading.Lock()
|
|
14
|
+
self.flush_fn = flush_fn
|
|
15
|
+
|
|
16
|
+
def push(self, entry: LogEntry):
|
|
17
|
+
with self.lock:
|
|
18
|
+
self.buf[self.head] = entry
|
|
19
|
+
self.head = (self.head + 1) % self.capacity
|
|
20
|
+
self.count += 1
|
|
21
|
+
if self.count >= self.capacity * 0.8:
|
|
22
|
+
entries = self._drain()
|
|
23
|
+
if self.flush_fn:
|
|
24
|
+
self.flush_fn(entries)
|
|
25
|
+
|
|
26
|
+
def flush(self) -> List[LogEntry]:
|
|
27
|
+
with self.lock:
|
|
28
|
+
return self._drain()
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def length(self) -> int:
|
|
32
|
+
return self.count
|
|
33
|
+
|
|
34
|
+
def _drain(self) -> List[LogEntry]:
|
|
35
|
+
entries = []
|
|
36
|
+
while self.count > 0:
|
|
37
|
+
if self.buf[self.tail]:
|
|
38
|
+
entries.append(self.buf[self.tail])
|
|
39
|
+
self.buf[self.tail] = None
|
|
40
|
+
self.tail = (self.tail + 1) % self.capacity
|
|
41
|
+
self.count -= 1
|
|
42
|
+
return entries
|
logs_sdk/client.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""核心客户端 — 缓冲管理、定时刷新、HTTP 上报、离线缓存"""
|
|
2
|
+
import json, time, os, socket, threading, logging
|
|
3
|
+
import httpx
|
|
4
|
+
from .types import LogConfig, LogEntry, new_uuid
|
|
5
|
+
from .buffer import RingBuffer
|
|
6
|
+
from .offline import OfflineCache
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger("logs-sdk")
|
|
9
|
+
|
|
10
|
+
class LogSDK:
|
|
11
|
+
def __init__(self, **kwargs):
|
|
12
|
+
self.config = LogConfig(**{k: v for k, v in kwargs.items() if k in LogConfig.__dataclass_fields__})
|
|
13
|
+
self.hostname = socket.gethostname()
|
|
14
|
+
self.pid = str(os.getpid())
|
|
15
|
+
self.offline_cache = OfflineCache()
|
|
16
|
+
self._closed = False
|
|
17
|
+
self._lock = threading.Lock()
|
|
18
|
+
|
|
19
|
+
self.buffer = RingBuffer(self.config.buffer_size, self._flush_entries)
|
|
20
|
+
self._timer: threading.Timer | None = None
|
|
21
|
+
self._start_flush_timer()
|
|
22
|
+
|
|
23
|
+
def send(self, entry: LogEntry):
|
|
24
|
+
"""异步发送一条日志"""
|
|
25
|
+
if self._closed:
|
|
26
|
+
logger.warning("Client 已关闭,日志丢弃")
|
|
27
|
+
return
|
|
28
|
+
entry.host = self.hostname
|
|
29
|
+
entry.process_id = self.pid
|
|
30
|
+
entry.environment = self.config.environment
|
|
31
|
+
entry.project_slug = self.config.project_slug
|
|
32
|
+
entry.service_name = self.config.service_name
|
|
33
|
+
entry.timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
34
|
+
self.buffer.push(entry)
|
|
35
|
+
|
|
36
|
+
def close(self):
|
|
37
|
+
"""优雅关闭"""
|
|
38
|
+
self._closed = True
|
|
39
|
+
if self._timer:
|
|
40
|
+
self._timer.cancel()
|
|
41
|
+
remaining = self.buffer.flush()
|
|
42
|
+
if remaining:
|
|
43
|
+
try:
|
|
44
|
+
self._send_batch(remaining)
|
|
45
|
+
except Exception as e:
|
|
46
|
+
logger.error(f"关闭时上报失败: {e}")
|
|
47
|
+
self.offline_cache.save(remaining)
|
|
48
|
+
self.offline_cache.flush_all(self._send_batch)
|
|
49
|
+
|
|
50
|
+
def fastapi_middleware(self):
|
|
51
|
+
"""返回 FastAPI/Starlette 中间件"""
|
|
52
|
+
from .middleware import FastAPIMiddleware
|
|
53
|
+
return FastAPIMiddleware(self)
|
|
54
|
+
|
|
55
|
+
def flask_middleware(self):
|
|
56
|
+
"""返回 Flask 中间件(预留)"""
|
|
57
|
+
raise NotImplementedError("Flask 中间件将在下一个版本支持")
|
|
58
|
+
|
|
59
|
+
def _flush_entries(self, entries):
|
|
60
|
+
"""异步发送(在单独线程中执行)"""
|
|
61
|
+
threading.Thread(target=self._flush_sync, args=(entries,), daemon=True).start()
|
|
62
|
+
|
|
63
|
+
def _flush_sync(self, entries):
|
|
64
|
+
for attempt in range(self.config.max_retries + 1):
|
|
65
|
+
try:
|
|
66
|
+
self._send_batch(entries)
|
|
67
|
+
return
|
|
68
|
+
except Exception as e:
|
|
69
|
+
if attempt == self.config.max_retries:
|
|
70
|
+
logger.error(f"上报失败(重试{self.config.max_retries}次): {e}")
|
|
71
|
+
self.offline_cache.save(entries)
|
|
72
|
+
else:
|
|
73
|
+
time.sleep(0.5 * (2 ** attempt))
|
|
74
|
+
|
|
75
|
+
def _send_batch(self, entries):
|
|
76
|
+
body = json.dumps({"logs": [e.to_dict() for e in entries]})
|
|
77
|
+
resp = httpx.post(self.config.endpoint, content=body,
|
|
78
|
+
headers={"Content-Type": "application/json", "X-API-Key": self.config.api_key,
|
|
79
|
+
"X-SDK-Type": "python", "X-SDK-Version": "0.3.0"},
|
|
80
|
+
timeout=15)
|
|
81
|
+
if resp.status_code not in (200, 201):
|
|
82
|
+
raise Exception(f"服务端返回 {resp.status_code}")
|
|
83
|
+
|
|
84
|
+
def _start_flush_timer(self):
|
|
85
|
+
def _loop():
|
|
86
|
+
while not self._closed:
|
|
87
|
+
time.sleep(self.config.flush_interval)
|
|
88
|
+
if self._closed: break
|
|
89
|
+
entries = self.buffer.flush()
|
|
90
|
+
if entries:
|
|
91
|
+
self._flush_entries(entries)
|
|
92
|
+
threading.Thread(target=_loop, daemon=True).start()
|
logs_sdk/middleware.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""FastAPI/Starlette/Flask 中间件 — 自动采集所有 HTTP 请求日志"""
|
|
2
|
+
import time, traceback, sys, json
|
|
3
|
+
from .types import LogEntry, new_uuid, detect_client_type, detect_origin, sanitize_headers, extract_api_version
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class FastAPIMiddleware:
|
|
7
|
+
"""Starlette BaseHTTPMiddleware 实现,兼容 FastAPI"""
|
|
8
|
+
|
|
9
|
+
def __init__(self, sdk):
|
|
10
|
+
self.sdk = sdk
|
|
11
|
+
|
|
12
|
+
async def dispatch(self, request, call_next):
|
|
13
|
+
entry = LogEntry()
|
|
14
|
+
entry.uuid = new_uuid()
|
|
15
|
+
entry.request_id = entry.uuid[:8]
|
|
16
|
+
entry.trace_id = entry.uuid
|
|
17
|
+
entry.span_id = entry.uuid
|
|
18
|
+
start = time.time()
|
|
19
|
+
|
|
20
|
+
# 读取请求体(缓存以支持多次读取)
|
|
21
|
+
body_bytes = b""
|
|
22
|
+
try:
|
|
23
|
+
body_bytes = await request.body()
|
|
24
|
+
except Exception:
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
# 请求信息
|
|
28
|
+
headers = dict(request.headers)
|
|
29
|
+
entry.method = request.method
|
|
30
|
+
entry.scheme = request.url.scheme
|
|
31
|
+
entry.full_url = str(request.url)
|
|
32
|
+
entry.host_header = request.headers.get("host", "")
|
|
33
|
+
entry.path = request.url.path
|
|
34
|
+
entry.query_string = request.url.query
|
|
35
|
+
entry.content_type = request.headers.get("content-type", "")
|
|
36
|
+
entry.user_agent = request.headers.get("user-agent", "")
|
|
37
|
+
entry.client_ip = headers.get("x-forwarded-for", request.client.host if request.client else "")
|
|
38
|
+
entry.client_ip_chain = headers.get("x-forwarded-for", "")
|
|
39
|
+
entry.client_type = detect_client_type(entry.user_agent, headers)
|
|
40
|
+
entry.origin = detect_origin(headers, entry.user_agent)
|
|
41
|
+
entry.request_headers = sanitize_headers(headers)
|
|
42
|
+
entry.request_body = body_bytes[:self.sdk.config.max_body_size].decode("utf-8", errors="replace")
|
|
43
|
+
entry.request_body_size = len(body_bytes)
|
|
44
|
+
entry.referer = request.headers.get("referer", "")
|
|
45
|
+
entry.trace_id = headers.get("x-trace-id", entry.uuid)
|
|
46
|
+
entry.parent_span_id = headers.get("x-parent-span-id", "")
|
|
47
|
+
entry.user_id = headers.get("x-user-id", "")
|
|
48
|
+
entry.session_id = headers.get("x-session-id", "")
|
|
49
|
+
entry.api_version = extract_api_version(entry.path)
|
|
50
|
+
entry.proto = request.scope.get("http_version", "1.1")
|
|
51
|
+
entry.tls_version = getattr(request.scope.get("transport"), "get_extra_info", lambda x: "")(f"tls_version") or ""
|
|
52
|
+
|
|
53
|
+
# 执行业务
|
|
54
|
+
try:
|
|
55
|
+
response = await call_next(request)
|
|
56
|
+
except Exception as e:
|
|
57
|
+
entry.is_error = True
|
|
58
|
+
entry.error_type = "panic"
|
|
59
|
+
entry.error_message = str(e)
|
|
60
|
+
entry.error_stack = traceback.format_exc()
|
|
61
|
+
raise
|
|
62
|
+
finally:
|
|
63
|
+
entry.duration_ms = int((time.time() - start) * 1000)
|
|
64
|
+
|
|
65
|
+
# 响应信息
|
|
66
|
+
entry.status_code = getattr(response, "status_code", 200)
|
|
67
|
+
resp_headers = dict(getattr(response, "headers", {}))
|
|
68
|
+
entry.response_headers = sanitize_headers(resp_headers)
|
|
69
|
+
entry.response_body_size = int(resp_headers.get("content-length", 0)) if resp_headers.get("content-length") else 0
|
|
70
|
+
|
|
71
|
+
if entry.status_code >= 500:
|
|
72
|
+
entry.is_error = True
|
|
73
|
+
entry.error_type = "http_error"
|
|
74
|
+
|
|
75
|
+
self.sdk.send(entry)
|
|
76
|
+
return response
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class FlaskMiddleware:
|
|
80
|
+
"""Flask WSGI 中间件"""
|
|
81
|
+
|
|
82
|
+
def __init__(self, sdk):
|
|
83
|
+
self.sdk = sdk
|
|
84
|
+
|
|
85
|
+
def __call__(self, environ, start_response):
|
|
86
|
+
entry = LogEntry()
|
|
87
|
+
entry.uuid = new_uuid()
|
|
88
|
+
entry.request_id = entry.uuid[:8]
|
|
89
|
+
entry.trace_id = entry.uuid
|
|
90
|
+
entry.span_id = entry.uuid
|
|
91
|
+
start = time.time()
|
|
92
|
+
|
|
93
|
+
entry.method = environ.get("REQUEST_METHOD", "")
|
|
94
|
+
entry.scheme = environ.get("wsgi.url_scheme", "http")
|
|
95
|
+
entry.host_header = environ.get("HTTP_HOST", "")
|
|
96
|
+
entry.path = environ.get("PATH_INFO", "")
|
|
97
|
+
entry.query_string = environ.get("QUERY_STRING", "")
|
|
98
|
+
entry.full_url = f"{entry.scheme}://{entry.host_header}{entry.path}" + (f"?{entry.query_string}" if entry.query_string else "")
|
|
99
|
+
entry.user_agent = environ.get("HTTP_USER_AGENT", "")
|
|
100
|
+
entry.client_ip = environ.get("HTTP_X_FORWARDED_FOR", environ.get("REMOTE_ADDR", ""))
|
|
101
|
+
entry.content_type = environ.get("CONTENT_TYPE", "")
|
|
102
|
+
|
|
103
|
+
headers = {k.replace("HTTP_", "").lower().replace("_", "-"): v for k, v in environ.items() if k.startswith("HTTP_")}
|
|
104
|
+
entry.client_type = detect_client_type(entry.user_agent, headers)
|
|
105
|
+
entry.origin = detect_origin(headers, entry.user_agent)
|
|
106
|
+
entry.request_headers = sanitize_headers(headers)
|
|
107
|
+
entry.api_version = extract_api_version(entry.path)
|
|
108
|
+
|
|
109
|
+
# 读取请求体
|
|
110
|
+
try:
|
|
111
|
+
body = environ["wsgi.input"].read(self.sdk.config.max_body_size)
|
|
112
|
+
entry.request_body = body.decode("utf-8", errors="replace")
|
|
113
|
+
entry.request_body_size = len(body)
|
|
114
|
+
except Exception:
|
|
115
|
+
pass
|
|
116
|
+
|
|
117
|
+
status_code = 200
|
|
118
|
+
|
|
119
|
+
def _start_response(status, response_headers, exc_info=None):
|
|
120
|
+
nonlocal status_code
|
|
121
|
+
status_code = int(status.split()[0])
|
|
122
|
+
resp_headers = {k.lower(): v for k, v in response_headers}
|
|
123
|
+
entry.response_headers = sanitize_headers(resp_headers)
|
|
124
|
+
return start_response(status, response_headers, exc_info)
|
|
125
|
+
|
|
126
|
+
try:
|
|
127
|
+
response_iter = self.sdk.app(environ, _start_response)
|
|
128
|
+
entry.duration_ms = int((time.time() - start) * 1000)
|
|
129
|
+
entry.status_code = status_code
|
|
130
|
+
if status_code >= 500:
|
|
131
|
+
entry.is_error = True
|
|
132
|
+
entry.error_type = "http_error"
|
|
133
|
+
self.sdk.send(entry)
|
|
134
|
+
return response_iter
|
|
135
|
+
except Exception as e:
|
|
136
|
+
entry.is_error = True
|
|
137
|
+
entry.error_type = "panic"
|
|
138
|
+
entry.error_message = str(e)
|
|
139
|
+
entry.error_stack = traceback.format_exc()
|
|
140
|
+
entry.duration_ms = int((time.time() - start) * 1000)
|
|
141
|
+
self.sdk.send(entry)
|
|
142
|
+
raise
|
logs_sdk/offline.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""离线缓存 — 网络故障时缓存到本地文件,恢复后自动重传"""
|
|
2
|
+
import json, os, tempfile, time, logging
|
|
3
|
+
from typing import List, Callable
|
|
4
|
+
from .types import LogEntry
|
|
5
|
+
|
|
6
|
+
logger = logging.getLogger("logs-sdk")
|
|
7
|
+
|
|
8
|
+
class OfflineCache:
|
|
9
|
+
def __init__(self, directory: str = ""):
|
|
10
|
+
self.dir = directory or os.path.join(tempfile.gettempdir(), "logs-sdk-offline")
|
|
11
|
+
self.max_size = 50 * 1024 * 1024
|
|
12
|
+
self.max_age = 24 * 3600
|
|
13
|
+
self.enabled = True
|
|
14
|
+
os.makedirs(self.dir, exist_ok=True)
|
|
15
|
+
|
|
16
|
+
def save(self, entries: List[LogEntry]):
|
|
17
|
+
if not self.enabled or not entries: return
|
|
18
|
+
self._cleanup()
|
|
19
|
+
filename = os.path.join(self.dir, f"offline-{time.strftime('%Y%m%dT%H%M%S')}.json")
|
|
20
|
+
try:
|
|
21
|
+
with open(filename, "w") as f:
|
|
22
|
+
json.dump([e.to_dict() for e in entries], f)
|
|
23
|
+
logger.info(f"离线缓存已保存: {filename} ({len(entries)} 条)")
|
|
24
|
+
except Exception as e:
|
|
25
|
+
logger.error(f"离线缓存保存失败: {e}")
|
|
26
|
+
|
|
27
|
+
def flush_all(self, send_fn: Callable):
|
|
28
|
+
files = [f for f in os.listdir(self.dir) if f.startswith("offline-") and f.endswith(".json")]
|
|
29
|
+
if not files: return
|
|
30
|
+
for fn in sorted(files):
|
|
31
|
+
filepath = os.path.join(self.dir, fn)
|
|
32
|
+
try:
|
|
33
|
+
if time.time() - os.path.getmtime(filepath) > self.max_age:
|
|
34
|
+
os.remove(filepath)
|
|
35
|
+
continue
|
|
36
|
+
with open(filepath) as f:
|
|
37
|
+
data = json.load(f)
|
|
38
|
+
entries = [LogEntry(**d) for d in data]
|
|
39
|
+
send_fn(entries)
|
|
40
|
+
os.remove(filepath)
|
|
41
|
+
logger.info(f"离线缓存已重传: {fn} ({len(entries)} 条)")
|
|
42
|
+
except Exception as e:
|
|
43
|
+
logger.error(f"离线缓存重传失败: {e}")
|
|
44
|
+
return
|
|
45
|
+
|
|
46
|
+
def pending_count(self) -> int:
|
|
47
|
+
return len([f for f in os.listdir(self.dir) if f.startswith("offline-")])
|
|
48
|
+
|
|
49
|
+
def _cleanup(self):
|
|
50
|
+
files = sorted([os.path.join(self.dir, f) for f in os.listdir(self.dir) if f.startswith("offline-")],
|
|
51
|
+
key=lambda x: os.path.getmtime(x))
|
|
52
|
+
total = sum(os.path.getsize(f) for f in files if os.path.exists(f))
|
|
53
|
+
for f in files:
|
|
54
|
+
if total <= self.max_size: break
|
|
55
|
+
if os.path.exists(f):
|
|
56
|
+
total -= os.path.getsize(f)
|
|
57
|
+
os.remove(f)
|
logs_sdk/types.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""SDK 核心类型定义 — 与 Go/Node.js SDK 完全对齐"""
|
|
2
|
+
from dataclasses import dataclass, field, asdict
|
|
3
|
+
from typing import Optional, Any
|
|
4
|
+
import json, uuid, time, socket, os, re
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class LogConfig:
|
|
8
|
+
endpoint: str
|
|
9
|
+
api_key: str
|
|
10
|
+
api_secret: str
|
|
11
|
+
project_slug: str
|
|
12
|
+
environment: str = "production"
|
|
13
|
+
service_name: str = ""
|
|
14
|
+
buffer_size: int = 1000
|
|
15
|
+
flush_interval: int = 5
|
|
16
|
+
max_retries: int = 3
|
|
17
|
+
max_body_size: int = 4096
|
|
18
|
+
max_stack_size: int = 8192
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class LogEntry:
|
|
22
|
+
uuid: str = ""
|
|
23
|
+
uid: int = 0
|
|
24
|
+
timestamp: str = ""
|
|
25
|
+
duration_ms: int = 0
|
|
26
|
+
project_slug: str = ""
|
|
27
|
+
environment: str = ""
|
|
28
|
+
service_name: str = ""
|
|
29
|
+
host: str = ""
|
|
30
|
+
process_id: str = ""
|
|
31
|
+
method: str = ""
|
|
32
|
+
scheme: str = ""
|
|
33
|
+
full_url: str = ""
|
|
34
|
+
host_header: str = ""
|
|
35
|
+
path: str = ""
|
|
36
|
+
query_string: str = ""
|
|
37
|
+
origin: str = ""
|
|
38
|
+
request_headers: str = "{}"
|
|
39
|
+
request_body: str = ""
|
|
40
|
+
request_body_size: int = 0
|
|
41
|
+
content_type: str = ""
|
|
42
|
+
status_code: int = 0
|
|
43
|
+
response_headers: str = "{}"
|
|
44
|
+
response_body: str = ""
|
|
45
|
+
response_body_size: int = 0
|
|
46
|
+
client_ip: str = ""
|
|
47
|
+
client_ip_chain: str = ""
|
|
48
|
+
client_type: str = "other"
|
|
49
|
+
client_port: int = 0
|
|
50
|
+
client_country: str = ""
|
|
51
|
+
client_province: str = ""
|
|
52
|
+
client_city: str = ""
|
|
53
|
+
client_isp: str = ""
|
|
54
|
+
user_agent: str = ""
|
|
55
|
+
device_type: str = ""
|
|
56
|
+
browser: str = ""
|
|
57
|
+
browser_version: str = ""
|
|
58
|
+
os_name: str = ""
|
|
59
|
+
os_version: str = ""
|
|
60
|
+
tls_version: str = ""
|
|
61
|
+
tls_cipher: str = ""
|
|
62
|
+
proto: str = ""
|
|
63
|
+
api_version: str = ""
|
|
64
|
+
referer: str = ""
|
|
65
|
+
upstream_status: int = 0
|
|
66
|
+
latency_breakdown: str = "{}"
|
|
67
|
+
request_id: str = ""
|
|
68
|
+
trace_id: str = ""
|
|
69
|
+
span_id: str = ""
|
|
70
|
+
parent_span_id: str = ""
|
|
71
|
+
user_id: str = ""
|
|
72
|
+
session_id: str = ""
|
|
73
|
+
is_error: bool = False
|
|
74
|
+
error_message: str = ""
|
|
75
|
+
error_type: str = ""
|
|
76
|
+
error_stack: str = ""
|
|
77
|
+
panic_location: str = ""
|
|
78
|
+
tags: dict = field(default_factory=dict)
|
|
79
|
+
|
|
80
|
+
def to_dict(self) -> dict:
|
|
81
|
+
d = asdict(self)
|
|
82
|
+
d["tags"] = json.dumps(self.tags) if self.tags else "{}"
|
|
83
|
+
return d
|
|
84
|
+
|
|
85
|
+
def new_uuid() -> str:
|
|
86
|
+
"""生成 UUID v7,32 位十六进制无连字符"""
|
|
87
|
+
return uuid.uuid4().hex # Python uuid4 无原生 v7
|
|
88
|
+
|
|
89
|
+
def detect_client_type(ua: str, headers: dict) -> str:
|
|
90
|
+
"""根据 User-Agent 和请求头识别客户端类型"""
|
|
91
|
+
ct = headers.get("x-client-type", "")
|
|
92
|
+
if ct: return ct
|
|
93
|
+
ua_lower = ua.lower()
|
|
94
|
+
if "micromessenger" in ua_lower or "miniprogram" in ua_lower: return "miniprogram"
|
|
95
|
+
if headers.get("x-caller-service"): return "server"
|
|
96
|
+
ref = headers.get("referer", "") or headers.get("origin", "")
|
|
97
|
+
if ref and any(b in ua_lower for b in ("mozilla", "chrome", "safari", "firefox")): return "web"
|
|
98
|
+
return "other"
|
|
99
|
+
|
|
100
|
+
def detect_origin(headers: dict, ua: str) -> str:
|
|
101
|
+
ct = detect_client_type(ua, headers)
|
|
102
|
+
if ct == "web": return headers.get("referer", "") or headers.get("origin", "")
|
|
103
|
+
if ct == "miniprogram": return f"miniprogram:{headers.get('x-miniprogram-appid','')}{headers.get('x-miniprogram-path','')}"
|
|
104
|
+
if ct == "app": return f"app:{headers.get('x-app-name','')}/{headers.get('x-app-version','')}/{headers.get('x-app-scene','')}"
|
|
105
|
+
if ct == "server": return f"server:{headers.get('x-caller-service','')}/{headers.get('x-caller-version','')}"
|
|
106
|
+
return "unknown"
|
|
107
|
+
|
|
108
|
+
def sanitize_headers(headers: dict) -> str:
|
|
109
|
+
safe = {}
|
|
110
|
+
for k, v in headers.items():
|
|
111
|
+
v = v[0] if isinstance(v, list) else str(v) if v else ""
|
|
112
|
+
if k.lower() in ("authorization", "cookie", "set-cookie"):
|
|
113
|
+
safe[k] = v[:15] + "..." if len(v) > 20 else "***"
|
|
114
|
+
else:
|
|
115
|
+
safe[k] = v
|
|
116
|
+
return json.dumps(safe)
|
|
117
|
+
|
|
118
|
+
def extract_api_version(path: str) -> str:
|
|
119
|
+
m = re.match(r"/api/(v\d+)(/|$)", path)
|
|
120
|
+
return m.group(1) if m else ""
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: logs-sdk
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: 日志管理平台 Python SDK — FastAPI/Starlette/Flask 中间件,自动采集 HTTP 日志
|
|
5
|
+
Project-URL: Homepage, https://github.com/xiaohao0725/logs-sdk-python
|
|
6
|
+
Project-URL: Repository, https://github.com/xiaohao0725/logs-sdk-python
|
|
7
|
+
Author: xiaohao0725
|
|
8
|
+
License: UNLICENSED
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Requires-Dist: httpx>=0.27
|
|
19
|
+
Requires-Dist: uuid6>=2024
|
|
20
|
+
Provides-Extra: fastapi
|
|
21
|
+
Requires-Dist: fastapi; extra == 'fastapi'
|
|
22
|
+
Requires-Dist: starlette; extra == 'fastapi'
|
|
23
|
+
Provides-Extra: flask
|
|
24
|
+
Requires-Dist: flask; extra == 'flask'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# 日志管理平台 Python SDK
|
|
28
|
+
|
|
29
|
+
[English Documentation](https://github.com/xiaohao0725/logs-sdk-python/blob/main/README_EN.md) | [PyPI](https://pypi.org/project/logs-sdk/)
|
|
30
|
+
|
|
31
|
+
`logs-sdk` 是日志管理平台的 Python SDK,提供 FastAPI/Starlette 和 Flask 中间件,一行代码即可自动采集 HTTP 请求的完整日志,异步批量上报。
|
|
32
|
+
|
|
33
|
+
## 功能特性
|
|
34
|
+
|
|
35
|
+
- ✅ **一行代码接入**:`app.add_middleware(logger.fastapi_middleware)`
|
|
36
|
+
- ✅ **完整采集**:60+ 字段——请求/响应头体、客户端信息、设备信息、TLS 版本
|
|
37
|
+
- ✅ **自动识别**:客户端类型(Web / 小程序 / App / 服务端)、请求来源
|
|
38
|
+
- ✅ **异常捕获**:HTTP 5xx + Python 异常堆栈自动采集
|
|
39
|
+
- ✅ **UUID v7**:32 位十六进制无连字符
|
|
40
|
+
- ✅ **敏感脱敏**:Authorization / Cookie 自动脱敏
|
|
41
|
+
- ✅ **异步非阻塞**:环形缓冲区 + 后台定时刷新
|
|
42
|
+
- ✅ **离线缓存**:断网本地存储,恢复自动重传
|
|
43
|
+
- ✅ **优雅关闭**:`close()` 确保缓冲日志全部上报
|
|
44
|
+
|
|
45
|
+
## 安装
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install logs-sdk
|
|
49
|
+
|
|
50
|
+
# 或安装 FastAPI 集成
|
|
51
|
+
pip install logs-sdk[fastapi]
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
要求 Python 3.9+。
|
|
55
|
+
|
|
56
|
+
## 快速开始
|
|
57
|
+
|
|
58
|
+
### FastAPI / Starlette
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from fastapi import FastAPI
|
|
62
|
+
from logs_sdk import LogSDK
|
|
63
|
+
|
|
64
|
+
app = FastAPI()
|
|
65
|
+
|
|
66
|
+
logger = LogSDK(
|
|
67
|
+
endpoint="https://api.logs.codexs.cn/api/v1/ingest/logs",
|
|
68
|
+
api_key="clog_pk_xxx",
|
|
69
|
+
api_secret="clog_sk_xxx",
|
|
70
|
+
project_slug="my-project",
|
|
71
|
+
environment="production",
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
# 一行代码接入
|
|
75
|
+
app.add_middleware(logger.fastapi_middleware)
|
|
76
|
+
|
|
77
|
+
@app.get("/api/hello")
|
|
78
|
+
def hello():
|
|
79
|
+
return {"message": "hello"}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Flask
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from flask import Flask
|
|
86
|
+
from logs_sdk import LogSDK
|
|
87
|
+
|
|
88
|
+
app = Flask(__name__)
|
|
89
|
+
|
|
90
|
+
logger = LogSDK(
|
|
91
|
+
endpoint="https://api.logs.codexs.cn/api/v1/ingest/logs",
|
|
92
|
+
api_key="clog_pk_xxx",
|
|
93
|
+
api_secret="clog_sk_xxx",
|
|
94
|
+
project_slug="my-project",
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# Flask 中间件
|
|
98
|
+
app.wsgi_app = logger.flask_middleware
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## 配置参数
|
|
102
|
+
|
|
103
|
+
| 参数 | 类型 | 默认值 | 说明 |
|
|
104
|
+
|------|------|--------|------|
|
|
105
|
+
| `endpoint` | `str` | **必填** | 日志上报地址 |
|
|
106
|
+
| `api_key` | `str` | **必填** | SDK 认证密钥(公钥) |
|
|
107
|
+
| `api_secret` | `str` | **必填** | SDK 认证密钥(私钥) |
|
|
108
|
+
| `project_slug` | `str` | **必填** | 项目短标识 |
|
|
109
|
+
| `environment` | `str` | `"production"` | 运行环境 |
|
|
110
|
+
| `service_name` | `str` | `""` | 微服务名称 |
|
|
111
|
+
| `buffer_size` | `int` | `1000` | 缓冲区容量 |
|
|
112
|
+
| `flush_interval` | `int` | `5` | 刷新间隔(秒) |
|
|
113
|
+
| `max_retries` | `int` | `3` | 最大重试次数 |
|
|
114
|
+
| `max_body_size` | `int` | `4096` | 请求/响应体最大采集大小 |
|
|
115
|
+
|
|
116
|
+
## 采集字段一览
|
|
117
|
+
|
|
118
|
+
与 Go/Node.js/Java SDK 完全对齐,详见 [LogEntry 类型定义](./src/logs_sdk/types.py)。
|
|
119
|
+
|
|
120
|
+
## 架构设计
|
|
121
|
+
|
|
122
|
+
```
|
|
123
|
+
HTTP 请求进入
|
|
124
|
+
│
|
|
125
|
+
├─ ① FastAPIMiddleware.dispatch()
|
|
126
|
+
│ ├─ 生成 UUID v7
|
|
127
|
+
│ ├─ 读取请求体
|
|
128
|
+
│ └─ 记录开始时间
|
|
129
|
+
│
|
|
130
|
+
├─ ② await call_next(request) # 业务处理
|
|
131
|
+
│
|
|
132
|
+
├─ ③ 构建 LogEntry(60+ 字段)
|
|
133
|
+
│
|
|
134
|
+
├─ ④ buffer.push(entry) # 非阻塞
|
|
135
|
+
│
|
|
136
|
+
└─ ⑤ 后台定时刷新 → POST → 重试 → 离线缓存
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
## 离线缓存
|
|
140
|
+
|
|
141
|
+
断网时自动缓存到 `$TMPDIR/logs-sdk-offline/`,恢复后自动重传。
|
|
142
|
+
|
|
143
|
+
## 版本历史
|
|
144
|
+
|
|
145
|
+
| 版本 | 日期 | 变更 |
|
|
146
|
+
|------|------|------|
|
|
147
|
+
| v0.3.0 | 2026-06-21 | 初始版本:FastAPI/Flask 中间件、异步缓冲、重试、离线缓存 |
|
|
148
|
+
|
|
149
|
+
## License
|
|
150
|
+
|
|
151
|
+
UNLICENSED — 内部使用
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
logs_sdk/__init__.py,sha256=kWcOhv8M3uuWu7BSPixt-_lFxTSsZ3SAK_vSamzCDQs,549
|
|
2
|
+
logs_sdk/buffer.py,sha256=hMXa8dtxAwKrwjKO5IQKRrkseg89qGhR4T12oEuXSWo,1274
|
|
3
|
+
logs_sdk/client.py,sha256=izvv2zFHW005KnDS7fjkjD4iyRtcN30wrJTliY7OEII,3639
|
|
4
|
+
logs_sdk/middleware.py,sha256=McY4j5F0Mqia_2bcthLhy8cg6gvmoNIG5Fcc-IY30f4,5983
|
|
5
|
+
logs_sdk/offline.py,sha256=zU8XADs6iZflSqBD-oZfPxnQfzRp6NhbEaZDFJNRGqo,2425
|
|
6
|
+
logs_sdk/types.py,sha256=XQE2v_i5euMUy0Or-ZoCeuCJcseF7mHN4zpFY1yPp0I,3864
|
|
7
|
+
logs_sdk-0.3.0.dist-info/METADATA,sha256=9ef-qqwp2TZWoCvkdGvPbfFTfDGYS7uPavFl252dvqg,4522
|
|
8
|
+
logs_sdk-0.3.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
9
|
+
logs_sdk-0.3.0.dist-info/RECORD,,
|