wimotek-chromium-sync 1.3.14 → 1.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -3
- package/package.json +1 -1
- package/wimotek_chromium_sync/__init__.py +9 -1
- package/wimotek_chromium_sync/audit.py +176 -0
- package/wimotek_chromium_sync/core.py +129 -1
package/README.md
CHANGED
|
@@ -29,6 +29,10 @@
|
|
|
29
29
|
* 仅对实质变更的 Cookie 执行落盘与热重载,避免无效 IO 读写,**大幅延长从属端 SSD 寿命**。
|
|
30
30
|
6. **全量纯函数工具方法开箱即用**:
|
|
31
31
|
* 模块顶级导出 `merge_cookies`、`filter_cookies`、`compute_cookie_delta` 等独立纯函数工具。
|
|
32
|
+
7. **开箱即用的结构化审计日志与自动大小轮转 (Structured Audit Logging & Auto-Rotation)**:
|
|
33
|
+
* **触发机制**:每次写入时自动检测目标文件大小,当达到阈值(默认 10MB / 10485760 字节)时,自动重命名归档为 `audit.jsonl.1`、`audit.jsonl.2` ... 并创建新的日志文件。
|
|
34
|
+
* **自动淘汰**:超出保留上限(默认最多保留 5 个归档备份,即 `audit.jsonl.1` 至 `audit.jsonl.5`)的最旧日志会被自动剔除,从根源杜绝磁盘打满。
|
|
35
|
+
* **JSON Lines 格式**:标准 JSONL 格式 (`.jsonl`),便于 ELK、Loki、Splunk 或云端日志系统直接采集与追踪。
|
|
32
36
|
|
|
33
37
|
---
|
|
34
38
|
|
|
@@ -125,6 +129,67 @@ CDP Hub 在管理大规模分布式浏览器节点时,节点所处的网络拓
|
|
|
125
129
|
|
|
126
130
|
---
|
|
127
131
|
|
|
132
|
+
## 📋 审计日志与开箱即用的自动大小轮转 (Auto-Rotation)
|
|
133
|
+
|
|
134
|
+
生产环境下长期运行多节点同步引擎,审计日志必须兼顾**追溯性**与**磁盘安全性**。`wimotek-chromium-sync` 内置了开箱即用的线程安全 JSON Lines (`.jsonl`) 审计日志记录器与自动大小轮转机制:
|
|
135
|
+
|
|
136
|
+
### 1. 工作原理与触发机制
|
|
137
|
+
* **触发机制 (Trigger Mechanism)**:每次写入审计事件前,自动检测主文件(默认 `audit.jsonl`)的体积大小。当写入后总大小达到阈值(默认 `10MB` / `10 * 1024 * 1024` 字节)时,自动将主日志重命名归档为 `audit.jsonl.1`、`audit.jsonl.2` ... 并初始化全新的 `audit.jsonl` 继续写入。
|
|
138
|
+
* **自动淘汰 (Auto-Eviction)**:严格控制归档保留数量(默认最多保留 `5` 个归档备份,即 `audit.jsonl.1` ~ `audit.jsonl.5`)。发生轮转时,超出上限的最旧归档(`audit.jsonl.5`)会被自动剔除删除,**从根源上杜绝磁盘打满或存储溢出风险**。
|
|
139
|
+
* **线程安全 (Thread Safety)**:内置全局排他锁,并发批量调度时写入不串行、保证每行均为标准合法的 JSON 对象。
|
|
140
|
+
|
|
141
|
+
### 2. 标准 JSON Lines 数据格式
|
|
142
|
+
每条日志独立成行,字段标准统一:
|
|
143
|
+
```json
|
|
144
|
+
{"timestamp": "2026-11-08T19:20:01.123Z", "level": "INFO", "event": "sync_start", "details": {"mode": "sync", "source": "node-master", "targets": ["node-worker-1", "node-worker-2"], "nodes": ["node-master", "node-worker-1", "node-worker-2"]}}
|
|
145
|
+
{"timestamp": "2026-11-08T19:20:01.345Z", "level": "INFO", "event": "node_connected", "details": {"node_id": "node-master", "name": "主控机", "mode": "direct", "cdp_url": "http://127.0.0.1:9222"}}
|
|
146
|
+
{"timestamp": "2026-11-08T19:20:01.450Z", "level": "WARNING", "event": "node_skipped", "details": {"node_id": "node-worker-2", "name": "从属机2", "mode": "ssh_tunnel", "cdp_url": "http://127.0.0.1:19223", "reason": "Connection refused"}}
|
|
147
|
+
{"timestamp": "2026-11-08T19:20:01.789Z", "level": "INFO", "event": "sync_finish", "details": {"status": "success", "mode": "sync", "source": "node-master", "transferred_count": 8, "diff_skipped_count": 92, "source_total": 100, "target_total": 100, "nodes_synced": ["node-worker-1"], "nodes_skipped": ["node-worker-2"], "duration": 0.67}}
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### 3. 支持的核心审计事件 (Events)
|
|
151
|
+
| 事件名称 (`event`) | 级别 (`level`) | 触发时机与包含字段 |
|
|
152
|
+
| :--- | :--- | :--- |
|
|
153
|
+
| `sync_start` | `INFO` | 同步引擎启动,记录模式 `mode`、源节点 `source` 与参与节点列表 |
|
|
154
|
+
| `node_connected` | `INFO` | 节点成功接入并建立 CDP 会话 |
|
|
155
|
+
| `node_skipped` | `WARNING` | 节点离线、握手失败或异常跳过,记录详细错误原因 `reason` |
|
|
156
|
+
| `source_offline` | `ERROR` | 单向同步指定的源节点离线,记录终止通知 |
|
|
157
|
+
| `cookies_merged` | `INFO` | `merge` 模式下凭据汇总去重完成,记录原始数与融合数 |
|
|
158
|
+
| `sync_finish` | `INFO` | 同步成功完成,记录真正写入条数、差量跳过条数与耗时 |
|
|
159
|
+
| `sync_failed` | `ERROR` | 全部节点离线或源节点离线导致同步流程无法继续 |
|
|
160
|
+
| `sync_exception` | `ERROR` | 同步过程中发生未预期异常捕获 |
|
|
161
|
+
|
|
162
|
+
### 4. 审计日志配置示例
|
|
163
|
+
```python
|
|
164
|
+
from wimotek_chromium_sync import ChromiumSync, AuditLogger
|
|
165
|
+
|
|
166
|
+
syncer = ChromiumSync(
|
|
167
|
+
nodes=[
|
|
168
|
+
{"id": "master", "cdp_port": 9222},
|
|
169
|
+
{"id": "slave", "cdp_port": 9223}
|
|
170
|
+
],
|
|
171
|
+
# 审计日志与自动轮转设置 (全部内置合理默认值,开箱即用)
|
|
172
|
+
audit_file="logs/chromium_sync_audit.jsonl", # 日志写入路径 (默认: "audit.jsonl")
|
|
173
|
+
audit_max_bytes=10 * 1024 * 1024, # 单文件最大容量,达到后轮转 (默认: 10MB)
|
|
174
|
+
audit_backup_count=5, # 最多保留归档文件数 (默认: 5 个)
|
|
175
|
+
audit_enabled=True # 是否开启审计日志 (默认: True)
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
# 执行同步 (自动记录审计日志并按需轮转)
|
|
179
|
+
report = syncer.run()
|
|
180
|
+
|
|
181
|
+
# 快速读取最近生成的结构化审计日志
|
|
182
|
+
recent_logs = syncer.get_audit_records(limit=20)
|
|
183
|
+
for entry in recent_logs:
|
|
184
|
+
print(f"[{entry['timestamp']}] {entry['event']} -> {entry['details']}")
|
|
185
|
+
|
|
186
|
+
# 查看当前存在的所有归档日志文件
|
|
187
|
+
print("当前审计归档链:", syncer.get_audit_files())
|
|
188
|
+
# ['logs/chromium_sync_audit.jsonl', 'logs/chromium_sync_audit.jsonl.1', ...]
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
128
193
|
## 🚀 极简开发示例
|
|
129
194
|
|
|
130
195
|
### 1. Hub 多模式混合集群融合 (全节点最新凭证合并)
|
|
@@ -153,7 +218,8 @@ syncer = ChromiumSync(
|
|
|
153
218
|
mode="merge",
|
|
154
219
|
include_domains=["google.com", "github.com"], # 仅同步核心业务站点
|
|
155
220
|
exclude_domains=["doubleclick.net"], # 剔除垃圾广告追踪
|
|
156
|
-
diff=True
|
|
221
|
+
diff=True, # 开启差量指纹比对保护磁盘
|
|
222
|
+
audit_file="audit.jsonl" # 开启结构化审计自动轮转
|
|
157
223
|
)
|
|
158
224
|
|
|
159
225
|
report = syncer.run()
|
|
@@ -201,7 +267,11 @@ with SSHTunnelManager(start_port=19222) as tm:
|
|
|
201
267
|
| `diff` | bool | `True` | **指纹差量同步**:相同凭证跳过写入,避免频繁刷盘保护 SSD 寿命 |
|
|
202
268
|
| `source` | str | 第一个节点 | `'sync'` 模式下的基准源节点 ID (离线时打印错误通知并直接终止同步) |
|
|
203
269
|
| `targets` | list | 其余节点 | `'sync'` 模式下的目标节点 ID 列表 |
|
|
204
|
-
| `
|
|
270
|
+
| `audit_file` | str | `'audit.jsonl'` | **审计日志文件路径** (支持相对或绝对路径) |
|
|
271
|
+
| `audit_max_bytes` | int | `10485760` (10MB) | **自动轮转阈值**:文件大小达到该字节数时触发归档 |
|
|
272
|
+
| `audit_backup_count` | int | `5` | **归档保留上限**:最多保留 5 个 `.1` ~ `.5` 归档备份,超出自动淘汰 |
|
|
273
|
+
| `audit_enabled` | bool | `True` | **开箱即用**:是否开启结构化审计日志记录 |
|
|
274
|
+
| `verbose` | bool | `True` | 是否输出详细控制台日志 (`⚠️ [SKIP]`, `⚠️ [SOURCE_OFFLINE]`) |
|
|
205
275
|
|
|
206
276
|
---
|
|
207
277
|
|
|
@@ -213,7 +283,8 @@ from wimotek_chromium_sync import (
|
|
|
213
283
|
NodeConfig, # 统一节点配置数据模型 (支持 4 种 CDP 连接模式)
|
|
214
284
|
merge_cookies, # 多节点 Cookie 智能融合纯函数
|
|
215
285
|
filter_cookies, # 域名白名单/黑名单过滤纯函数
|
|
216
|
-
compute_cookie_delta
|
|
286
|
+
compute_cookie_delta, # Cookie 指纹变更差量计算纯函数
|
|
287
|
+
AuditLogger # 开箱即用的 JSON Lines 审计日志与自动轮转器
|
|
217
288
|
)
|
|
218
289
|
```
|
|
219
290
|
|
package/package.json
CHANGED
|
@@ -3,6 +3,14 @@ wimotek-chromium-sync Package
|
|
|
3
3
|
Universal Chromium Session & Cookie Synchronizer with SSH Tunnel and Cluster Support
|
|
4
4
|
"""
|
|
5
5
|
from .core import ChromiumSync, NodeConfig, merge_cookies, filter_cookies, compute_cookie_delta
|
|
6
|
+
from .audit import AuditLogger
|
|
6
7
|
|
|
7
8
|
__version__ = "1.3.0"
|
|
8
|
-
__all__ = [
|
|
9
|
+
__all__ = [
|
|
10
|
+
"ChromiumSync",
|
|
11
|
+
"NodeConfig",
|
|
12
|
+
"merge_cookies",
|
|
13
|
+
"filter_cookies",
|
|
14
|
+
"compute_cookie_delta",
|
|
15
|
+
"AuditLogger",
|
|
16
|
+
]
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Structured Audit Logging & Auto-Rotation Engine for wimotek-chromium-sync
|
|
5
|
+
开箱即用的结构化 JSON Lines 审计日志与文件大小自动轮转引擎
|
|
6
|
+
- 触发机制: 每次写入时检测目标文件大小,达到阈值 (默认 10MB) 时,自动归档为 audit.jsonl.1, audit.jsonl.2...
|
|
7
|
+
- 自动淘汰: 超出保留上限 (默认最多保留 5 个归档备份) 的最旧日志自动被剔除,从根源杜绝磁盘打满
|
|
8
|
+
- 线程安全: 内置锁机制保障多线程并发写入不串行
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import json
|
|
13
|
+
import time
|
|
14
|
+
import threading
|
|
15
|
+
from datetime import datetime
|
|
16
|
+
from typing import Optional, Dict, Any, Union, List
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class AuditLogger:
|
|
20
|
+
"""
|
|
21
|
+
生产级 JSON Lines 审计日志记录器
|
|
22
|
+
"""
|
|
23
|
+
DEFAULT_FILE: str = "audit.jsonl"
|
|
24
|
+
DEFAULT_MAX_BYTES: int = 10 * 1024 * 1024 # 默认 10MB
|
|
25
|
+
DEFAULT_BACKUP_COUNT: int = 5 # 默认保留 5 个历史归档
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
file_path: Optional[str] = "audit.jsonl",
|
|
30
|
+
max_bytes: int = 10 * 1024 * 1024,
|
|
31
|
+
backup_count: int = 5,
|
|
32
|
+
enabled: bool = True
|
|
33
|
+
):
|
|
34
|
+
self.file_path = file_path or self.DEFAULT_FILE
|
|
35
|
+
self.max_bytes = max_bytes if max_bytes > 0 else self.DEFAULT_MAX_BYTES
|
|
36
|
+
self.backup_count = backup_count if backup_count >= 0 else self.DEFAULT_BACKUP_COUNT
|
|
37
|
+
self.enabled = enabled
|
|
38
|
+
self._lock = threading.Lock()
|
|
39
|
+
|
|
40
|
+
def should_rotate(self, new_bytes_len: int = 0) -> bool:
|
|
41
|
+
"""检查写入新数据后是否达到轮转阈值"""
|
|
42
|
+
if not self.file_path or self.max_bytes <= 0:
|
|
43
|
+
return False
|
|
44
|
+
try:
|
|
45
|
+
if os.path.exists(self.file_path):
|
|
46
|
+
current_size = os.path.getsize(self.file_path)
|
|
47
|
+
return (current_size + new_bytes_len) >= self.max_bytes
|
|
48
|
+
except OSError:
|
|
49
|
+
pass
|
|
50
|
+
return False
|
|
51
|
+
|
|
52
|
+
def rotate(self) -> None:
|
|
53
|
+
"""
|
|
54
|
+
执行文件大小自动轮转与淘汰:
|
|
55
|
+
1. 若已存在 audit.jsonl.{backup_count},自动删除最旧归档;
|
|
56
|
+
2. 将历史归档依次向后推进: audit.jsonl.{N-1} -> audit.jsonl.{N};
|
|
57
|
+
3. 将主文件 audit.jsonl 重命名为 audit.jsonl.1;
|
|
58
|
+
4. 随后写入自动写入新的主文件。
|
|
59
|
+
"""
|
|
60
|
+
if not self.file_path or not os.path.exists(self.file_path):
|
|
61
|
+
return
|
|
62
|
+
|
|
63
|
+
if self.backup_count <= 0:
|
|
64
|
+
# 不保留备份,直接清空主文件
|
|
65
|
+
try:
|
|
66
|
+
os.remove(self.file_path)
|
|
67
|
+
except OSError:
|
|
68
|
+
pass
|
|
69
|
+
return
|
|
70
|
+
|
|
71
|
+
# 1. 淘汰超出保留上限的最旧备份文件
|
|
72
|
+
oldest_backup = f"{self.file_path}.{self.backup_count}"
|
|
73
|
+
if os.path.exists(oldest_backup):
|
|
74
|
+
try:
|
|
75
|
+
os.remove(oldest_backup)
|
|
76
|
+
except OSError:
|
|
77
|
+
pass
|
|
78
|
+
|
|
79
|
+
# 2. 从 backup_count - 1 递减到 1 依次向后重命名
|
|
80
|
+
for i in range(self.backup_count - 1, 0, -1):
|
|
81
|
+
src = f"{self.file_path}.{i}"
|
|
82
|
+
dst = f"{self.file_path}.{i + 1}"
|
|
83
|
+
if os.path.exists(src):
|
|
84
|
+
try:
|
|
85
|
+
if os.path.exists(dst):
|
|
86
|
+
os.remove(dst)
|
|
87
|
+
os.rename(src, dst)
|
|
88
|
+
except OSError:
|
|
89
|
+
pass
|
|
90
|
+
|
|
91
|
+
# 3. 将当前主文件重命名为 .1
|
|
92
|
+
first_backup = f"{self.file_path}.1"
|
|
93
|
+
try:
|
|
94
|
+
if os.path.exists(first_backup):
|
|
95
|
+
os.remove(first_backup)
|
|
96
|
+
os.rename(self.file_path, first_backup)
|
|
97
|
+
except OSError:
|
|
98
|
+
pass
|
|
99
|
+
|
|
100
|
+
def log(
|
|
101
|
+
self,
|
|
102
|
+
event_type: str,
|
|
103
|
+
details: Optional[Dict[str, Any]] = None,
|
|
104
|
+
level: str = "INFO",
|
|
105
|
+
**kwargs
|
|
106
|
+
) -> Dict[str, Any]:
|
|
107
|
+
"""
|
|
108
|
+
写入一条结构化审计日志 (JSON Lines 格式)
|
|
109
|
+
自动触发大小检测、轮转与淘汰
|
|
110
|
+
"""
|
|
111
|
+
if not self.enabled or not self.file_path:
|
|
112
|
+
return {}
|
|
113
|
+
|
|
114
|
+
now = datetime.utcnow()
|
|
115
|
+
record = {
|
|
116
|
+
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
|
|
117
|
+
"level": level.upper(),
|
|
118
|
+
"event": event_type,
|
|
119
|
+
"details": details.copy() if details else {}
|
|
120
|
+
}
|
|
121
|
+
if kwargs:
|
|
122
|
+
record["details"].update(kwargs)
|
|
123
|
+
|
|
124
|
+
line = json.dumps(record, ensure_ascii=False) + "\n"
|
|
125
|
+
encoded = line.encode("utf-8")
|
|
126
|
+
|
|
127
|
+
with self._lock:
|
|
128
|
+
# 写入前检测是否需轮转
|
|
129
|
+
if self.should_rotate(len(encoded)):
|
|
130
|
+
self.rotate()
|
|
131
|
+
|
|
132
|
+
parent_dir = os.path.dirname(self.file_path)
|
|
133
|
+
if parent_dir and not os.path.exists(parent_dir):
|
|
134
|
+
try:
|
|
135
|
+
os.makedirs(parent_dir, exist_ok=True)
|
|
136
|
+
except OSError:
|
|
137
|
+
pass
|
|
138
|
+
|
|
139
|
+
with open(self.file_path, "a", encoding="utf-8") as f:
|
|
140
|
+
f.write(line)
|
|
141
|
+
|
|
142
|
+
return record
|
|
143
|
+
|
|
144
|
+
def get_audit_files(self) -> List[str]:
|
|
145
|
+
"""获取当前存在的主日志及全部轮转归档文件列表 (按新到旧排序)"""
|
|
146
|
+
files = []
|
|
147
|
+
if os.path.exists(self.file_path):
|
|
148
|
+
files.append(self.file_path)
|
|
149
|
+
for i in range(1, self.backup_count + 1):
|
|
150
|
+
rotated = f"{self.file_path}.{i}"
|
|
151
|
+
if os.path.exists(rotated):
|
|
152
|
+
files.append(rotated)
|
|
153
|
+
return files
|
|
154
|
+
|
|
155
|
+
def read_recent_records(self, limit: int = 50) -> List[Dict[str, Any]]:
|
|
156
|
+
"""便捷读取最近的审计记录 (主要用于控制台展示或调试)"""
|
|
157
|
+
records: List[Dict[str, Any]] = []
|
|
158
|
+
if not self.file_path or not os.path.exists(self.file_path):
|
|
159
|
+
return records
|
|
160
|
+
|
|
161
|
+
try:
|
|
162
|
+
with open(self.file_path, "r", encoding="utf-8") as f:
|
|
163
|
+
lines = f.readlines()
|
|
164
|
+
for line in reversed(lines):
|
|
165
|
+
line = line.strip()
|
|
166
|
+
if line:
|
|
167
|
+
try:
|
|
168
|
+
records.append(json.loads(line))
|
|
169
|
+
if len(records) >= limit:
|
|
170
|
+
break
|
|
171
|
+
except json.JSONDecodeError:
|
|
172
|
+
continue
|
|
173
|
+
except OSError:
|
|
174
|
+
pass
|
|
175
|
+
|
|
176
|
+
return records
|
|
@@ -23,6 +23,7 @@ import time
|
|
|
23
23
|
import urllib.request
|
|
24
24
|
from dataclasses import dataclass, field, asdict
|
|
25
25
|
from typing import Dict, List, Optional, Union, Any, Tuple, TYPE_CHECKING
|
|
26
|
+
from .audit import AuditLogger
|
|
26
27
|
|
|
27
28
|
if TYPE_CHECKING:
|
|
28
29
|
from playwright.async_api import BrowserContext, Playwright, Browser
|
|
@@ -285,7 +286,11 @@ class ChromiumSync:
|
|
|
285
286
|
ssh_key: Optional[str] = None, # 集群通用私钥 (路径 或 PEM String)
|
|
286
287
|
ssh_password: Optional[str] = None, # 集群通用 SSH 密码
|
|
287
288
|
auto_close_tunnel: bool = False,
|
|
288
|
-
verbose: bool = True
|
|
289
|
+
verbose: bool = True,
|
|
290
|
+
audit_file: Optional[str] = "audit.jsonl",
|
|
291
|
+
audit_max_bytes: int = 10 * 1024 * 1024,
|
|
292
|
+
audit_backup_count: int = 5,
|
|
293
|
+
audit_enabled: bool = True
|
|
289
294
|
):
|
|
290
295
|
if not nodes:
|
|
291
296
|
raise ValueError("[-] 必须至少提供一个节点配置!")
|
|
@@ -301,6 +306,22 @@ class ChromiumSync:
|
|
|
301
306
|
self.diff = diff
|
|
302
307
|
self.auto_close_tunnel = auto_close_tunnel
|
|
303
308
|
|
|
309
|
+
# 审计日志与自动轮转参数 (开箱即用)
|
|
310
|
+
self.audit_enabled: bool = audit_enabled
|
|
311
|
+
self.audit_file: str = audit_file or "audit.jsonl"
|
|
312
|
+
self.audit_max_bytes: int = audit_max_bytes
|
|
313
|
+
self.audit_backup_count: int = audit_backup_count
|
|
314
|
+
self.audit_logger: Optional[AuditLogger] = (
|
|
315
|
+
AuditLogger(
|
|
316
|
+
file_path=self.audit_file,
|
|
317
|
+
max_bytes=self.audit_max_bytes,
|
|
318
|
+
backup_count=self.audit_backup_count,
|
|
319
|
+
enabled=self.audit_enabled
|
|
320
|
+
)
|
|
321
|
+
if self.audit_enabled and self.audit_file
|
|
322
|
+
else None
|
|
323
|
+
)
|
|
324
|
+
|
|
304
325
|
log_level = logging.INFO if verbose else logging.WARNING
|
|
305
326
|
logging.basicConfig(level=log_level, format="%(asctime)s [%(levelname)s] %(message)s")
|
|
306
327
|
|
|
@@ -336,6 +357,30 @@ class ChromiumSync:
|
|
|
336
357
|
self._pooled_browsers: Dict[str, Any] = {}
|
|
337
358
|
self._pooled_contexts: Dict[str, BrowserContext] = {}
|
|
338
359
|
|
|
360
|
+
def _audit(
|
|
361
|
+
self,
|
|
362
|
+
event_type: str,
|
|
363
|
+
details: Optional[Dict[str, Any]] = None,
|
|
364
|
+
level: str = "INFO",
|
|
365
|
+
**kwargs
|
|
366
|
+
) -> Optional[Dict[str, Any]]:
|
|
367
|
+
"""写入一条结构化审计日志 (自动大小轮转与淘汰)"""
|
|
368
|
+
if self.audit_logger:
|
|
369
|
+
return self.audit_logger.log(event_type, details=details, level=level, **kwargs)
|
|
370
|
+
return None
|
|
371
|
+
|
|
372
|
+
def get_audit_records(self, limit: int = 50) -> List[Dict[str, Any]]:
|
|
373
|
+
"""读取最近生成的结构化审计日志记录"""
|
|
374
|
+
if self.audit_logger:
|
|
375
|
+
return self.audit_logger.read_recent_records(limit=limit)
|
|
376
|
+
return []
|
|
377
|
+
|
|
378
|
+
def get_audit_files(self) -> List[str]:
|
|
379
|
+
"""获取当前存在的主日志及轮转归档文件列表"""
|
|
380
|
+
if self.audit_logger:
|
|
381
|
+
return self.audit_logger.get_audit_files()
|
|
382
|
+
return []
|
|
383
|
+
|
|
339
384
|
# =========================================================================
|
|
340
385
|
# 会话级连接池复用 (Connection Pooling 上下文管理器)
|
|
341
386
|
# =========================================================================
|
|
@@ -834,10 +879,18 @@ class ChromiumSync:
|
|
|
834
879
|
active_source = source or self.source
|
|
835
880
|
active_targets = targets or self.targets
|
|
836
881
|
|
|
882
|
+
self._audit("sync_start", {
|
|
883
|
+
"mode": active_mode,
|
|
884
|
+
"source": active_source,
|
|
885
|
+
"targets": active_targets,
|
|
886
|
+
"nodes": [n.id for n in self.nodes]
|
|
887
|
+
})
|
|
888
|
+
|
|
837
889
|
try:
|
|
838
890
|
from playwright.async_api import async_playwright
|
|
839
891
|
except ImportError:
|
|
840
892
|
elapsed = round(time.time() - start_time, 2)
|
|
893
|
+
self._audit("sync_failed", {"reason": "playwright not installed"}, level="ERROR")
|
|
841
894
|
return {
|
|
842
895
|
"status": "failed",
|
|
843
896
|
"mode": active_mode,
|
|
@@ -869,6 +922,12 @@ class ChromiumSync:
|
|
|
869
922
|
for node in self.nodes:
|
|
870
923
|
if node.id in contexts:
|
|
871
924
|
nodes_data[node.id] = await self._fetch_node_data(contexts[node.id], node)
|
|
925
|
+
self._audit("node_connected", {
|
|
926
|
+
"node_id": node.id,
|
|
927
|
+
"name": node.name,
|
|
928
|
+
"mode": getattr(node, "mode", "direct"),
|
|
929
|
+
"pooled": True
|
|
930
|
+
})
|
|
872
931
|
else:
|
|
873
932
|
logging.warning(f"⚠️ [SKIP] 连接池中节点 [{node.name}] 不可用,自动跳过。")
|
|
874
933
|
skipped_nodes.append({
|
|
@@ -878,6 +937,12 @@ class ChromiumSync:
|
|
|
878
937
|
"cdp_url": node.cdp_url,
|
|
879
938
|
"reason": "Not connected in pool"
|
|
880
939
|
})
|
|
940
|
+
self._audit("node_skipped", {
|
|
941
|
+
"node_id": node.id,
|
|
942
|
+
"name": node.name,
|
|
943
|
+
"mode": getattr(node, "mode", "direct"),
|
|
944
|
+
"reason": "Not connected in pool"
|
|
945
|
+
}, level="WARNING")
|
|
881
946
|
else:
|
|
882
947
|
async with async_playwright() as p:
|
|
883
948
|
for node in self.nodes:
|
|
@@ -891,6 +956,12 @@ class ChromiumSync:
|
|
|
891
956
|
browsers[node.id] = b
|
|
892
957
|
contexts[node.id] = ctx
|
|
893
958
|
nodes_data[node.id] = await self._fetch_node_data(ctx, node)
|
|
959
|
+
self._audit("node_connected", {
|
|
960
|
+
"node_id": node.id,
|
|
961
|
+
"name": node.name,
|
|
962
|
+
"mode": getattr(node, "mode", "direct"),
|
|
963
|
+
"cdp_url": cdp_url
|
|
964
|
+
})
|
|
894
965
|
except Exception as e:
|
|
895
966
|
logging.warning(f"⚠️ [SKIP] 节点 [{node.name}] 离线或连接异常 ({cdp_url}),自动跳过: {e}")
|
|
896
967
|
skipped_nodes.append({
|
|
@@ -900,9 +971,21 @@ class ChromiumSync:
|
|
|
900
971
|
"cdp_url": cdp_url,
|
|
901
972
|
"reason": str(e)
|
|
902
973
|
})
|
|
974
|
+
self._audit("node_skipped", {
|
|
975
|
+
"node_id": node.id,
|
|
976
|
+
"name": node.name,
|
|
977
|
+
"mode": getattr(node, "mode", "direct"),
|
|
978
|
+
"cdp_url": cdp_url,
|
|
979
|
+
"reason": str(e)
|
|
980
|
+
}, level="WARNING")
|
|
903
981
|
|
|
904
982
|
if not contexts:
|
|
905
983
|
elapsed = round(time.time() - start_time, 2)
|
|
984
|
+
self._audit("sync_failed", {
|
|
985
|
+
"mode": active_mode,
|
|
986
|
+
"reason": "No available online nodes",
|
|
987
|
+
"skipped_nodes": [n["id"] for n in skipped_nodes]
|
|
988
|
+
}, level="ERROR")
|
|
906
989
|
return {
|
|
907
990
|
"status": "failed",
|
|
908
991
|
"mode": active_mode,
|
|
@@ -928,6 +1011,11 @@ class ChromiumSync:
|
|
|
928
1011
|
# 连接池模式下执行核心逻辑
|
|
929
1012
|
if not contexts:
|
|
930
1013
|
elapsed = round(time.time() - start_time, 2)
|
|
1014
|
+
self._audit("sync_failed", {
|
|
1015
|
+
"mode": active_mode,
|
|
1016
|
+
"reason": "No available online nodes in pool",
|
|
1017
|
+
"skipped_nodes": [n["id"] for n in skipped_nodes]
|
|
1018
|
+
}, level="ERROR")
|
|
931
1019
|
return {
|
|
932
1020
|
"status": "failed",
|
|
933
1021
|
"mode": active_mode,
|
|
@@ -950,6 +1038,7 @@ class ChromiumSync:
|
|
|
950
1038
|
except Exception as e:
|
|
951
1039
|
elapsed = round(time.time() - start_time, 2)
|
|
952
1040
|
logging.error(f"[-] 同步发生异常: {e}")
|
|
1041
|
+
self._audit("sync_exception", {"mode": active_mode, "error": str(e)}, level="ERROR")
|
|
953
1042
|
return {
|
|
954
1043
|
"status": "failed",
|
|
955
1044
|
"mode": active_mode,
|
|
@@ -1006,6 +1095,12 @@ class ChromiumSync:
|
|
|
1006
1095
|
if active_targets:
|
|
1007
1096
|
dest_nodes = [n for n in dest_nodes if n.id in active_targets]
|
|
1008
1097
|
|
|
1098
|
+
self._audit("cookies_merged", {
|
|
1099
|
+
"raw_total": source_total,
|
|
1100
|
+
"merged_total": len(merged),
|
|
1101
|
+
"dest_nodes": [n.id for n in dest_nodes]
|
|
1102
|
+
})
|
|
1103
|
+
|
|
1009
1104
|
for dest in dest_nodes:
|
|
1010
1105
|
logging.info(f"[*] 注入合并凭据至节点: [{dest.name}]")
|
|
1011
1106
|
w, s = await self._apply_cookies_and_storage(
|
|
@@ -1018,6 +1113,17 @@ class ChromiumSync:
|
|
|
1018
1113
|
total_skipped += s
|
|
1019
1114
|
|
|
1020
1115
|
elapsed = round(time.time() - start_time, 2)
|
|
1116
|
+
self._audit("sync_finish", {
|
|
1117
|
+
"status": "success",
|
|
1118
|
+
"mode": "merge",
|
|
1119
|
+
"transferred_count": total_transferred,
|
|
1120
|
+
"diff_skipped_count": total_skipped,
|
|
1121
|
+
"source_total": source_total,
|
|
1122
|
+
"target_total": len(merged),
|
|
1123
|
+
"nodes_synced": [n.id for n in dest_nodes],
|
|
1124
|
+
"nodes_skipped": [n["id"] for n in skips_list],
|
|
1125
|
+
"duration": elapsed
|
|
1126
|
+
})
|
|
1021
1127
|
return {
|
|
1022
1128
|
"status": "success",
|
|
1023
1129
|
"mode": "merge",
|
|
@@ -1044,6 +1150,16 @@ class ChromiumSync:
|
|
|
1044
1150
|
elapsed = round(time.time() - start_time, 2)
|
|
1045
1151
|
msg = f"⚠️ [SOURCE_OFFLINE] 单向同步指定的源节点 [{source_name}] ({active_source}) 离线或连接异常,无法提取基准凭据,同步流程终止。"
|
|
1046
1152
|
logging.error(msg)
|
|
1153
|
+
self._audit("source_offline", {
|
|
1154
|
+
"source": active_source,
|
|
1155
|
+
"source_name": source_name,
|
|
1156
|
+
"details": msg
|
|
1157
|
+
}, level="ERROR")
|
|
1158
|
+
self._audit("sync_failed", {
|
|
1159
|
+
"mode": "sync",
|
|
1160
|
+
"reason": f"Source node '{active_source}' is offline",
|
|
1161
|
+
"source": active_source
|
|
1162
|
+
}, level="ERROR")
|
|
1047
1163
|
return {
|
|
1048
1164
|
"status": "failed",
|
|
1049
1165
|
"mode": "sync",
|
|
@@ -1082,6 +1198,18 @@ class ChromiumSync:
|
|
|
1082
1198
|
total_skipped += s
|
|
1083
1199
|
|
|
1084
1200
|
elapsed = round(time.time() - start_time, 2)
|
|
1201
|
+
self._audit("sync_finish", {
|
|
1202
|
+
"status": "success",
|
|
1203
|
+
"mode": "sync",
|
|
1204
|
+
"source": active_source,
|
|
1205
|
+
"transferred_count": total_transferred,
|
|
1206
|
+
"diff_skipped_count": total_skipped,
|
|
1207
|
+
"source_total": len(src_cookies),
|
|
1208
|
+
"target_total": len(src_cookies),
|
|
1209
|
+
"nodes_synced": [n.id for n in dest_nodes],
|
|
1210
|
+
"nodes_skipped": [n["id"] for n in skips_list],
|
|
1211
|
+
"duration": elapsed
|
|
1212
|
+
})
|
|
1085
1213
|
return {
|
|
1086
1214
|
"status": "success",
|
|
1087
1215
|
"mode": "sync",
|