crawlerkeeper-sdk 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawlerkeeper_sdk-0.2.2/.gitignore +16 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/PKG-INFO +1 -1
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/crawlerkeeper_sdk/__init__.py +1 -1
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/crawlerkeeper_sdk/output.py +2 -1
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/crawlerkeeper_sdk/stealth_session.py +21 -4
- crawlerkeeper_sdk-0.2.1/.gitignore +0 -80
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/LICENSE +0 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/crawlerkeeper_sdk/README.md +0 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/crawlerkeeper_sdk/env.py +0 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/crawlerkeeper_sdk/params.py +0 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/crawlerkeeper_sdk/pool.py +0 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/crawlerkeeper_sdk/proxy.py +0 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/crawlerkeeper_sdk/py.typed +0 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/pyproject.toml +0 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/tests/__init__.py +0 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/tests/test_pool.py +0 -0
- {crawlerkeeper_sdk-0.2.1 → crawlerkeeper_sdk-0.2.2}/tests/test_stealth_session.py +0 -0
|
@@ -18,10 +18,11 @@ def log(message: str, level: str = "INFO") -> None:
|
|
|
18
18
|
在 Agent 环境中写入 LOG_FILE_PATH 文件,否则输出到控制台。
|
|
19
19
|
|
|
20
20
|
Args:
|
|
21
|
-
message:
|
|
21
|
+
message: 日志内容,换行会被转义为 \\n,保证一条日志只占一行
|
|
22
22
|
level: 日志级别,可选 DEBUG, INFO, WARNING, ERROR
|
|
23
23
|
"""
|
|
24
24
|
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
25
|
+
message = message.replace("\r\n", "\\n").replace("\r", "\\n").replace("\n", "\\n")
|
|
25
26
|
line = f"[{timestamp}] [{level}] {message}\n"
|
|
26
27
|
|
|
27
28
|
if env.LOG_FILE_PATH:
|
|
@@ -56,6 +56,23 @@ TLS_NAMES = [
|
|
|
56
56
|
"firefox147",
|
|
57
57
|
]
|
|
58
58
|
|
|
59
|
+
def _preview(text: Optional[str], limit: int = 120) -> str:
|
|
60
|
+
"""把响应体压缩成单行摘要,避免多行内容刷屏日志。"""
|
|
61
|
+
if not text:
|
|
62
|
+
return ""
|
|
63
|
+
|
|
64
|
+
lines = [" ".join(line.split()) for line in text.strip().splitlines()]
|
|
65
|
+
lines = [line for line in lines if line]
|
|
66
|
+
|
|
67
|
+
# 跳过纯 HTML 标签行,优先取有实际内容的一行
|
|
68
|
+
for line in lines:
|
|
69
|
+
if not line.startswith("<"):
|
|
70
|
+
return line[:limit] + "..." if len(line) > limit else line
|
|
71
|
+
|
|
72
|
+
first = lines[0] if lines else ""
|
|
73
|
+
return first[:limit] + "..." if len(first) > limit else first
|
|
74
|
+
|
|
75
|
+
|
|
59
76
|
class StealthSession:
|
|
60
77
|
"""
|
|
61
78
|
支持 TLS 指纹伪装的 HTTP Session。
|
|
@@ -133,9 +150,9 @@ class StealthSession:
|
|
|
133
150
|
error(f"[tunnel] 请求异常 (attempt {attempt}/3): {e}")
|
|
134
151
|
continue
|
|
135
152
|
if resp.status_code != 403:
|
|
136
|
-
|
|
153
|
+
# 成功请求不打印日志,避免日志刷屏
|
|
137
154
|
return resp
|
|
138
|
-
info(f"[tunnel] 请求返回 403 (attempt {attempt}/3):
|
|
155
|
+
info(f"[tunnel] 请求返回 403 (attempt {attempt}/3): {_preview(resp.text)}")
|
|
139
156
|
if attempt == 3:
|
|
140
157
|
break
|
|
141
158
|
raise ProxyExhaustedError("tunnel 模式请求达到最大重试次数")
|
|
@@ -153,11 +170,11 @@ class StealthSession:
|
|
|
153
170
|
continue
|
|
154
171
|
|
|
155
172
|
if resp.status_code != 403:
|
|
156
|
-
|
|
173
|
+
# 成功请求不打印日志,避免日志刷屏
|
|
157
174
|
return resp
|
|
158
175
|
|
|
159
176
|
# 403: 上报并切换代理
|
|
160
|
-
info(f"[private] 请求返回 403 (attempt {attempt}/3):
|
|
177
|
+
info(f"[private] 请求返回 403 (attempt {attempt}/3): {_preview(resp.text)}")
|
|
161
178
|
proxy_info = self._extract_proxy_info(proxy)
|
|
162
179
|
if proxy_info:
|
|
163
180
|
self._proxy_manager.report_block(proxy_info)
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
# Build artifacts
|
|
2
|
-
target/
|
|
3
|
-
*.class
|
|
4
|
-
*.jar
|
|
5
|
-
*.war
|
|
6
|
-
|
|
7
|
-
# Maven
|
|
8
|
-
.mvn/
|
|
9
|
-
mvnw
|
|
10
|
-
mvnw.cmd
|
|
11
|
-
|
|
12
|
-
# Node
|
|
13
|
-
node_modules/
|
|
14
|
-
dist/
|
|
15
|
-
pnpm-lock.yaml
|
|
16
|
-
yarn.lock
|
|
17
|
-
package-lock.json
|
|
18
|
-
npm-debug.log*
|
|
19
|
-
|
|
20
|
-
# IDE
|
|
21
|
-
.idea/
|
|
22
|
-
.vscode/
|
|
23
|
-
*.iml
|
|
24
|
-
*.ipr
|
|
25
|
-
*.iws
|
|
26
|
-
.project
|
|
27
|
-
.classpath
|
|
28
|
-
.settings/
|
|
29
|
-
*.swp
|
|
30
|
-
*.swo
|
|
31
|
-
|
|
32
|
-
# OS
|
|
33
|
-
.DS_Store
|
|
34
|
-
Thumbs.db
|
|
35
|
-
*.tmp
|
|
36
|
-
|
|
37
|
-
# Logs
|
|
38
|
-
*.log
|
|
39
|
-
logs/
|
|
40
|
-
!openspec/specs/logs/
|
|
41
|
-
|
|
42
|
-
# Config (sensitive)
|
|
43
|
-
*.env
|
|
44
|
-
*.env.local
|
|
45
|
-
application-local.yml
|
|
46
|
-
application-*.properties
|
|
47
|
-
|
|
48
|
-
# Local config (generated by CLI)
|
|
49
|
-
CLAUDE.local.md
|
|
50
|
-
|
|
51
|
-
# Worktrees
|
|
52
|
-
.worktrees/
|
|
53
|
-
worktrees/
|
|
54
|
-
|
|
55
|
-
# Temp
|
|
56
|
-
tmp/
|
|
57
|
-
temp/
|
|
58
|
-
crawlerkeeper-agent/config/config.yaml
|
|
59
|
-
|
|
60
|
-
# Agent data
|
|
61
|
-
crawlerkeeper-agent/data/
|
|
62
|
-
|
|
63
|
-
.playwright-mcp
|
|
64
|
-
*/.playwright-cli/
|
|
65
|
-
|
|
66
|
-
__pycache__/
|
|
67
|
-
settings.local.json.worktrees/
|
|
68
|
-
|
|
69
|
-
findings.md
|
|
70
|
-
progress.md
|
|
71
|
-
task_plan.md
|
|
72
|
-
*/superpowers/
|
|
73
|
-
.claude/settings.local.json
|
|
74
|
-
uv.lock
|
|
75
|
-
|
|
76
|
-
out/
|
|
77
|
-
crawlerkeeper-agent/.last_daemon_args
|
|
78
|
-
crawlerkeeper-agent/agent.pid
|
|
79
|
-
.gstack/
|
|
80
|
-
.omo/
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|