crawlerkeeper-sdk 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,16 @@
1
+ # Python
2
+ __pycache__/
3
+ *.pyc
4
+ .venv/
5
+
6
+ # 构建产物
7
+ dist/
8
+
9
+ # 运行时数据(本地部署产生,勿提交)
10
+ data/
11
+ logs/
12
+ agent.pid
13
+ .last_daemon_args
14
+
15
+ # 本地配置(含节点 Token,勿提交)
16
+ config/config.yaml
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: crawlerkeeper-sdk
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: CrawlerKeeper 爬虫 SDK:日志上报、结果写入、任务参数解析、TLS 指纹请求、代理管理与并发线程池
5
5
  Author: CrawlerKeeper Team
6
6
  License-Expression: MIT
@@ -15,7 +15,7 @@ from .stealth_session import StealthSession
15
15
  from .proxy import ProxyExhaustedError
16
16
  from .pool import ThreadPool
17
17
 
18
- __version__ = "0.2.1"
18
+ __version__ = "0.2.2"
19
19
  __all__ = [
20
20
  "info",
21
21
  "warning",
@@ -18,10 +18,11 @@ def log(message: str, level: str = "INFO") -> None:
18
18
  在 Agent 环境中写入 LOG_FILE_PATH 文件,否则输出到控制台。
19
19
 
20
20
  Args:
21
- message: 日志内容
21
+ message: 日志内容,换行会被转义为 \\n,保证一条日志只占一行
22
22
  level: 日志级别,可选 DEBUG, INFO, WARNING, ERROR
23
23
  """
24
24
  timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
25
+ message = message.replace("\r\n", "\\n").replace("\r", "\\n").replace("\n", "\\n")
25
26
  line = f"[{timestamp}] [{level}] {message}\n"
26
27
 
27
28
  if env.LOG_FILE_PATH:
@@ -56,6 +56,23 @@ TLS_NAMES = [
56
56
  "firefox147",
57
57
  ]
58
58
 
59
+ def _preview(text: Optional[str], limit: int = 120) -> str:
60
+ """把响应体压缩成单行摘要,避免多行内容刷屏日志。"""
61
+ if not text:
62
+ return ""
63
+
64
+ lines = [" ".join(line.split()) for line in text.strip().splitlines()]
65
+ lines = [line for line in lines if line]
66
+
67
+ # 跳过纯 HTML 标签行,优先取有实际内容的一行
68
+ for line in lines:
69
+ if not line.startswith("<"):
70
+ return line[:limit] + "..." if len(line) > limit else line
71
+
72
+ first = lines[0] if lines else ""
73
+ return first[:limit] + "..." if len(first) > limit else first
74
+
75
+
59
76
  class StealthSession:
60
77
  """
61
78
  支持 TLS 指纹伪装的 HTTP Session。
@@ -133,9 +150,9 @@ class StealthSession:
133
150
  error(f"[tunnel] 请求异常 (attempt {attempt}/3): {e}")
134
151
  continue
135
152
  if resp.status_code != 403:
136
- info(f"[tunnel] 请求成功 (attempt {attempt}/3): status={resp.status_code}, resp={resp.text[:200] if resp.text else ''}")
153
+ # 成功请求不打印日志,避免日志刷屏
137
154
  return resp
138
- info(f"[tunnel] 请求返回 403 (attempt {attempt}/3): resp={resp.text[:200] if resp.text else ''}")
155
+ info(f"[tunnel] 请求返回 403 (attempt {attempt}/3): {_preview(resp.text)}")
139
156
  if attempt == 3:
140
157
  break
141
158
  raise ProxyExhaustedError("tunnel 模式请求达到最大重试次数")
@@ -153,11 +170,11 @@ class StealthSession:
153
170
  continue
154
171
 
155
172
  if resp.status_code != 403:
156
- info(f"[private] 请求成功 (attempt {attempt}/3): status={resp.status_code}, resp={resp.text[:200] if resp.text else ''}")
173
+ # 成功请求不打印日志,避免日志刷屏
157
174
  return resp
158
175
 
159
176
  # 403: 上报并切换代理
160
- info(f"[private] 请求返回 403 (attempt {attempt}/3): resp={resp.text[:200] if resp.text else ''}")
177
+ info(f"[private] 请求返回 403 (attempt {attempt}/3): {_preview(resp.text)}")
161
178
  proxy_info = self._extract_proxy_info(proxy)
162
179
  if proxy_info:
163
180
  self._proxy_manager.report_block(proxy_info)
@@ -1,80 +0,0 @@
1
- # Build artifacts
2
- target/
3
- *.class
4
- *.jar
5
- *.war
6
-
7
- # Maven
8
- .mvn/
9
- mvnw
10
- mvnw.cmd
11
-
12
- # Node
13
- node_modules/
14
- dist/
15
- pnpm-lock.yaml
16
- yarn.lock
17
- package-lock.json
18
- npm-debug.log*
19
-
20
- # IDE
21
- .idea/
22
- .vscode/
23
- *.iml
24
- *.ipr
25
- *.iws
26
- .project
27
- .classpath
28
- .settings/
29
- *.swp
30
- *.swo
31
-
32
- # OS
33
- .DS_Store
34
- Thumbs.db
35
- *.tmp
36
-
37
- # Logs
38
- *.log
39
- logs/
40
- !openspec/specs/logs/
41
-
42
- # Config (sensitive)
43
- *.env
44
- *.env.local
45
- application-local.yml
46
- application-*.properties
47
-
48
- # Local config (generated by CLI)
49
- CLAUDE.local.md
50
-
51
- # Worktrees
52
- .worktrees/
53
- worktrees/
54
-
55
- # Temp
56
- tmp/
57
- temp/
58
- crawlerkeeper-agent/config/config.yaml
59
-
60
- # Agent data
61
- crawlerkeeper-agent/data/
62
-
63
- .playwright-mcp
64
- */.playwright-cli/
65
-
66
- __pycache__/
67
- settings.local.json.worktrees/
68
-
69
- findings.md
70
- progress.md
71
- task_plan.md
72
- */superpowers/
73
- .claude/settings.local.json
74
- uv.lock
75
-
76
- out/
77
- crawlerkeeper-agent/.last_daemon_args
78
- crawlerkeeper-agent/agent.pid
79
- .gstack/
80
- .omo/