crawlerkeeper-sdk 0.2.1__tar.gz

crawlerkeeper_sdk-0.2.1/.gitignore

@@ -0,0 +1,80 @@
+# Build artifacts
+target/
+*.class
+*.jar
+*.war
+
+# Maven
+.mvn/
+mvnw
+mvnw.cmd
+
+# Node
+node_modules/
+dist/
+pnpm-lock.yaml
+yarn.lock
+package-lock.json
+npm-debug.log*
+
+# IDE
+.idea/
+.vscode/
+*.iml
+*.ipr
+*.iws
+.project
+.classpath
+.settings/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+*.tmp
+
+# Logs
+*.log
+logs/
+!openspec/specs/logs/
+
+# Config (sensitive)
+*.env
+*.env.local
+application-local.yml
+application-*.properties
+
+# Local config (generated by CLI)
+CLAUDE.local.md
+
+# Worktrees
+.worktrees/
+worktrees/
+
+# Temp
+tmp/
+temp/
+crawlerkeeper-agent/config/config.yaml
+
+# Agent data
+crawlerkeeper-agent/data/
+
+.playwright-mcp
+*/.playwright-cli/
+
+__pycache__/
+settings.local.json.worktrees/
+
+findings.md
+progress.md
+task_plan.md
+*/superpowers/
+.claude/settings.local.json
+uv.lock
+
+out/
+crawlerkeeper-agent/.last_daemon_args
+crawlerkeeper-agent/agent.pid
+.gstack/
+.omo/

crawlerkeeper_sdk-0.2.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 CrawlerKeeper Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crawlerkeeper_sdk-0.2.1/PKG-INFO

@@ -0,0 +1,264 @@
+Metadata-Version: 2.5
+Name: crawlerkeeper-sdk
+Version: 0.2.1
+Summary: CrawlerKeeper 爬虫 SDK：日志上报、结果写入、任务参数解析、TLS 指纹请求、代理管理与并发线程池
+Author: CrawlerKeeper Team
+License-Expression: MIT
+License-File: LICENSE
+Keywords: crawler,crawlerkeeper,scraping,spider,tls-fingerprint
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: curl-cffi>=0.7.0
+Description-Content-Type: text/markdown
+
+# CrawlerKeeper SDK for Python
+
+CrawlerKeeper Python SDK 提供了爬虫与 Agent 通信的封装，支持日志记录、结果写入、参数获取、代理管理和并发爬取。
+
+## 安装
+
+```bash
+pip install crawlerkeeper-sdk
+```
+
+## 快速开始
+
+```python
+from crawlerkeeper_sdk import params, info, result, StealthSession
+
+# 获取任务参数
+config = params()
+info(f"收到参数: {config}")
+
+# 使用 StealthSession 发送请求
+session = StealthSession()
+resp = session.get("https://example.com/api/data")
+data = resp.json()
+
+# 写入结果
+result({"data": data, "source": "example.com"})
+
+info("爬取完成")
+session.close()
+```
+
+## 核心功能
+
+### 日志记录
+
+```python
+from crawlerkeeper_sdk import info, warning, error
+
+info("信息日志")
+warning("警告日志")
+error("错误日志")
+```
+
+### 结果写入
+
+```python
+from crawlerkeeper_sdk import result
+
+result({"count": 100, "items": [...]})
+```
+
+### 任务参数
+
+```python
+from crawlerkeeper_sdk import params
+
+# python spider.py --params '{"keyword": "python", "page": 1}'
+config = params()  # {"keyword": "python", "page": 1}
+```
+
+## StealthSession
+
+`StealthSession` 提供隐式浏览器指纹的 HTTP 请求能力，自动管理 TLS 指纹和代理。
+
+```python
+from crawlerkeeper_sdk import StealthSession, result
+
+session = StealthSession()
+
+# GET 请求
+resp = session.get("https://example.com/api/data")
+print(resp.status_code)
+print(resp.json())
+
+# POST 请求
+resp = session.post("https://example.com/api/submit", json={"key": "value"})
+
+session.close()
+```
+
+### 主要方法
+
+| 方法 | 说明 |
+|------|------|
+| `get(url, **kwargs)` | 发送 GET 请求 |
+| `post(url, **kwargs)` | 发送 POST 请求 |
+| `close()` | 关闭会话 |
+
+## ThreadPool 并发爬取
+
+`ThreadPool` 提供并发爬取能力，内部管理 `StealthSession` 的创建和关闭。
+
+### 推荐用法：使用上下文管理器
+
+```python
+from crawlerkeeper_sdk import ThreadPool, StealthSession, result
+
+def crawl(session: StealthSession, keyword: str, page: int):
+    """爬虫函数：session 会自动传入"""
+    resp = session.get(f"https://example.com/search?kw={keyword}&page={page}")
+    return resp.json()
+
+# 使用 with 语句自动管理生命周期，不会遗漏 close()
+with ThreadPool(max_workers=5) as pool:
+    pool.submit(crawl, keyword="python", page=1)
+    pool.submit(crawl, keyword="java", page=1)
+    pool.submit(crawl, keyword="go", page=1)
+
+    # 阻塞等待所有任务完成
+    results = pool.wait()
+
+    # 处理结果
+    for data in results:
+        if data is not None:
+            result(data)
+# with 块结束时自动调用 wait() 和 close()
+```
+
+### 传统用法
+
+```python
+pool = ThreadPool(max_workers=5)
+pool.submit(crawl, keyword="python", page=1)
+pool.submit(crawl, keyword="java", page=1)
+
+results = pool.wait()
+pool.close()  # 需要手动调用
+```
+
+### 环境变量配置并发数
+
+```bash
+export CONCURRENT_NUM=10
+```
+
+```python
+# 不指定 max_workers 时使用 CONCURRENT_NUM 环境变量
+with ThreadPool() as pool:
+    pool.submit(crawl, keyword="python", page=1)
+```
+
+### ThreadPool 方法
+
+| 方法 | 说明 |
+|------|------|
+| `submit(crawl_fn, **kwargs)` | 提交任务，worker 全满时阻塞等待 |
+| `wait()` | 阻塞等待所有任务完成，返回结果列表 |
+| `close()` | 等待所有任务完成后关闭线程池 |
+
+### 异常处理
+
+爬虫函数抛出的异常会被捕获并通过 `output.error()` 记录，任务结果记为 `None`，不中断其他任务。
+
+```python
+def failing_crawl(session):
+    raise ValueError("some error")
+
+pool = ThreadPool(max_workers=2)
+pool.submit(failing_crawl)
+results = pool.wait()  # results[0] is None
+pool.close()
+```
+
+## 代理配置
+
+`StealthSession` 的代理行为由环境变量控制（通常由 Agent 下发），不支持按请求传入代理参数：
+
+| 环境变量 | 说明 |
+|---------|------|
+| `PROXY_ENABLED` | `on` 开启代理，其他值直连 |
+| `PROXY_MODE` | `tunnel` 静态代理 / `private` 管理端动态分配 |
+| `PROXY_URL` | tunnel 模式的静态代理 URL |
+| `PROXY_VENDOR` | private 模式的供应商标识 |
+| `PROXY_TARGET_SITE` | private 模式的目标站点标识 |
+| `ADMIN_SERVER_URL` | private 模式下管理端地址 |
+| `AGENT_API_KEY` | private 模式下调用管理端的第三方 Token（`td_` 前缀；代理分配/上报接口仅对第三方 Token 开放，节点 `ck_` Key 无权调用） |
+
+```bash
+# tunnel 模式示例
+export PROXY_ENABLED=on
+export PROXY_MODE=tunnel
+export PROXY_URL=http://proxy:8080
+```
+
+private 模式下代理用尽或持续被目标站封禁时，`get`/`post` 抛出 `ProxyExhaustedError`。
+
+## 运行模式
+
+SDK 自动检测运行模式：
+
+- **Agent 模式**: 当 `LOG_FILE_PATH` 和 `DATA_FILE_PATH` 环境变量存在时，日志和结果写入文件
+- **开发模式**: 环境变量不存在时，输出到控制台
+
+```bash
+# 开发模式
+python spider.py
+
+# Agent 模式
+export LOG_FILE_PATH=/tmp/test.log
+export DATA_FILE_PATH=/tmp/test.json
+python spider.py
+```
+
+## 完整示例
+
+```python
+#!/usr/bin/env python3
+from crawlerkeeper_sdk import params, info, result, StealthSession, ThreadPool
+
+def crawl(session: StealthSession, keyword: str, page: int):
+    """爬虫函数"""
+    info(f"爬取 {keyword} 第 {page} 页")
+    resp = session.get(f"https://example.com/api?kw={keyword}&page={page}")
+    return resp.json()
+
+def main():
+    config = params()
+    keywords = config.get("keywords", ["python"])
+    max_workers = config.get("max_workers", 3)
+
+    with ThreadPool(max_workers=max_workers) as pool:
+        # 提交所有任务
+        for keyword in keywords:
+            for page in range(1, 4):
+                pool.submit(crawl, keyword=keyword, page=page)
+
+        # 阻塞等待所有任务完成
+        results = pool.wait()
+
+        # 输出结果
+        for data in results:
+            if data is not None:
+                result(data)
+
+    info("爬取完成")
+
+if __name__ == "__main__":
+    main()
+```
+
+## 许可证
+
+MIT License

crawlerkeeper_sdk-0.2.1/crawlerkeeper_sdk/README.md

@@ -0,0 +1,243 @@
+# CrawlerKeeper SDK for Python
+
+CrawlerKeeper Python SDK 提供了爬虫与 Agent 通信的封装，支持日志记录、结果写入、参数获取、代理管理和并发爬取。
+
+## 安装
+
+```bash
+pip install crawlerkeeper-sdk
+```
+
+## 快速开始
+
+```python
+from crawlerkeeper_sdk import params, info, result, StealthSession
+
+# 获取任务参数
+config = params()
+info(f"收到参数: {config}")
+
+# 使用 StealthSession 发送请求
+session = StealthSession()
+resp = session.get("https://example.com/api/data")
+data = resp.json()
+
+# 写入结果
+result({"data": data, "source": "example.com"})
+
+info("爬取完成")
+session.close()
+```
+
+## 核心功能
+
+### 日志记录
+
+```python
+from crawlerkeeper_sdk import info, warning, error
+
+info("信息日志")
+warning("警告日志")
+error("错误日志")
+```
+
+### 结果写入
+
+```python
+from crawlerkeeper_sdk import result
+
+result({"count": 100, "items": [...]})
+```
+
+### 任务参数
+
+```python
+from crawlerkeeper_sdk import params
+
+# python spider.py --params '{"keyword": "python", "page": 1}'
+config = params()  # {"keyword": "python", "page": 1}
+```
+
+## StealthSession
+
+`StealthSession` 提供隐式浏览器指纹的 HTTP 请求能力，自动管理 TLS 指纹和代理。
+
+```python
+from crawlerkeeper_sdk import StealthSession, result
+
+session = StealthSession()
+
+# GET 请求
+resp = session.get("https://example.com/api/data")
+print(resp.status_code)
+print(resp.json())
+
+# POST 请求
+resp = session.post("https://example.com/api/submit", json={"key": "value"})
+
+session.close()
+```
+
+### 主要方法
+
+| 方法 | 说明 |
+|------|------|
+| `get(url, **kwargs)` | 发送 GET 请求 |
+| `post(url, **kwargs)` | 发送 POST 请求 |
+| `close()` | 关闭会话 |
+
+## ThreadPool 并发爬取
+
+`ThreadPool` 提供并发爬取能力，内部管理 `StealthSession` 的创建和关闭。
+
+### 推荐用法：使用上下文管理器
+
+```python
+from crawlerkeeper_sdk import ThreadPool, StealthSession, result
+
+def crawl(session: StealthSession, keyword: str, page: int):
+    """爬虫函数：session 会自动传入"""
+    resp = session.get(f"https://example.com/search?kw={keyword}&page={page}")
+    return resp.json()
+
+# 使用 with 语句自动管理生命周期，不会遗漏 close()
+with ThreadPool(max_workers=5) as pool:
+    pool.submit(crawl, keyword="python", page=1)
+    pool.submit(crawl, keyword="java", page=1)
+    pool.submit(crawl, keyword="go", page=1)
+
+    # 阻塞等待所有任务完成
+    results = pool.wait()
+
+    # 处理结果
+    for data in results:
+        if data is not None:
+            result(data)
+# with 块结束时自动调用 wait() 和 close()
+```
+
+### 传统用法
+
+```python
+pool = ThreadPool(max_workers=5)
+pool.submit(crawl, keyword="python", page=1)
+pool.submit(crawl, keyword="java", page=1)
+
+results = pool.wait()
+pool.close()  # 需要手动调用
+```
+
+### 环境变量配置并发数
+
+```bash
+export CONCURRENT_NUM=10
+```
+
+```python
+# 不指定 max_workers 时使用 CONCURRENT_NUM 环境变量
+with ThreadPool() as pool:
+    pool.submit(crawl, keyword="python", page=1)
+```
+
+### ThreadPool 方法
+
+| 方法 | 说明 |
+|------|------|
+| `submit(crawl_fn, **kwargs)` | 提交任务，worker 全满时阻塞等待 |
+| `wait()` | 阻塞等待所有任务完成，返回结果列表 |
+| `close()` | 等待所有任务完成后关闭线程池 |
+
+### 异常处理
+
+爬虫函数抛出的异常会被捕获并通过 `output.error()` 记录，任务结果记为 `None`，不中断其他任务。
+
+```python
+def failing_crawl(session):
+    raise ValueError("some error")
+
+pool = ThreadPool(max_workers=2)
+pool.submit(failing_crawl)
+results = pool.wait()  # results[0] is None
+pool.close()
+```
+
+## 代理配置
+
+`StealthSession` 的代理行为由环境变量控制（通常由 Agent 下发），不支持按请求传入代理参数：
+
+| 环境变量 | 说明 |
+|---------|------|
+| `PROXY_ENABLED` | `on` 开启代理，其他值直连 |
+| `PROXY_MODE` | `tunnel` 静态代理 / `private` 管理端动态分配 |
+| `PROXY_URL` | tunnel 模式的静态代理 URL |
+| `PROXY_VENDOR` | private 模式的供应商标识 |
+| `PROXY_TARGET_SITE` | private 模式的目标站点标识 |
+| `ADMIN_SERVER_URL` | private 模式下管理端地址 |
+| `AGENT_API_KEY` | private 模式下调用管理端的第三方 Token（`td_` 前缀；代理分配/上报接口仅对第三方 Token 开放，节点 `ck_` Key 无权调用） |
+
+```bash
+# tunnel 模式示例
+export PROXY_ENABLED=on
+export PROXY_MODE=tunnel
+export PROXY_URL=http://proxy:8080
+```
+
+private 模式下代理用尽或持续被目标站封禁时，`get`/`post` 抛出 `ProxyExhaustedError`。
+
+## 运行模式
+
+SDK 自动检测运行模式：
+
+- **Agent 模式**: 当 `LOG_FILE_PATH` 和 `DATA_FILE_PATH` 环境变量存在时，日志和结果写入文件
+- **开发模式**: 环境变量不存在时，输出到控制台
+
+```bash
+# 开发模式
+python spider.py
+
+# Agent 模式
+export LOG_FILE_PATH=/tmp/test.log
+export DATA_FILE_PATH=/tmp/test.json
+python spider.py
+```
+
+## 完整示例
+
+```python
+#!/usr/bin/env python3
+from crawlerkeeper_sdk import params, info, result, StealthSession, ThreadPool
+
+def crawl(session: StealthSession, keyword: str, page: int):
+    """爬虫函数"""
+    info(f"爬取 {keyword} 第 {page} 页")
+    resp = session.get(f"https://example.com/api?kw={keyword}&page={page}")
+    return resp.json()
+
+def main():
+    config = params()
+    keywords = config.get("keywords", ["python"])
+    max_workers = config.get("max_workers", 3)
+
+    with ThreadPool(max_workers=max_workers) as pool:
+        # 提交所有任务
+        for keyword in keywords:
+            for page in range(1, 4):
+                pool.submit(crawl, keyword=keyword, page=page)
+
+        # 阻塞等待所有任务完成
+        results = pool.wait()
+
+        # 输出结果
+        for data in results:
+            if data is not None:
+                result(data)
+
+    info("爬取完成")
+
+if __name__ == "__main__":
+    main()
+```
+
+## 许可证
+
+MIT License

crawlerkeeper_sdk-0.2.1/crawlerkeeper_sdk/__init__.py

@@ -0,0 +1,28 @@
+"""
+CrawlerKeeper SDK for Python
+
+提供爬虫与 Agent 通信的封装，支持日志记录、结果写入、参数解析和代理管理。
+"""
+
+from .output import (
+    info,
+    warning,
+    error,
+    result,
+)
+from .params import params
+from .stealth_session import StealthSession
+from .proxy import ProxyExhaustedError
+from .pool import ThreadPool
+
+__version__ = "0.2.1"
+__all__ = [
+    "info",
+    "warning",
+    "error",
+    "result",
+    "params",
+    "StealthSession",
+    "ProxyExhaustedError",
+    "ThreadPool",
+]

crawlerkeeper_sdk-0.2.1/crawlerkeeper_sdk/env.py

@@ -0,0 +1,29 @@
+"""
+环境变量管理模块
+
+统一管理所有环境变量访问。
+"""
+
+import os
+from typing import Optional
+
+# Agent 环境
+LOG_FILE_PATH: Optional[str] = os.environ.get("LOG_FILE_PATH")
+DATA_FILE_PATH: Optional[str] = os.environ.get("DATA_FILE_PATH")
+
+# Admin Server
+ADMIN_SERVER_URL: str = os.environ.get("ADMIN_SERVER_URL", "http://localhost:8080")
+AGENT_API_KEY: str = os.environ.get("AGENT_API_KEY", "")
+
+# 代理配置
+PROXY_ENABLED: bool = os.environ.get("PROXY_ENABLED", "off").lower() == "on"
+PROXY_MODE: str = os.environ.get("PROXY_MODE", "private").lower()
+PROXY_URL: Optional[str] = os.environ.get("PROXY_URL")
+PROXY_VENDOR: str = os.environ.get("PROXY_VENDOR", "default")
+PROXY_TARGET_SITE: str = os.environ.get("PROXY_TARGET_SITE", "default")
+
+# 并发配置
+CONCURRENT_NUM: int = max(1, int(os.environ.get("CONCURRENT_NUM", "1")))
+
+# 请求超时配置（秒）
+CRAWL_TIMEOUT: int = max(1, int(os.environ.get("CRAWL_TIMEOUT", "30")))