crawlerkeeper-sdk 0.2.1__py3-none-any.whl

crawlerkeeper_sdk/README.md

@@ -0,0 +1,243 @@
+# CrawlerKeeper SDK for Python
+
+CrawlerKeeper Python SDK 提供了爬虫与 Agent 通信的封装，支持日志记录、结果写入、参数获取、代理管理和并发爬取。
+
+## 安装
+
+```bash
+pip install crawlerkeeper-sdk
+```
+
+## 快速开始
+
+```python
+from crawlerkeeper_sdk import params, info, result, StealthSession
+
+# 获取任务参数
+config = params()
+info(f"收到参数: {config}")
+
+# 使用 StealthSession 发送请求
+session = StealthSession()
+resp = session.get("https://example.com/api/data")
+data = resp.json()
+
+# 写入结果
+result({"data": data, "source": "example.com"})
+
+info("爬取完成")
+session.close()
+```
+
+## 核心功能
+
+### 日志记录
+
+```python
+from crawlerkeeper_sdk import info, warning, error
+
+info("信息日志")
+warning("警告日志")
+error("错误日志")
+```
+
+### 结果写入
+
+```python
+from crawlerkeeper_sdk import result
+
+result({"count": 100, "items": [...]})
+```
+
+### 任务参数
+
+```python
+from crawlerkeeper_sdk import params
+
+# python spider.py --params '{"keyword": "python", "page": 1}'
+config = params()  # {"keyword": "python", "page": 1}
+```
+
+## StealthSession
+
+`StealthSession` 提供隐式浏览器指纹的 HTTP 请求能力，自动管理 TLS 指纹和代理。
+
+```python
+from crawlerkeeper_sdk import StealthSession, result
+
+session = StealthSession()
+
+# GET 请求
+resp = session.get("https://example.com/api/data")
+print(resp.status_code)
+print(resp.json())
+
+# POST 请求
+resp = session.post("https://example.com/api/submit", json={"key": "value"})
+
+session.close()
+```
+
+### 主要方法
+
+| 方法 | 说明 |
+|------|------|
+| `get(url, **kwargs)` | 发送 GET 请求 |
+| `post(url, **kwargs)` | 发送 POST 请求 |
+| `close()` | 关闭会话 |
+
+## ThreadPool 并发爬取
+
+`ThreadPool` 提供并发爬取能力，内部管理 `StealthSession` 的创建和关闭。
+
+### 推荐用法：使用上下文管理器
+
+```python
+from crawlerkeeper_sdk import ThreadPool, StealthSession, result
+
+def crawl(session: StealthSession, keyword: str, page: int):
+    """爬虫函数：session 会自动传入"""
+    resp = session.get(f"https://example.com/search?kw={keyword}&page={page}")
+    return resp.json()
+
+# 使用 with 语句自动管理生命周期，不会遗漏 close()
+with ThreadPool(max_workers=5) as pool:
+    pool.submit(crawl, keyword="python", page=1)
+    pool.submit(crawl, keyword="java", page=1)
+    pool.submit(crawl, keyword="go", page=1)
+
+    # 阻塞等待所有任务完成
+    results = pool.wait()
+
+    # 处理结果
+    for data in results:
+        if data is not None:
+            result(data)
+# with 块结束时自动调用 wait() 和 close()
+```
+
+### 传统用法
+
+```python
+pool = ThreadPool(max_workers=5)
+pool.submit(crawl, keyword="python", page=1)
+pool.submit(crawl, keyword="java", page=1)
+
+results = pool.wait()
+pool.close()  # 需要手动调用
+```
+
+### 环境变量配置并发数
+
+```bash
+export CONCURRENT_NUM=10
+```
+
+```python
+# 不指定 max_workers 时使用 CONCURRENT_NUM 环境变量
+with ThreadPool() as pool:
+    pool.submit(crawl, keyword="python", page=1)
+```
+
+### ThreadPool 方法
+
+| 方法 | 说明 |
+|------|------|
+| `submit(crawl_fn, **kwargs)` | 提交任务，worker 全满时阻塞等待 |
+| `wait()` | 阻塞等待所有任务完成，返回结果列表 |
+| `close()` | 等待所有任务完成后关闭线程池 |
+
+### 异常处理
+
+爬虫函数抛出的异常会被捕获并通过 `output.error()` 记录，任务结果记为 `None`，不中断其他任务。
+
+```python
+def failing_crawl(session):
+    raise ValueError("some error")
+
+pool = ThreadPool(max_workers=2)
+pool.submit(failing_crawl)
+results = pool.wait()  # results[0] is None
+pool.close()
+```
+
+## 代理配置
+
+`StealthSession` 的代理行为由环境变量控制（通常由 Agent 下发），不支持按请求传入代理参数：
+
+| 环境变量 | 说明 |
+|---------|------|
+| `PROXY_ENABLED` | `on` 开启代理，其他值直连 |
+| `PROXY_MODE` | `tunnel` 静态代理 / `private` 管理端动态分配 |
+| `PROXY_URL` | tunnel 模式的静态代理 URL |
+| `PROXY_VENDOR` | private 模式的供应商标识 |
+| `PROXY_TARGET_SITE` | private 模式的目标站点标识 |
+| `ADMIN_SERVER_URL` | private 模式下管理端地址 |
+| `AGENT_API_KEY` | private 模式下调用管理端的第三方 Token（`td_` 前缀；代理分配/上报接口仅对第三方 Token 开放，节点 `ck_` Key 无权调用） |
+
+```bash
+# tunnel 模式示例
+export PROXY_ENABLED=on
+export PROXY_MODE=tunnel
+export PROXY_URL=http://proxy:8080
+```
+
+private 模式下代理用尽或持续被目标站封禁时，`get`/`post` 抛出 `ProxyExhaustedError`。
+
+## 运行模式
+
+SDK 自动检测运行模式：
+
+- **Agent 模式**: 当 `LOG_FILE_PATH` 和 `DATA_FILE_PATH` 环境变量存在时，日志和结果写入文件
+- **开发模式**: 环境变量不存在时，输出到控制台
+
+```bash
+# 开发模式
+python spider.py
+
+# Agent 模式
+export LOG_FILE_PATH=/tmp/test.log
+export DATA_FILE_PATH=/tmp/test.json
+python spider.py
+```
+
+## 完整示例
+
+```python
+#!/usr/bin/env python3
+from crawlerkeeper_sdk import params, info, result, StealthSession, ThreadPool
+
+def crawl(session: StealthSession, keyword: str, page: int):
+    """爬虫函数"""
+    info(f"爬取 {keyword} 第 {page} 页")
+    resp = session.get(f"https://example.com/api?kw={keyword}&page={page}")
+    return resp.json()
+
+def main():
+    config = params()
+    keywords = config.get("keywords", ["python"])
+    max_workers = config.get("max_workers", 3)
+
+    with ThreadPool(max_workers=max_workers) as pool:
+        # 提交所有任务
+        for keyword in keywords:
+            for page in range(1, 4):
+                pool.submit(crawl, keyword=keyword, page=page)
+
+        # 阻塞等待所有任务完成
+        results = pool.wait()
+
+        # 输出结果
+        for data in results:
+            if data is not None:
+                result(data)
+
+    info("爬取完成")
+
+if __name__ == "__main__":
+    main()
+```
+
+## 许可证
+
+MIT License

crawlerkeeper_sdk/__init__.py

@@ -0,0 +1,28 @@
+"""
+CrawlerKeeper SDK for Python
+
+提供爬虫与 Agent 通信的封装，支持日志记录、结果写入、参数解析和代理管理。
+"""
+
+from .output import (
+    info,
+    warning,
+    error,
+    result,
+)
+from .params import params
+from .stealth_session import StealthSession
+from .proxy import ProxyExhaustedError
+from .pool import ThreadPool
+
+__version__ = "0.2.1"
+__all__ = [
+    "info",
+    "warning",
+    "error",
+    "result",
+    "params",
+    "StealthSession",
+    "ProxyExhaustedError",
+    "ThreadPool",
+]

crawlerkeeper_sdk/env.py

@@ -0,0 +1,29 @@
+"""
+环境变量管理模块
+
+统一管理所有环境变量访问。
+"""
+
+import os
+from typing import Optional
+
+# Agent 环境
+LOG_FILE_PATH: Optional[str] = os.environ.get("LOG_FILE_PATH")
+DATA_FILE_PATH: Optional[str] = os.environ.get("DATA_FILE_PATH")
+
+# Admin Server
+ADMIN_SERVER_URL: str = os.environ.get("ADMIN_SERVER_URL", "http://localhost:8080")
+AGENT_API_KEY: str = os.environ.get("AGENT_API_KEY", "")
+
+# 代理配置
+PROXY_ENABLED: bool = os.environ.get("PROXY_ENABLED", "off").lower() == "on"
+PROXY_MODE: str = os.environ.get("PROXY_MODE", "private").lower()
+PROXY_URL: Optional[str] = os.environ.get("PROXY_URL")
+PROXY_VENDOR: str = os.environ.get("PROXY_VENDOR", "default")
+PROXY_TARGET_SITE: str = os.environ.get("PROXY_TARGET_SITE", "default")
+
+# 并发配置
+CONCURRENT_NUM: int = max(1, int(os.environ.get("CONCURRENT_NUM", "1")))
+
+# 请求超时配置（秒）
+CRAWL_TIMEOUT: int = max(1, int(os.environ.get("CRAWL_TIMEOUT", "30")))

crawlerkeeper_sdk/output.py

@@ -0,0 +1,104 @@
+"""
+日志模块
+
+提供日志记录功能，支持 Agent 模式和开发模式。
+"""
+
+import sys
+from datetime import datetime
+from typing import Any
+
+from . import env
+
+
+def log(message: str, level: str = "INFO") -> None:
+    """
+    写入日志
+
+    在 Agent 环境中写入 LOG_FILE_PATH 文件，否则输出到控制台。
+
+    Args:
+        message: 日志内容
+        level: 日志级别，可选 DEBUG, INFO, WARNING, ERROR
+    """
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    line = f"[{timestamp}] [{level}] {message}\n"
+
+    if env.LOG_FILE_PATH:
+        # Agent 模式：写入文件
+        try:
+            with open(env.LOG_FILE_PATH, "a", encoding="utf-8") as f:
+                f.write(line)
+        except Exception:
+            # 静默失败，不中断爬虫
+            pass
+    else:
+        # 开发模式：输出到控制台
+        print(line, end="")
+
+
+def info(message: str) -> None:
+    """
+    写入 INFO 级别日志
+
+    Args:
+        message: 日志内容
+    """
+    log(message, level="INFO")
+
+
+def warning(message: str) -> None:
+    """
+    写入 WARNING 级别日志
+
+    Args:
+        message: 日志内容
+    """
+    log(message, level="WARNING")
+
+
+def error(message: str) -> None:
+    """
+    写入 ERROR 级别日志
+
+    Args:
+        message: 日志内容
+    """
+    log(message, level="ERROR")
+
+
+def result(data: Any) -> None:
+    """
+    写入结果数据
+
+    数据自动序列化为 JSON 格式写入 DATA_FILE_PATH 文件。
+    在开发模式下，输出到控制台并格式化。
+    Agent 模式下直接写入文件，不格式化。
+
+    Args:
+        data: 任意可 JSON 序列化的数据
+    """
+    import json
+
+    try:
+        if env.DATA_FILE_PATH:
+            # Agent 模式：不格式化，紧凑写入
+            content = json.dumps(data, ensure_ascii=False, separators=(",", ":"))
+        else:
+            # 开发模式：格式化输出
+            content = json.dumps(data, ensure_ascii=False, indent=2)
+
+    except (TypeError, ValueError) as e:
+        log(f"结果序列化失败: {e}", level="ERROR")
+        return
+
+    if env.DATA_FILE_PATH:
+        # Agent 模式：写入文件
+        try:
+            with open(env.DATA_FILE_PATH, "w", encoding="utf-8") as f:
+                f.write(content)
+        except Exception as e:
+            log(f"写入结果失败: {e}", level="ERROR")
+    else:
+        # 开发模式：输出到控制台
+        print("RESULT:", content)

crawlerkeeper_sdk/params.py

@@ -0,0 +1,42 @@
+"""
+CrawlerKeeper SDK 核心实现
+
+提供任务参数解析功能。
+"""
+
+import json
+import sys
+from typing import Any, Dict, Optional
+
+# 缓存参数解析结果
+_params_cache: Optional[Dict[str, Any]] = None
+
+
+def params() -> Dict[str, Any]:
+    """
+    获取任务参数
+
+    自动解析命令行 `--params` 参数，返回 JSON 解析后的字典。
+    结果会被缓存，多次调用只解析一次。
+
+    Returns:
+        参数字典，无参数时返回空字典
+    """
+    global _params_cache
+
+    if _params_cache is not None:
+        return _params_cache
+
+    _params_cache = {}
+    args = sys.argv[1:]
+
+    for i, arg in enumerate(args):
+        if arg == "--params" and i + 1 < len(args):
+            try:
+                _params_cache = json.loads(args[i + 1])
+            except json.JSONDecodeError:
+                from .output import log
+                log("参数解析失败，使用空参数", level="WARNING")
+            break
+
+    return _params_cache

crawlerkeeper_sdk/pool.py

@@ -0,0 +1,114 @@
+"""
+ThreadPool 模块
+
+提供并发爬取能力，管理 StealthSession 生命周期。
+"""
+
+import threading
+from typing import Any, Callable
+
+from .output import error
+from .stealth_session import StealthSession
+
+
+class ThreadPool:
+    """
+    线程池管理器
+
+    提供并发爬取能力，内部管理 StealthSession 的创建和关闭。
+    爬虫函数签名：crawl_fn(session: StealthSession, **kwargs) -> Any
+    """
+
+    def __init__(self, max_workers: int | None = None):
+        """
+        初始化线程池
+
+        Args:
+            max_workers: 最大 worker 数量，默认使用 env.CONCURRENT_NUM
+        """
+        from . import env
+
+        if max_workers is None:
+            max_workers = env.CONCURRENT_NUM
+        self._max_workers = max(1, max_workers)
+        self._semaphore = threading.Semaphore(self._max_workers)
+        self._results: list[Any] = []
+        self._results_lock = threading.Lock()
+        self._done_cv = threading.Condition()
+        self._tasks_done = 0
+        self._tasks_total = 0
+
+    def submit(self, crawl_fn: Callable, **kwargs) -> None:
+        """
+        提交任务。当 worker 全满时，主线程在此阻塞等待空 slot。
+
+        Args:
+            crawl_fn: 爬虫函数，签名 (session, **kwargs) -> Any
+            **kwargs: 传递给爬虫函数的关键字参数
+        """
+        self._semaphore.acquire()
+        # 预占结果槽位，保证 wait() 的返回顺序与 submit() 调用顺序一致
+        with self._results_lock:
+            index = self._tasks_total
+            self._tasks_total += 1
+            self._results.append(None)
+        t = threading.Thread(target=self._worker, args=(crawl_fn, kwargs, index))
+        t.start()
+
+    def _worker(self, crawl_fn: Callable, kwargs: dict, index: int) -> None:
+        """
+        Worker 线程：创建 Session，执行爬虫，关闭 Session。
+
+        Session 创建失败（如缺少 curl-cffi）也必须走 finally 归还信号量并通知等待者，
+        否则 wait() 会永久阻塞。
+
+        Args:
+            crawl_fn: 爬虫函数
+            kwargs: 爬虫函数参数
+            index: 结果槽位下标，保证返回顺序与提交顺序一致
+        """
+        session = None
+        try:
+            session = StealthSession()
+            data = crawl_fn(session, **kwargs)
+        except Exception as e:
+            error(f"crawl error: {e}")
+            data = None
+        finally:
+            if session is not None:
+                session.close()
+            with self._results_lock:
+                self._results[index] = data
+            with self._done_cv:
+                self._tasks_done += 1
+                self._done_cv.notify_all()
+            self._semaphore.release()
+
+    def wait(self) -> list[Any]:
+        """
+        阻塞等待所有任务完成，返回结果列表。
+
+        Returns:
+            包含所有任务返回值的列表，顺序与 submit() 调用顺序一致
+        """
+        with self._done_cv:
+            while self._tasks_done < self._tasks_total:
+                self._done_cv.wait()
+        with self._results_lock:
+            return list(self._results)
+
+    def close(self) -> None:
+        """
+        等待所有任务完成后关闭线程池（wait() 之后调用）。
+        """
+        self._semaphore.acquire()  # 确保所有任务已提交
+        self._semaphore.release()
+
+    def __enter__(self) -> "ThreadPool":
+        """支持上下文管理器：with ThreadPool() as pool:"""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        """上下文管理器退出时自动调用 close()"""
+        self.wait()
+        self.close()

crawlerkeeper_sdk/proxy.py

@@ -0,0 +1,173 @@
+"""
+ProxyManager - 代理管理模块
+
+统一管理代理配置和生命周期，支持两种模式：
+- tunnel: 静态配置代理 URL
+- private: 从管理端动态分配代理
+"""
+
+import json
+from typing import Any, Dict, Optional
+
+from . import env
+
+
+def _http_post(endpoint: str, data: Dict[str, Any] = None) -> Dict[str, Any]:
+    """发送 POST 请求到 Admin Server"""
+    try:
+        import urllib.request
+
+        url = env.ADMIN_SERVER_URL + endpoint
+        json_data = json.dumps(data or {}).encode("utf-8")
+        api_key = env.AGENT_API_KEY
+        headers = {"Content-Type": "application/json"}
+        if api_key:
+            headers["Authorization"] = f"ApiKey {api_key}"
+        req = urllib.request.Request(
+            url,
+            data=json_data,
+            headers=headers
+        )
+
+        with urllib.request.urlopen(req, timeout=30) as response:
+            response_body = response.read().decode("utf-8")
+            return json.loads(response_body)
+    except Exception as e:
+        from .output import log
+        log(f"HTTP POST 请求失败: {endpoint}, 错误: {e}", level="ERROR")
+        return {"code": -1, "message": str(e)}
+
+
+def allocate_proxy(vendor: str, target_site: str) -> Dict[str, Any]:
+    """
+    从管理端获取可用代理
+
+    Args:
+        vendor: 供应商标识 (由管理端代理配置统一下发)
+        target_site: 被爬取网站标识 (如 example.com)
+
+    Returns:
+        代理信息字典，包含 ip, port, username, password, protocol
+        如果获取失败，返回空字典
+    """
+    from .output import log
+
+    result = _http_post("/api/proxies/allocate", {
+        "vendor": vendor,
+        "targetSite": target_site
+    })
+
+    if result.get("code") == 200 and result.get("data"):
+        proxy_info = result["data"]
+        log(f"获取代理成功: {vendor} -> {proxy_info.get('ip')}:{proxy_info.get('port')}", level="INFO")
+        return proxy_info
+    else:
+        log(f"获取代理失败: {result.get('message', '未知错误')}", level="ERROR")
+        return {}
+
+
+def report_proxy_block(vendor: str, ip: str, target_site: str) -> bool:
+    """
+    上报代理被目标站封禁
+
+    Args:
+        vendor: 供应商标识
+        ip: 代理IP
+        target_site: 被爬取网站标识
+
+    Returns:
+        True-上报成功, False-上报失败
+    """
+    from .output import log
+
+    result = _http_post("/api/proxies/block", {
+        "vendor": vendor,
+        "ip": ip,
+        "targetSite": target_site
+    })
+
+    if result.get("code") == 200:
+        log(f"上报屏蔽成功: {vendor} {ip} -> {target_site}", level="INFO")
+        return True
+    else:
+        log(f"上报屏蔽失败: {result.get('message', '未知错误')}", level="WARNING")
+        return False
+
+
+class ProxyExhaustedError(Exception):
+    """代理用尽异常"""
+
+    def __init__(self, reason: str):
+        self.reason = reason
+        super().__init__(f"代理用尽: {reason}")
+
+
+class ProxyManager:
+    """
+    代理管理器，根据环境变量配置决定代理策略
+    """
+
+    def __init__(self):
+        self.enabled = env.PROXY_ENABLED
+        self.mode = env.PROXY_MODE
+        self.proxy_url = env.PROXY_URL
+        self.vendor = env.PROXY_VENDOR
+        self.target_site = env.PROXY_TARGET_SITE
+
+    def resolve(self) -> Optional[Dict[str, str]]:
+        """
+        获取当前代理配置
+
+        Returns:
+            代理 dict: {"http": url, "https": url} 或 None(直连)
+        """
+        if not self.enabled:
+            return None
+
+        if self.mode == "tunnel":
+            return self._resolve_tunnel()
+        elif self.mode == "private":
+            return self._resolve_private()
+        else:
+            return None
+
+    def _resolve_tunnel(self) -> Optional[Dict[str, str]]:
+        """Tunnel 模式: 解析静态代理 URL"""
+        if not self.proxy_url:
+            return None
+        return {"http": self.proxy_url, "https": self.proxy_url}
+
+    def _resolve_private(self) -> Optional[Dict[str, str]]:
+        """Private 模式: 从管理端分配代理,失败重试3次"""
+        for attempt in range(1, 4):
+            proxy_info = allocate_proxy(self.vendor, self.target_site)
+            if proxy_info:
+                return self._build_proxy_url(proxy_info)
+            if attempt < 3:
+                from .output import log
+                log(f"分配代理失败,重试第{attempt + 1}次", level="WARNING")
+        raise ProxyExhaustedError("allocate_proxy 重试3次后仍失败")
+
+    def _build_proxy_url(self, proxy_info: Dict[str, Any]) -> Dict[str, str]:
+        """将代理信息构建为 URL 格式"""
+        ip = proxy_info.get("ip", "")
+        port = proxy_info.get("port", "")
+        username = proxy_info.get("username", "")
+        password = proxy_info.get("password", "")
+
+        if username and password:
+            proxy_url = f"http://{username}:{password}@{ip}:{port}"
+        else:
+            proxy_url = f"http://{ip}:{port}"
+
+        return {"http": proxy_url, "https": proxy_url}
+
+    def report_block(self, proxy_info: Dict[str, Any]) -> None:
+        """
+        上报代理被封
+
+        Args:
+            proxy_info: 代理信息字典
+        """
+        ip = proxy_info.get("ip", "") if isinstance(proxy_info, dict) else str(proxy_info)
+        report_proxy_block(self.vendor, ip, self.target_site)

crawlerkeeper_sdk/stealth_session.py

@@ -0,0 +1,196 @@
+"""
+StealthSession - TLS 指纹伪装和代理注入
+
+基于 curl-cffi 实现浏览器级 TLS 指纹，自动从 ProxyManager 读取代理配置。
+"""
+
+import random
+from typing import Any, Dict, Optional
+
+try:
+    from curl_cffi import requests as curl_requests
+except ImportError:
+    curl_requests = None
+
+from .env import CRAWL_TIMEOUT
+from .output import error, info
+from .proxy import ProxyExhaustedError, ProxyManager
+
+# TLS 指纹池：由 SDK 维护，随库发版更新
+TLS_NAMES = [
+    "edge99",
+    "edge101",
+    "chrome99",
+    "chrome100",
+    "chrome101",
+    "chrome104",
+    "chrome107",
+    "chrome110",
+    "chrome116",
+    "chrome119",
+    "chrome120",
+    "chrome123",
+    "chrome124",
+    "chrome131",
+    "chrome133a",
+    "chrome136",
+    "chrome142",
+    "chrome145",
+    "chrome146",
+    "chrome99_android",
+    "chrome131_android",
+    "safari153",
+    "safari155",
+    "safari170",
+    "safari172_ios",
+    "safari180",
+    "safari180_ios",
+    "safari184",
+    "safari184_ios",
+    "safari260",
+    "safari2601",
+    "safari260_ios",
+    "firefox133",
+    "firefox135",
+    "firefox144",
+    "firefox147",
+]
+
+class StealthSession:
+    """
+    支持 TLS 指纹伪装的 HTTP Session。
+
+    构造时随机选择 TLS 指纹和 User-Agent，通过 ProxyManager 管理代理配置。
+    默认超时时间为 CRAWL_TIMEOUT 环境变量指定（默认 30 秒）。
+    对外暴露 get/post 接口，使用方式与 requests.Session 类似。
+    """
+
+    def __init__(self):
+        if curl_requests is None:
+            raise ImportError("请先安装 curl-cffi: uv add curl-cffi")
+
+        self._tls_name = random.choice(TLS_NAMES)
+        self._proxy_manager = ProxyManager()
+        self._session = curl_requests.Session(timeout=CRAWL_TIMEOUT)
+
+    def get(self, url: str, **kwargs) -> "curl_requests.Response":
+        """
+        发送 GET 请求，自动注入 TLS 指纹。
+
+        Args:
+            url: 请求 URL
+            **kwargs: 透传给 curl-cffi Session.get（如 headers, timeout 等）
+
+        Returns:
+            curl_cffi.requests.Response 对象
+        """
+        headers = kwargs.pop("headers", {})
+
+        if not self._proxy_manager.enabled:
+            # 代理关闭: 直连，不重试
+            return self._do_request("get", url, proxies=None, headers=headers, **kwargs)
+
+        if self._proxy_manager.mode == "tunnel":
+            # Tunnel 模式: 使用固定代理,403 重试
+            return self._tunnel_request("get", url, headers=headers, **kwargs)
+        else:
+            # Private 模式: 动态代理,403 切换
+            return self._private_request("get", url, headers=headers, **kwargs)
+
+    def post(self, url: str, **kwargs) -> "curl_requests.Response":
+        """
+        发送 POST 请求，自动注入 TLS 指纹。
+
+        Args:
+            url: 请求 URL
+            **kwargs: 透传给 curl-cffi Session.post（如 data, json, timeout 等）
+
+        Returns:
+            curl_cffi.requests.Response 对象
+        """
+        headers = kwargs.pop("headers", {})
+
+        if not self._proxy_manager.enabled:
+            # 代理关闭: 直连，不重试
+            return self._do_request("post", url, proxies=None, headers=headers, **kwargs)
+
+        if self._proxy_manager.mode == "tunnel":
+            # Tunnel 模式: 使用固定代理,403 重试
+            return self._tunnel_request("post", url, headers=headers, **kwargs)
+        else:
+            # Private 模式: 动态代理,403 切换
+            return self._private_request("post", url, headers=headers, **kwargs)
+
+    def _tunnel_request(self, method: str, url: str, headers: Dict, **kwargs):
+        """Tunnel 模式请求: 固定代理,异常或403重试，共用3次重试"""
+        for attempt in range(1, 4):
+            proxy = self._proxy_manager.resolve()
+            try:
+                resp = self._do_request(method, url, proxies=proxy, headers=headers, **kwargs)
+            except Exception as e:
+                if attempt == 3:
+                    raise
+                error(f"[tunnel] 请求异常 (attempt {attempt}/3): {e}")
+                continue
+            if resp.status_code != 403:
+                info(f"[tunnel] 请求成功 (attempt {attempt}/3): status={resp.status_code}, resp={resp.text[:200] if resp.text else ''}")
+                return resp
+            info(f"[tunnel] 请求返回 403 (attempt {attempt}/3): resp={resp.text[:200] if resp.text else ''}")
+            if attempt == 3:
+                break
+        raise ProxyExhaustedError("tunnel 模式请求达到最大重试次数")
+
+    def _private_request(self, method: str, url: str, headers: Dict, **kwargs):
+        """Private 模式请求: 动态代理,异常或403切换，共用3次重试"""
+        for attempt in range(1, 4):
+            proxy = self._proxy_manager.resolve()
+            try:
+                resp = self._do_request(method, url, proxies=proxy, headers=headers, **kwargs)
+            except Exception as e:
+                if attempt == 3:
+                    raise
+                error(f"[private] 请求异常 (attempt {attempt}/3): {e}")
+                continue
+
+            if resp.status_code != 403:
+                info(f"[private] 请求成功 (attempt {attempt}/3): status={resp.status_code}, resp={resp.text[:200] if resp.text else ''}")
+                return resp
+
+            # 403: 上报并切换代理
+            info(f"[private] 请求返回 403 (attempt {attempt}/3): resp={resp.text[:200] if resp.text else ''}")
+            proxy_info = self._extract_proxy_info(proxy)
+            if proxy_info:
+                self._proxy_manager.report_block(proxy_info)
+
+            if attempt == 3:
+                break
+
+        raise ProxyExhaustedError("private 模式请求达到最大重试次数")
+
+    def _do_request(self, method: str, url: str, proxies: Optional[Dict[str, str]], headers: Dict, **kwargs):
+        """执行 HTTP 请求"""
+        if method == "get":
+            return self._session.get(url, impersonate=self._tls_name, headers=headers, proxies=proxies, **kwargs)
+        else:
+            return self._session.post(url, impersonate=self._tls_name, headers=headers, proxies=proxies, **kwargs)
+
+    def _extract_proxy_info(self, proxy: Optional[Dict[str, str]]) -> Optional[Dict[str, Any]]:
+        """从代理 URL 中提取 ip 和 port 信息"""
+        if not proxy:
+            return None
+        http_proxy = proxy.get("http", "")
+        if not http_proxy:
+            return None
+        # 格式: http://user:pass@ip:port 或 http://ip:port
+        import re
+        match = re.search(r"@([\d.]+):(\d+)", http_proxy)
+        if match:
+            return {"ip": match.group(1), "port": match.group(2)}
+        match = re.search(r"://([\d.]+):(\d+)", http_proxy)
+        if match:
+            return {"ip": match.group(1), "port": match.group(2)}
+        return None
+
+    def close(self) -> None:
+        """关闭底层 Session。"""
+        self._session.close()

crawlerkeeper_sdk-0.2.1.dist-info/METADATA

@@ -0,0 +1,264 @@
+Metadata-Version: 2.5
+Name: crawlerkeeper-sdk
+Version: 0.2.1
+Summary: CrawlerKeeper 爬虫 SDK：日志上报、结果写入、任务参数解析、TLS 指纹请求、代理管理与并发线程池
+Author: CrawlerKeeper Team
+License-Expression: MIT
+License-File: LICENSE
+Keywords: crawler,crawlerkeeper,scraping,spider,tls-fingerprint
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: curl-cffi>=0.7.0
+Description-Content-Type: text/markdown
+
+# CrawlerKeeper SDK for Python
+
+CrawlerKeeper Python SDK 提供了爬虫与 Agent 通信的封装，支持日志记录、结果写入、参数获取、代理管理和并发爬取。
+
+## 安装
+
+```bash
+pip install crawlerkeeper-sdk
+```
+
+## 快速开始
+
+```python
+from crawlerkeeper_sdk import params, info, result, StealthSession
+
+# 获取任务参数
+config = params()
+info(f"收到参数: {config}")
+
+# 使用 StealthSession 发送请求
+session = StealthSession()
+resp = session.get("https://example.com/api/data")
+data = resp.json()
+
+# 写入结果
+result({"data": data, "source": "example.com"})
+
+info("爬取完成")
+session.close()
+```
+
+## 核心功能
+
+### 日志记录
+
+```python
+from crawlerkeeper_sdk import info, warning, error
+
+info("信息日志")
+warning("警告日志")
+error("错误日志")
+```
+
+### 结果写入
+
+```python
+from crawlerkeeper_sdk import result
+
+result({"count": 100, "items": [...]})
+```
+
+### 任务参数
+
+```python
+from crawlerkeeper_sdk import params
+
+# python spider.py --params '{"keyword": "python", "page": 1}'
+config = params()  # {"keyword": "python", "page": 1}
+```
+
+## StealthSession
+
+`StealthSession` 提供隐式浏览器指纹的 HTTP 请求能力，自动管理 TLS 指纹和代理。
+
+```python
+from crawlerkeeper_sdk import StealthSession, result
+
+session = StealthSession()
+
+# GET 请求
+resp = session.get("https://example.com/api/data")
+print(resp.status_code)
+print(resp.json())
+
+# POST 请求
+resp = session.post("https://example.com/api/submit", json={"key": "value"})
+
+session.close()
+```
+
+### 主要方法
+
+| 方法 | 说明 |
+|------|------|
+| `get(url, **kwargs)` | 发送 GET 请求 |
+| `post(url, **kwargs)` | 发送 POST 请求 |
+| `close()` | 关闭会话 |
+
+## ThreadPool 并发爬取
+
+`ThreadPool` 提供并发爬取能力，内部管理 `StealthSession` 的创建和关闭。
+
+### 推荐用法：使用上下文管理器
+
+```python
+from crawlerkeeper_sdk import ThreadPool, StealthSession, result
+
+def crawl(session: StealthSession, keyword: str, page: int):
+    """爬虫函数：session 会自动传入"""
+    resp = session.get(f"https://example.com/search?kw={keyword}&page={page}")
+    return resp.json()
+
+# 使用 with 语句自动管理生命周期，不会遗漏 close()
+with ThreadPool(max_workers=5) as pool:
+    pool.submit(crawl, keyword="python", page=1)
+    pool.submit(crawl, keyword="java", page=1)
+    pool.submit(crawl, keyword="go", page=1)
+
+    # 阻塞等待所有任务完成
+    results = pool.wait()
+
+    # 处理结果
+    for data in results:
+        if data is not None:
+            result(data)
+# with 块结束时自动调用 wait() 和 close()
+```
+
+### 传统用法
+
+```python
+pool = ThreadPool(max_workers=5)
+pool.submit(crawl, keyword="python", page=1)
+pool.submit(crawl, keyword="java", page=1)
+
+results = pool.wait()
+pool.close()  # 需要手动调用
+```
+
+### 环境变量配置并发数
+
+```bash
+export CONCURRENT_NUM=10
+```
+
+```python
+# 不指定 max_workers 时使用 CONCURRENT_NUM 环境变量
+with ThreadPool() as pool:
+    pool.submit(crawl, keyword="python", page=1)
+```
+
+### ThreadPool 方法
+
+| 方法 | 说明 |
+|------|------|
+| `submit(crawl_fn, **kwargs)` | 提交任务，worker 全满时阻塞等待 |
+| `wait()` | 阻塞等待所有任务完成，返回结果列表 |
+| `close()` | 等待所有任务完成后关闭线程池 |
+
+### 异常处理
+
+爬虫函数抛出的异常会被捕获并通过 `output.error()` 记录，任务结果记为 `None`，不中断其他任务。
+
+```python
+def failing_crawl(session):
+    raise ValueError("some error")
+
+pool = ThreadPool(max_workers=2)
+pool.submit(failing_crawl)
+results = pool.wait()  # results[0] is None
+pool.close()
+```
+
+## 代理配置
+
+`StealthSession` 的代理行为由环境变量控制（通常由 Agent 下发），不支持按请求传入代理参数：
+
+| 环境变量 | 说明 |
+|---------|------|
+| `PROXY_ENABLED` | `on` 开启代理，其他值直连 |
+| `PROXY_MODE` | `tunnel` 静态代理 / `private` 管理端动态分配 |
+| `PROXY_URL` | tunnel 模式的静态代理 URL |
+| `PROXY_VENDOR` | private 模式的供应商标识 |
+| `PROXY_TARGET_SITE` | private 模式的目标站点标识 |
+| `ADMIN_SERVER_URL` | private 模式下管理端地址 |
+| `AGENT_API_KEY` | private 模式下调用管理端的第三方 Token（`td_` 前缀；代理分配/上报接口仅对第三方 Token 开放，节点 `ck_` Key 无权调用） |
+
+```bash
+# tunnel 模式示例
+export PROXY_ENABLED=on
+export PROXY_MODE=tunnel
+export PROXY_URL=http://proxy:8080
+```
+
+private 模式下代理用尽或持续被目标站封禁时，`get`/`post` 抛出 `ProxyExhaustedError`。
+
+## 运行模式
+
+SDK 自动检测运行模式：
+
+- **Agent 模式**: 当 `LOG_FILE_PATH` 和 `DATA_FILE_PATH` 环境变量存在时，日志和结果写入文件
+- **开发模式**: 环境变量不存在时，输出到控制台
+
+```bash
+# 开发模式
+python spider.py
+
+# Agent 模式
+export LOG_FILE_PATH=/tmp/test.log
+export DATA_FILE_PATH=/tmp/test.json
+python spider.py
+```
+
+## 完整示例
+
+```python
+#!/usr/bin/env python3
+from crawlerkeeper_sdk import params, info, result, StealthSession, ThreadPool
+
+def crawl(session: StealthSession, keyword: str, page: int):
+    """爬虫函数"""
+    info(f"爬取 {keyword} 第 {page} 页")
+    resp = session.get(f"https://example.com/api?kw={keyword}&page={page}")
+    return resp.json()
+
+def main():
+    config = params()
+    keywords = config.get("keywords", ["python"])
+    max_workers = config.get("max_workers", 3)
+
+    with ThreadPool(max_workers=max_workers) as pool:
+        # 提交所有任务
+        for keyword in keywords:
+            for page in range(1, 4):
+                pool.submit(crawl, keyword=keyword, page=page)
+
+        # 阻塞等待所有任务完成
+        results = pool.wait()
+
+        # 输出结果
+        for data in results:
+            if data is not None:
+                result(data)
+
+    info("爬取完成")
+
+if __name__ == "__main__":
+    main()
+```
+
+## 许可证
+
+MIT License

crawlerkeeper_sdk-0.2.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+crawlerkeeper_sdk/README.md,sha256=HGmr25aTEbBqINiCKd4DT70IdYQd9ZyCBthP0cGkjBQ,6014
+crawlerkeeper_sdk/__init__.py,sha256=T5gFCn8l0MfLMwcH4oeaXyJjXKxskJQO9OA6yJgSuFo,527
+crawlerkeeper_sdk/env.py,sha256=LeZhobgb4lxAtx4lAWsNt1pQ4pFCTBPLJMFlKFfkFrc,954
+crawlerkeeper_sdk/output.py,sha256=NmcRPkr8BiU6p-fvO-ehD3Tbfja8DZpID1arslSDTRI,2489
+crawlerkeeper_sdk/params.py,sha256=pv3t9JrL54WZEIzKKf6Kf9Mueoc5v3By7JbgfA8Cbx8,987
+crawlerkeeper_sdk/pool.py,sha256=LhkRnnY_yB05_r_JRqH7KcbxjlCxwU_5UxJztv2KnOM,3706
+crawlerkeeper_sdk/proxy.py,sha256=0CG9Ot0OKiKJFxuNxFziHZmBkgVOv6BViCFQ7DfG9ZU,5384
+crawlerkeeper_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlerkeeper_sdk/stealth_session.py,sha256=XfvQCh5s9bQ9F-nPD6Zid5A0shmgEZTpktm1oE1j1F0,7003
+crawlerkeeper_sdk-0.2.1.dist-info/METADATA,sha256=LwpTrJfiOjT0oIXRKQ-rdnuYPa2CBtAp3ud9QYDzn7E,6900
+crawlerkeeper_sdk-0.2.1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+crawlerkeeper_sdk-0.2.1.dist-info/licenses/LICENSE,sha256=QM-S3_7nggIKygDQWfaysv1PqPGsKjHYzC_Q1p_zVLo,1075
+crawlerkeeper_sdk-0.2.1.dist-info/RECORD,,

crawlerkeeper_sdk-0.2.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

crawlerkeeper_sdk-0.2.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 CrawlerKeeper Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.