crawlerkeeper-agent 1.0.0__py3-none-any.whl

crawlerkeeper/__init__.py

@@ -0,0 +1,2 @@
+"""CrawlerKeeper Agent - 节点代理程序"""
+__version__ = "1.0.0"

crawlerkeeper/__main__.py

@@ -0,0 +1,65 @@
+"""CrawlerKeeper Agent 命令行入口。
+
+提供 pip 安装后的 ``crawlerkeeper-agent`` 命令与 ``python -m crawlerkeeper`` 两种启动方式。
+本地源码运行走 ``src/main.py``（薄委托到本模块）。
+"""
+import os
+import sys
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from crawlerkeeper import __version__
+from crawlerkeeper.agent import Agent
+
+
+def main() -> None:
+    """主函数"""
+    # 版本查询：只打印版本号，不启动 Agent
+    if len(sys.argv) > 1 and sys.argv[1] in ("-V", "--version"):
+        print(f"crawlerkeeper-agent {__version__}")
+        return
+
+    print("正在启动 CrawlerKeeper Agent...", flush=True)
+
+    # 解析命令行参数
+    config_file: Optional[str] = None
+    extra_config: Dict[str, Any] = {"cleanup_temp_files": True}  # 默认开启清理
+    i = 1
+    while i < len(sys.argv):
+        if sys.argv[i] in ['-c', '--config'] and i + 1 < len(sys.argv):
+            config_file = sys.argv[i + 1]
+            i += 2
+        elif sys.argv[i].startswith('--cleanup_temp_files='):
+            _, value = sys.argv[i].split('=', 1)
+            extra_config['cleanup_temp_files'] = value.lower() == 'on'
+            i += 1
+        elif sys.argv[i] == '--cleanup_temp_files':
+            # 单独出现默认为 on，或取下一个参数值
+            if i + 1 < len(sys.argv) and sys.argv[i + 1] in ('on', 'off'):
+                extra_config['cleanup_temp_files'] = sys.argv[i + 1].lower() == 'on'
+                i += 2
+            else:
+                extra_config['cleanup_temp_files'] = True
+                i += 1
+        else:
+            i += 1
+
+    # 如果没有指定配置文件，使用当前工作目录下的默认路径
+    if not config_file:
+        config_file = str(Path.cwd() / "config" / "config.yaml")
+
+    print(f"配置文件: {config_file}", flush=True)
+    print(f"额外配置: {extra_config}", flush=True)
+
+    # 创建并启动 Agent
+    agent = Agent(config_file if os.path.exists(config_file) else None, extra_config=extra_config)
+
+    try:
+        agent.start()
+    except Exception as e:
+        print(f"Agent 启动失败: {e}", flush=True)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

crawlerkeeper/agent.py

@@ -0,0 +1,281 @@
+"""Agent 主程序模块"""
+import os
+import sys
+import signal
+import logging
+import time
+import threading
+from pathlib import Path
+from typing import Optional, Dict, Any
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from .config import Config
+from .client import HttpClient
+from .heartbeat import HeartbeatManager
+from .task import TaskManager
+from .executor import SpiderExecutor
+from .deployment import DeploymentManager
+from .logger import setup_logger
+from .metrics import get_metrics_server
+
+logger = logging.getLogger("crawlerkeeper-agent")
+
+
+class Agent:
+    """CrawlerKeeper Agent 主类"""
+
+    def __init__(self, config_file: str = None, extra_config: Optional[Dict[str, Any]] = None):
+        self.config = Config(config_file)
+        self.extra_config = extra_config or {}
+        self.client: Optional[HttpClient] = None
+        self.heartbeat_manager: Optional[HeartbeatManager] = None
+        self.task_manager: Optional[TaskManager] = None
+        self.executor: Optional[SpiderExecutor] = None
+        self.deployment_manager: Optional[DeploymentManager] = None
+        self.running = False
+        self.metrics_server = None
+        self._http_server = None
+        self._http_thread = None
+
+    def _start_http_server(self):
+        """启动 HTTP 服务器用于 /metrics 端点"""
+        metrics_server = get_metrics_server()
+        metrics_server.set_thread_pool_size(self.task_manager.max_workers if self.task_manager else 4)
+
+        MetricsHandler = type('MetricsHandler', (BaseHTTPRequestHandler,), {
+            'do_GET': lambda self: (
+                self.send_response(200),
+                self.send_header('Content-Type', metrics_server.get_content_type()),
+                self.end_headers(),
+                self.wfile.write(metrics_server.get_metrics())
+            )
+        })
+
+        server_address = ('', self.config.agent_port)
+        self._http_server = HTTPServer(server_address, MetricsHandler)
+        self._http_thread = threading.Thread(target=self._http_server.serve_forever, daemon=True)
+        self._http_thread.start()
+        logger.info(f"指标 HTTP 服务器已启动，端口: {self.config.agent_port}")
+
+    def start(self):
+        """启动 Agent"""
+        # 设置日志
+        setup_logger(
+            level=self.config.log_level,
+            log_file=self.config.log_file
+        )
+
+        logger.info("=" * 50)
+        logger.info("CrawlerKeeper Agent 启动")
+        logger.info("=" * 50)
+        logger.info(f"配置: {self.config}")
+
+        # 创建 HTTP 客户端
+        self.client = HttpClient(
+            base_url=self.config.server_url,
+            api_key=self.config.api_key,
+            timeout=self.config.heartbeat_timeout
+        )
+
+        # 注册节点
+        if not self._register_node():
+            logger.error("节点注册失败，Agent 退出")
+            return False
+
+        # 初始化组件
+        self._init_components()
+
+        # 初始化指标服务器
+        self.metrics_server = get_metrics_server()
+
+        # 启动 HTTP 服务器（用于 /metrics）
+        self._start_http_server()
+
+        # 注册信号处理
+        self._register_signal_handlers()
+
+        # 启动组件
+        self._start_components()
+
+        # 主循环
+        self._main_loop()
+
+        return True
+
+    def _register_node(self) -> bool:
+        """注册节点"""
+        # 检查是否已有节点 ID
+        if self.config.node_id and self.config.api_key:
+            logger.info(f"使用已有节点配置: node_id={self.config.node_id}")
+            return True
+
+        # 注册新节点
+        logger.info("开始注册节点...")
+        logger.info(f"注册配置: name={self.config.agent_name}, ip={self.config.agent_ip}, port={self.config.agent_port}")
+
+        url = "/api/nodes/register"
+        data = {
+            "name": self.config.agent_name,
+            "ip": self.config.agent_ip,
+            "port": self.config.agent_port,
+            "description": f"CrawlerKeeper Agent - {self.config.agent_name}"
+        }
+        logger.debug(f"注册请求数据: {data}")
+
+        response = self.client.post(url, data)
+
+        if not response:
+            logger.error("节点注册请求失败")
+            return False
+
+        if response.get("code") != 200:
+            logger.error(f"节点注册失败: {response.get('message')}")
+            return False
+
+        # 保存节点信息
+        data = response.get("data", {})
+        self.config.node_id = data.get("id")
+
+        if not self.config.node_id:
+            logger.error("节点注册响应数据不完整")
+            return False
+
+        logger.info(f"节点注册成功: id={self.config.node_id}")
+
+        # 保存配置
+        config_file = os.getenv("AGENT_CONFIG_FILE", "./config/config.yaml")
+        self.config.save_to_file(config_file)
+
+        return True
+
+    def _init_components(self):
+        """初始化组件"""
+        # 心跳管理器
+        self.heartbeat_manager = HeartbeatManager(
+            client=self.client,
+            node_id=self.config.node_id,
+            interval=self.config.heartbeat_interval,
+            timeout=self.config.heartbeat_timeout
+        )
+
+        # 任务执行器
+        self.executor = SpiderExecutor(
+            client=self.client,
+            work_dir=self.config.work_dir,
+            task_execute_timeout=self.config.task_execute_timeout
+        )
+
+        # 任务管理器
+        self.task_manager = TaskManager(
+            client=self.client,
+            node_id=self.config.node_id,
+            poll_interval=self.config.task_poll_interval,
+            work_dir=self.config.work_dir,
+            extra_config=self.extra_config
+        )
+        self.task_manager.set_executor(self.executor)
+        self.executor.set_task_manager(self.task_manager)
+
+        # 部署管理器
+        self.deployment_manager = DeploymentManager(
+            client=self.client,
+            node_id=self.config.node_id,
+            poll_interval=self.config.deploy_poll_interval,
+            work_dir=self.config.work_dir
+        )
+
+        logger.info("组件初始化完成")
+
+    def _start_components(self):
+        """启动组件"""
+        # 启动心跳
+        self.heartbeat_manager.start()
+
+        # 启动任务拉取
+        self.task_manager.start()
+
+        # 启动部署拉取
+        self.deployment_manager.start()
+
+        self.running = True
+        logger.info("所有组件已启动")
+
+    def _stop_components(self):
+        """停止组件"""
+        logger.info("正在停止组件...")
+
+        if self.task_manager:
+            self.task_manager.stop()
+
+        if self.heartbeat_manager:
+            self.heartbeat_manager.stop()
+
+        if self.deployment_manager:
+            self.deployment_manager.stop()
+
+        logger.info("所有组件已停止")
+
+    def _register_signal_handlers(self):
+        """注册信号处理器"""
+        def sigterm_handler(signum, frame):
+            """SIGTERM 信号处理器 - 优雅关闭"""
+            logger.info(f"收到 SIGTERM 信号: {signum}，将进行优雅关闭")
+            self.stop(graceful=True)
+
+        def sigint_handler(signum, frame):
+            """SIGINT 信号处理器 - 强制关闭"""
+            logger.info(f"收到 SIGINT 信号: {signum}，将强制关闭")
+            self.stop(graceful=False)
+
+        signal.signal(signal.SIGINT, sigint_handler)
+        signal.signal(signal.SIGTERM, sigterm_handler)
+
+    def _main_loop(self):
+        """主循环"""
+        logger.info("Agent 运行中，按 Ctrl+C 停止")
+
+        try:
+            while self.running:
+                # 检查心跳状态
+                if self.heartbeat_manager and not self.heartbeat_manager.is_alive():
+                    logger.warning("心跳检测失败，尝试重新注册...")
+                    if not self._register_node():
+                        logger.error("重新注册失败")
+
+                time.sleep(10)
+
+        except KeyboardInterrupt:
+            logger.info("收到键盘中断")
+
+    def stop(self, graceful: bool = True):
+        """停止 Agent
+
+        Args:
+            graceful: True 则优雅关闭（等待任务完成），False 则强制关闭
+        """
+        logger.info(f"正在停止 Agent (graceful={graceful})...")
+        self.running = False
+
+        # 停止 HTTP 服务器
+        if self._http_server:
+            self._http_server.shutdown()
+            logger.info("指标 HTTP 服务器已关闭")
+
+        # 停止任务管理器（传入优雅关闭参数）
+        if self.task_manager:
+            self.task_manager.stop(graceful=graceful)
+
+        # 停止其他组件
+        if self.heartbeat_manager:
+            self.heartbeat_manager.stop()
+
+        if self.deployment_manager:
+            self.deployment_manager.stop()
+
+        logger.info("Agent 已停止")
+
+    def restart(self):
+        """重启 Agent"""
+        logger.info("正在重启 Agent...")
+        self.stop()
+        time.sleep(2)
+        self.start()

crawlerkeeper/client.py

@@ -0,0 +1,112 @@
+"""HTTP 客户端模块"""
+import logging
+import requests
+import time
+from typing import Optional, Dict, Any
+
+# 使用根 logger，确保能继承全局配置
+logger = logging.getLogger("crawlerkeeper-agent")
+logger.setLevel(logging.DEBUG)
+logger.propagate = True
+
+
+class HttpClient:
+    """HTTP 客户端封装"""
+
+    def __init__(self, base_url: str, api_key: str = None, timeout: int = 30):
+        self.base_url = base_url.rstrip('/')
+        self.api_key = api_key
+        self.timeout = timeout
+        self.session = requests.Session()
+
+    def _get_headers(self) -> Dict[str, str]:
+        """获取请求头"""
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json"
+        }
+        if self.api_key:
+            headers["Authorization"] = f"ApiKey {self.api_key}"
+        return headers
+
+    def _build_url(self, path: str) -> str:
+        """构建完整 URL"""
+        if path.startswith('http'):
+            return path
+        return f"{self.base_url}{path}"
+
+    def get(self, path: str, params: Optional[Dict] = None, retry: int = 3) -> Optional[Dict]:
+        """发送 GET 请求"""
+        url = self._build_url(path)
+        headers = self._get_headers()
+
+        for attempt in range(retry):
+            try:
+                response = self.session.get(
+                    url,
+                    params=params,
+                    headers=headers,
+                    timeout=self.timeout
+                )
+                response.raise_for_status()
+                result = response.json()
+                return result
+            except requests.exceptions.RequestException as e:
+                logger.warning(f"GET 请求失败 (尝试 {attempt + 1}/{retry}): {url}, 错误: {e}")
+                if attempt < retry - 1:
+                    time.sleep(2 ** attempt)  # 指数退避
+                else:
+                    logger.error(f"GET 请求最终失败: {url}")
+                    return None
+        return None
+
+    def post(self, path: str, data: Optional[Dict] = None, retry: int = 3) -> Optional[Dict]:
+        """发送 POST 请求"""
+        url = self._build_url(path)
+        headers = self._get_headers()
+
+        for attempt in range(retry):
+            try:
+                response = self.session.post(
+                    url,
+                    json=data,
+                    headers=headers,
+                    timeout=self.timeout
+                )
+                response.raise_for_status()
+                result = response.json()
+                return result
+            except requests.exceptions.RequestException as e:
+                logger.warning(f"POST 请求失败 (尝试 {attempt + 1}/{retry}): {url}, 错误: {e}")
+                if attempt < retry - 1:
+                    time.sleep(2 ** attempt)
+                else:
+                    logger.error(f"POST 请求最终失败: {url}")
+                    return None
+        return None
+
+    def get_raw(self, path: str, params: Optional[Dict] = None, retry: int = 3):
+        """发送 GET 请求并返回原始响应（用于下载文件）"""
+        url = self._build_url(path)
+        headers = self._get_headers()
+        # 下载文件时不需要 Accept: application/json
+        headers.pop("Accept", None)
+
+        for attempt in range(retry):
+            try:
+                response = self.session.get(
+                    url,
+                    params=params,
+                    headers=headers,
+                    timeout=self.timeout
+                )
+                response.raise_for_status()
+                return response
+            except requests.exceptions.RequestException as e:
+                logger.warning(f"GET raw 请求失败 (尝试 {attempt + 1}/{retry}): {url}, 错误: {e}")
+                if attempt < retry - 1:
+                    time.sleep(2 ** attempt)
+                else:
+                    logger.error(f"GET raw 请求最终失败: {url}")
+                    return None
+        return None

crawlerkeeper/code_package.py

@@ -0,0 +1,45 @@
+"""爬虫代码包（ZIP）下载校验工具。
+
+后端业务异常（如 Git 克隆失败、归属校验失败）会以 HTTP 200 + JSON 响应体返回，
+Agent 若直接把响应体当 ZIP 解压，真实失败原因会被掩盖成「解压失败」。
+下载后先按 ZIP 魔数校验响应体，非 ZIP 时输出可定位的错误提示并按下载失败处理。
+"""
+import logging
+from typing import Optional
+
+logger = logging.getLogger("crawlerkeeper-agent")
+
+# ZIP 本地文件头魔数（PK\x03\x04）
+ZIP_MAGIC = b"PK\x03\x04"
+
+# 错误响应体日志摘录长度上限
+SNIPPET_LIMIT = 200
+
+
+def is_zip_content(content: Optional[bytes]) -> bool:
+    """判断响应体是否为 ZIP 包
+
+    Args:
+        content: 响应体字节
+
+    Returns:
+        True 表示响应体以 ZIP 魔数开头
+    """
+    if not content:
+        return False
+    return bool(content.startswith(ZIP_MAGIC))
+
+
+def body_snippet(content: Optional[bytes]) -> str:
+    """提取响应体文本摘录（用于非 ZIP 响应时的错误定位）
+
+    Args:
+        content: 响应体字节
+
+    Returns:
+        可读文本摘录（最多 200 字符），空响应体返回占位说明
+    """
+    if not content:
+        return "<空响应体>"
+    text = content.decode("utf-8", errors="replace").strip()
+    return text[:SNIPPET_LIMIT]

crawlerkeeper/config.py

@@ -0,0 +1,175 @@
+"""配置管理模块"""
+import os
+import json
+from pathlib import Path
+from typing import Optional
+import yaml
+import socket
+import logging
+
+logger = logging.getLogger("crawlerkeeper-agent")
+
+
+class Config:
+    """Agent 配置类"""
+
+    def __init__(self, config_file: str = None):
+        # 默认配置
+        self.server_url: str = os.getenv("AGENT_SERVER_URL", "http://localhost:8080")
+        self.api_key: str = os.getenv("AGENT_API_KEY", "")
+
+        self.agent_name: str = os.getenv("AGENT_NAME", socket.gethostname())
+        self.agent_ip: str = os.getenv("AGENT_IP", self._get_local_ip())
+        self.agent_port: int = int(os.getenv("AGENT_PORT", "9000"))
+
+        self.heartbeat_interval: int = int(os.getenv("AGENT_HEARTBEAT_INTERVAL", "30"))
+        self.heartbeat_timeout: int = int(os.getenv("AGENT_HEARTBEAT_TIMEOUT", "90"))
+
+        self.task_poll_interval: int = int(os.getenv("AGENT_TASK_POLL_INTERVAL", "10"))
+        self.deploy_poll_interval: int = int(os.getenv("AGENT_DEPLOY_POLL_INTERVAL", "30"))
+        # 任务执行超时（秒）：0 表示不限时；负值视为配置错误，按 0 处理（避免立即超时误杀任务）
+        self.task_execute_timeout: int = int(os.getenv("AGENT_TASK_EXECUTE_TIMEOUT", "3600"))
+        if self.task_execute_timeout < 0:
+            logger.warning(f"AGENT_TASK_EXECUTE_TIMEOUT 配置为负数({self.task_execute_timeout})，按 0（不限时）处理")
+            self.task_execute_timeout = 0
+        self.work_dir: str = os.getenv("AGENT_WORK_DIR", "./data/spiders")
+        # 注意：最大并行任务数不通过配置文件设置，由 TaskManager 根据
+        # AGENT_MAX_WORKERS 环境变量或容器 CPU 限制自动计算
+
+        self.log_level: str = os.getenv("AGENT_LOG_LEVEL", "INFO")
+        self.log_file: str = os.getenv("AGENT_LOG_FILE", "./logs/agent.log")
+
+        self.node_id: Optional[int] = None
+        self.server_url_from_register: Optional[str] = None
+
+        # 加载配置文件
+        if config_file:
+            self._load_from_file(config_file)
+
+    def _get_local_ip(self) -> str:
+        """获取本机 IP 地址"""
+        try:
+            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            s.settimeout(3)  # 3秒超时
+            s.connect(("8.8.8.8", 80))
+            ip = s.getsockname()[0]
+            s.close()
+            return ip
+        except Exception:
+            return "127.0.0.1"
+
+    def _load_from_file(self, config_file: str):
+        """从 YAML 配置文件加载配置"""
+        config_path = Path(config_file)
+        if not config_path.exists():
+            logger.warning(f"配置文件不存在: {config_file}")
+            return
+
+        try:
+            with open(config_path, 'r', encoding='utf-8') as f:
+                config = yaml.safe_load(f)
+
+            if not config:
+                return
+
+            # 主控系统配置
+            if 'server' in config:
+                server = config['server']
+                if 'url' in server and not os.getenv("AGENT_SERVER_URL"):
+                    self.server_url = server['url']
+
+            # Agent 配置
+            if 'agent' in config:
+                agent = config['agent']
+                # 只有当环境变量未设置且配置文件值不为空时才使用配置文件的值
+                if 'name' in agent and agent['name'] and not os.getenv("AGENT_NAME"):
+                    self.agent_name = agent['name']
+                if 'ip' in agent and agent.get('ip') and not os.getenv("AGENT_IP"):
+                    self.agent_ip = agent.get('ip')
+                if 'port' in agent and agent.get('port') and not os.getenv("AGENT_PORT"):
+                    try:
+                        self.agent_port = int(agent['port'])
+                    except (ValueError, TypeError):
+                        pass  # 保持默认值
+                if 'api_key' in agent and agent['api_key'] and not os.getenv("AGENT_API_KEY"):
+                    self.api_key = agent['api_key']
+
+            # 心跳配置
+            if 'heartbeat' in config:
+                hb = config['heartbeat']
+                if 'interval' in hb and not os.getenv("AGENT_HEARTBEAT_INTERVAL"):
+                    self.heartbeat_interval = hb['interval']
+                if 'timeout' in hb and not os.getenv("AGENT_HEARTBEAT_TIMEOUT"):
+                    self.heartbeat_timeout = hb['timeout']
+
+            # 任务配置
+            if 'task' in config:
+                task = config['task']
+                if 'poll_interval' in task and not os.getenv("AGENT_TASK_POLL_INTERVAL"):
+                    self.task_poll_interval = task['poll_interval']
+                if 'deploy_poll_interval' in task and not os.getenv("AGENT_DEPLOY_POLL_INTERVAL"):
+                    self.deploy_poll_interval = task['deploy_poll_interval']
+                if 'work_dir' in task and not os.getenv("AGENT_WORK_DIR"):
+                    self.work_dir = task['work_dir']
+
+            # 日志配置
+            if 'logging' in config:
+                log = config['logging']
+                if 'level' in log and not os.getenv("AGENT_LOG_LEVEL"):
+                    self.log_level = log['level']
+                if 'file' in log and not os.getenv("AGENT_LOG_FILE"):
+                    self.log_file = log['file']
+
+            if self.api_key and not os.getenv("AGENT_API_KEY"):
+                os.environ["AGENT_API_KEY"] = self.api_key
+
+            logger.info(f"已从配置文件加载配置: {config_file}")
+        except Exception as e:
+            logger.error(f"加载配置文件失败: {e}")
+
+    def save_to_file(self, config_file: str):
+        """保存配置到文件"""
+        config_path = Path(config_file)
+        config_path.parent.mkdir(parents=True, exist_ok=True)
+
+        config = {
+            'server': {
+                'url': self.server_url
+            },
+            'agent': {
+                'name': self.agent_name,
+                'ip': self.agent_ip,
+                'port': self.agent_port,
+                'api_key': self.api_key
+            },
+            'heartbeat': {
+                'interval': self.heartbeat_interval,
+                'timeout': self.heartbeat_timeout
+            },
+            'task': {
+                'poll_interval': self.task_poll_interval,
+                'deploy_poll_interval': self.deploy_poll_interval,
+                'work_dir': self.work_dir
+            },
+            'logging': {
+                'level': self.log_level,
+                'file': self.log_file
+            }
+        }
+
+        try:
+            with open(config_path, 'w', encoding='utf-8') as f:
+                yaml.dump(config, f, allow_unicode=True, default_flow_style=False)
+            logger.info(f"配置已保存到: {config_file}")
+        except Exception as e:
+            logger.error(f"保存配置文件失败: {e}")
+
+    def get_auth_headers(self) -> dict:
+        """获取认证 Header"""
+        if self.api_key:
+            return {"Authorization": f"ApiKey {self.api_key}"}
+        return {}
+
+    def __repr__(self):
+        return (f"Config(server_url={self.server_url}, agent_name={self.agent_name}, "
+                f"agent_ip={self.agent_ip}, node_id={self.node_id})")