crawlo 1.4.5__py3-none-any.whl → 1.4.6__py3-none-any.whl

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = '1.4.5'
+__version__ = '1.4.6'

crawlo/downloader/cffi_downloader.py

@@ -22,7 +22,9 @@ class CurlCffiDownloader(DownloaderBase):
     """
 
     def __init__(self, crawler):
-        self.crawler = crawler
+        # 调用父类初始化方法，确保 _closed 等属性被正确初始化
+        super().__init__(crawler)
+        
         self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
         self._active_requests = set()
 

crawlo/middleware/proxy.py

@@ -1,386 +1,209 @@
 #!/usr/bin/python
 # -*- coding: UTF-8 -*-
-import time
-import asyncio
-import socket
+"""
+通用代理中间件
+支持静态代理列表和动态代理API两种模式
+"""
+import random
 from urllib.parse import urlparse
-from typing import Optional, Dict, Any, Callable, Union, TYPE_CHECKING, List
+from typing import Optional, List
 
-from crawlo import Request, Response
-from crawlo.exceptions import NotConfiguredError
+from crawlo.network import Request, Response
 from crawlo.utils.log import get_logger
 
-if TYPE_CHECKING:
-    import aiohttp
-
-try:
-    import httpx
-
-    HTTPX_EXCEPTIONS = (httpx.NetworkError, httpx.TimeoutException, httpx.ReadError, httpx.ConnectError)
-except ImportError:
-    HTTPX_EXCEPTIONS = ()
-    httpx = None
-
-try:
-    import aiohttp
-
-    AIOHTTP_EXCEPTIONS = (
-        aiohttp.ClientError, aiohttp.ClientConnectorError, aiohttp.ClientResponseError, aiohttp.ServerTimeoutError,
-        aiohttp.ServerDisconnectedError)
-except ImportError:
-    AIOHTTP_EXCEPTIONS = ()
-    aiohttp = None
-
-try:
-    from curl_cffi import requests as cffi_requests
-
-    CURL_CFFI_EXCEPTIONS = (cffi_requests.RequestsError,)
-except (ImportError, AttributeError):
-    CURL_CFFI_EXCEPTIONS = ()
-    cffi_requests = None
-
-NETWORK_EXCEPTIONS = (
-                         asyncio.TimeoutError,
-                         socket.gaierror,
-                         ConnectionError,
-                         TimeoutError,
-                     ) + HTTPX_EXCEPTIONS + AIOHTTP_EXCEPTIONS + CURL_CFFI_EXCEPTIONS
-
-ProxyExtractor = Callable[[Dict[str, Any]], Union[None, str, Dict[str, str]]]
-
-
-class Proxy:
-    """代理对象，包含代理信息和统计数据"""
-
-    def __init__(self, proxy_str: str):
-        self.proxy_str = proxy_str
-        self.success_count = 0
-        self.failure_count = 0
-        self.last_used_time = 0.0
-        self.is_healthy = True
-
-    @property
-    def success_rate(self) -> float:
-        """计算代理成功率"""
-        total = self.success_count + self.failure_count
-        if total == 0:
-            return 1.0
-        return self.success_count / total
-
-    def mark_success(self):
-        """标记代理使用成功"""
-        self.success_count += 1
-        self.last_used_time = time.time()
-        self.is_healthy = True
-
-    def mark_failure(self):
-        """标记代理使用失败"""
-        self.failure_count += 1
-        self.last_used_time = time.time()
-        # 如果失败率过高，标记为不健康
-        if self.failure_count > 3 and self.success_rate < 0.5:
-            self.is_healthy = False
-
 
 class ProxyMiddleware:
+    """通用代理中间件"""
+
     def __init__(self, settings, log_level):
         self.logger = get_logger(self.__class__.__name__, log_level)
 
-        self._session: Optional[Any] = None  # aiohttp.ClientSession when aiohttp is available
-        # 将单个代理改为代理池
-        self._proxy_pool: List[Proxy] = []
-        self._current_proxy_index: int = 0
-        self._last_fetch_time: float = 0
-
-        self.proxy_extractor = settings.get("PROXY_EXTRACTOR", "proxy")
-        self.refresh_interval = settings.get_float("PROXY_REFRESH_INTERVAL", 60)
-        self.timeout = settings.get_float("PROXY_API_TIMEOUT", 10)
-        # 新增配置：代理池大小
-        self.proxy_pool_size = settings.get_int("PROXY_POOL_SIZE", 5)
-        # 新增配置：健康检查阈值
-        self.health_check_threshold = settings.get_float("PROXY_HEALTH_CHECK_THRESHOLD", 0.5)
-
-        self.enabled = settings.get_bool("PROXY_ENABLED", True)
-
-        if not self.enabled:
-            self.logger.info("ProxyMiddleware disabled")
-            return
-
+        # 获取代理列表和API URL
+        self.proxies: List[str] = settings.get("PROXY_LIST", [])
         self.api_url = settings.get("PROXY_API_URL")
-        if not self.api_url:
-            raise NotConfiguredError("PROXY_API_URL not configured, ProxyMiddleware disabled")
-
-        self.logger.info(
-            f"Proxy middleware enabled | API: {self.api_url} | Refresh interval: {self.refresh_interval}s | Proxy pool size: {self.proxy_pool_size}")
+        # 获取代理提取配置
+        self.proxy_extractor = settings.get("PROXY_EXTRACTOR", "proxy")  # 默认从"proxy"字段提取
+        
+        # 记录失败的代理，避免重复使用
+        self.failed_proxies = set()
+        # 最大失败次数，超过这个次数的代理将被标记为失效
+        self.max_failed_attempts = settings.get("PROXY_MAX_FAILED_ATTEMPTS", 3)
+        # 失效代理记录
+        self.proxy_failure_count = {}
+        
+        # 根据配置决定启用哪种模式
+        if self.proxies:
+            self.mode = "static"  # 静态代理模式
+            self.enabled = True
+            self.logger.info(f"ProxyMiddleware enabled (static mode) with {len(self.proxies)} proxies")
+        elif self.api_url:
+            self.mode = "dynamic"  # 动态代理模式
+            self.enabled = True
+            self.logger.info(f"ProxyMiddleware enabled (dynamic mode) | API: {self.api_url}")
+        else:
+            self.mode = None
+            self.enabled = False
+            self.logger.info("ProxyMiddleware disabled (no proxy configuration)")
 
     @classmethod
     def create_instance(cls, crawler):
         return cls(settings=crawler.settings, log_level=crawler.settings.get("LOG_LEVEL"))
 
-    def _compile_extractor(self) -> ProxyExtractor:
-        if callable(self.proxy_extractor):
-            return self.proxy_extractor
-
-        if isinstance(self.proxy_extractor, str):
-            keys = self.proxy_extractor.split(".")
-
-            def extract(data: Dict[str, Any]) -> Union[None, str, Dict[str, str]]:
-                for k in keys:
-                    if isinstance(data, dict):
-                        data = data.get(k)
-                    else:
-                        return None
-                    if data is None:
-                        break
-                return data
-
-            return extract
-
-        raise ValueError(f"PROXY_EXTRACTOR 必须是 str 或 callable，当前类型: {type(self.proxy_extractor)}")
-
-    async def _close_session(self):
-        if self._session:
-            try:
-                await self._session.close()
-                self.logger.debug("aiohttp session closed.")
-            except Exception as e:
-                self.logger.warning(f"Error closing aiohttp session: {e}")
-            finally:
-                self._session = None
-
-    async def _get_session(self) -> Any:  # returns aiohttp.ClientSession when aiohttp is available
-        if aiohttp is None:
-            raise RuntimeError("aiohttp not installed, cannot use ProxyMiddleware")
-
-        if self._session is None or self._session.closed:
-            if self._session and self._session.closed:
-                self.logger.debug("Existing session closed, creating new session...")
-            timeout = aiohttp.ClientTimeout(total=self.timeout)
-            self._session = aiohttp.ClientSession(timeout=timeout)
-            self.logger.debug("New aiohttp session created.")
-        return self._session
-
-    async def _fetch_raw_data(self) -> Optional[Dict[str, Any]]:
-        max_retries = 2
-        retry_count = 0
-
-        while retry_count <= max_retries:
-            session = await self._get_session()
-            try:
-                async with session.get(self.api_url) as resp:
-                    content_type = resp.content_type.lower()
-                    if 'application/json' not in content_type:
-                        self.logger.warning(
-                            f"Proxy API returned non-JSON content type: {content_type} (URL: {self.api_url})")
-                        try:
-                            text = await resp.text()
-                            return {"__raw_text__": text.strip(), "__content_type__": content_type}
-                        except Exception as e:
-                            self.logger.error(f"Failed to read non-JSON response body: {repr(e)}")
-                            return None
-
-                    if resp.status != 200:
-                        try:
-                            error_text = await resp.text()
-                        except:
-                            error_text = "<Unable to read response body>"
-                        self.logger.error(f"Proxy API status code error: {resp.status}, Response body: {error_text}")
-                        if 400 <= resp.status < 500:
-                            return None
-                        return None
-
-                    return await resp.json()
-
-            except NETWORK_EXCEPTIONS as e:
-                retry_count += 1
-                self.logger.warning(f"Failed to request proxy API (attempt {retry_count}/{max_retries + 1}): {repr(e)}")
-                if retry_count <= max_retries:
-                    self.logger.info("Closing and rebuilding session for retry...")
-                    await self._close_session()
-                else:
-                    self.logger.error(
-                        f"Failed to request proxy API, maximum retry attempts reached ({max_retries + 1}): {repr(e)}")
-                    return None
-
-            except aiohttp.ContentTypeError as e:
-                self.logger.error(f"Proxy API response content type error: {repr(e)}")
-                return None
-
-            except Exception as e:
-                self.logger.critical(f"Unexpected error occurred while requesting proxy API: {repr(e)}", exc_info=True)
-                return None
-
-        return None
-
-    async def _extract_proxy(self, data: Dict[str, Any]) -> Optional[Union[str, Dict[str, str]]]:
-        extractor = self._compile_extractor()
+    async def _fetch_proxy_from_api(self) -> Optional[str]:
+        """从代理API获取代理"""
         try:
-            result = extractor(data)
-            if isinstance(result, str) and result.strip():
-                return result.strip()
-            elif isinstance(result, dict):
-                cleaned = {k: v.strip() if isinstance(v, str) else v for k, v in result.items()}
-                return cleaned if cleaned else None
-            return None
+            import aiohttp
+            async with aiohttp.ClientSession() as session:
+                async with session.get(self.api_url) as resp:
+                    if resp.status == 200:
+                        data = await resp.json()
+                        # 支持多种代理提取方式
+                        proxy = self._extract_proxy_from_data(data)
+                        if proxy and isinstance(proxy, str) and (proxy.startswith("http://") or proxy.startswith("https://")):
+                            return proxy
+                    else:
+                        self.logger.warning(f"Proxy API returned status {resp.status}")
         except Exception as e:
-            self.logger.error(f"Error executing PROXY_EXTRACTOR: {repr(e)}")
-            return None
-
-    async def _get_proxy_from_api(self) -> Optional[Union[str, Dict[str, str]]]:
-        raw_data = await self._fetch_raw_data()
-        if not raw_data:
-            return None
-
-        if "__raw_text__" in raw_data:
-            text = raw_data["__raw_text__"]
-            if text.startswith("http://") or text.startswith("https://"):
-                return text
-
-        return await self._extract_proxy(raw_data)
-
-    def _parse_proxy_data(self, proxy_data: Union[str, Dict[str, Any]]) -> List[str]:
-        """解析代理数据，提取代理URL列表"""
-        new_proxies = []
-        if isinstance(proxy_data, str):
-            # 单个代理
-            if proxy_data.startswith("http://") or proxy_data.startswith("https://"):
-                new_proxies = [proxy_data]
-        elif isinstance(proxy_data, dict):
-            # 如果是字典，尝试提取代理列表
-            for key, value in proxy_data.items():
-                if isinstance(value, str) and (value.startswith("http://") or value.startswith("https://")):
-                    new_proxies.append(value)
-                elif isinstance(value, list):
-                    # 如果值是列表，添加所有有效的代理
-                    for item in value:
-                        if isinstance(item, str) and (item.startswith("http://") or item.startswith("https://")):
-                            new_proxies.append(item)
-        return new_proxies
-
-    def _get_healthy_proxies(self) -> List[Proxy]:
-        """获取所有健康的代理"""
-        return [p for p in self._proxy_pool if p.is_healthy and p.success_rate >= self.health_check_threshold]
-
-    async def _update_proxy_pool(self):
-        """更新代理池"""
-        if not self.enabled:
-            self.logger.debug("ProxyMiddleware disabled, skipping proxy fetch.")
-            return
-
-        now = asyncio.get_event_loop().time()
-        if (now - self._last_fetch_time) < self.refresh_interval:
-            return
-
-        # 获取新的代理列表
-        proxy_data = await self._get_proxy_from_api()
-        if not proxy_data:
-            self.logger.warning("Failed to get new proxies, proxy pool will remain unchanged.")
-            return
-
-        # 解析代理数据
-        new_proxies = self._parse_proxy_data(proxy_data)
-
-        # 创建新的代理池
-        if new_proxies:
-            self._proxy_pool = [Proxy(proxy_str) for proxy_str in new_proxies[:self.proxy_pool_size]]
-            self._current_proxy_index = 0
-            self._last_fetch_time = now
-            self.logger.info(f"Updated proxy pool, added {len(self._proxy_pool)} proxies")
-        else:
-            self.logger.warning("No valid proxies parsed, proxy pool will remain unchanged.")
-
-    async def _get_healthy_proxy(self) -> Optional[Proxy]:
-        """从代理池中获取一个健康的代理"""
-        if not self._proxy_pool:
-            await self._update_proxy_pool()
-
-        if not self._proxy_pool:
-            return None
-
-        # 查找健康的代理
-        healthy_proxies = self._get_healthy_proxies()
-
-        if not healthy_proxies:
-            # 如果没有健康的代理，尝试更新代理池
-            await self._update_proxy_pool()
-            healthy_proxies = self._get_healthy_proxies()
-
-        if not healthy_proxies:
-            return None
-
-        # 使用轮询方式选择代理
-        self._current_proxy_index = (self._current_proxy_index + 1) % len(healthy_proxies)
-        selected_proxy = healthy_proxies[self._current_proxy_index]
-        return selected_proxy
+            self.logger.warning(f"Failed to fetch proxy from API: {repr(e)}")
+        return None
 
-    @staticmethod
-    def _is_https(request: Request) -> bool:
-        return urlparse(request.url).scheme == "https"
+    def _extract_proxy_from_data(self, data) -> Optional[str]:
+        """
+        从API返回的数据中提取代理
+        
+        支持多种提取方式：
+        1. 字符串: 直接作为字段名使用
+        2. 字典: 包含type和value字段，type支持"field"和"jsonpath"
+        3. 函数: 用户自定义提取函数
+        """
+        if isinstance(self.proxy_extractor, str):
+            # 简单字段名提取（向后兼容）
+            if self.proxy_extractor in data:
+                proxy_value = data[self.proxy_extractor]
+                # 如果返回的是字典，尝试提取http或https字段
+                if isinstance(proxy_value, dict):
+                    if "http" in proxy_value:
+                        return proxy_value["http"]
+                    elif "https" in proxy_value:
+                        return proxy_value["https"]
+                return proxy_value
+        elif isinstance(self.proxy_extractor, dict):
+            # 复杂提取规则
+            extractor_type = self.proxy_extractor.get("type", "field")
+            extractor_value = self.proxy_extractor.get("value", "proxy")
+            
+            if extractor_type == "field":
+                # 字段提取
+                if extractor_value in data:
+                    proxy_value = data[extractor_value]
+                    # 如果返回的是字典，尝试提取http或https字段
+                    if isinstance(proxy_value, dict):
+                        if "http" in proxy_value:
+                            return proxy_value["http"]
+                        elif "https" in proxy_value:
+                            return proxy_value["https"]
+                    return proxy_value
+            elif extractor_type == "jsonpath":
+                # JSON路径提取（需要安装jsonpath库）
+                try:
+                    import jsonpath
+                    matches = jsonpath.jsonpath(data, extractor_value)
+                    if matches:
+                        return matches[0]
+                except ImportError:
+                    self.logger.warning("jsonpath library not installed, falling back to default extraction")
+                    if "proxy" in data:
+                        proxy_value = data["proxy"]
+                        if isinstance(proxy_value, dict):
+                            if "http" in proxy_value:
+                                return proxy_value["http"]
+                            elif "https" in proxy_value:
+                                return proxy_value["https"]
+                        return proxy_value
+            elif extractor_type == "custom":
+                # 自定义提取函数（需要用户提供）
+                custom_func = self.proxy_extractor.get("function")
+                if callable(custom_func):
+                    return custom_func(data)
+        elif callable(self.proxy_extractor):
+            # 直接调用用户提供的函数
+            return self.proxy_extractor(data)
+        
+        # 默认提取方式（向后兼容）
+        if "proxy" in data:
+            proxy_value = data["proxy"]
+            # 如果返回的是字典，尝试提取http或https字段
+            if isinstance(proxy_value, dict):
+                if "http" in proxy_value:
+                    return proxy_value["http"]
+                elif "https" in proxy_value:
+                    return proxy_value["https"]
+            return proxy_value
+        
+        return None
 
     async def process_request(self, request: Request, spider) -> Optional[Request]:
+        """为请求分配代理"""
         if not self.enabled:
-            self.logger.debug(f"ProxyMiddleware disabled, request will connect directly: {request.url}")
             return None
 
         if request.proxy:
+            # 请求已指定代理，不覆盖
             return None
 
-        proxy_obj = await self._get_healthy_proxy()
-        if proxy_obj:
-            proxy = proxy_obj.proxy_str
-            # 处理带认证的代理URL
-            if isinstance(proxy, str) and "@" in proxy and "://" in proxy:
-                # 解析带认证的代理URL
-                parsed = urlparse(proxy)
-                if parsed.username and parsed.password:
-                    # 对于AioHttp下载器，需要特殊处理认证信息
-                    downloader_type = spider.crawler.settings.get("DOWNLOADER_TYPE", "aiohttp")
-                    if downloader_type == "aiohttp":
-                        # 将认证信息存储在meta中，由下载器处理
-                        request.meta["proxy_auth"] = {
-                            "username": parsed.username,
-                            "password": parsed.password
-                        }
-                        # 清理URL中的认证信息
-                        clean_proxy = f"{parsed.scheme}://{parsed.hostname}"
-                        if parsed.port:
-                            clean_proxy += f":{parsed.port}"
-                        request.proxy = clean_proxy
-                    else:
-                        # 其他下载器可以直接使用带认证的URL
-                        request.proxy = proxy
-                else:
-                    request.proxy = proxy
+        proxy = None
+        if self.mode == "static" and self.proxies:
+            # 静态代理模式：随机选择一个代理，排除已知失败的代理
+            available_proxies = [p for p in self.proxies if p not in self.failed_proxies]
+            if available_proxies:
+                proxy = random.choice(available_proxies)
             else:
-                request.proxy = proxy
-
-            # 记录使用的代理
-            request.meta["_used_proxy"] = proxy_obj
-            self.logger.info(f"Assigned proxy → {proxy} | {request.url}")
+                self.logger.warning("所有静态代理都已失效，将使用直连")
+        elif self.mode == "dynamic" and self.api_url:
+            # 动态代理模式：从API获取代理
+            proxy = await self._fetch_proxy_from_api()
+
+        if proxy:
+            # 检查代理是否在失败列表中
+            if proxy in self.failed_proxies:
+                self.logger.warning(f"尝试使用已知失败的代理: {proxy}，但仍会尝试")
+            
+            request.proxy = proxy
+            self.logger.debug(f"Assigned proxy {proxy} to {request.url}")
         else:
-            self.logger.warning(f"No proxy obtained, request connecting directly: {request.url}")
+            self.logger.warning(f"No proxy available, request connecting directly: {request.url}")
 
         return None
 
     def process_response(self, request: Request, response: Response, spider) -> Response:
-        proxy_obj = request.meta.get("_used_proxy")
-        if proxy_obj and isinstance(proxy_obj, Proxy):
-            proxy_obj.mark_success()
-            status_code = getattr(response, 'status_code', 'N/A')
-            self.logger.debug(f"Proxy success: {proxy_obj.proxy_str} | {request.url} | Status: {status_code}")
-        elif request.proxy:
-            status_code = getattr(response, 'status_code', 'N/A')
-            self.logger.debug(f"Proxy success: {request.proxy} | {request.url} | Status: {status_code}")
+        """处理响应"""
+        if request.proxy:
+            self.logger.debug(f"Proxy request successful: {request.proxy} | {request.url}")
+            # 代理请求成功，从失败列表中移除（如果存在）
+            self.failed_proxies.discard(request.proxy)
+            # 重置失败计数
+            if request.proxy in self.proxy_failure_count:
+                del self.proxy_failure_count[request.proxy]
         return response
 
     def process_exception(self, request: Request, exception: Exception, spider) -> Optional[Request]:
-        proxy_obj = request.meta.get("_used_proxy")
-        if proxy_obj and isinstance(proxy_obj, Proxy):
-            proxy_obj.mark_failure()
-            self.logger.warning(f"Proxy request failed: {proxy_obj.proxy_str} | {request.url} | {repr(exception)}")
-        elif request.proxy:
-            self.logger.warning(f"Proxy request failed: {request.proxy} | {request.url} | {repr(exception)}")
+        """处理异常"""
+        if request.proxy:
+            error_msg = f"Proxy request failed: {request.proxy} | {request.url} | {repr(exception)}"
+            self.logger.warning(error_msg)
+            
+            # 记录代理失败次数
+            if request.proxy not in self.proxy_failure_count:
+                self.proxy_failure_count[request.proxy] = 0
+            self.proxy_failure_count[request.proxy] += 1
+            
+            # 如果失败次数超过阈值，将代理标记为失效
+            if self.proxy_failure_count[request.proxy] >= self.max_failed_attempts:
+                self.failed_proxies.add(request.proxy)
+                self.logger.warning(f"代理 {request.proxy} 已失败 {self.max_failed_attempts} 次，标记为失效")
+                
+                # 从代理列表中移除（仅适用于静态代理模式）
+                if self.mode == "static" and request.proxy in self.proxies:
+                    self.proxies.remove(request.proxy)
+                    self.logger.info(f"已从静态代理列表中移除失效代理: {request.proxy}")
         return None
-
-    async def close(self):
-        await self._close_session()