crawlo 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

crawlo/tools/anti_crawler.py

@@ -0,0 +1,269 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-10 22:00
+# @Author  : crawl-coder
+# @Desc    : 反爬虫应对工具
+"""
+
+import asyncio
+import random
+import time
+from typing import Dict, Any, Optional, List, Callable
+
+
+class ProxyPoolManager:
+    """代理池管理器类"""
+
+    def __init__(self, proxies: Optional[List[Dict[str, str]]] = None):
+        """
+        初始化代理池管理器
+        
+        Args:
+            proxies (Optional[List[Dict[str, str]]]): 代理列表
+        """
+        self.proxies = proxies or [
+            {"http": "http://proxy1.example.com:8080", "https": "https://proxy1.example.com:8080"},
+            {"http": "http://proxy2.example.com:8080", "https": "https://proxy2.example.com:8080"},
+            {"http": "http://proxy3.example.com:8080", "https": "https://proxy3.example.com:8080"}
+        ]
+        self.proxy_status = {id(proxy): {"last_used": 0, "success_count": 0, "fail_count": 0} 
+                           for proxy in self.proxies}
+
+    def get_random_proxy(self) -> Dict[str, str]:
+        """
+        获取随机代理
+        
+        Returns:
+            Dict[str, str]: 代理配置
+        """
+        return random.choice(self.proxies)
+
+    def get_best_proxy(self) -> Dict[str, str]:
+        """
+        根据成功率获取最佳代理
+        
+        Returns:
+            Dict[str, str]: 代理配置
+        """
+        if not self.proxy_status:
+            return self.get_random_proxy()
+            
+        # 计算每个代理的成功率
+        proxy_scores = []
+        for proxy in self.proxies:
+            proxy_id = id(proxy)
+            status = self.proxy_status.get(proxy_id, {"success_count": 0, "fail_count": 0})
+            total = status["success_count"] + status["fail_count"]
+            
+            if total == 0:
+                score = 0.5  # 默认成功率
+            else:
+                score = status["success_count"] / total
+                
+            proxy_scores.append((proxy, score))
+            
+        # 按成功率排序，返回成功率最高的代理
+        proxy_scores.sort(key=lambda x: x[1], reverse=True)
+        return proxy_scores[0][0]
+
+    def report_proxy_result(self, proxy: Dict[str, str], success: bool) -> None:
+        """
+        报告代理使用结果
+        
+        Args:
+            proxy (Dict[str, str]): 代理配置
+            success (bool): 是否成功
+        """
+        proxy_id = id(proxy)
+        if proxy_id not in self.proxy_status:
+            self.proxy_status[proxy_id] = {"last_used": 0, "success_count": 0, "fail_count": 0}
+            
+        status = self.proxy_status[proxy_id]
+        status["last_used"] = time.time()
+        
+        if success:
+            status["success_count"] += 1
+        else:
+            status["fail_count"] += 1
+
+    def remove_invalid_proxy(self, proxy: Dict[str, str]) -> None:
+        """
+        移除无效代理
+        
+        Args:
+            proxy (Dict[str, str]): 代理配置
+        """
+        if proxy in self.proxies:
+            self.proxies.remove(proxy)
+            proxy_id = id(proxy)
+            if proxy_id in self.proxy_status:
+                del self.proxy_status[proxy_id]
+
+
+class CaptchaHandler:
+    """验证码处理器类"""
+
+    def __init__(self, captcha_service: Optional[Callable] = None):
+        """
+        初始化验证码处理器
+        
+        Args:
+            captcha_service (Optional[Callable]): 验证码识别服务
+        """
+        self.captcha_service = captcha_service
+
+    async def recognize_captcha(self, image_data: bytes, 
+                               captcha_type: str = "image") -> Optional[str]:
+        """
+        识别验证码
+        
+        Args:
+            image_data (bytes): 验证码图片数据
+            captcha_type (str): 验证码类型
+            
+        Returns:
+            Optional[str]: 识别结果
+        """
+        if self.captcha_service:
+            try:
+                return await self.captcha_service(image_data, captcha_type)
+            except Exception:
+                return None
+        else:
+            # 如果没有配置验证码服务，返回None
+            return None
+
+    async def handle_manual_captcha(self, prompt: str = "请输入验证码: ") -> str:
+        """
+        处理手动验证码输入
+        
+        Args:
+            prompt (str): 提示信息
+            
+        Returns:
+            str: 用户输入的验证码
+        """
+        # 在实际应用中，这里可能需要与用户界面交互
+        # 为了演示目的，我们模拟用户输入
+        print(prompt)
+        return input() if not asyncio.get_event_loop().is_running() else ""
+
+
+class AntiCrawler:
+    """反爬虫应对工具类"""
+
+    def __init__(self, proxies: Optional[List[Dict[str, str]]] = None,
+                 captcha_service: Optional[Callable] = None):
+        """
+        初始化反爬虫应对工具
+        
+        Args:
+            proxies (Optional[List[Dict[str, str]]]): 代理列表
+            captcha_service (Optional[Callable]): 验证码识别服务
+        """
+        self.proxy_manager = ProxyPoolManager(proxies)
+        self.captcha_handler = CaptchaHandler(captcha_service)
+
+    def get_random_user_agent(self) -> str:
+        """
+        获取随机User-Agent
+        
+        Returns:
+            str: 随机User-Agent
+        """
+        user_agents = [
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0",
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59",
+            "Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Mobile Safari/537.36"
+        ]
+        return random.choice(user_agents)
+
+    def rotate_proxy(self) -> Dict[str, str]:
+        """
+        轮换代理
+        
+        Returns:
+            Dict[str, str]: 代理配置
+        """
+        return self.proxy_manager.get_best_proxy()
+
+    def handle_captcha(self, response_text: str) -> bool:
+        """
+        检测是否遇到验证码
+        
+        Args:
+            response_text (str): 响应文本
+            
+        Returns:
+            bool: 是否遇到验证码
+        """
+        captcha_keywords = ["captcha", "verify", "验证", "验证码", "human verification"]
+        return any(keyword in response_text.lower() for keyword in captcha_keywords)
+
+    def detect_rate_limiting(self, status_code: int, response_headers: Dict[str, Any]) -> bool:
+        """
+        检测是否遇到频率限制
+        
+        Args:
+            status_code (int): HTTP状态码
+            response_headers (Dict[str, Any]): 响应头
+            
+        Returns:
+            bool: 是否遇到频率限制
+        """
+        # 检查状态码
+        if status_code in [429, 503]:
+            return True
+            
+        # 检查响应头
+        rate_limit_headers = ["x-ratelimit-remaining", "retry-after", "x-ratelimit-reset"]
+        return any(header.lower() in [k.lower() for k in response_headers.keys()] 
+                  for header in rate_limit_headers)
+
+    def random_delay(self, min_delay: float = 1.0, max_delay: float = 3.0) -> None:
+        """
+        随机延迟，避免请求过于频繁
+        
+        Args:
+            min_delay (float): 最小延迟时间（秒）
+            max_delay (float): 最大延迟时间（秒）
+        """
+        delay = random.uniform(min_delay, max_delay)
+        time.sleep(delay)
+
+    async def async_random_delay(self, min_delay: float = 1.0, max_delay: float = 3.0) -> None:
+        """
+        异步随机延迟，避免请求过于频繁
+        
+        Args:
+            min_delay (float): 最小延迟时间（秒）
+            max_delay (float): 最大延迟时间（秒）
+        """
+        delay = random.uniform(min_delay, max_delay)
+        await asyncio.sleep(delay)
+
+
+# 便捷函数
+def get_random_user_agent() -> str:
+    """获取随机User-Agent"""
+    return AntiCrawler().get_random_user_agent()
+
+
+def rotate_proxy(proxies: Optional[List[Dict[str, str]]] = None) -> Dict[str, str]:
+    """轮换代理"""
+    return AntiCrawler(proxies).rotate_proxy()
+
+
+def handle_captcha(response_text: str) -> bool:
+    """检测是否遇到验证码"""
+    return AntiCrawler().handle_captcha(response_text)
+
+
+def detect_rate_limiting(status_code: int, response_headers: Dict[str, Any]) -> bool:
+    """检测是否遇到频率限制"""
+    return AntiCrawler().detect_rate_limiting(status_code, response_headers)

crawlo/tools/authenticated_proxy.py

@@ -0,0 +1,241 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+带认证代理工具
+=============
+支持带用户名密码认证的代理和非认证代理的统一处理工具
+
+功能特性:
+- 支持HTTP/HTTPS/SOCKS代理
+- 支持带认证和不带认证的代理
+- 统一的代理格式处理
+- 代理有效性检测
+"""
+
+from typing import Dict, Any, Optional, Union
+from urllib.parse import urlparse, urlunparse
+
+
+class AuthenticatedProxy:
+    """带认证代理处理类"""
+
+    def __init__(self, proxy_url: str):
+        """
+        初始化代理对象
+        
+        Args:
+            proxy_url (str): 代理URL，支持带认证信息的格式
+                - 带认证: http://username:password@proxy.example.com:8080
+                - 无认证: http://proxy.example.com:8080
+        """
+        self.proxy_url = proxy_url
+        self.parsed = urlparse(proxy_url)
+        
+        # 提取认证信息
+        self.username = self.parsed.username
+        self.password = self.parsed.password
+        
+        # 构造不带认证信息的URL
+        self.clean_url = urlunparse((
+            self.parsed.scheme,
+            f"{self.parsed.hostname}:{self.parsed.port}" if self.parsed.port else self.parsed.hostname,
+            self.parsed.path,
+            self.parsed.params,
+            self.parsed.query,
+            self.parsed.fragment
+        ))
+        
+        # 构造下载器兼容的代理字典
+        self.proxy_dict = {
+            "http": self.clean_url,
+            "https": self.clean_url
+        }
+        
+        # 如果有认证信息，构造认证字符串
+        if self.username and self.password:
+            self.auth_string = f"{self.username}:{self.password}"
+        else:
+            self.auth_string = None
+
+    def get_proxy_for_downloader(self) -> Union[str, Dict[str, str]]:
+        """
+        获取适用于下载器的代理配置
+        
+        Returns:
+            Union[str, Dict[str, str]]: 代理配置
+                - 对于AioHttp/CurlCffi: 返回字典格式 {"http": "...", "https": "..."}
+                - 对于HttpX: 可以直接使用字符串或字典格式
+        """
+        return self.proxy_dict
+
+    def get_auth_credentials(self) -> Optional[Dict[str, str]]:
+        """
+        获取认证凭据
+        
+        Returns:
+            Optional[Dict[str, str]]: 认证凭据 {"username": "...", "password": "..."}
+        """
+        if self.username and self.password:
+            return {
+                "username": self.username,
+                "password": self.password
+            }
+        return None
+
+    def get_auth_header(self) -> Optional[str]:
+        """
+        获取Basic Auth认证头
+        
+        Returns:
+            Optional[str]: Basic Auth头信息
+        """
+        if self.username and self.password:
+            import base64
+            credentials = f"{self.username}:{self.password}"
+            encoded = base64.b64encode(credentials.encode()).decode()
+            return f"Basic {encoded}"
+        return None
+
+    def is_valid(self) -> bool:
+        """
+        检查代理URL是否有效
+        
+        Returns:
+            bool: 代理URL是否有效
+        """
+        # 检查协议
+        if self.parsed.scheme not in ["http", "https", "socks4", "socks5"]:
+            return False
+            
+        # 检查主机名
+        if not self.parsed.hostname:
+            return False
+            
+        # 检查端口（如果指定）
+        if self.parsed.port and (self.parsed.port < 1 or self.parsed.port > 65535):
+            return False
+            
+        return True
+
+    def __str__(self) -> str:
+        return self.proxy_url
+
+    def __repr__(self) -> str:
+        return f"AuthenticatedProxy(url='{self.proxy_url}', username={self.username is not None})"
+
+
+def create_proxy_config(proxy_url: str) -> Dict[str, Any]:
+    """
+    创建代理配置，兼容各种下载器
+    
+    Args:
+        proxy_url (str): 代理URL
+        
+    Returns:
+        Dict[str, Any]: 代理配置字典
+    """
+    proxy = AuthenticatedProxy(proxy_url)
+    
+    if not proxy.is_valid():
+        raise ValueError(f"Invalid proxy URL: {proxy_url}")
+    
+    config = {
+        "url": proxy.clean_url,
+        "proxy_dict": proxy.proxy_dict,
+        "has_auth": proxy.auth_string is not None
+    }
+    
+    if proxy.auth_string:
+        config["auth"] = proxy.get_auth_credentials()
+        config["auth_header"] = proxy.get_auth_header()
+        
+    return config
+
+
+def format_proxy_for_request(proxy_config: Dict[str, Any], downloader_type: str = "aiohttp") -> Dict[str, Any]:
+    """
+    格式化代理配置以适配特定下载器
+    
+    Args:
+        proxy_config (Dict[str, Any]): 代理配置
+        downloader_type (str): 下载器类型 (aiohttp, httpx, curl_cffi)
+        
+    Returns:
+        Dict[str, Any]: 适配下载器的代理配置
+    """
+    formatted = {}
+    
+    if downloader_type.lower() == "aiohttp":
+        # AioHttp使用proxy和proxy_auth参数
+        formatted["proxy"] = proxy_config["url"]
+        if proxy_config.get("has_auth") and proxy_config.get("auth"):
+            from aiohttp import BasicAuth
+            auth = proxy_config["auth"]
+            formatted["proxy_auth"] = BasicAuth(auth["username"], auth["password"])
+            
+    elif downloader_type.lower() == "httpx":
+        # HttpX可以直接使用代理URL字符串或字典
+        formatted["proxy"] = proxy_config["url"]
+        
+    elif downloader_type.lower() == "curl_cffi":
+        # CurlCffi使用proxies字典
+        formatted["proxies"] = proxy_config["proxy_dict"]
+        # 认证信息包含在URL中或通过headers传递
+        if proxy_config.get("auth_header"):
+            formatted["headers"] = {"Proxy-Authorization": proxy_config["auth_header"]}
+            
+    return formatted
+
+
+# 便捷函数
+def parse_proxy_url(proxy_url: str) -> Dict[str, Any]:
+    """
+    解析代理URL并返回详细信息
+    
+    Args:
+        proxy_url (str): 代理URL
+        
+    Returns:
+        Dict[str, Any]: 代理详细信息
+    """
+    return create_proxy_config(proxy_url)
+
+
+def validate_proxy_url(proxy_url: str) -> bool:
+    """
+    验证代理URL是否有效
+    
+    Args:
+        proxy_url (str): 代理URL
+        
+    Returns:
+        bool: 是否有效
+    """
+    try:
+        proxy = AuthenticatedProxy(proxy_url)
+        return proxy.is_valid()
+    except:
+        return False
+
+
+def get_proxy_info(proxy_url: str) -> Dict[str, Any]:
+    """
+    获取代理详细信息
+    
+    Args:
+        proxy_url (str): 代理URL
+        
+    Returns:
+        Dict[str, Any]: 代理详细信息
+    """
+    proxy = AuthenticatedProxy(proxy_url)
+    return {
+        "original_url": proxy.proxy_url,
+        "clean_url": proxy.clean_url,
+        "scheme": proxy.parsed.scheme,
+        "hostname": proxy.parsed.hostname,
+        "port": proxy.parsed.port,
+        "has_auth": proxy.auth_string is not None,
+        "username": proxy.username,
+        "is_valid": proxy.is_valid()
+    }

crawlo/tools/data_validator.py

@@ -0,0 +1,181 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-10 22:00
+# @Author  : crawl-coder
+# @Desc    : 数据验证工具
+"""
+
+import re
+from typing import Any, Union, Dict, List
+from datetime import datetime
+from urllib.parse import urlparse
+
+
+class DataValidator:
+    """数据验证工具类"""
+
+    @staticmethod
+    def validate_email(email: str) -> bool:
+        """
+        验证邮箱地址格式
+        
+        Args:
+            email (str): 邮箱地址
+            
+        Returns:
+            bool: 验证结果
+        """
+        pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+        return bool(re.match(pattern, email))
+
+    @staticmethod
+    def validate_phone(phone: str, country_code: str = "CN") -> bool:
+        """
+        验证电话号码格式
+        
+        Args:
+            phone (str): 电话号码
+            country_code (str): 国家代码，默认为"CN"
+            
+        Returns:
+            bool: 验证结果
+        """
+        if country_code == "CN":
+            # 中国手机号码格式
+            pattern = r'^1[3-9]\d{9}$'
+            return bool(re.match(pattern, phone))
+        else:
+            # 通用格式，只检查是否全为数字且长度在7-15之间
+            pattern = r'^\d{7,15}$'
+            return bool(re.match(pattern, phone))
+
+    @staticmethod
+    def validate_url(url: str) -> bool:
+        """
+        验证URL格式
+        
+        Args:
+            url (str): URL地址
+            
+        Returns:
+            bool: 验证结果
+        """
+        try:
+            result = urlparse(url)
+            return all([result.scheme, result.netloc])
+        except Exception:
+            return False
+
+    @staticmethod
+    def validate_chinese_id_card(id_card: str) -> bool:
+        """
+        验证中国身份证号码格式
+        
+        Args:
+            id_card (str): 身份证号码
+            
+        Returns:
+            bool: 验证结果
+        """
+        # 18位身份证号码格式
+        pattern = r'^[1-9]\d{5}(18|19|20)\d{2}((0[1-9])|(1[0-2]))(([0-2][1-9])|10|20|30|31)\d{3}[0-9Xx]$'
+        return bool(re.match(pattern, id_card))
+
+    @staticmethod
+    def validate_date(date_str: str, date_format: str = "%Y-%m-%d") -> bool:
+        """
+        验证日期格式
+        
+        Args:
+            date_str (str): 日期字符串
+            date_format (str): 日期格式，默认为"%Y-%m-%d"
+            
+        Returns:
+            bool: 验证结果
+        """
+        try:
+            datetime.strptime(date_str, date_format)
+            return True
+        except ValueError:
+            return False
+
+    @staticmethod
+    def validate_number_range(value: Union[int, float], min_val: Union[int, float], 
+                             max_val: Union[int, float]) -> bool:
+        """
+        验证数值是否在指定范围内
+        
+        Args:
+            value (Union[int, float]): 要验证的数值
+            min_val (Union[int, float]): 最小值
+            max_val (Union[int, float]): 最大值
+            
+        Returns:
+            bool: 验证结果
+        """
+        return min_val <= value <= max_val
+
+    @staticmethod
+    def check_data_integrity(data: Dict[str, Any], required_fields: List[str]) -> Dict[str, Any]:
+        """
+        检查数据完整性，确保关键字段不为空
+        
+        Args:
+            data (Dict[str, Any]): 要检查的数据
+            required_fields (List[str]): 必需字段列表
+            
+        Returns:
+            Dict[str, Any]: 检查结果，包含缺失字段和空值字段
+        """
+        missing_fields = []
+        empty_fields = []
+        
+        for field in required_fields:
+            if field not in data:
+                missing_fields.append(field)
+            elif data[field] is None or data[field] == "":
+                empty_fields.append(field)
+                
+        return {
+            "is_valid": len(missing_fields) == 0 and len(empty_fields) == 0,
+            "missing_fields": missing_fields,
+            "empty_fields": empty_fields
+        }
+
+
+# 便捷函数
+def validate_email(email: str) -> bool:
+    """验证邮箱地址格式"""
+    return DataValidator.validate_email(email)
+
+
+def validate_phone(phone: str, country_code: str = "CN") -> bool:
+    """验证电话号码格式"""
+    return DataValidator.validate_phone(phone, country_code)
+
+
+def validate_url(url: str) -> bool:
+    """验证URL格式"""
+    return DataValidator.validate_url(url)
+
+
+def validate_chinese_id_card(id_card: str) -> bool:
+    """验证中国身份证号码格式"""
+    return DataValidator.validate_chinese_id_card(id_card)
+
+
+def validate_date(date_str: str, date_format: str = "%Y-%m-%d") -> bool:
+    """验证日期格式"""
+    return DataValidator.validate_date(date_str, date_format)
+
+
+def validate_number_range(value: Union[int, float], min_val: Union[int, float], 
+                         max_val: Union[int, float]) -> bool:
+    """验证数值是否在指定范围内"""
+    return DataValidator.validate_number_range(value, min_val, max_val)
+
+
+def check_data_integrity(data: Dict[str, Any], required_fields: List[str]) -> Dict[str, Any]:
+    """检查数据完整性"""
+    return DataValidator.check_data_integrity(data, required_fields)