crawlo 1.4.4__py3-none-any.whl → 1.4.6__py3-none-any.whl

tests/test_spider_loader_comprehensive.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+综合测试SpiderLoader功能
+"""
+
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.spider_loader import SpiderLoader
+from crawlo.crawler import CrawlerProcess
+from crawlo.settings.setting_manager import SettingManager
+
+
+def test_spider_loader_comprehensive():
+    """综合测试SpiderLoader功能"""
+    print("综合测试SpiderLoader功能...")
+    
+    # 1. 测试基本的SpiderLoader功能
+    print("\n1. 测试基本的SpiderLoader功能")
+    settings = SettingManager({
+        'SPIDER_MODULES': ['tests.test_spiders'],
+        'SPIDER_LOADER_WARN_ONLY': True
+    })
+    
+    loader = SpiderLoader.from_settings(settings)
+    spider_names = loader.list()
+    print(f"  发现的爬虫: {spider_names}")
+    
+    if spider_names:
+        spider_name = spider_names[0]
+        spider_class = loader.load(spider_name)
+        print(f"  成功加载爬虫: {spider_name} -> {spider_class}")
+    
+    # 2. 测试CrawlerProcess与SPIDER_MODULES的集成
+    print("\n2. 测试CrawlerProcess与SPIDER_MODULES的集成")
+    process = CrawlerProcess(settings=settings)
+    process_spider_names = process.get_spider_names()
+    print(f"  CrawlerProcess发现的爬虫: {process_spider_names}")
+    
+    is_registered = process.is_spider_registered('test_spider')
+    print(f"  爬虫'test_spider'是否已注册: {is_registered}")
+    
+    spider_class = process.get_spider_class('test_spider')
+    print(f"  爬虫'test_spider'的类: {spider_class}")
+    
+    # 3. 测试接口规范
+    print("\n3. 测试接口规范")
+    # 检查SpiderLoader是否实现了ISpiderLoader接口所需的方法
+    from crawlo.interfaces import ISpiderLoader
+    # 由于ISpiderLoader是Protocol，我们不能直接使用isinstance检查
+    # 而是检查是否实现了所需的方法
+    required_methods = ['load', 'list', 'find_by_request']
+    implements_interface = all(hasattr(loader, method) for method in required_methods)
+    print(f"  SpiderLoader是否实现了ISpiderLoader接口: {implements_interface}")
+    
+    # 4. 测试方法存在性
+    print("\n4. 测试方法存在性")
+    required_methods = ['load', 'list', 'find_by_request', 'get_all']
+    for method in required_methods:
+        has_method = hasattr(loader, method)
+        print(f"  SpiderLoader是否有{method}方法: {has_method}")
+    
+    print("\n综合测试完成!")
+
+
+if __name__ == '__main__':
+    test_spider_loader_comprehensive()

tests/test_spiders/__init__.py

@@ -0,0 +1 @@
+# 测试爬虫模块

tests/test_spiders/test_spider.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+
+from crawlo.spider import Spider
+
+
+class TestSpider(Spider):
+    name = 'test_spider'
+    
+    def parse(self, response):
+        pass

tests/verify_mysql_warnings.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+"""
+验证 MySQL 警告是否已解决
+通过模拟实际运行环境来检查
+"""
+import asyncio
+import sys
+import os
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.db_helper import SQLBuilder
+from crawlo.pipelines.mysql_pipeline import BaseMySQLPipeline, AsyncmyMySQLPipeline, AiomysqlMySQLPipeline
+
+
+def verify_mysql_syntax():
+    """验证 MySQL 语法是否正确，不会产生警告"""
+    print("=== 验证 MySQL 语法是否正确 ===\n")
+    
+    # 模拟实际使用的数据
+    test_data = {
+        'title': '新一代OLED屏下光谱颜色传感技术：解锁显示新密码，重塑视觉新体验',
+        'publish_time': '2025-10-09 09:57',
+        'url': 'https://ee.ofweek.com/2025-10/ART-8460-2806-30671544.html',
+        'source': '',
+        'content': '在全球智能手机市场竞争日趋白热化的当下，消费者对手机屏幕显示效果的要求愈发严苛...'
+    }
+    
+    # 模拟 ofweek_standalone 项目的配置
+    update_columns = ('title', 'publish_time')
+    
+    print("1. 检查 SQLBuilder 生成的语法...")
+    sql = SQLBuilder.make_insert(
+        table="news_items",
+        data=test_data,
+        auto_update=False,
+        update_columns=update_columns,
+        insert_ignore=False
+    )
+    
+    print("生成的 SQL:")
+    print(sql[:200] + "..." if len(sql) > 200 else sql)
+    print()
+    
+    # 检查是否包含弃用的 VALUES() 函数用法
+    if "VALUES(`title`)" in sql or "VALUES(`publish_time`)" in sql:
+        print("✗ 发现弃用的 VALUES() 函数用法，会产生警告")
+        return False
+    else:
+        print("✓ 未发现弃用的 VALUES() 函数用法")
+    
+    if "AS `excluded`" in sql and "ON DUPLICATE KEY UPDATE" in sql:
+        print("✓ 正确使用了新的 MySQL 语法")
+    else:
+        print("✗ 未正确使用新的 MySQL 语法")
+        return False
+    
+    # 检查更新子句
+    if "`title`=`excluded`.`title`" in sql and "`publish_time`=`excluded`.`publish_time`" in sql:
+        print("✓ 更新子句正确使用了 excluded 别名")
+    else:
+        print("✗ 更新子句语法不正确")
+        return False
+    
+    print("\n2. 检查批量插入语法...")
+    batch_result = SQLBuilder.make_batch(
+        table="news_items",
+        datas=[test_data, test_data],
+        auto_update=False,
+        update_columns=update_columns
+    )
+    
+    if batch_result:
+        batch_sql, _ = batch_result
+        print("生成的批量 SQL:")
+        print(batch_sql[:200] + "..." if len(batch_sql) > 200 else batch_sql)
+        print()
+        
+        # 检查批量插入语法
+        if "VALUES(`title`)" in batch_sql or "VALUES(`publish_time`)" in batch_sql:
+            print("✗ 批量插入中发现弃用的 VALUES() 函数用法，会产生警告")
+            return False
+        else:
+            print("✓ 批量插入未发现弃用的 VALUES() 函数用法")
+        
+        if "AS `excluded`" in batch_sql and "ON DUPLICATE KEY UPDATE" in batch_sql:
+            print("✓ 批量插入正确使用了新的 MySQL 语法")
+        else:
+            print("✗ 批量插入未正确使用新的 MySQL 语法")
+            return False
+        
+        # 检查批量更新子句
+        if "`title`=`excluded`.`title`" in batch_sql and "`publish_time`=`excluded`.`publish_time`" in batch_sql:
+            print("✓ 批量插入更新子句正确使用了 excluded 别名")
+        else:
+            print("✗ 批量插入更新子句语法不正确")
+            return False
+    
+    print("\n=== 验证完成 ===")
+    print("✓ 所有语法检查通过，应该不会再出现 MySQL 的 VALUES() 函数弃用警告")
+    return True
+
+
+if __name__ == "__main__":
+    success = verify_mysql_syntax()
+    if success:
+        print("\n🎉 MySQL 语法问题已解决！")
+    else:
+        print("\n❌ 仍存在 MySQL 语法问题需要修复")

crawlo/middleware/simple_proxy.py

@@ -1,65 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-简化版代理中间件
-提供基础的代理功能，避免过度复杂的实现
-"""
-import random
-from typing import Optional, List
-
-from crawlo import Request, Response
-from crawlo.exceptions import NotConfiguredError
-from crawlo.utils.log import get_logger
-
-
-class SimpleProxyMiddleware:
-    """简化版代理中间件"""
-
-    def __init__(self, settings, log_level):
-        self.logger = get_logger(self.__class__.__name__, log_level)
-
-        # 获取代理列表
-        self.proxies: List[str] = settings.get("PROXY_LIST", [])
-        self.enabled = settings.get_bool("PROXY_ENABLED", False)
-
-        if not self.enabled:
-            self.logger.info("SimpleProxyMiddleware disabled")
-            return
-
-        if not self.proxies:
-            raise NotConfiguredError("PROXY_LIST not configured, SimpleProxyMiddleware disabled")
-
-        self.logger.info(f"SimpleProxyMiddleware enabled with {len(self.proxies)} proxies")
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls(settings=crawler.settings, log_level=crawler.settings.get("LOG_LEVEL"))
-
-    async def process_request(self, request: Request, spider) -> Optional[Request]:
-        """为请求分配代理"""
-        if not self.enabled:
-            return None
-
-        if request.proxy:
-            # 请求已指定代理，不覆盖
-            return None
-
-        if self.proxies:
-            # 随机选择一个代理
-            proxy = random.choice(self.proxies)
-            request.proxy = proxy
-            self.logger.debug(f"Assigned proxy {proxy} to {request.url}")
-
-        return None
-
-    def process_response(self, request: Request, response: Response, spider) -> Response:
-        """处理响应"""
-        if request.proxy:
-            self.logger.debug(f"Proxy request successful: {request.proxy} | {request.url}")
-        return response
-
-    def process_exception(self, request: Request, exception: Exception, spider) -> Optional[Request]:
-        """处理异常"""
-        if request.proxy:
-            self.logger.warning(f"Proxy request failed: {request.proxy} | {request.url} | {repr(exception)}")
-        return None

crawlo/tools/anti_crawler.py

@@ -1,269 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 反爬虫应对工具
-"""
-
-import asyncio
-import random
-import time
-from typing import Dict, Any, Optional, List, Callable
-
-
-class ProxyPoolManager:
-    """代理池管理器类"""
-
-    def __init__(self, proxies: Optional[List[Dict[str, str]]] = None):
-        """
-        初始化代理池管理器
-        
-        Args:
-            proxies (Optional[List[Dict[str, str]]]): 代理列表
-        """
-        self.proxies = proxies or [
-            {"http": "http://proxy1.example.com:8080", "https": "https://proxy1.example.com:8080"},
-            {"http": "http://proxy2.example.com:8080", "https": "https://proxy2.example.com:8080"},
-            {"http": "http://proxy3.example.com:8080", "https": "https://proxy3.example.com:8080"}
-        ]
-        self.proxy_status = {id(proxy): {"last_used": 0, "success_count": 0, "fail_count": 0} 
-                           for proxy in self.proxies}
-
-    def get_random_proxy(self) -> Dict[str, str]:
-        """
-        获取随机代理
-        
-        Returns:
-            Dict[str, str]: 代理配置
-        """
-        return random.choice(self.proxies)
-
-    def get_best_proxy(self) -> Dict[str, str]:
-        """
-        根据成功率获取最佳代理
-        
-        Returns:
-            Dict[str, str]: 代理配置
-        """
-        if not self.proxy_status:
-            return self.get_random_proxy()
-            
-        # 计算每个代理的成功率
-        proxy_scores = []
-        for proxy in self.proxies:
-            proxy_id = id(proxy)
-            status = self.proxy_status.get(proxy_id, {"success_count": 0, "fail_count": 0})
-            total = status["success_count"] + status["fail_count"]
-            
-            if total == 0:
-                score = 0.5  # 默认成功率
-            else:
-                score = status["success_count"] / total
-                
-            proxy_scores.append((proxy, score))
-            
-        # 按成功率排序，返回成功率最高的代理
-        proxy_scores.sort(key=lambda x: x[1], reverse=True)
-        return proxy_scores[0][0]
-
-    def report_proxy_result(self, proxy: Dict[str, str], success: bool) -> None:
-        """
-        报告代理使用结果
-        
-        Args:
-            proxy (Dict[str, str]): 代理配置
-            success (bool): 是否成功
-        """
-        proxy_id = id(proxy)
-        if proxy_id not in self.proxy_status:
-            self.proxy_status[proxy_id] = {"last_used": 0, "success_count": 0, "fail_count": 0}
-            
-        status = self.proxy_status[proxy_id]
-        status["last_used"] = time.time()
-        
-        if success:
-            status["success_count"] += 1
-        else:
-            status["fail_count"] += 1
-
-    def remove_invalid_proxy(self, proxy: Dict[str, str]) -> None:
-        """
-        移除无效代理
-        
-        Args:
-            proxy (Dict[str, str]): 代理配置
-        """
-        if proxy in self.proxies:
-            self.proxies.remove(proxy)
-            proxy_id = id(proxy)
-            if proxy_id in self.proxy_status:
-                del self.proxy_status[proxy_id]
-
-
-class CaptchaHandler:
-    """验证码处理器类"""
-
-    def __init__(self, captcha_service: Optional[Callable] = None):
-        """
-        初始化验证码处理器
-        
-        Args:
-            captcha_service (Optional[Callable]): 验证码识别服务
-        """
-        self.captcha_service = captcha_service
-
-    async def recognize_captcha(self, image_data: bytes, 
-                               captcha_type: str = "image") -> Optional[str]:
-        """
-        识别验证码
-        
-        Args:
-            image_data (bytes): 验证码图片数据
-            captcha_type (str): 验证码类型
-            
-        Returns:
-            Optional[str]: 识别结果
-        """
-        if self.captcha_service:
-            try:
-                return await self.captcha_service(image_data, captcha_type)
-            except Exception:
-                return None
-        else:
-            # 如果没有配置验证码服务，返回None
-            return None
-
-    async def handle_manual_captcha(self, prompt: str = "请输入验证码: ") -> str:
-        """
-        处理手动验证码输入
-        
-        Args:
-            prompt (str): 提示信息
-            
-        Returns:
-            str: 用户输入的验证码
-        """
-        # 在实际应用中，这里可能需要与用户界面交互
-        # 为了演示目的，我们模拟用户输入
-        print(prompt)
-        return input() if not asyncio.get_event_loop().is_running() else ""
-
-
-class AntiCrawler:
-    """反爬虫应对工具类"""
-
-    def __init__(self, proxies: Optional[List[Dict[str, str]]] = None,
-                 captcha_service: Optional[Callable] = None):
-        """
-        初始化反爬虫应对工具
-        
-        Args:
-            proxies (Optional[List[Dict[str, str]]]): 代理列表
-            captcha_service (Optional[Callable]): 验证码识别服务
-        """
-        self.proxy_manager = ProxyPoolManager(proxies)
-        self.captcha_handler = CaptchaHandler(captcha_service)
-
-    def get_random_user_agent(self) -> str:
-        """
-        获取随机User-Agent
-        
-        Returns:
-            str: 随机User-Agent
-        """
-        user_agents = [
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0",
-            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59",
-            "Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Mobile Safari/537.36"
-        ]
-        return random.choice(user_agents)
-
-    def rotate_proxy(self) -> Dict[str, str]:
-        """
-        轮换代理
-        
-        Returns:
-            Dict[str, str]: 代理配置
-        """
-        return self.proxy_manager.get_best_proxy()
-
-    def handle_captcha(self, response_text: str) -> bool:
-        """
-        检测是否遇到验证码
-        
-        Args:
-            response_text (str): 响应文本
-            
-        Returns:
-            bool: 是否遇到验证码
-        """
-        captcha_keywords = ["captcha", "verify", "验证", "验证码", "human verification"]
-        return any(keyword in response_text.lower() for keyword in captcha_keywords)
-
-    def detect_rate_limiting(self, status_code: int, response_headers: Dict[str, Any]) -> bool:
-        """
-        检测是否遇到频率限制
-        
-        Args:
-            status_code (int): HTTP状态码
-            response_headers (Dict[str, Any]): 响应头
-            
-        Returns:
-            bool: 是否遇到频率限制
-        """
-        # 检查状态码
-        if status_code in [429, 503]:
-            return True
-            
-        # 检查响应头
-        rate_limit_headers = ["x-ratelimit-remaining", "retry-after", "x-ratelimit-reset"]
-        return any(header.lower() in [k.lower() for k in response_headers.keys()] 
-                  for header in rate_limit_headers)
-
-    def random_delay(self, min_delay: float = 1.0, max_delay: float = 3.0) -> None:
-        """
-        随机延迟，避免请求过于频繁
-        
-        Args:
-            min_delay (float): 最小延迟时间（秒）
-            max_delay (float): 最大延迟时间（秒）
-        """
-        delay = random.uniform(min_delay, max_delay)
-        time.sleep(delay)
-
-    async def async_random_delay(self, min_delay: float = 1.0, max_delay: float = 3.0) -> None:
-        """
-        异步随机延迟，避免请求过于频繁
-        
-        Args:
-            min_delay (float): 最小延迟时间（秒）
-            max_delay (float): 最大延迟时间（秒）
-        """
-        delay = random.uniform(min_delay, max_delay)
-        await asyncio.sleep(delay)
-
-
-# 便捷函数
-def get_random_user_agent() -> str:
-    """获取随机User-Agent"""
-    return AntiCrawler().get_random_user_agent()
-
-
-def rotate_proxy(proxies: Optional[List[Dict[str, str]]] = None) -> Dict[str, str]:
-    """轮换代理"""
-    return AntiCrawler(proxies).rotate_proxy()
-
-
-def handle_captcha(response_text: str) -> bool:
-    """检测是否遇到验证码"""
-    return AntiCrawler().handle_captcha(response_text)
-
-
-def detect_rate_limiting(status_code: int, response_headers: Dict[str, Any]) -> bool:
-    """检测是否遇到频率限制"""
-    return AntiCrawler().detect_rate_limiting(status_code, response_headers)

crawlo/utils/class_loader.py

@@ -1,26 +0,0 @@
-# -*- coding: UTF-8 -*-
-"""
-类加载器工具模块
-==============
-提供动态类加载功能，避免循环依赖问题。
-"""
-import importlib
-from typing import Any
-
-
-def load_class(path: str) -> Any:
-    """
-    动态加载类
-    
-    Args:
-        path: 类的完整路径，如 'package.module.ClassName'
-        
-    Returns:
-        加载的类对象
-    """
-    try:
-        module_path, class_name = path.rsplit('.', 1)
-        module = importlib.import_module(module_path)
-        return getattr(module, class_name)
-    except (ValueError, ImportError, AttributeError) as e:
-        raise ImportError(f"无法加载类 '{path}': {e}")