crawlo 1.4.3__py3-none-any.whl → 1.4.5__py3-none-any.whl

tests/test_encoding_detection.py

@@ -0,0 +1,127 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response 编码检测优化测试
+"""
+import unittest
+
+# 模拟 Response 类的部分功能用于测试
+class MockResponse:
+    def __init__(self, body, headers=None, request=None):
+        self.body = body
+        self.headers = headers or {}
+        self.request = request
+        self._DEFAULT_ENCODING = "ascii"
+    
+    def _determine_encoding(self):
+        """简化版编码检测"""
+        # 1. 优先使用声明的编码
+        declared_encoding = self._declared_encoding()
+        if declared_encoding:
+            return declared_encoding
+            
+        # 2. 默认使用 utf-8
+        return 'utf-8'
+        
+    def _declared_encoding(self):
+        """获取声明的编码"""
+        # 1. Request 中指定的编码
+        if self.request and getattr(self.request, 'encoding', None):
+            return self.request.encoding
+            
+        # 2. 从 Content-Type 头中检测
+        content_type = self.headers.get("content-type", "") or self.headers.get("Content-Type", "")
+        if content_type:
+            import re
+            charset_match = re.search(r"charset=([\w-]+)", content_type, re.I)
+            if charset_match:
+                return charset_match.group(1).lower()
+        
+        return None
+
+
+class TestDetermineEncoding(unittest.TestCase):
+    """编码检测测试类"""
+
+    def test_request_encoding_priority(self):
+        """测试 Request 编码优先级"""
+        class MockRequest:
+            encoding = 'gbk'
+        
+        response = MockResponse(b'', request=MockRequest())
+        encoding = response._determine_encoding()
+        self.assertEqual(encoding, 'gbk')
+
+    def test_content_type_encoding(self):
+        """测试 Content-Type 头部编码检测"""
+        response = MockResponse(
+            b'',
+            headers={"content-type": "text/html; charset=iso-8859-1"}
+        )
+        encoding = response._determine_encoding()
+        self.assertEqual(encoding, 'iso-8859-1')
+
+    def test_default_encoding(self):
+        """测试默认编码"""
+        response = MockResponse(b'')
+        encoding = response._determine_encoding()
+        self.assertEqual(encoding, 'utf-8')
+
+    def test_case_insensitive_content_type(self):
+        """测试 Content-Type 头部大小写不敏感"""
+        response = MockResponse(
+            b'',
+            headers={"Content-Type": "text/html; CHARSET=UTF-8"}
+        )
+        encoding = response._determine_encoding()
+        self.assertEqual(encoding, 'utf-8')
+        
+    def test_declared_encoding_with_request(self):
+        """测试声明编码 - Request优先级"""
+        class MockRequest:
+            encoding = 'gbk'
+        
+        response = MockResponse(b'', request=MockRequest())
+        declared_encoding = response._declared_encoding()
+        self.assertEqual(declared_encoding, 'gbk')
+        
+    def test_declared_encoding_with_content_type(self):
+        """测试声明编码 - Content-Type"""
+        response = MockResponse(
+            b'',
+            headers={"content-type": "text/html; charset=iso-8859-1"}
+        )
+        declared_encoding = response._declared_encoding()
+        self.assertEqual(declared_encoding, 'iso-8859-1')
+
+
+def test_encoding_detection():
+    """简单测试编码检测功能"""
+    print("测试编码检测功能...")
+    
+    # 测试 Request 编码优先级
+    class MockRequest:
+        encoding = 'gbk'
+    
+    response1 = MockResponse(b'', request=MockRequest())
+    encoding1 = response1._determine_encoding()
+    print(f"Request 编码优先级: {encoding1}")
+    
+    # 测试 Content-Type 头部编码
+    response2 = MockResponse(
+        b'',
+        headers={"content-type": "text/html; charset=iso-8859-1"}
+    )
+    encoding2 = response2._determine_encoding()
+    print(f"Content-Type 编码: {encoding2}")
+    
+    # 测试默认编码
+    response3 = MockResponse(b'')
+    encoding3 = response3._determine_encoding()
+    print(f"默认编码: {encoding3}")
+    
+    print("编码检测测试完成！")
+
+
+if __name__ == '__main__':
+    test_encoding_detection()

tests/test_factory_compatibility.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+测试CrawloConfig工厂模式兼容性
+"""
+
+import sys
+import os
+import traceback
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.config import CrawloConfig
+
+
+def test_standalone_factory():
+    """测试单机模式工厂函数"""
+    print("测试单机模式工厂函数...")
+    
+    try:
+        # 创建单机模式配置
+        config = CrawloConfig.standalone(
+            project_name='ofweek_standalone',
+            concurrency=8,
+            download_delay=1.0
+        )
+        
+        print(f"配置创建成功")
+        print(f"RUN_MODE: {config.get('RUN_MODE')}")
+        print(f"QUEUE_TYPE: {config.get('QUEUE_TYPE')}")
+        print(f"PROJECT_NAME: {config.get('PROJECT_NAME')}")
+        print(f"CONCURRENCY: {config.get('CONCURRENCY')}")
+        print(f"DOWNLOAD_DELAY: {config.get('DOWNLOAD_DELAY')}")
+        
+        # 验证配置是否正确
+        assert config.get('RUN_MODE') == 'standalone'
+        assert config.get('QUEUE_TYPE') == 'memory'
+        assert config.get('PROJECT_NAME') == 'ofweek_standalone'
+        assert config.get('CONCURRENCY') == 8
+        assert config.get('DOWNLOAD_DELAY') == 1.0
+        
+        print("✅ 单机模式工厂函数测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 单机模式工厂函数测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def test_distributed_factory():
+    """测试分布式模式工厂函数"""
+    print("\n测试分布式模式工厂函数...")
+    
+    try:
+        # 创建分布式模式配置
+        config = CrawloConfig.distributed(
+            redis_host='127.0.0.1',
+            redis_port=6379,
+            project_name='ofweek_distributed',
+            concurrency=16,
+            download_delay=0.5
+        )
+        
+        print(f"配置创建成功")
+        print(f"RUN_MODE: {config.get('RUN_MODE')}")
+        print(f"QUEUE_TYPE: {config.get('QUEUE_TYPE')}")
+        print(f"PROJECT_NAME: {config.get('PROJECT_NAME')}")
+        print(f"CONCURRENCY: {config.get('CONCURRENCY')}")
+        print(f"DOWNLOAD_DELAY: {config.get('DOWNLOAD_DELAY')}")
+        print(f"REDIS_HOST: {config.get('REDIS_HOST')}")
+        print(f"REDIS_PORT: {config.get('REDIS_PORT')}")
+        
+        # 验证配置是否正确
+        assert config.get('RUN_MODE') == 'distributed'
+        assert config.get('QUEUE_TYPE') == 'redis'
+        assert config.get('PROJECT_NAME') == 'ofweek_distributed'
+        assert config.get('CONCURRENCY') == 16
+        assert config.get('DOWNLOAD_DELAY') == 0.5
+        assert config.get('REDIS_HOST') == '127.0.0.1'
+        assert config.get('REDIS_PORT') == 6379
+        
+        print("✅ 分布式模式工厂函数测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 分布式模式工厂函数测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def test_auto_factory():
+    """测试自动模式工厂函数"""
+    print("\n测试自动模式工厂函数...")
+    
+    try:
+        # 创建自动模式配置
+        config = CrawloConfig.auto(
+            project_name='ofweek_auto',
+            concurrency=12,
+            download_delay=0.8
+        )
+        
+        print(f"配置创建成功")
+        print(f"RUN_MODE: {config.get('RUN_MODE')}")
+        print(f"QUEUE_TYPE: {config.get('QUEUE_TYPE')}")
+        print(f"PROJECT_NAME: {config.get('PROJECT_NAME')}")
+        print(f"CONCURRENCY: {config.get('CONCURRENCY')}")
+        print(f"DOWNLOAD_DELAY: {config.get('DOWNLOAD_DELAY')}")
+        
+        # 验证配置是否正确
+        assert config.get('RUN_MODE') == 'auto'
+        assert config.get('QUEUE_TYPE') == 'auto'
+        assert config.get('PROJECT_NAME') == 'ofweek_auto'
+        assert config.get('CONCURRENCY') == 12
+        assert config.get('DOWNLOAD_DELAY') == 0.8
+        
+        print("✅ 自动模式工厂函数测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 自动模式工厂函数测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def test_config_to_dict():
+    """测试配置转换为字典"""
+    print("\n测试配置转换为字典...")
+    
+    try:
+        # 创建配置
+        config = CrawloConfig.standalone(
+            project_name='test_project',
+            concurrency=4
+        )
+        
+        # 转换为字典
+        config_dict = config.to_dict()
+        
+        print(f"字典转换成功")
+        print(f"字典键数量: {len(config_dict)}")
+        
+        # 验证关键配置项
+        assert 'RUN_MODE' in config_dict
+        assert 'QUEUE_TYPE' in config_dict
+        assert 'PROJECT_NAME' in config_dict
+        assert 'CONCURRENCY' in config_dict
+        
+        print("✅ 配置转换为字典测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 配置转换为字典测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def main():
+    """主函数"""
+    print("开始测试CrawloConfig工厂模式兼容性...")
+    print("=" * 50)
+    
+    tests = [
+        test_standalone_factory,
+        test_distributed_factory,
+        test_auto_factory,
+        test_config_to_dict,
+    ]
+    
+    passed = 0
+    total = len(tests)
+    
+    for test_func in tests:
+        try:
+            if test_func():
+                passed += 1
+                print(f"✓ {test_func.__name__} 通过")
+            else:
+                print(f"✗ {test_func.__name__} 失败")
+        except Exception as e:
+            print(f"✗ {test_func.__name__} 异常: {e}")
+        print()
+    
+    print("=" * 50)
+    print(f"测试结果: {passed}/{total} 通过")
+    
+    if passed == total:
+        print("所有测试通过！CrawloConfig工厂模式兼容性正常。")
+        return 0
+    else:
+        print("部分测试失败，请检查实现。")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = main()
+    exit(exit_code)

tests/test_multi_directory.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+"""
+测试多个爬虫目录的支持
+"""
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.dirname(__file__))
+
+# 添加ofweek_standalone到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'examples', 'ofweek_standalone'))
+
+def test_multiple_spider_directories():
+    """测试多个爬虫目录的支持"""
+    print("测试多个爬虫目录的支持...")
+    
+    # 导入设置
+    import examples.ofweek_standalone.ofweek_standalone.settings as settings_module
+    
+    # 创建设置管理器
+    from crawlo.settings.setting_manager import SettingManager
+    settings = SettingManager()
+    settings.set_settings(settings_module)
+    
+    # 检查SPIDER_MODULES配置
+    spider_modules = settings.get('SPIDER_MODULES')
+    print(f"SPIDER_MODULES配置: {spider_modules}")
+    
+    # 创建CrawlerProcess实例
+    from crawlo.crawler import CrawlerProcess
+    process = CrawlerProcess(settings=settings)
+    
+    # 检查是否注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    # 验证期望的爬虫是否已注册
+    expected_spiders = ['of_week_standalone', 'test_spider']
+    registered_spiders = []
+    
+    for spider_name in expected_spiders:
+        if spider_name in spider_names:
+            print(f"✅ 成功: 爬虫 '{spider_name}' 已注册")
+            registered_spiders.append(spider_name)
+        else:
+            print(f"❌ 失败: 爬虫 '{spider_name}' 未找到")
+    
+    if len(registered_spiders) == len(expected_spiders):
+        print(f"🎉 所有爬虫都已成功注册!")
+        return True
+    else:
+        print(f"⚠️  部分爬虫未注册: {set(expected_spiders) - set(registered_spiders)}")
+        return False
+
+
+if __name__ == '__main__':
+    print("开始测试多个爬虫目录的支持...\n")
+    
+    success = test_multiple_spider_directories()
+    
+    if success:
+        print("\n🎉 测试通过!")
+        sys.exit(0)
+    else:
+        print("\n❌ 测试失败!")
+        sys.exit(1)

tests/test_multiple_spider_modules.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+"""
+测试多个SPIDER_MODULES目录的支持
+"""
+import sys
+import os
+import asyncio
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.dirname(__file__))
+
+# 添加ofweek_standalone到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'examples', 'ofweek_standalone'))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import get_spider_names
+
+
+def test_multiple_spider_modules():
+    """测试多个SPIDER_MODULES目录的支持"""
+    print("测试多个SPIDER_MODULES目录的支持...")
+    
+    # 模拟包含多个目录的SPIDER_MODULES配置
+    spider_modules = ['ofweek_standalone.spiders', 'ofweek_standalone.new_spiders']
+    
+    # 创建CrawlerProcess实例
+    process = CrawlerProcess(spider_modules=spider_modules)
+    
+    # 检查是否注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    # 验证期望的爬虫是否已注册
+    expected_spider = 'of_week_standalone'
+    if expected_spider in spider_names:
+        print(f"✅ 成功: 爬虫 '{expected_spider}' 已注册")
+        return True
+    else:
+        print(f"❌ 失败: 爬虫 '{expected_spider}' 未找到")
+        return False
+
+
+def test_settings_with_multiple_spider_modules():
+    """测试settings中配置多个SPIDER_MODULES目录"""
+    print("\n测试settings中配置多个SPIDER_MODULES目录...")
+    
+    # 创建模拟的settings对象
+    class MockSettings:
+        def get(self, key, default=None):
+            if key == 'SPIDER_MODULES':
+                return ['ofweek_standalone.spiders', 'ofweek_standalone.new_spiders']
+            return default
+    
+    settings = MockSettings()
+    
+    # 创建CrawlerProcess实例
+    process = CrawlerProcess(settings=settings)
+    
+    # 检查是否注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    return True
+
+
+if __name__ == '__main__':
+    print("开始测试多个SPIDER_MODULES目录的支持...\n")
+    
+    # 测试显式传递多个spider_modules参数
+    success1 = test_multiple_spider_modules()
+    
+    # 测试从settings中读取多个spider_modules配置
+    success2 = test_settings_with_multiple_spider_modules()
+    
+    if success1 and success2:
+        print("\n🎉 所有测试通过!")
+        sys.exit(0)
+    else:
+        print("\n❌ 部分测试失败!")
+        sys.exit(1)

tests/test_optimized_selector_naming.py

@@ -0,0 +1,101 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+优化后的选择器命名测试
+"""
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils import (
+    extract_text,
+    extract_texts,
+    extract_attr,
+    extract_attrs,
+    is_xpath
+)
+from parsel import Selector
+
+
+def test_optimized_naming():
+    """测试优化后的命名"""
+    print("测试优化后的选择器命名...")
+    print("=" * 50)
+    
+    # 创建测试HTML
+    html_content = """
+    <html>
+    <head>
+        <title>测试页面</title>
+    </head>
+    <body>
+        <div class="content">
+            <h1>主标题</h1>
+            <p class="intro">介绍段落</p>
+            <ul class="list">
+                <li>项目1</li>
+                <li>项目2</li>
+                <li>项目3</li>
+            </ul>
+            <a href="https://example.com" class="link">链接文本</a>
+            <img src="image.jpg" alt="图片描述" class="image">
+        </div>
+    </body>
+    </html>
+    """
+    
+    selector = Selector(text=html_content)
+    
+    # 测试 is_xpath
+    print("1. 测试 is_xpath:")
+    print(f"   '/' 开头: {is_xpath('/')}")
+    print(f"   '//' 开头: {is_xpath('//title')}")
+    print(f"   './' 开头: {is_xpath('./div')}")
+    print(f"   'title' 开头: {is_xpath('title')}")
+    print()
+    
+    # 测试 extract_text
+    print("2. 测试 extract_text:")
+    title_elements = selector.css('title')
+    title_text = extract_text(title_elements)
+    print(f"   标题文本: {title_text}")
+    
+    h1_elements = selector.css('.content h1')
+    h1_text = extract_text(h1_elements)
+    print(f"   H1文本: {h1_text}")
+    print()
+    
+    # 测试 extract_texts
+    print("3. 测试 extract_texts:")
+    li_elements = selector.css('.list li')
+    li_texts = extract_texts(li_elements)
+    print(f"   列表项文本: {li_texts}")
+    print()
+    
+    # 测试 extract_attr
+    print("4. 测试 extract_attr:")
+    link_elements = selector.css('.link')
+    link_href = extract_attr(link_elements, 'href')
+    print(f"   链接href: {link_href}")
+    
+    img_elements = selector.css('.image')
+    img_alt = extract_attr(img_elements, 'alt')
+    print(f"   图片alt: {img_alt}")
+    print()
+    
+    # 测试 extract_attrs
+    print("5. 测试 extract_attrs:")
+    all_links = selector.css('a')
+    all_hrefs = extract_attrs(all_links, 'href')
+    print(f"   所有链接href: {all_hrefs}")
+    
+    all_images = selector.css('img')
+    all_srcs = extract_attrs(all_images, 'src')
+    print(f"   所有图片src: {all_srcs}")
+    print()
+    
+    print("所有测试完成！")
+
+
+if __name__ == '__main__':
+    test_optimized_naming()

tests/test_priority_behavior.py

@@ -66,22 +66,22 @@ async def test_redis_queue_priority():
         await queue._redis.delete(f"{queue.queue_name}:data")
         
         # 创建不同优先级的请求
-        # 注意：Redis队列中，score = -priority
-        # 所以priority=-100的请求score=100，priority=100的请求score=-100
-        # zpopmin会弹出score最小的元素，所以priority=100的请求会先出队
-        request_low_priority = Request(url="https://low-priority.com", priority=100)   # 低优先级（数值大）
-        request_high_priority = Request(url="https://high-priority.com", priority=-100)  # 高优先级（数值小）
-        request_normal_priority = Request(url="https://normal-priority.com", priority=0)  # 正常优先级
+        # 注意：Request构造函数会将传入的priority值取反存储
+        # 所以priority=100的请求实际存储为-100，priority=-100的请求实际存储为100
+        request_low_priority = Request(url="https://low-priority.com", priority=100)   # 实际存储为-100（高优先级）
+        request_high_priority = Request(url="https://high-priority.com", priority=-100)  # 实际存储为100（低优先级）
+        request_normal_priority = Request(url="https://normal-priority.com", priority=0)  # 实际存储为0（正常优先级）
         
         # 按照正确的顺序入队以验证优先级行为
-        await queue.put(request_high_priority, priority=-100)  # 高优先级，score=100
-        await queue.put(request_normal_priority, priority=0)   # 正常优先级，score=0
-        await queue.put(request_low_priority, priority=100)    # 低优先级，score=-100
+        # 使用实际存储的priority值
+        await queue.put(request_low_priority, priority=request_low_priority.priority)    # 实际score=-100
+        await queue.put(request_normal_priority, priority=request_normal_priority.priority)   # 实际score=0
+        await queue.put(request_high_priority, priority=request_high_priority.priority)  # 实际score=100
         
         print(f"  队列大小: {await queue.qsize()}")
         
-        # 出队顺序应该按照score从小到大（priority从大到小）
-        # 所以低优先级先出队，高优先级最后出队
+        # 出队顺序应该按照score从小到大（priority从小到大）
+        # 所以request_low_priority先出队（score=-100），request_normal_priority第二个出队（score=0），request_high_priority最后出队（score=100）
         item1 = await queue.get(timeout=2.0)
         item2 = await queue.get(timeout=2.0)
         item3 = await queue.get(timeout=2.0)
@@ -91,13 +91,13 @@ async def test_redis_queue_priority():
         print(f"  第二个出队: {item2.url if item2 else None}")
         print(f"  第三个出队: {item3.url if item3 else None}")
         
-        # Redis队列中，score小的先出队，所以priority大的先出队
-        assert item1 is not None and item1.url == "https://low-priority.com", f"低优先级应该先出队，实际: {item1.url if item1 else None}"
-        assert item2 is not None and item2.url == "https://normal-priority.com", f"正常优先级应该第二个出队，实际: {item2.url if item2 else None}"
-        assert item3 is not None and item3.url == "https://high-priority.com", f"高优先级应该最后出队，实际: {item3.url if item3 else None}"
+        # Redis队列中，score小的先出队，所以priority小的先出队
+        assert item1 is not None and item1.url == "https://low-priority.com", f"低优先级请求应该先出队，实际: {item1.url if item1 else None}"
+        assert item2 is not None and item2.url == "https://normal-priority.com", f"正常优先级请求应该第二个出队，实际: {item2.url if item2 else None}"
+        assert item3 is not None and item3.url == "https://high-priority.com", f"高优先级请求应该最后出队，实际: {item3.url if item3 else None}"
         
         print("  ✅ Redis队列优先级测试通过（确认了score越小越优先的规则）")
-        print("  注意：Redis队列中score = -priority，所以priority值大的请求score小，会先出队")
+        print("  注意：Redis队列中score = priority，所以priority值小的请求score小，会先出队")
         
     except Exception as e:
         print(f"  ❌ Redis队列优先级测试失败: {e}")
@@ -196,8 +196,8 @@ async def main():
         print("\n总结:")
         print("1. 请求优先级遵循'数值越小越优先'的原则")
         print("2. 内存队列: 直接使用(priority, request)元组，priority小的先出队")
-        print("3. Redis队列: 使用score = -priority，score小的先出队，所以priority大的先出队")
-        print("   这是一个已知的行为差异，需要在使用时注意")
+        print("3. Redis队列: 使用score = priority，score小的先出队，所以priority小的先出队")
+        print("   现在内存队列和Redis队列行为一致")
         print("4. 重试中间件会根据RETRY_PRIORITY配置调整请求优先级")
         print("5. 系统内置的优先级常量: URGENT(-200) < HIGH(-100) < NORMAL(0) < LOW(100) < BACKGROUND(200)")
         print("6. Request对象构造时会将传入的priority值取反存储")