crawlo 1.4.4__py3-none-any.whl → 1.4.5__py3-none-any.whl

tests/test_factory_compatibility.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+测试CrawloConfig工厂模式兼容性
+"""
+
+import sys
+import os
+import traceback
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.config import CrawloConfig
+
+
+def test_standalone_factory():
+    """测试单机模式工厂函数"""
+    print("测试单机模式工厂函数...")
+    
+    try:
+        # 创建单机模式配置
+        config = CrawloConfig.standalone(
+            project_name='ofweek_standalone',
+            concurrency=8,
+            download_delay=1.0
+        )
+        
+        print(f"配置创建成功")
+        print(f"RUN_MODE: {config.get('RUN_MODE')}")
+        print(f"QUEUE_TYPE: {config.get('QUEUE_TYPE')}")
+        print(f"PROJECT_NAME: {config.get('PROJECT_NAME')}")
+        print(f"CONCURRENCY: {config.get('CONCURRENCY')}")
+        print(f"DOWNLOAD_DELAY: {config.get('DOWNLOAD_DELAY')}")
+        
+        # 验证配置是否正确
+        assert config.get('RUN_MODE') == 'standalone'
+        assert config.get('QUEUE_TYPE') == 'memory'
+        assert config.get('PROJECT_NAME') == 'ofweek_standalone'
+        assert config.get('CONCURRENCY') == 8
+        assert config.get('DOWNLOAD_DELAY') == 1.0
+        
+        print("✅ 单机模式工厂函数测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 单机模式工厂函数测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def test_distributed_factory():
+    """测试分布式模式工厂函数"""
+    print("\n测试分布式模式工厂函数...")
+    
+    try:
+        # 创建分布式模式配置
+        config = CrawloConfig.distributed(
+            redis_host='127.0.0.1',
+            redis_port=6379,
+            project_name='ofweek_distributed',
+            concurrency=16,
+            download_delay=0.5
+        )
+        
+        print(f"配置创建成功")
+        print(f"RUN_MODE: {config.get('RUN_MODE')}")
+        print(f"QUEUE_TYPE: {config.get('QUEUE_TYPE')}")
+        print(f"PROJECT_NAME: {config.get('PROJECT_NAME')}")
+        print(f"CONCURRENCY: {config.get('CONCURRENCY')}")
+        print(f"DOWNLOAD_DELAY: {config.get('DOWNLOAD_DELAY')}")
+        print(f"REDIS_HOST: {config.get('REDIS_HOST')}")
+        print(f"REDIS_PORT: {config.get('REDIS_PORT')}")
+        
+        # 验证配置是否正确
+        assert config.get('RUN_MODE') == 'distributed'
+        assert config.get('QUEUE_TYPE') == 'redis'
+        assert config.get('PROJECT_NAME') == 'ofweek_distributed'
+        assert config.get('CONCURRENCY') == 16
+        assert config.get('DOWNLOAD_DELAY') == 0.5
+        assert config.get('REDIS_HOST') == '127.0.0.1'
+        assert config.get('REDIS_PORT') == 6379
+        
+        print("✅ 分布式模式工厂函数测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 分布式模式工厂函数测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def test_auto_factory():
+    """测试自动模式工厂函数"""
+    print("\n测试自动模式工厂函数...")
+    
+    try:
+        # 创建自动模式配置
+        config = CrawloConfig.auto(
+            project_name='ofweek_auto',
+            concurrency=12,
+            download_delay=0.8
+        )
+        
+        print(f"配置创建成功")
+        print(f"RUN_MODE: {config.get('RUN_MODE')}")
+        print(f"QUEUE_TYPE: {config.get('QUEUE_TYPE')}")
+        print(f"PROJECT_NAME: {config.get('PROJECT_NAME')}")
+        print(f"CONCURRENCY: {config.get('CONCURRENCY')}")
+        print(f"DOWNLOAD_DELAY: {config.get('DOWNLOAD_DELAY')}")
+        
+        # 验证配置是否正确
+        assert config.get('RUN_MODE') == 'auto'
+        assert config.get('QUEUE_TYPE') == 'auto'
+        assert config.get('PROJECT_NAME') == 'ofweek_auto'
+        assert config.get('CONCURRENCY') == 12
+        assert config.get('DOWNLOAD_DELAY') == 0.8
+        
+        print("✅ 自动模式工厂函数测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 自动模式工厂函数测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def test_config_to_dict():
+    """测试配置转换为字典"""
+    print("\n测试配置转换为字典...")
+    
+    try:
+        # 创建配置
+        config = CrawloConfig.standalone(
+            project_name='test_project',
+            concurrency=4
+        )
+        
+        # 转换为字典
+        config_dict = config.to_dict()
+        
+        print(f"字典转换成功")
+        print(f"字典键数量: {len(config_dict)}")
+        
+        # 验证关键配置项
+        assert 'RUN_MODE' in config_dict
+        assert 'QUEUE_TYPE' in config_dict
+        assert 'PROJECT_NAME' in config_dict
+        assert 'CONCURRENCY' in config_dict
+        
+        print("✅ 配置转换为字典测试通过")
+        return True
+        
+    except Exception as e:
+        print(f"❌ 配置转换为字典测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+def main():
+    """主函数"""
+    print("开始测试CrawloConfig工厂模式兼容性...")
+    print("=" * 50)
+    
+    tests = [
+        test_standalone_factory,
+        test_distributed_factory,
+        test_auto_factory,
+        test_config_to_dict,
+    ]
+    
+    passed = 0
+    total = len(tests)
+    
+    for test_func in tests:
+        try:
+            if test_func():
+                passed += 1
+                print(f"✓ {test_func.__name__} 通过")
+            else:
+                print(f"✗ {test_func.__name__} 失败")
+        except Exception as e:
+            print(f"✗ {test_func.__name__} 异常: {e}")
+        print()
+    
+    print("=" * 50)
+    print(f"测试结果: {passed}/{total} 通过")
+    
+    if passed == total:
+        print("所有测试通过！CrawloConfig工厂模式兼容性正常。")
+        return 0
+    else:
+        print("部分测试失败，请检查实现。")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = main()
+    exit(exit_code)

tests/test_optimized_selector_naming.py

@@ -0,0 +1,101 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+优化后的选择器命名测试
+"""
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils import (
+    extract_text,
+    extract_texts,
+    extract_attr,
+    extract_attrs,
+    is_xpath
+)
+from parsel import Selector
+
+
+def test_optimized_naming():
+    """测试优化后的命名"""
+    print("测试优化后的选择器命名...")
+    print("=" * 50)
+    
+    # 创建测试HTML
+    html_content = """
+    <html>
+    <head>
+        <title>测试页面</title>
+    </head>
+    <body>
+        <div class="content">
+            <h1>主标题</h1>
+            <p class="intro">介绍段落</p>
+            <ul class="list">
+                <li>项目1</li>
+                <li>项目2</li>
+                <li>项目3</li>
+            </ul>
+            <a href="https://example.com" class="link">链接文本</a>
+            <img src="image.jpg" alt="图片描述" class="image">
+        </div>
+    </body>
+    </html>
+    """
+    
+    selector = Selector(text=html_content)
+    
+    # 测试 is_xpath
+    print("1. 测试 is_xpath:")
+    print(f"   '/' 开头: {is_xpath('/')}")
+    print(f"   '//' 开头: {is_xpath('//title')}")
+    print(f"   './' 开头: {is_xpath('./div')}")
+    print(f"   'title' 开头: {is_xpath('title')}")
+    print()
+    
+    # 测试 extract_text
+    print("2. 测试 extract_text:")
+    title_elements = selector.css('title')
+    title_text = extract_text(title_elements)
+    print(f"   标题文本: {title_text}")
+    
+    h1_elements = selector.css('.content h1')
+    h1_text = extract_text(h1_elements)
+    print(f"   H1文本: {h1_text}")
+    print()
+    
+    # 测试 extract_texts
+    print("3. 测试 extract_texts:")
+    li_elements = selector.css('.list li')
+    li_texts = extract_texts(li_elements)
+    print(f"   列表项文本: {li_texts}")
+    print()
+    
+    # 测试 extract_attr
+    print("4. 测试 extract_attr:")
+    link_elements = selector.css('.link')
+    link_href = extract_attr(link_elements, 'href')
+    print(f"   链接href: {link_href}")
+    
+    img_elements = selector.css('.image')
+    img_alt = extract_attr(img_elements, 'alt')
+    print(f"   图片alt: {img_alt}")
+    print()
+    
+    # 测试 extract_attrs
+    print("5. 测试 extract_attrs:")
+    all_links = selector.css('a')
+    all_hrefs = extract_attrs(all_links, 'href')
+    print(f"   所有链接href: {all_hrefs}")
+    
+    all_images = selector.css('img')
+    all_srcs = extract_attrs(all_images, 'src')
+    print(f"   所有图片src: {all_srcs}")
+    print()
+    
+    print("所有测试完成！")
+
+
+if __name__ == '__main__':
+    test_optimized_naming()

tests/test_priority_behavior.py

@@ -66,22 +66,22 @@ async def test_redis_queue_priority():
         await queue._redis.delete(f"{queue.queue_name}:data")
         
         # 创建不同优先级的请求
-        # 注意：Redis队列中，score = -priority
-        # 所以priority=-100的请求score=100，priority=100的请求score=-100
-        # zpopmin会弹出score最小的元素，所以priority=100的请求会先出队
-        request_low_priority = Request(url="https://low-priority.com", priority=100)   # 低优先级（数值大）
-        request_high_priority = Request(url="https://high-priority.com", priority=-100)  # 高优先级（数值小）
-        request_normal_priority = Request(url="https://normal-priority.com", priority=0)  # 正常优先级
+        # 注意：Request构造函数会将传入的priority值取反存储
+        # 所以priority=100的请求实际存储为-100，priority=-100的请求实际存储为100
+        request_low_priority = Request(url="https://low-priority.com", priority=100)   # 实际存储为-100（高优先级）
+        request_high_priority = Request(url="https://high-priority.com", priority=-100)  # 实际存储为100（低优先级）
+        request_normal_priority = Request(url="https://normal-priority.com", priority=0)  # 实际存储为0（正常优先级）
         
         # 按照正确的顺序入队以验证优先级行为
-        await queue.put(request_high_priority, priority=-100)  # 高优先级，score=100
-        await queue.put(request_normal_priority, priority=0)   # 正常优先级，score=0
-        await queue.put(request_low_priority, priority=100)    # 低优先级，score=-100
+        # 使用实际存储的priority值
+        await queue.put(request_low_priority, priority=request_low_priority.priority)    # 实际score=-100
+        await queue.put(request_normal_priority, priority=request_normal_priority.priority)   # 实际score=0
+        await queue.put(request_high_priority, priority=request_high_priority.priority)  # 实际score=100
         
         print(f"  队列大小: {await queue.qsize()}")
         
-        # 出队顺序应该按照score从小到大（priority从大到小）
-        # 所以低优先级先出队，高优先级最后出队
+        # 出队顺序应该按照score从小到大（priority从小到大）
+        # 所以request_low_priority先出队（score=-100），request_normal_priority第二个出队（score=0），request_high_priority最后出队（score=100）
         item1 = await queue.get(timeout=2.0)
         item2 = await queue.get(timeout=2.0)
         item3 = await queue.get(timeout=2.0)
@@ -91,13 +91,13 @@ async def test_redis_queue_priority():
         print(f"  第二个出队: {item2.url if item2 else None}")
         print(f"  第三个出队: {item3.url if item3 else None}")
         
-        # Redis队列中，score小的先出队，所以priority大的先出队
-        assert item1 is not None and item1.url == "https://low-priority.com", f"低优先级应该先出队，实际: {item1.url if item1 else None}"
-        assert item2 is not None and item2.url == "https://normal-priority.com", f"正常优先级应该第二个出队，实际: {item2.url if item2 else None}"
-        assert item3 is not None and item3.url == "https://high-priority.com", f"高优先级应该最后出队，实际: {item3.url if item3 else None}"
+        # Redis队列中，score小的先出队，所以priority小的先出队
+        assert item1 is not None and item1.url == "https://low-priority.com", f"低优先级请求应该先出队，实际: {item1.url if item1 else None}"
+        assert item2 is not None and item2.url == "https://normal-priority.com", f"正常优先级请求应该第二个出队，实际: {item2.url if item2 else None}"
+        assert item3 is not None and item3.url == "https://high-priority.com", f"高优先级请求应该最后出队，实际: {item3.url if item3 else None}"
         
         print("  ✅ Redis队列优先级测试通过（确认了score越小越优先的规则）")
-        print("  注意：Redis队列中score = -priority，所以priority值大的请求score小，会先出队")
+        print("  注意：Redis队列中score = priority，所以priority值小的请求score小，会先出队")
         
     except Exception as e:
         print(f"  ❌ Redis队列优先级测试失败: {e}")
@@ -196,8 +196,8 @@ async def main():
         print("\n总结:")
         print("1. 请求优先级遵循'数值越小越优先'的原则")
         print("2. 内存队列: 直接使用(priority, request)元组，priority小的先出队")
-        print("3. Redis队列: 使用score = -priority，score小的先出队，所以priority大的先出队")
-        print("   这是一个已知的行为差异，需要在使用时注意")
+        print("3. Redis队列: 使用score = priority，score小的先出队，所以priority小的先出队")
+        print("   现在内存队列和Redis队列行为一致")
         print("4. 重试中间件会根据RETRY_PRIORITY配置调整请求优先级")
         print("5. 系统内置的优先级常量: URGENT(-200) < HIGH(-100) < NORMAL(0) < LOW(100) < BACKGROUND(200)")
         print("6. Request对象构造时会将传入的priority值取反存储")

tests/test_response_follow.py

@@ -0,0 +1,105 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response.follow 方法测试
+"""
+import unittest
+from unittest.mock import Mock
+
+# 模拟 Request 类
+class MockRequest:
+    def __init__(self, url, callback=None, **kwargs):
+        self.url = url
+        self.callback = callback
+        self.kwargs = kwargs
+
+# 模拟 crawlo.Request
+import sys
+sys.modules['crawlo'] = Mock()
+sys.modules['crawlo'].Request = MockRequest
+
+from crawlo.network.response import Response
+
+
+class TestResponseFollow(unittest.TestCase):
+    """Response.follow 方法测试类"""
+
+    def setUp(self):
+        """测试前准备"""
+        # 创建一个模拟的HTML响应
+        html_content = """
+        <html>
+        <head>
+            <title>测试页面</title>
+        </head>
+        <body>
+            <div class="content">
+                <h1>主标题</h1>
+                <p class="intro">这是介绍段落</p>
+                <ul class="list">
+                    <li>项目1</li>
+                    <li>项目2</li>
+                    <li>项目3</li>
+                </ul>
+                <a href="https://example.com" class="link">链接文本</a>
+                <a href="/relative/path" class="relative-link">相对链接</a>
+                <img src="image.jpg" alt="图片描述" class="image">
+            </div>
+        </body>
+        </html>
+        """
+        
+        # 创建模拟的请求对象
+        mock_request = Mock()
+        mock_request.callback = None
+        
+        self.response = Response(
+            url="https://example.com/test",
+            body=html_content.encode('utf-8'),
+            headers={"content-type": "text/html; charset=utf-8"},
+            request=mock_request
+        )
+
+    def test_follow_absolute_url(self):
+        """测试处理绝对URL"""
+        request = self.response.follow("https://other.com/page", callback=lambda r: None)
+        self.assertEqual(request.url, "https://other.com/page")
+        self.assertIsNotNone(request.callback)
+
+    def test_follow_relative_url(self):
+        """测试处理相对URL"""
+        request = self.response.follow("/relative/path", callback=lambda r: None)
+        self.assertEqual(request.url, "https://example.com/relative/path")
+        self.assertIsNotNone(request.callback)
+
+    def test_follow_complex_relative_url(self):
+        """测试处理复杂的相对URL"""
+        request = self.response.follow("../other/path", callback=lambda r: None)
+        self.assertEqual(request.url, "https://example.com/other/path")
+        
+        request2 = self.response.follow("./another/path", callback=lambda r: None)
+        self.assertEqual(request2.url, "https://example.com/another/path")
+
+    def test_follow_with_query_params(self):
+        """测试处理带查询参数的URL"""
+        request = self.response.follow("/path?param=value", callback=lambda r: None)
+        self.assertEqual(request.url, "https://example.com/path?param=value")
+        
+        request2 = self.response.follow("/path#section", callback=lambda r: None)
+        self.assertEqual(request2.url, "https://example.com/path#section")
+
+    def test_follow_with_additional_kwargs(self):
+        """测试传递额外参数"""
+        request = self.response.follow(
+            "/path", 
+            callback=lambda r: None,
+            method="POST",
+            headers={"User-Agent": "test"}
+        )
+        self.assertEqual(request.url, "https://example.com/path")
+        self.assertEqual(request.kwargs.get("method"), "POST")
+        self.assertEqual(request.kwargs.get("headers"), {"User-Agent": "test"})
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_response_selector_methods.py

@@ -0,0 +1,93 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+测试Response类中的选择器方法
+"""
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.network.response import Response
+
+
+def test_response_selector_methods():
+    """测试Response类中的选择器方法"""
+    print("测试Response类中的选择器方法...")
+    print("=" * 50)
+    
+    # 创建测试HTML响应
+    html_content = """
+    <html>
+    <head>
+        <title>测试页面</title>
+    </head>
+    <body>
+        <div class="content">
+            <h1>主标题</h1>
+            <p class="intro">介绍段落</p>
+            <ul class="list">
+                <li>项目1</li>
+                <li>项目2</li>
+                <li>项目3</li>
+            </ul>
+            <a href="https://example.com" class="link">链接文本</a>
+            <img src="image.jpg" alt="图片描述" class="image">
+        </div>
+    </body>
+    </html>
+    """
+    
+    # 创建Response对象
+    response = Response(
+        url="https://example.com/test",
+        body=html_content.encode('utf-8'),
+        headers={"content-type": "text/html; charset=utf-8"}
+    )
+    
+    # 测试 extract_text (CSS选择器)
+    print("1. 测试 extract_text (CSS选择器):")
+    title_text = response.extract_text('title')
+    print(f"   标题文本: {title_text}")
+    
+    h1_text = response.extract_text('.content h1')
+    print(f"   H1文本: {h1_text}")
+    print()
+    
+    # 测试 extract_text (XPath选择器)
+    print("2. 测试 extract_text (XPath选择器):")
+    title_text_xpath = response.extract_text('//title')
+    print(f"   标题文本: {title_text_xpath}")
+    
+    h1_text_xpath = response.extract_text('//div[@class="content"]/h1')
+    print(f"   H1文本: {h1_text_xpath}")
+    print()
+    
+    # 测试 extract_texts
+    print("3. 测试 extract_texts:")
+    li_texts = response.extract_texts('.list li')
+    print(f"   列表项文本: {li_texts}")
+    print()
+    
+    # 测试 extract_attr
+    print("4. 测试 extract_attr:")
+    link_href = response.extract_attr('.link', 'href')
+    print(f"   链接href: {link_href}")
+    
+    img_alt = response.extract_attr('.image', 'alt')
+    print(f"   图片alt: {img_alt}")
+    print()
+    
+    # 测试 extract_attrs
+    print("5. 测试 extract_attrs:")
+    all_links = response.extract_attrs('a', 'href')
+    print(f"   所有链接href: {all_links}")
+    
+    all_images = response.extract_attrs('img', 'src')
+    print(f"   所有图片src: {all_images}")
+    print()
+    
+    print("所有测试完成！")
+
+
+if __name__ == '__main__':
+    test_response_selector_methods()

tests/test_response_url_methods.py

@@ -0,0 +1,71 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response URL 处理方法测试
+"""
+import unittest
+from urllib.parse import urlparse, urlsplit, parse_qs, urlencode, quote, unquote, urldefrag
+
+
+class TestUrlMethods(unittest.TestCase):
+    """URL 处理方法测试类"""
+
+    def setUp(self):
+        """测试前准备"""
+        self.test_url = "https://example.com/test?param1=value1&param2=value2#section1"
+
+    def test_urlparse(self):
+        """测试 urlparse 方法"""
+        parsed = urlparse(self.test_url)
+        self.assertEqual(parsed.scheme, "https")
+        self.assertEqual(parsed.netloc, "example.com")
+        self.assertEqual(parsed.path, "/test")
+        self.assertEqual(parsed.query, "param1=value1&param2=value2")
+        self.assertEqual(parsed.fragment, "section1")
+
+    def test_urlsplit(self):
+        """测试 urlsplit 方法"""
+        split_result = urlsplit(self.test_url)
+        self.assertEqual(split_result.scheme, "https")
+        self.assertEqual(split_result.netloc, "example.com")
+        self.assertEqual(split_result.path, "/test")
+        self.assertEqual(split_result.query, "param1=value1&param2=value2")
+        self.assertEqual(split_result.fragment, "section1")
+
+    def test_parse_qs(self):
+        """测试 parse_qs 方法"""
+        query_dict = parse_qs("param1=value1&param2=value2&param2=value3")
+        self.assertIn("param1", query_dict)
+        self.assertIn("param2", query_dict)
+        self.assertEqual(query_dict["param1"], ["value1"])
+        self.assertEqual(query_dict["param2"], ["value2", "value3"])
+
+    def test_urlencode(self):
+        """测试 urlencode 方法"""
+        query_dict = {"name": "张三", "age": 25, "city": "北京"}
+        encoded = urlencode(query_dict)
+        # 注意：urlencode 的顺序可能不同，所以我们检查是否包含所有键值对
+        self.assertIn("name=%E5%BC%A0%E4%B8%89", encoded)
+        self.assertIn("age=25", encoded)
+        self.assertIn("city=%E5%8C%97%E4%BA%AC", encoded)
+
+    def test_quote_unquote(self):
+        """测试 quote 和 unquote 方法"""
+        # 测试 quote
+        original = "hello world 你好"
+        quoted = quote(original)
+        self.assertEqual(quoted, "hello%20world%20%E4%BD%A0%E5%A5%BD")
+        
+        # 测试 unquote
+        unquoted = unquote(quoted)
+        self.assertEqual(unquoted, original)
+
+    def test_urldefrag(self):
+        """测试 urldefrag 方法"""
+        url_without_frag, fragment = urldefrag(self.test_url)
+        self.assertEqual(url_without_frag, "https://example.com/test?param1=value1&param2=value2")
+        self.assertEqual(fragment, "section1")
+
+
+if __name__ == '__main__':
+    unittest.main()