crawlo 1.4.0__py3-none-any.whl → 1.4.2__py3-none-any.whl

examples/test_project/test_project/spiders/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: UTF-8 -*-
+"""
+test_project.spiders
+========================
+存放所有的爬虫。
+"""
+
+# 自动导入所有爬虫以确保它们被注册
+# 示例：
+# from .YourSpider import YourSpider

examples/test_project/test_project/spiders/of_week_dis.py

@@ -0,0 +1,144 @@
+# -*- coding: UTF-8 -*-
+"""
+test_project.spiders.of_week_dis
+=======================================
+由 `crawlo genspider` 命令生成的爬虫。
+基于 Crawlo 框架，支持异步并发、分布式爬取等功能。
+
+使用示例：
+    crawlo crawl of_week_dis
+"""
+
+from crawlo.spider import Spider
+from crawlo import Request
+from ..items import ExampleItem
+
+
+class OfweekdisSpider(Spider):
+    """
+    爬虫：of_week_dis
+    
+    功能说明：
+    - 支持并发爬取
+    - 自动去重过滤
+    - 错误重试机制
+    - 数据管道处理
+    """
+    name = 'of_week_dis'
+    allowed_domains = ['ee.ofweek.com']
+    start_urls = ['https://ee.ofweek.com/']
+    
+    # 高级配置（可选）
+    # custom_settings = {
+    #     'DOWNLOAD_DELAY': 2.0,
+    #     'CONCURRENCY': 4,
+    #     'RETRY_HTTP_CODES': [500, 502, 503, 504, 408, 429],
+    #     'ALLOWED_RESPONSE_CODES': [200, 301, 302],  # 只允许特定状态码
+    #     'DENIED_RESPONSE_CODES': [403, 404],        # 拒绝特定状态码
+    # }
+
+    def start_requests(self):
+        """
+        生成初始请求。
+        
+        支持自定义请求头、代理、优先级等。
+        """
+        headers = {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
+        }
+        
+        for url in self.start_urls:
+            yield Request(
+                url=url,
+                callback=self.parse,
+                headers=headers,
+                # meta={'proxy': 'http://proxy.example.com:8080'},  # 自定义代理
+                # priority=10,  # 请求优先级（数字越大优先级越高）
+            )
+
+    def parse(self, response):
+        """
+        解析响应的主方法。
+        
+        Args:
+            response: 响应对象，包含页面内容和元数据
+            
+        Yields:
+            Request: 新的请求对象（用于深度爬取）
+            Item: 数据项对象（用于数据存储）
+        """
+        self.logger.info(f'正在解析页面: {response.url}')
+        
+        # ================== 数据提取示例 ==================
+        
+        # 提取数据并创建 Item
+        # item = Item()
+        # item['title'] = response.xpath('//title/text()').get(default='')
+        # item['url'] = response.url
+        # item['content'] = response.xpath('//div[@class="content"]//text()').getall()
+        # yield item
+        
+        # 直接返回字典（简单数据）
+        yield {
+            'title': response.xpath('//title/text()').get(default=''),
+            'url': response.url,
+            'status_code': response.status_code,
+            # 'description': response.xpath('//meta[@name="description"]/@content').get(),
+            # 'keywords': response.xpath('//meta[@name="keywords"]/@content').get(),
+        }
+        
+        # ================== 链接提取示例 ==================
+        
+        # 提取并跟进链接
+        # links = response.xpath('//a/@href').getall()
+        # for link in links:
+        #     # 过滤有效链接
+        #     if link and not link.startswith(('javascript:', 'mailto:', '#')):
+        #         yield response.follow(
+        #             link,
+        #             callback=self.parse_detail,  # 或者 self.parse 继续递归
+        #             meta={'parent_url': response.url}  # 传递父页面信息
+        #         )
+        
+        # 用 CSS 选择器提取链接
+        # for link in response.css('a.item-link::attr(href)').getall():
+        #     yield response.follow(link, callback=self.parse_detail)
+        
+        # ================== 分页处理示例 ==================
+        
+        # 处理分页
+        # next_page = response.xpath('//a[@class="next"]/@href').get()
+        # if next_page:
+        #     yield response.follow(next_page, callback=self.parse)
+        
+        # 数字分页
+        # current_page = int(response.meta.get('page', 1))
+        # max_pages = 100  # 设置最大页数
+        # if current_page < max_pages:
+        #     next_url = f'https://ee.ofweek.com/page/{current_page + 1}'
+        #     yield Request(
+        #         url=next_url,
+        #         callback=self.parse,
+        #         meta={'page': current_page + 1}
+        #     )
+
+    def parse_detail(self, response):
+        """
+        解析详情页面的方法（可选）。
+        
+        用于处理从列表页跳转而来的详情页。
+        """
+        self.logger.info(f'正在解析详情页: {response.url}')
+        
+        # parent_url = response.meta.get('parent_url', '')
+        # 
+        # yield {
+        #     'title': response.xpath('//h1/text()').get(default=''),
+        #     'content': '\n'.join(response.xpath('//div[@class="content"]//text()').getall()),
+        #     'url': response.url,
+        #     'parent_url': parent_url,
+        #     'publish_time': response.xpath('//time/@datetime').get(),
+        # }
+        
+        pass

tests/debug_framework_logger.py

@@ -7,7 +7,7 @@ import sys
 import os
 sys.path.insert(0, '/')
 
-from crawlo.core.framework_initializer import initialize_framework, get_framework_initializer
+from crawlo.initialization import initialize_framework, get_framework_initializer
 from crawlo.utils.log import get_logger, LoggerManager
 import logging
 

tests/debug_log_levels.py

@@ -7,7 +7,7 @@ import sys
 import os
 sys.path.insert(0, '/')
 
-from crawlo.core.framework_initializer import initialize_framework
+from crawlo.initialization import initialize_framework
 from crawlo.utils.log import LoggerManager, get_logger
 import logging
 

tests/test_all_pipeline_fingerprints.py

@@ -0,0 +1,134 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+所有去重管道指纹一致性测试
+====================
+验证所有去重管道对相同数据生成一致的指纹
+"""
+
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.fingerprint import FingerprintGenerator
+
+
+class MockItem:
+    """模拟数据项类"""
+    
+    def __init__(self, **kwargs):
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+    
+    def to_dict(self):
+        """转换为字典"""
+        return {k: v for k, v in self.__dict__.items() if not k.startswith('_')}
+
+
+def test_all_pipeline_fingerprints():
+    """测试所有管道指纹一致性"""
+    # 创建测试数据项
+    test_item = MockItem(
+        title="Test Title",
+        url="https://example.com",
+        content="Test content",
+        price=99.99
+    )
+    
+    # 使用指纹生成器生成指纹
+    expected_fingerprint = FingerprintGenerator.item_fingerprint(test_item)
+    
+    print(f"Expected fingerprint: {expected_fingerprint}")
+    
+    # 测试内存去重管道指纹生成方法
+    try:
+        from crawlo.pipelines.memory_dedup_pipeline import MemoryDedupPipeline
+        memory_pipeline = MemoryDedupPipeline()
+        memory_fingerprint = memory_pipeline._generate_item_fingerprint(test_item)
+        print(f"Memory pipeline fingerprint: {memory_fingerprint}")
+        assert memory_fingerprint == expected_fingerprint, "Memory pipeline fingerprint mismatch"
+        print("✓ Memory pipeline fingerprint一致")
+    except Exception as e:
+        print(f"✗ Memory pipeline test failed: {e}")
+    
+    # 测试Redis去重管道指纹生成方法
+    try:
+        from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
+        redis_pipeline = RedisDedupPipeline()
+        redis_fingerprint = redis_pipeline._generate_item_fingerprint(test_item)
+        print(f"Redis pipeline fingerprint: {redis_fingerprint}")
+        assert redis_fingerprint == expected_fingerprint, "Redis pipeline fingerprint mismatch"
+        print("✓ Redis pipeline fingerprint一致")
+    except Exception as e:
+        print(f"✗ Redis pipeline test failed: {e}")
+    
+    # 测试Bloom去重管道指纹生成方法
+    try:
+        from crawlo.pipelines.bloom_dedup_pipeline import BloomDedupPipeline
+        bloom_pipeline = BloomDedupPipeline()
+        bloom_fingerprint = bloom_pipeline._generate_item_fingerprint(test_item)
+        print(f"Bloom pipeline fingerprint: {bloom_fingerprint}")
+        assert bloom_fingerprint == expected_fingerprint, "Bloom pipeline fingerprint mismatch"
+        print("✓ Bloom pipeline fingerprint一致")
+    except Exception as e:
+        print(f"✗ Bloom pipeline test failed: {e}")
+    
+    # 测试数据库去重管道指纹生成方法
+    try:
+        from crawlo.pipelines.database_dedup_pipeline import DatabaseDedupPipeline
+        database_pipeline = DatabaseDedupPipeline()
+        database_fingerprint = database_pipeline._generate_item_fingerprint(test_item)
+        print(f"Database pipeline fingerprint: {database_fingerprint}")
+        assert database_fingerprint == expected_fingerprint, "Database pipeline fingerprint mismatch"
+        print("✓ Database pipeline fingerprint一致")
+    except Exception as e:
+        print(f"✗ Database pipeline test failed: {e}")
+    
+    # 测试分布式协调工具指纹生成方法
+    try:
+        from crawlo.tools.distributed_coordinator import DeduplicationTool
+        dedup_tool = DeduplicationTool()
+        tool_fingerprint = dedup_tool.generate_fingerprint(test_item.to_dict())
+        print(f"Deduplication tool fingerprint: {tool_fingerprint}")
+        # 注意：这里我们传入的是字典，因为工具类的generate_fingerprint方法直接处理数据
+        expected_tool_fingerprint = FingerprintGenerator.data_fingerprint(test_item.to_dict())
+        assert tool_fingerprint == expected_tool_fingerprint, "Deduplication tool fingerprint mismatch"
+        print("✓ Deduplication tool fingerprint一致")
+    except Exception as e:
+        print(f"✗ Deduplication tool test failed: {e}")
+
+
+def test_fingerprint_stability():
+    """测试指纹稳定性"""
+    # 创建相同的测试数据项多次
+    item1 = MockItem(
+        title="Test Title",
+        url="https://example.com",
+        content="Test content",
+        price=99.99
+    )
+    
+    item2 = MockItem(
+        title="Test Title",
+        url="https://example.com",
+        content="Test content",
+        price=99.99
+    )
+    
+    # 生成指纹
+    fingerprint1 = FingerprintGenerator.item_fingerprint(item1)
+    fingerprint2 = FingerprintGenerator.item_fingerprint(item2)
+    
+    # 验证相同数据生成相同指纹
+    print(f"\nFirst fingerprint: {fingerprint1}")
+    print(f"Second fingerprint: {fingerprint2}")
+    assert fingerprint1 == fingerprint2, "Same items should generate same fingerprints"
+    print("✓ 相同数据生成相同指纹")
+
+
+if __name__ == '__main__':
+    test_all_pipeline_fingerprints()
+    test_fingerprint_stability()
+    print("\n🎉 所有测试通过!")