crawlo 1.4.4__py3-none-any.whl → 1.4.6__py3-none-any.whl

tests/bug_check_test.py

@@ -0,0 +1,251 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+框架bug检查测试脚本
+全面测试框架的核心功能，查找潜在的bug
+"""
+
+import sys
+import os
+import asyncio
+import traceback
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+from crawlo.network.response import Response
+
+
+class TestSpider(Spider):
+    """测试爬虫"""
+    name = 'bug_check_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        # 生成一些测试请求
+        yield Request('https://httpbin.org/get', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        # 测试Response的各种方法
+        self.test_response_methods(response)
+        return []
+    
+    def test_response_methods(self, response):
+        """测试Response的各种方法"""
+        print("测试Response方法...")
+        
+        # 测试URL处理方法
+        try:
+            joined_url = response.urljoin('/test')
+            print(f"  urljoin测试通过: {joined_url}")
+        except Exception as e:
+            print(f"  urljoin测试失败: {e}")
+        
+        try:
+            parsed = response.urlparse()
+            print(f"  urlparse测试通过: {parsed}")
+        except Exception as e:
+            print(f"  urlparse测试失败: {e}")
+        
+        # 测试选择器方法
+        try:
+            # 测试提取文本
+            text = response.extract_text('title', default='No title')
+            print(f"  extract_text测试通过: {text}")
+        except Exception as e:
+            print(f"  extract_text测试失败: {e}")
+        
+        try:
+            # 测试提取多个文本
+            texts = response.extract_texts('h1,h2,h3', default=[])
+            print(f"  extract_texts测试通过: {texts}")
+        except Exception as e:
+            print(f"  extract_texts测试失败: {e}")
+        
+        try:
+            # 测试提取属性
+            attr = response.extract_attr('a', 'href', default='')
+            print(f"  extract_attr测试通过: {attr}")
+        except Exception as e:
+            print(f"  extract_attr测试失败: {e}")
+        
+        try:
+            # 测试提取多个属性
+            attrs = response.extract_attrs('a', 'href', default=[])
+            print(f"  extract_attrs测试通过: {attrs}")
+        except Exception as e:
+            print(f"  extract_attrs测试失败: {e}")
+
+
+async def test_request_serialization():
+    """测试Request序列化"""
+    print("测试Request序列化...")
+    
+    try:
+        # 创建一个复杂的Request对象
+        request = Request(
+            url='https://example.com/test',
+            method='POST',
+            headers={'User-Agent': 'Test'},
+            json_body={'key': 'value'},
+            meta={'test': 'data'},
+            priority=5
+        )
+        
+        # 测试pickle序列化
+        import pickle
+        serialized = pickle.dumps(request)
+        deserialized = pickle.loads(serialized)
+        print(f"  Request序列化测试通过: {deserialized.url}")
+        return True
+    except Exception as e:
+        print(f"  Request序列化测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_queue_operations():
+    """测试队列操作"""
+    print("测试队列操作...")
+    
+    try:
+        from crawlo.queue.queue_manager import QueueConfig, QueueManager, QueueType
+        
+        # 创建内存队列配置
+        queue_config = QueueConfig(
+            queue_type=QueueType.MEMORY,
+            max_queue_size=10
+        )
+        
+        # 创建队列管理器
+        queue_manager = QueueManager(queue_config)
+        await queue_manager.initialize()
+        
+        # 测试添加请求
+        request = Request('https://example.com/test')
+        success = await queue_manager.put(request)
+        print(f"  队列添加请求: {'成功' if success else '失败'}")
+        
+        # 测试获取请求
+        retrieved = await queue_manager.get(timeout=1.0)
+        print(f"  队列获取请求: {'成功' if retrieved else '失败'}")
+        
+        # 关闭队列
+        await queue_manager.close()
+        print("  队列操作测试完成")
+        return True
+    except Exception as e:
+        print(f"  队列操作测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_redis_queue_operations():
+    """测试Redis队列操作"""
+    print("测试Redis队列操作...")
+    
+    try:
+        from crawlo.queue.queue_manager import QueueConfig, QueueManager, QueueType
+        
+        # 创建Redis队列配置
+        queue_config = QueueConfig(
+            queue_type=QueueType.REDIS,
+            redis_url='redis://127.0.0.1:6379/15',  # 使用测试数据库
+            queue_name='test:bug_check',
+            max_queue_size=10
+        )
+        
+        # 创建队列管理器
+        queue_manager = QueueManager(queue_config)
+        initialized = await queue_manager.initialize()
+        if not initialized:
+            print("  Redis队列初始化失败，跳过测试")
+            return True
+        
+        # 测试添加请求
+        request = Request('https://example.com/test_redis')
+        success = await queue_manager.put(request)
+        print(f"  Redis队列添加请求: {'成功' if success else '失败'}")
+        
+        # 测试获取请求
+        retrieved = await queue_manager.get(timeout=1.0)
+        print(f"  Redis队列获取请求: {'成功' if retrieved else '失败'}")
+        
+        # 关闭队列
+        await queue_manager.close()
+        print("  Redis队列操作测试完成")
+        return True
+    except Exception as e:
+        print(f"  Redis队列操作测试失败: {e}")
+        # 不将Redis连接失败视为测试失败
+        return True
+
+
+async def test_spider_registration():
+    """测试爬虫注册"""
+    print("测试爬虫注册...")
+    
+    try:
+        from crawlo.spider import get_spider_by_name, get_spider_names
+        
+        # 检查测试爬虫是否已注册
+        spider_class = get_spider_by_name('bug_check_spider')
+        if spider_class:
+            print(f"  爬虫注册测试通过: {spider_class.__name__}")
+        else:
+            print("  爬虫注册测试失败: 未找到注册的爬虫")
+            return False
+            
+        # 检查所有注册的爬虫
+        spider_names = get_spider_names()
+        print(f"  已注册的爬虫: {spider_names}")
+        return True
+    except Exception as e:
+        print(f"  爬虫注册测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def main():
+    """主函数"""
+    print("开始框架bug检查测试...")
+    print("=" * 50)
+    
+    tests = [
+        test_request_serialization,
+        test_queue_operations,
+        test_redis_queue_operations,
+        test_spider_registration,
+    ]
+    
+    passed = 0
+    total = len(tests)
+    
+    for test_func in tests:
+        try:
+            if await test_func():
+                passed += 1
+                print(f"✓ {test_func.__name__} 通过")
+            else:
+                print(f"✗ {test_func.__name__} 失败")
+        except Exception as e:
+            print(f"✗ {test_func.__name__} 异常: {e}")
+        print()
+    
+    print("=" * 50)
+    print(f"测试结果: {passed}/{total} 通过")
+    
+    if passed == total:
+        print("所有测试通过！框架核心功能正常。")
+        return 0
+    else:
+        print("部分测试失败，请检查框架实现。")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
+    exit(exit_code)

tests/direct_selector_helper_test.py

@@ -0,0 +1,97 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+直接测试选择器辅助工具
+"""
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+# 直接导入工具模块
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'crawlo', 'utils'))
+import selector_helper as sh
+from parsel import Selector
+
+
+def test_direct_selector_helper():
+    """直接测试选择器辅助工具"""
+    print("直接测试选择器辅助工具...")
+    print("=" * 50)
+    
+    # 创建测试HTML
+    html_content = """
+    <html>
+    <head>
+        <title>测试页面</title>
+    </head>
+    <body>
+        <div class="content">
+            <h1>主标题</h1>
+            <p class="intro">介绍段落</p>
+            <ul class="list">
+                <li>项目1</li>
+                <li>项目2</li>
+                <li>项目3</li>
+            </ul>
+            <a href="https://example.com" class="link">链接文本</a>
+            <img src="image.jpg" alt="图片描述" class="image">
+        </div>
+    </body>
+    </html>
+    """
+    
+    selector = Selector(text=html_content)
+    
+    # 测试 is_xpath
+    print("1. 测试 is_xpath:")
+    print(f"   '/' 开头: {sh.is_xpath('/')}")
+    print(f"   '//' 开头: {sh.is_xpath('//title')}")
+    print(f"   './' 开头: {sh.is_xpath('./div')}")
+    print(f"   'title' 开头: {sh.is_xpath('title')}")
+    print()
+    
+    # 测试 extract_text
+    print("2. 测试 extract_text:")
+    title_elements = selector.css('title')
+    title_text = sh.extract_text(title_elements)
+    print(f"   标题文本: {title_text}")
+    
+    h1_elements = selector.css('.content h1')
+    h1_text = sh.extract_text(h1_elements)
+    print(f"   H1文本: {h1_text}")
+    print()
+    
+    # 测试 extract_texts
+    print("3. 测试 extract_texts:")
+    li_elements = selector.css('.list li')
+    li_texts = sh.extract_texts(li_elements)
+    print(f"   列表项文本: {li_texts}")
+    print()
+    
+    # 测试 extract_attr
+    print("4. 测试 extract_attr:")
+    link_elements = selector.css('.link')
+    link_href = sh.extract_attr(link_elements, 'href')
+    print(f"   链接href: {link_href}")
+    
+    img_elements = selector.css('.image')
+    img_alt = sh.extract_attr(img_elements, 'alt')
+    print(f"   图片alt: {img_alt}")
+    print()
+    
+    # 测试 extract_attrs
+    print("5. 测试 extract_attrs:")
+    all_links = selector.css('a')
+    all_hrefs = sh.extract_attrs(all_links, 'href')
+    print(f"   所有链接href: {all_hrefs}")
+    
+    all_images = selector.css('img')
+    all_srcs = sh.extract_attrs(all_images, 'src')
+    print(f"   所有图片src: {all_srcs}")
+    print()
+    
+    print("所有测试完成！")
+
+
+if __name__ == '__main__':
+    test_direct_selector_helper()

tests/explain_mysql_update_behavior.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""
+解释 MySQL ON DUPLICATE KEY UPDATE 行为
+帮助理解为什么在使用 MYSQL_UPDATE_COLUMNS 时可能显示"未插入新记录"
+"""
+import sys
+import os
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.db_helper import SQLBuilder
+
+
+def explain_mysql_behavior():
+    """解释 MySQL ON DUPLICATE KEY UPDATE 的行为"""
+    print("=== MySQL ON DUPLICATE KEY UPDATE 行为解释 ===\n")
+    
+    # 模拟实际使用的数据
+    table = "news_items"
+    item_data = {
+        'title': '新一代OLED屏下光谱颜色传感技术：解锁显示新密码，重塑视觉新体验',
+        'publish_time': '2025-10-09 09:57',
+        'url': 'https://ee.ofweek.com/2025-10/ART-8460-2806-30671544.html',
+        'source': '',
+        'content': '在全球智能手机市场竞争日趋白热化的当下，消费者对手机屏幕显示效果的要求愈发严苛...'
+    }
+    
+    print("当使用 MYSQL_UPDATE_COLUMNS 配置时:")
+    print("MYSQL_UPDATE_COLUMNS = ('title', 'publish_time')")
+    print()
+    
+    # 生成 SQL
+    sql = SQLBuilder.make_insert(
+        table=table,
+        data=item_data,
+        auto_update=False,
+        update_columns=('title', 'publish_time'),
+        insert_ignore=False
+    )
+    
+    print("生成的 SQL 语句:")
+    print(sql)
+    print()
+    
+    print("MySQL 行为说明:")
+    print("1. 如果这是一条新记录（没有主键或唯一键冲突）:")
+    print("   - MySQL 会正常插入记录")
+    print("   - 返回影响行数为 1")
+    print()
+    print("2. 如果遇到主键或唯一键冲突:")
+    print("   - MySQL 会执行 ON DUPLICATE KEY UPDATE 子句")
+    print("   - 更新指定的字段: title 和 publish_time")
+    print()
+    print("3. 关键点 - 如果更新的字段值与现有记录完全相同:")
+    print("   - MySQL 不会实际更新任何数据")
+    print("   - 返回影响行数为 0")
+    print("   - 这就是你看到 'SQL执行成功但未插入新记录' 的原因")
+    print()
+    
+    print("如何验证是否真的更新了数据:")
+    print("1. 检查数据库中的记录是否发生变化")
+    print("2. 如果内容字段有变化但未在 update_columns 中指定，则不会更新")
+    print("3. 可以在 update_columns 中添加更多字段以确保更新")
+    print()
+    
+    print("建议的配置:")
+    print("# 如果希望在冲突时更新更多字段，可以这样配置:")
+    print("MYSQL_UPDATE_COLUMNS = ('title', 'publish_time', 'content')")
+    print()
+    print("# 或者如果希望完全替换记录:")
+    print("MYSQL_AUTO_UPDATE = True")
+    print("MYSQL_UPDATE_COLUMNS = ()  # 空元组")
+
+
+if __name__ == "__main__":
+    explain_mysql_behavior()

tests/ofweek_scrapy/ofweek_scrapy/items.py

@@ -0,0 +1,12 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class OfweekScrapyItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass

tests/ofweek_scrapy/ofweek_scrapy/middlewares.py

@@ -0,0 +1,100 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
+class OfweekScrapySpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    async def process_start(self, start):
+        # Called with an async iterator over the spider start() method or the
+        # maching method of an earlier spider middleware.
+        async for item_or_request in start:
+            yield item_or_request
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)
+
+
+class OfweekScrapyDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)

tests/ofweek_scrapy/ofweek_scrapy/pipelines.py

@@ -0,0 +1,13 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
+class OfweekScrapyPipeline:
+    def process_item(self, item, spider):
+        return item

tests/ofweek_scrapy/ofweek_scrapy/settings.py

@@ -0,0 +1,85 @@
+# Scrapy settings for ofweek_scrapy project
+
+BOT_NAME = 'ofweek_scrapy'
+
+SPIDER_MODULES = ['ofweek_scrapy.spiders']
+NEWSPIDER_MODULE = 'ofweek_scrapy.spiders'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+# 使用与Crawlo相同的并发数
+CONCURRENT_REQUESTS = 8
+
+# Configure a delay for requests for the same website (default: 0)
+# 使用与Crawlo相同的下载延迟
+DOWNLOAD_DELAY = 1.0
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#    'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'ofweek_scrapy.middlewares.OfweekScrapySpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'ofweek_scrapy.middlewares.OfweekScrapyDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    'ofweek_scrapy.pipelines.OfweekScrapyPipeline': 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
+TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+FEED_EXPORT_ENCODING = 'utf-8'
+
+# 日志配置
+LOG_LEVEL = 'INFO'

tests/ofweek_scrapy/ofweek_scrapy/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.