crawlo 1.4.4__py3-none-any.whl → 1.4.6__py3-none-any.whl

tests/test_mysql_pipeline_config.py

@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+import asyncio
+import unittest
+from unittest.mock import Mock, patch
+
+from crawlo.pipelines.mysql_pipeline import BaseMySQLPipeline, AsyncmyMySQLPipeline, AiomysqlMySQLPipeline
+
+
+class TestMySQLPipelineConfig(unittest.TestCase):
+    """测试MySQL管道配置"""
+
+    def setUp(self):
+        """设置测试环境"""
+        self.mock_crawler = Mock()
+        self.mock_crawler.settings = Mock()
+        self.mock_crawler.settings.get = Mock(return_value=None)
+        self.mock_crawler.settings.get_int = Mock(return_value=100)
+        self.mock_crawler.settings.get_bool = Mock(return_value=False)
+        self.mock_crawler.subscriber = Mock()
+        self.mock_crawler.subscriber.subscribe = Mock()
+        
+        # 模拟爬虫对象
+        self.mock_spider = Mock()
+        self.mock_spider.name = "test_spider"
+        self.mock_spider.custom_settings = {}
+        self.mock_spider.mysql_table = None
+        self.mock_crawler.spider = self.mock_spider
+
+    def test_default_config_values(self):
+        """测试默认配置值"""
+        # 设置默认返回值（模拟默认配置文件中的值）
+        self.mock_crawler.settings.get_bool = Mock(side_effect=lambda key, default: {
+            'MYSQL_AUTO_UPDATE': False,
+            'MYSQL_INSERT_IGNORE': False,
+            'MYSQL_USE_BATCH': False
+        }.get(key, default))
+        
+        self.mock_crawler.settings.get = Mock(side_effect=lambda key, default=None: {
+            'MYSQL_UPDATE_COLUMNS': ()
+        }.get(key, default))
+        
+        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
+        
+        # 验证默认配置值
+        self.assertEqual(pipeline.auto_update, False)
+        self.assertEqual(pipeline.insert_ignore, False)
+        self.assertEqual(pipeline.update_columns, ())
+
+    def test_custom_config_values(self):
+        """测试自定义配置值"""
+        # 设置自定义配置值
+        self.mock_crawler.settings.get_bool = Mock(side_effect=lambda key, default: {
+            'MYSQL_AUTO_UPDATE': True,
+            'MYSQL_INSERT_IGNORE': True
+        }.get(key, default))
+        
+        self.mock_crawler.settings.get = Mock(side_effect=lambda key, default=None: {
+            'MYSQL_UPDATE_COLUMNS': ('updated_at', 'view_count')
+        }.get(key, default))
+        
+        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
+        
+        # 验证自定义配置值
+        self.assertEqual(pipeline.auto_update, True)
+        self.assertEqual(pipeline.insert_ignore, True)
+        self.assertEqual(pipeline.update_columns, ('updated_at', 'view_count'))
+
+    def test_sql_generation_with_config(self):
+        """测试使用配置生成SQL"""
+        # 设置配置
+        self.mock_crawler.settings.get_bool = Mock(side_effect=lambda key, default: {
+            'MYSQL_AUTO_UPDATE': True,
+            'MYSQL_INSERT_IGNORE': False
+        }.get(key, default))
+        
+        self.mock_crawler.settings.get = Mock(side_effect=lambda key, default=None: {
+            'MYSQL_UPDATE_COLUMNS': ()
+        }.get(key, default))
+        
+        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
+        
+        # 测试数据
+        item_dict = {"id": 1, "name": "test"}
+        
+        async def test_async():
+            with patch('crawlo.pipelines.mysql_pipeline.SQLBuilder.make_insert') as mock_make_insert:
+                mock_make_insert.return_value = "TEST SQL"
+                
+                # 调用_make_insert_sql方法
+                result = await pipeline._make_insert_sql(item_dict)
+                
+                # 验证调用了SQLBuilder.make_insert并传递了正确的参数
+                mock_make_insert.assert_called_once()
+                call_args = mock_make_insert.call_args
+                self.assertEqual(call_args[1]['auto_update'], True)
+                self.assertEqual(call_args[1]['insert_ignore'], False)
+                self.assertEqual(call_args[1]['update_columns'], ())
+                
+        asyncio.run(test_async())
+
+    def test_sql_generation_with_kwargs_override(self):
+        """测试使用kwargs覆盖配置生成SQL"""
+        # 设置配置
+        self.mock_crawler.settings.get_bool = Mock(side_effect=lambda key, default: {
+            'MYSQL_AUTO_UPDATE': False,
+            'MYSQL_INSERT_IGNORE': False
+        }.get(key, default))
+        
+        self.mock_crawler.settings.get = Mock(side_effect=lambda key, default=None: {
+            'MYSQL_UPDATE_COLUMNS': ()
+        }.get(key, default))
+        
+        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
+        
+        # 测试数据
+        item_dict = {"id": 1, "name": "test"}
+        
+        async def test_async():
+            with patch('crawlo.pipelines.mysql_pipeline.SQLBuilder.make_insert') as mock_make_insert:
+                mock_make_insert.return_value = "TEST SQL"
+                
+                # 调用_make_insert_sql方法并传递kwargs
+                result = await pipeline._make_insert_sql(
+                    item_dict, 
+                    auto_update=True, 
+                    insert_ignore=True
+                )
+                
+                # 验证kwargs覆盖了配置
+                mock_make_insert.assert_called_once()
+                call_args = mock_make_insert.call_args
+                self.assertEqual(call_args[1]['auto_update'], True)
+                self.assertEqual(call_args[1]['insert_ignore'], True)
+                self.assertEqual(call_args[1]['update_columns'], ())
+                
+        asyncio.run(test_async())
+
+    def test_batch_config_passing(self):
+        """测试批量处理中配置的传递"""
+        # 设置配置
+        self.mock_crawler.settings.get_bool = Mock(side_effect=lambda key, default: {
+            'MYSQL_AUTO_UPDATE': True,
+            'MYSQL_INSERT_IGNORE': False,
+            'MYSQL_USE_BATCH': True
+        }.get(key, default))
+        
+        self.mock_crawler.settings.get = Mock(side_effect=lambda key, default=None: {
+            'MYSQL_UPDATE_COLUMNS': ('updated_at',)
+        }.get(key, default))
+        
+        self.mock_crawler.settings.get_int = Mock(side_effect=lambda key, default=100: {
+            'MYSQL_BATCH_SIZE': 2
+        }.get(key, default))
+        
+        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
+        
+        # 验证配置已正确设置
+        self.assertEqual(pipeline.auto_update, True)
+        self.assertEqual(pipeline.update_columns, ('updated_at',))
+        self.assertEqual(pipeline.use_batch, True)
+        self.assertEqual(pipeline.batch_size, 2)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mysql_pipeline_error.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+import asyncio
+import unittest
+from unittest.mock import Mock, patch, AsyncMock
+
+from crawlo.pipelines.mysql_pipeline import AsyncmyMySQLPipeline, AiomysqlMySQLPipeline
+from crawlo.exceptions import ItemDiscard
+
+
+class TestMySQLPipelineError(unittest.TestCase):
+    """测试MySQL管道错误处理"""
+
+    def setUp(self):
+        """设置测试环境"""
+        self.mock_crawler = Mock()
+        self.mock_crawler.settings = Mock()
+        self.mock_crawler.settings.get = Mock(return_value=None)
+        self.mock_crawler.settings.get_int = Mock(return_value=100)
+        self.mock_crawler.settings.get_bool = Mock(return_value=False)
+        self.mock_crawler.subscriber = Mock()
+        self.mock_crawler.subscriber.subscribe = Mock()
+        self.mock_crawler.stats = Mock()
+        self.mock_crawler.stats.inc_value = Mock()
+        
+        # 模拟爬虫对象
+        self.mock_spider = Mock()
+        self.mock_spider.name = "test_spider"
+        self.mock_spider.custom_settings = {}
+        self.mock_spider.mysql_table = None
+        self.mock_crawler.spider = self.mock_spider
+
+    def test_asyncmy_process_item_with_connection_error(self):
+        """测试AsyncmyMySQLPipeline处理连接错误"""
+        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
+        
+        # 模拟连接池和数据库操作
+        mock_pool = AsyncMock()
+        mock_conn = AsyncMock()
+        mock_cursor = AsyncMock()
+        
+        # 模拟acquire方法返回连接
+        mock_pool.acquire.return_value = mock_conn
+        
+        # 模拟cursor方法返回游标
+        mock_conn.cursor.return_value = mock_cursor
+        
+        # 模拟execute方法抛出异常
+        mock_cursor.execute.side_effect = Exception("测试异常")
+        
+        # 设置管道的连接池
+        pipeline.pool = mock_pool
+        pipeline._pool_initialized = True
+        
+        # 测试数据
+        test_item = {"id": 1, "name": "test"}
+        
+        async def test_async():
+            with self.assertRaises(ItemDiscard) as context:
+                await pipeline.process_item(test_item, self.mock_spider)
+            
+            # 验证错误信息
+            self.assertIn("MySQL插入失败", str(context.exception))
+            
+        asyncio.run(test_async())
+
+    def test_execute_sql_with_exception(self):
+        """测试_execute_sql方法处理异常"""
+        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
+        
+        # 模拟连接池和数据库操作
+        mock_pool = AsyncMock()
+        mock_conn = AsyncMock()
+        mock_cursor = AsyncMock()
+        
+        # 模拟acquire方法返回连接
+        mock_pool.acquire.return_value = mock_conn
+        
+        # 模拟cursor方法返回游标
+        mock_conn.cursor.return_value = mock_cursor
+        
+        # 模拟execute方法抛出异常
+        mock_cursor.execute.side_effect = Exception("测试异常")
+        
+        # 设置管道的连接池
+        pipeline.pool = mock_pool
+        pipeline._pool_initialized = True
+        
+        async def test_async():
+            with self.assertRaises(ItemDiscard) as context:
+                await pipeline._execute_sql("SELECT 1")
+            
+            # 验证错误信息
+            self.assertIn("MySQL插入失败", str(context.exception))
+            
+        asyncio.run(test_async())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mysql_pipeline_init_log.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+"""
+测试 MySQL 管道初始化日志
+验证在管道初始化时是否正确打印日志
+"""
+import sys
+import os
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.pipelines.mysql_pipeline import AsyncmyMySQLPipeline, AiomysqlMySQLPipeline
+
+
+# 创建一个简单的爬虫模拟类
+class MockSpider:
+    name = "test_spider"
+
+
+# 创建一个简单的设置模拟类
+class MockSettings:
+    def get(self, key, default=None):
+        # 简化的设置获取
+        settings_map = {
+            'MYSQL_HOST': 'localhost',
+            'MYSQL_PORT': 3306,
+            'MYSQL_USER': 'root',
+            'MYSQL_PASSWORD': '',
+            'MYSQL_DB': 'test_db',
+            'MYSQL_TABLE': 'test_table',
+            'LOG_LEVEL': 'INFO'
+        }
+        return settings_map.get(key, default)
+    
+    def get_int(self, key, default=0):
+        return int(self.get(key, default))
+    
+    def get_bool(self, key, default=False):
+        return bool(self.get(key, default))
+
+
+# 创建一个简单的订阅者模拟类
+class MockSubscriber:
+    def subscribe(self, func, event):
+        # 简化的订阅
+        pass
+
+
+# 创建一个简单的爬虫模拟类
+class MockCrawler:
+    def __init__(self):
+        self.settings = MockSettings()
+        self.subscriber = MockSubscriber()
+        self.spider = MockSpider()
+
+
+def test_pipeline_init_logs():
+    """测试管道初始化日志"""
+    print("=== Testing MySQL Pipeline Initialization Logs ===")
+    
+    # 创建模拟的爬虫
+    crawler = MockCrawler()
+    
+    print("1. Testing AsyncmyMySQLPipeline initialization...")
+    try:
+        asyncmy_pipeline = AsyncmyMySQLPipeline.from_crawler(crawler)
+        print("   ✓ AsyncmyMySQLPipeline initialized successfully")
+    except Exception as e:
+        print(f"   ✗ AsyncmyMySQLPipeline initialization failed: {e}")
+    
+    print("\n2. Testing AiomysqlMySQLPipeline initialization...")
+    try:
+        aiomysql_pipeline = AiomysqlMySQLPipeline.from_crawler(crawler)
+        print("   ✓ AiomysqlMySQLPipeline initialized successfully")
+    except Exception as e:
+        print(f"   ✗ AiomysqlMySQLPipeline initialization failed: {e}")
+    
+    print("\n=== Test completed ===")
+    print("Note: Actual log output can be seen when running in a full crawler environment")
+
+
+if __name__ == "__main__":
+    test_pipeline_init_logs()

tests/test_mysql_pipeline_integration.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+"""
+MySQL 管道集成测试
+验证 MYSQL_UPDATE_COLUMNS 配置在实际使用中是否解决了 MySQL 警告问题
+"""
+import asyncio
+import sys
+import os
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.db_helper import SQLBuilder
+
+
+def test_complete_workflow():
+    """测试完整的数据插入工作流程"""
+    print("=== MySQL 管道集成测试 ===\n")
+    
+    # 模拟实际使用的配置
+    table = "news_items"
+    item_data = {
+        'title': '新一代OLED屏下光谱颜色传感技术：解锁显示新密码，重塑视觉新体验',
+        'publish_time': '2025-10-09 09:57',
+        'url': 'https://ee.ofweek.com/2025-10/ART-8460-2806-30671544.html',
+        'source': '',
+        'content': '在全球智能手机市场竞争日趋白热化的当下，消费者对手机屏幕显示效果的要求愈发严苛...'
+    }
+    
+    print("1. 测试单条插入 SQL 生成...")
+    # 测试单条插入
+    single_sql = SQLBuilder.make_insert(
+        table=table,
+        data=item_data,
+        auto_update=False,  # 不使用 REPLACE INTO
+        insert_ignore=False,  # 不使用 INSERT IGNORE
+        update_columns=('title', 'publish_time')  # 冲突时更新指定列
+    )
+    
+    print("生成的单条插入 SQL:")
+    print(single_sql)
+    print()
+    
+    # 验证语法正确性
+    if "AS `excluded`" in single_sql and "`title`=`excluded`.`title`" in single_sql:
+        print("✓ 单条插入正确使用了新的 MySQL 语法")
+    else:
+        print("✗ 单条插入语法不正确")
+    
+    print("\n2. 测试批量插入 SQL 生成...")
+    # 测试批量插入
+    batch_data = [item_data, item_data]  # 模拟重复数据
+    batch_result = SQLBuilder.make_batch(
+        table=table,
+        datas=batch_data,
+        auto_update=False,
+        update_columns=('title', 'publish_time')
+    )
+    
+    if batch_result:
+        batch_sql, values_list = batch_result
+        print("生成的批量插入 SQL:")
+        print(batch_sql)
+        print(f"参数值列表数量: {len(values_list)}")
+        print()
+        
+        # 验证语法正确性
+        if "AS `excluded`" in batch_sql and "`title`=`excluded`.`title`" in batch_sql:
+            print("✓ 批量插入正确使用了新的 MySQL 语法")
+        else:
+            print("✗ 批量插入语法不正确")
+    
+    print("\n3. 测试不同配置组合...")
+    
+    # 测试仅使用 INSERT IGNORE
+    ignore_sql = SQLBuilder.make_insert(
+        table=table,
+        data=item_data,
+        auto_update=False,
+        insert_ignore=True,
+        update_columns=()  # 不指定更新列
+    )
+    
+    print("INSERT IGNORE SQL:")
+    print(ignore_sql)
+    print()
+    
+    if "INSERT IGNORE" in ignore_sql and "AS `excluded`" not in ignore_sql:
+        print("✓ INSERT IGNORE 模式正确")
+    else:
+        print("✗ INSERT IGNORE 模式不正确")
+    
+    # 测试使用 REPLACE INTO
+    replace_sql = SQLBuilder.make_insert(
+        table=table,
+        data=item_data,
+        auto_update=True,  # 使用 REPLACE INTO
+        insert_ignore=False,
+        update_columns=()  # 不指定更新列
+    )
+    
+    print("REPLACE INTO SQL:")
+    print(replace_sql)
+    print()
+    
+    if "REPLACE INTO" in replace_sql and "AS `excluded`" not in replace_sql:
+        print("✓ REPLACE INTO 模式正确")
+    else:
+        print("✗ REPLACE INTO 模式不正确")
+    
+    # 测试普通 INSERT
+    normal_sql = SQLBuilder.make_insert(
+        table=table,
+        data=item_data,
+        auto_update=False,
+        insert_ignore=False,
+        update_columns=()  # 不指定更新列
+    )
+    
+    print("普通 INSERT SQL:")
+    print(normal_sql)
+    print()
+    
+    if "INSERT INTO" in normal_sql and "AS `excluded`" not in normal_sql:
+        print("✓ 普通 INSERT 模式正确")
+    else:
+        print("✗ 普通 INSERT 模式不正确")
+    
+    print("\n=== 测试完成 ===")
+
+
+if __name__ == "__main__":
+    test_complete_workflow()

tests/test_mysql_pipeline_refactor.py

@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+import asyncio
+import unittest
+from unittest.mock import Mock, patch, AsyncMock
+
+from crawlo.pipelines.mysql_pipeline import BaseMySQLPipeline, AsyncmyMySQLPipeline, AiomysqlMySQLPipeline
+
+
+class TestBaseMySQLPipeline(unittest.TestCase):
+    """测试MySQL管道基类"""
+
+    def setUp(self):
+        """设置测试环境"""
+        self.mock_crawler = Mock()
+        self.mock_crawler.settings = Mock()
+        self.mock_crawler.settings.get = Mock(return_value=None)
+        self.mock_crawler.settings.get_int = Mock(return_value=100)
+        self.mock_crawler.settings.get_bool = Mock(return_value=False)
+        self.mock_crawler.subscriber = Mock()
+        self.mock_crawler.subscriber.subscribe = Mock()
+        
+        # 模拟爬虫对象
+        self.mock_spider = Mock()
+        self.mock_spider.name = "test_spider"
+        self.mock_spider.custom_settings = {}
+        self.mock_spider.mysql_table = None
+        self.mock_crawler.spider = self.mock_spider
+
+    def test_base_init(self):
+        """测试基类初始化"""
+        # 创建一个实现基类的测试类
+        class TestMySQLPipeline(BaseMySQLPipeline):
+            async def _ensure_pool(self):
+                pass
+        
+        pipeline = TestMySQLPipeline(self.mock_crawler)
+        
+        # 验证属性初始化
+        self.assertEqual(pipeline.crawler, self.mock_crawler)
+        self.assertEqual(pipeline.settings, self.mock_crawler.settings)
+        self.assertEqual(pipeline.table_name, "test_spider_items")
+        self.assertEqual(pipeline.batch_size, 100)
+        self.assertEqual(pipeline.use_batch, False)
+        self.assertEqual(pipeline.batch_buffer, [])
+        
+        # 验证订阅了关闭事件
+        self.mock_crawler.subscriber.subscribe.assert_called_once_with(
+            pipeline.spider_closed, event='spider_closed'
+        )
+
+    def test_make_insert_sql_default(self):
+        """测试默认的SQL生成方法"""
+        class TestMySQLPipeline(BaseMySQLPipeline):
+            async def _ensure_pool(self):
+                pass
+                
+        pipeline = TestMySQLPipeline(self.mock_crawler)
+        item_dict = {"name": "test", "value": 123}
+        
+        # 由于_make_insert_sql是异步方法，我们需要运行事件循环
+        async def test_async():
+            with patch('crawlo.pipelines.mysql_pipeline.SQLBuilder.make_insert') as mock_make_insert:
+                mock_make_insert.return_value = "TEST SQL"
+                result = await pipeline._make_insert_sql(item_dict)
+                mock_make_insert.assert_called_once_with(table=pipeline.table_name, data=item_dict)
+                self.assertEqual(result, "TEST SQL")
+                
+        asyncio.run(test_async())
+
+
+class TestAsyncmyMySQLPipeline(unittest.TestCase):
+    """测试AsyncmyMySQLPipeline实现"""
+
+    def setUp(self):
+        """设置测试环境"""
+        self.mock_crawler = Mock()
+        self.mock_crawler.settings = Mock()
+        self.mock_crawler.settings.get = Mock(return_value=None)
+        self.mock_crawler.settings.get_int = Mock(return_value=100)
+        self.mock_crawler.settings.get_bool = Mock(return_value=False)
+        self.mock_crawler.subscriber = Mock()
+        self.mock_crawler.subscriber.subscribe = Mock()
+        
+        # 模拟爬虫对象
+        self.mock_spider = Mock()
+        self.mock_spider.name = "test_spider"
+        self.mock_spider.custom_settings = {}
+        self.mock_spider.mysql_table = None
+        self.mock_crawler.spider = self.mock_spider
+
+    def test_init(self):
+        """测试初始化"""
+        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
+        
+        # 验证属性初始化
+        self.assertEqual(pipeline.crawler, self.mock_crawler)
+        self.assertEqual(pipeline.settings, self.mock_crawler.settings)
+        self.assertEqual(pipeline.table_name, "test_spider_items")
+
+    def test_from_crawler(self):
+        """测试from_crawler类方法"""
+        pipeline = AsyncmyMySQLPipeline.from_crawler(self.mock_crawler)
+        self.assertIsInstance(pipeline, AsyncmyMySQLPipeline)
+        self.assertEqual(pipeline.crawler, self.mock_crawler)
+
+
+class TestAiomysqlMySQLPipeline(unittest.TestCase):
+    """测试AiomysqlMySQLPipeline实现"""
+
+    def setUp(self):
+        """设置测试环境"""
+        self.mock_crawler = Mock()
+        self.mock_crawler.settings = Mock()
+        self.mock_crawler.settings.get = Mock(return_value=None)
+        self.mock_crawler.settings.get_int = Mock(return_value=100)
+        self.mock_crawler.settings.get_bool = Mock(return_value=False)
+        self.mock_crawler.subscriber = Mock()
+        self.mock_crawler.subscriber.subscribe = Mock()
+        
+        # 模拟爬虫对象
+        self.mock_spider = Mock()
+        self.mock_spider.name = "test_spider"
+        self.mock_spider.custom_settings = {}
+        self.mock_spider.mysql_table = None
+        self.mock_crawler.spider = self.mock_spider
+
+    def test_init(self):
+        """测试初始化"""
+        pipeline = AiomysqlMySQLPipeline(self.mock_crawler)
+        
+        # 验证属性初始化
+        self.assertEqual(pipeline.crawler, self.mock_crawler)
+        self.assertEqual(pipeline.settings, self.mock_crawler.settings)
+        self.assertEqual(pipeline.table_name, "test_spider_items")
+
+    def test_from_crawler(self):
+        """测试from_crawler类方法"""
+        pipeline = AiomysqlMySQLPipeline.from_crawler(self.mock_crawler)
+        self.assertIsInstance(pipeline, AiomysqlMySQLPipeline)
+        self.assertEqual(pipeline.crawler, self.mock_crawler)
+
+
+if __name__ == "__main__":
+    unittest.main()