crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl

tests/test_mysql_pipeline_error.py

@@ -1,99 +1,99 @@
-# -*- coding: utf-8 -*-
-import asyncio
-import unittest
-from unittest.mock import Mock, patch, AsyncMock
-
-from crawlo.pipelines.mysql_pipeline import AsyncmyMySQLPipeline, AiomysqlMySQLPipeline
-from crawlo.exceptions import ItemDiscard
-
-
-class TestMySQLPipelineError(unittest.TestCase):
-    """测试MySQL管道错误处理"""
-
-    def setUp(self):
-        """设置测试环境"""
-        self.mock_crawler = Mock()
-        self.mock_crawler.settings = Mock()
-        self.mock_crawler.settings.get = Mock(return_value=None)
-        self.mock_crawler.settings.get_int = Mock(return_value=100)
-        self.mock_crawler.settings.get_bool = Mock(return_value=False)
-        self.mock_crawler.subscriber = Mock()
-        self.mock_crawler.subscriber.subscribe = Mock()
-        self.mock_crawler.stats = Mock()
-        self.mock_crawler.stats.inc_value = Mock()
-        
-        # 模拟爬虫对象
-        self.mock_spider = Mock()
-        self.mock_spider.name = "test_spider"
-        self.mock_spider.custom_settings = {}
-        self.mock_spider.mysql_table = None
-        self.mock_crawler.spider = self.mock_spider
-
-    def test_asyncmy_process_item_with_connection_error(self):
-        """测试AsyncmyMySQLPipeline处理连接错误"""
-        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
-        
-        # 模拟连接池和数据库操作
-        mock_pool = AsyncMock()
-        mock_conn = AsyncMock()
-        mock_cursor = AsyncMock()
-        
-        # 模拟acquire方法返回连接
-        mock_pool.acquire.return_value = mock_conn
-        
-        # 模拟cursor方法返回游标
-        mock_conn.cursor.return_value = mock_cursor
-        
-        # 模拟execute方法抛出异常
-        mock_cursor.execute.side_effect = Exception("测试异常")
-        
-        # 设置管道的连接池
-        pipeline.pool = mock_pool
-        pipeline._pool_initialized = True
-        
-        # 测试数据
-        test_item = {"id": 1, "name": "test"}
-        
-        async def test_async():
-            with self.assertRaises(ItemDiscard) as context:
-                await pipeline.process_item(test_item, self.mock_spider)
-            
-            # 验证错误信息
-            self.assertIn("MySQL插入失败", str(context.exception))
-            
-        asyncio.run(test_async())
-
-    def test_execute_sql_with_exception(self):
-        """测试_execute_sql方法处理异常"""
-        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
-        
-        # 模拟连接池和数据库操作
-        mock_pool = AsyncMock()
-        mock_conn = AsyncMock()
-        mock_cursor = AsyncMock()
-        
-        # 模拟acquire方法返回连接
-        mock_pool.acquire.return_value = mock_conn
-        
-        # 模拟cursor方法返回游标
-        mock_conn.cursor.return_value = mock_cursor
-        
-        # 模拟execute方法抛出异常
-        mock_cursor.execute.side_effect = Exception("测试异常")
-        
-        # 设置管道的连接池
-        pipeline.pool = mock_pool
-        pipeline._pool_initialized = True
-        
-        async def test_async():
-            with self.assertRaises(ItemDiscard) as context:
-                await pipeline._execute_sql("SELECT 1")
-            
-            # 验证错误信息
-            self.assertIn("MySQL插入失败", str(context.exception))
-            
-        asyncio.run(test_async())
-
-
-if __name__ == "__main__":
+# -*- coding: utf-8 -*-
+import asyncio
+import unittest
+from unittest.mock import Mock, patch, AsyncMock
+
+from crawlo.pipelines.mysql_pipeline import AsyncmyMySQLPipeline, AiomysqlMySQLPipeline
+from crawlo.exceptions import ItemDiscard
+
+
+class TestMySQLPipelineError(unittest.TestCase):
+    """测试MySQL管道错误处理"""
+
+    def setUp(self):
+        """设置测试环境"""
+        self.mock_crawler = Mock()
+        self.mock_crawler.settings = Mock()
+        self.mock_crawler.settings.get = Mock(return_value=None)
+        self.mock_crawler.settings.get_int = Mock(return_value=100)
+        self.mock_crawler.settings.get_bool = Mock(return_value=False)
+        self.mock_crawler.subscriber = Mock()
+        self.mock_crawler.subscriber.subscribe = Mock()
+        self.mock_crawler.stats = Mock()
+        self.mock_crawler.stats.inc_value = Mock()
+        
+        # 模拟爬虫对象
+        self.mock_spider = Mock()
+        self.mock_spider.name = "test_spider"
+        self.mock_spider.custom_settings = {}
+        self.mock_spider.mysql_table = None
+        self.mock_crawler.spider = self.mock_spider
+
+    def test_asyncmy_process_item_with_connection_error(self):
+        """测试AsyncmyMySQLPipeline处理连接错误"""
+        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
+        
+        # 模拟连接池和数据库操作
+        mock_pool = AsyncMock()
+        mock_conn = AsyncMock()
+        mock_cursor = AsyncMock()
+        
+        # 模拟acquire方法返回连接
+        mock_pool.acquire.return_value = mock_conn
+        
+        # 模拟cursor方法返回游标
+        mock_conn.cursor.return_value = mock_cursor
+        
+        # 模拟execute方法抛出异常
+        mock_cursor.execute.side_effect = Exception("测试异常")
+        
+        # 设置管道的连接池
+        pipeline.pool = mock_pool
+        pipeline._pool_initialized = True
+        
+        # 测试数据
+        test_item = {"id": 1, "name": "test"}
+        
+        async def test_async():
+            with self.assertRaises(ItemDiscard) as context:
+                await pipeline.process_item(test_item, self.mock_spider)
+            
+            # 验证错误信息
+            self.assertIn("MySQL插入失败", str(context.exception))
+            
+        asyncio.run(test_async())
+
+    def test_execute_sql_with_exception(self):
+        """测试_execute_sql方法处理异常"""
+        pipeline = AsyncmyMySQLPipeline(self.mock_crawler)
+        
+        # 模拟连接池和数据库操作
+        mock_pool = AsyncMock()
+        mock_conn = AsyncMock()
+        mock_cursor = AsyncMock()
+        
+        # 模拟acquire方法返回连接
+        mock_pool.acquire.return_value = mock_conn
+        
+        # 模拟cursor方法返回游标
+        mock_conn.cursor.return_value = mock_cursor
+        
+        # 模拟execute方法抛出异常
+        mock_cursor.execute.side_effect = Exception("测试异常")
+        
+        # 设置管道的连接池
+        pipeline.pool = mock_pool
+        pipeline._pool_initialized = True
+        
+        async def test_async():
+            with self.assertRaises(ItemDiscard) as context:
+                await pipeline._execute_sql("SELECT 1")
+            
+            # 验证错误信息
+            self.assertIn("MySQL插入失败", str(context.exception))
+            
+        asyncio.run(test_async())
+
+
+if __name__ == "__main__":
     unittest.main()

tests/test_mysql_pipeline_init_log.py

@@ -1,83 +1,83 @@
-# -*- coding: utf-8 -*-
-"""
-测试 MySQL 管道初始化日志
-验证在管道初始化时是否正确打印日志
-"""
-import sys
-import os
-
-# 添加项目根目录到 Python 路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.pipelines.mysql_pipeline import AsyncmyMySQLPipeline, AiomysqlMySQLPipeline
-
-
-# 创建一个简单的爬虫模拟类
-class MockSpider:
-    name = "test_spider"
-
-
-# 创建一个简单的设置模拟类
-class MockSettings:
-    def get(self, key, default=None):
-        # 简化的设置获取
-        settings_map = {
-            'MYSQL_HOST': 'localhost',
-            'MYSQL_PORT': 3306,
-            'MYSQL_USER': 'root',
-            'MYSQL_PASSWORD': '',
-            'MYSQL_DB': 'test_db',
-            'MYSQL_TABLE': 'test_table',
-            'LOG_LEVEL': 'INFO'
-        }
-        return settings_map.get(key, default)
-    
-    def get_int(self, key, default=0):
-        return int(self.get(key, default))
-    
-    def get_bool(self, key, default=False):
-        return bool(self.get(key, default))
-
-
-# 创建一个简单的订阅者模拟类
-class MockSubscriber:
-    def subscribe(self, func, event):
-        # 简化的订阅
-        pass
-
-
-# 创建一个简单的爬虫模拟类
-class MockCrawler:
-    def __init__(self):
-        self.settings = MockSettings()
-        self.subscriber = MockSubscriber()
-        self.spider = MockSpider()
-
-
-def test_pipeline_init_logs():
-    """测试管道初始化日志"""
-    print("=== Testing MySQL Pipeline Initialization Logs ===")
-    
-    # 创建模拟的爬虫
-    crawler = MockCrawler()
-    
-    print("1. Testing AsyncmyMySQLPipeline initialization...")
-    try:
-        asyncmy_pipeline = AsyncmyMySQLPipeline.from_crawler(crawler)
-        print("   ✓ AsyncmyMySQLPipeline initialized successfully")
-    except Exception as e:
-        print(f"   ✗ AsyncmyMySQLPipeline initialization failed: {e}")
-    
-    print("\n2. Testing AiomysqlMySQLPipeline initialization...")
-    try:
-        aiomysql_pipeline = AiomysqlMySQLPipeline.from_crawler(crawler)
-        print("   ✓ AiomysqlMySQLPipeline initialized successfully")
-    except Exception as e:
-        print(f"   ✗ AiomysqlMySQLPipeline initialization failed: {e}")
-    
-    print("\n=== Test completed ===")
-    print("Note: Actual log output can be seen when running in a full crawler environment")
-
-
-if __name__ == "__main__":
+# -*- coding: utf-8 -*-
+"""
+测试 MySQL 管道初始化日志
+验证在管道初始化时是否正确打印日志
+"""
+import sys
+import os
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.pipelines.mysql_pipeline import AsyncmyMySQLPipeline, AiomysqlMySQLPipeline
+
+
+# 创建一个简单的爬虫模拟类
+class MockSpider:
+    name = "test_spider"
+
+
+# 创建一个简单的设置模拟类
+class MockSettings:
+    def get(self, key, default=None):
+        # 简化的设置获取
+        settings_map = {
+            'MYSQL_HOST': 'localhost',
+            'MYSQL_PORT': 3306,
+            'MYSQL_USER': 'root',
+            'MYSQL_PASSWORD': '',
+            'MYSQL_DB': 'test_db',
+            'MYSQL_TABLE': 'test_table',
+            'LOG_LEVEL': 'INFO'
+        }
+        return settings_map.get(key, default)
+    
+    def get_int(self, key, default=0):
+        return int(self.get(key, default))
+    
+    def get_bool(self, key, default=False):
+        return bool(self.get(key, default))
+
+
+# 创建一个简单的订阅者模拟类
+class MockSubscriber:
+    def subscribe(self, func, event):
+        # 简化的订阅
+        pass
+
+
+# 创建一个简单的爬虫模拟类
+class MockCrawler:
+    def __init__(self):
+        self.settings = MockSettings()
+        self.subscriber = MockSubscriber()
+        self.spider = MockSpider()
+
+
+def test_pipeline_init_logs():
+    """测试管道初始化日志"""
+    print("=== Testing MySQL Pipeline Initialization Logs ===")
+    
+    # 创建模拟的爬虫
+    crawler = MockCrawler()
+    
+    print("1. Testing AsyncmyMySQLPipeline initialization...")
+    try:
+        asyncmy_pipeline = AsyncmyMySQLPipeline.from_crawler(crawler)
+        print("   ✓ AsyncmyMySQLPipeline initialized successfully")
+    except Exception as e:
+        print(f"   ✗ AsyncmyMySQLPipeline initialization failed: {e}")
+    
+    print("\n2. Testing AiomysqlMySQLPipeline initialization...")
+    try:
+        aiomysql_pipeline = AiomysqlMySQLPipeline.from_crawler(crawler)
+        print("   ✓ AiomysqlMySQLPipeline initialized successfully")
+    except Exception as e:
+        print(f"   ✗ AiomysqlMySQLPipeline initialization failed: {e}")
+    
+    print("\n=== Test completed ===")
+    print("Note: Actual log output can be seen when running in a full crawler environment")
+
+
+if __name__ == "__main__":
     test_pipeline_init_logs()

tests/test_mysql_pipeline_integration.py

@@ -1,133 +1,133 @@
-# -*- coding: utf-8 -*-
-"""
-MySQL 管道集成测试
-验证 MYSQL_UPDATE_COLUMNS 配置在实际使用中是否解决了 MySQL 警告问题
-"""
-import asyncio
-import sys
-import os
-
-# 添加项目根目录到 Python 路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.utils.db_helper import SQLBuilder
-
-
-def test_complete_workflow():
-    """测试完整的数据插入工作流程"""
-    print("=== MySQL 管道集成测试 ===\n")
-    
-    # 模拟实际使用的配置
-    table = "news_items"
-    item_data = {
-        'title': '新一代OLED屏下光谱颜色传感技术：解锁显示新密码，重塑视觉新体验',
-        'publish_time': '2025-10-09 09:57',
-        'url': 'https://ee.ofweek.com/2025-10/ART-8460-2806-30671544.html',
-        'source': '',
-        'content': '在全球智能手机市场竞争日趋白热化的当下，消费者对手机屏幕显示效果的要求愈发严苛...'
-    }
-    
-    print("1. 测试单条插入 SQL 生成...")
-    # 测试单条插入
-    single_sql = SQLBuilder.make_insert(
-        table=table,
-        data=item_data,
-        auto_update=False,  # 不使用 REPLACE INTO
-        insert_ignore=False,  # 不使用 INSERT IGNORE
-        update_columns=('title', 'publish_time')  # 冲突时更新指定列
-    )
-    
-    print("生成的单条插入 SQL:")
-    print(single_sql)
-    print()
-    
-    # 验证语法正确性
-    if "AS `excluded`" in single_sql and "`title`=`excluded`.`title`" in single_sql:
-        print("✓ 单条插入正确使用了新的 MySQL 语法")
-    else:
-        print("✗ 单条插入语法不正确")
-    
-    print("\n2. 测试批量插入 SQL 生成...")
-    # 测试批量插入
-    batch_data = [item_data, item_data]  # 模拟重复数据
-    batch_result = SQLBuilder.make_batch(
-        table=table,
-        datas=batch_data,
-        auto_update=False,
-        update_columns=('title', 'publish_time')
-    )
-    
-    if batch_result:
-        batch_sql, values_list = batch_result
-        print("生成的批量插入 SQL:")
-        print(batch_sql)
-        print(f"参数值列表数量: {len(values_list)}")
-        print()
-        
-        # 验证语法正确性
-        if "AS `excluded`" in batch_sql and "`title`=`excluded`.`title`" in batch_sql:
-            print("✓ 批量插入正确使用了新的 MySQL 语法")
-        else:
-            print("✗ 批量插入语法不正确")
-    
-    print("\n3. 测试不同配置组合...")
-    
-    # 测试仅使用 INSERT IGNORE
-    ignore_sql = SQLBuilder.make_insert(
-        table=table,
-        data=item_data,
-        auto_update=False,
-        insert_ignore=True,
-        update_columns=()  # 不指定更新列
-    )
-    
-    print("INSERT IGNORE SQL:")
-    print(ignore_sql)
-    print()
-    
-    if "INSERT IGNORE" in ignore_sql and "AS `excluded`" not in ignore_sql:
-        print("✓ INSERT IGNORE 模式正确")
-    else:
-        print("✗ INSERT IGNORE 模式不正确")
-    
-    # 测试使用 REPLACE INTO
-    replace_sql = SQLBuilder.make_insert(
-        table=table,
-        data=item_data,
-        auto_update=True,  # 使用 REPLACE INTO
-        insert_ignore=False,
-        update_columns=()  # 不指定更新列
-    )
-    
-    print("REPLACE INTO SQL:")
-    print(replace_sql)
-    print()
-    
-    if "REPLACE INTO" in replace_sql and "AS `excluded`" not in replace_sql:
-        print("✓ REPLACE INTO 模式正确")
-    else:
-        print("✗ REPLACE INTO 模式不正确")
-    
-    # 测试普通 INSERT
-    normal_sql = SQLBuilder.make_insert(
-        table=table,
-        data=item_data,
-        auto_update=False,
-        insert_ignore=False,
-        update_columns=()  # 不指定更新列
-    )
-    
-    print("普通 INSERT SQL:")
-    print(normal_sql)
-    print()
-    
-    if "INSERT INTO" in normal_sql and "AS `excluded`" not in normal_sql:
-        print("✓ 普通 INSERT 模式正确")
-    else:
-        print("✗ 普通 INSERT 模式不正确")
-    
-    print("\n=== 测试完成 ===")
-
-
-if __name__ == "__main__":
+# -*- coding: utf-8 -*-
+"""
+MySQL 管道集成测试
+验证 MYSQL_UPDATE_COLUMNS 配置在实际使用中是否解决了 MySQL 警告问题
+"""
+import asyncio
+import sys
+import os
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.db_helper import SQLBuilder
+
+
+def test_complete_workflow():
+    """测试完整的数据插入工作流程"""
+    print("=== MySQL 管道集成测试 ===\n")
+    
+    # 模拟实际使用的配置
+    table = "news_items"
+    item_data = {
+        'title': '新一代OLED屏下光谱颜色传感技术：解锁显示新密码，重塑视觉新体验',
+        'publish_time': '2025-10-09 09:57',
+        'url': 'https://ee.ofweek.com/2025-10/ART-8460-2806-30671544.html',
+        'source': '',
+        'content': '在全球智能手机市场竞争日趋白热化的当下，消费者对手机屏幕显示效果的要求愈发严苛...'
+    }
+    
+    print("1. 测试单条插入 SQL 生成...")
+    # 测试单条插入
+    single_sql = SQLBuilder.make_insert(
+        table=table,
+        data=item_data,
+        auto_update=False,  # 不使用 REPLACE INTO
+        insert_ignore=False,  # 不使用 INSERT IGNORE
+        update_columns=('title', 'publish_time')  # 冲突时更新指定列
+    )
+    
+    print("生成的单条插入 SQL:")
+    print(single_sql)
+    print()
+    
+    # 验证语法正确性
+    if "AS `excluded`" in single_sql and "`title`=`excluded`.`title`" in single_sql:
+        print("✓ 单条插入正确使用了新的 MySQL 语法")
+    else:
+        print("✗ 单条插入语法不正确")
+    
+    print("\n2. 测试批量插入 SQL 生成...")
+    # 测试批量插入
+    batch_data = [item_data, item_data]  # 模拟重复数据
+    batch_result = SQLBuilder.make_batch(
+        table=table,
+        datas=batch_data,
+        auto_update=False,
+        update_columns=('title', 'publish_time')
+    )
+    
+    if batch_result:
+        batch_sql, values_list = batch_result
+        print("生成的批量插入 SQL:")
+        print(batch_sql)
+        print(f"参数值列表数量: {len(values_list)}")
+        print()
+        
+        # 验证语法正确性
+        if "AS `excluded`" in batch_sql and "`title`=`excluded`.`title`" in batch_sql:
+            print("✓ 批量插入正确使用了新的 MySQL 语法")
+        else:
+            print("✗ 批量插入语法不正确")
+    
+    print("\n3. 测试不同配置组合...")
+    
+    # 测试仅使用 INSERT IGNORE
+    ignore_sql = SQLBuilder.make_insert(
+        table=table,
+        data=item_data,
+        auto_update=False,
+        insert_ignore=True,
+        update_columns=()  # 不指定更新列
+    )
+    
+    print("INSERT IGNORE SQL:")
+    print(ignore_sql)
+    print()
+    
+    if "INSERT IGNORE" in ignore_sql and "AS `excluded`" not in ignore_sql:
+        print("✓ INSERT IGNORE 模式正确")
+    else:
+        print("✗ INSERT IGNORE 模式不正确")
+    
+    # 测试使用 REPLACE INTO
+    replace_sql = SQLBuilder.make_insert(
+        table=table,
+        data=item_data,
+        auto_update=True,  # 使用 REPLACE INTO
+        insert_ignore=False,
+        update_columns=()  # 不指定更新列
+    )
+    
+    print("REPLACE INTO SQL:")
+    print(replace_sql)
+    print()
+    
+    if "REPLACE INTO" in replace_sql and "AS `excluded`" not in replace_sql:
+        print("✓ REPLACE INTO 模式正确")
+    else:
+        print("✗ REPLACE INTO 模式不正确")
+    
+    # 测试普通 INSERT
+    normal_sql = SQLBuilder.make_insert(
+        table=table,
+        data=item_data,
+        auto_update=False,
+        insert_ignore=False,
+        update_columns=()  # 不指定更新列
+    )
+    
+    print("普通 INSERT SQL:")
+    print(normal_sql)
+    print()
+    
+    if "INSERT INTO" in normal_sql and "AS `excluded`" not in normal_sql:
+        print("✓ 普通 INSERT 模式正确")
+    else:
+        print("✗ 普通 INSERT 模式不正确")
+    
+    print("\n=== 测试完成 ===")
+
+
+if __name__ == "__main__":
     test_complete_workflow()