crawlo 1.4.4__py3-none-any.whl → 1.4.6__py3-none-any.whl

tests/simple_selector_test.py

@@ -0,0 +1,208 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+简化选择器测试
+"""
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+# 直接导入需要的模块
+from parsel import Selector, SelectorList
+
+
+class MockResponse:
+    """模拟Response类用于测试"""
+    
+    def __init__(self, text):
+        self._text = text
+        self._selector_instance = None
+    
+    @property
+    def text(self):
+        return self._text
+    
+    @property
+    def _selector(self):
+        if self._selector_instance is None:
+            self._selector_instance = Selector(self.text)
+        return self._selector_instance
+
+    def xpath(self, query):
+        return self._selector.xpath(query)
+
+    def css(self, query):
+        return self._selector.css(query)
+
+    def _is_xpath(self, query):
+        return query.startswith(('/', '//', './'))
+
+    def _extract_text_from_elements(self, elements, join_str=" "):
+        texts = []
+        for element in elements:
+            if hasattr(element, 'xpath'):
+                element_texts = element.xpath('.//text()').getall()
+            else:
+                element_texts = [str(element)]
+            for text in element_texts:
+                cleaned = text.strip()
+                if cleaned:
+                    texts.append(cleaned)
+        return join_str.join(texts)
+
+    def extract_text(self, xpath_or_css, join_str=" ", default=''):
+        try:
+            elements = self.xpath(xpath_or_css) if self._is_xpath(xpath_or_css) else self.css(xpath_or_css)
+            if not elements:
+                return default
+            return self._extract_text_from_elements(elements, join_str)
+        except Exception:
+            return default
+
+    def extract_texts(self, xpath_or_css, join_str=" ", default=None):
+        if default is None:
+            default = []
+            
+        try:
+            elements = self.xpath(xpath_or_css) if self._is_xpath(xpath_or_css) else self.css(xpath_or_css)
+            if not elements:
+                return default
+                
+            result = []
+            for element in elements:
+                if hasattr(element, 'xpath'):
+                    texts = element.xpath('.//text()').getall()
+                else:
+                    texts = [str(element)]
+                    
+                clean_texts = [text.strip() for text in texts if text.strip()]
+                if clean_texts:
+                    result.append(join_str.join(clean_texts))
+                    
+            return result if result else default
+        except Exception:
+            return default
+
+    def extract_attr(self, xpath_or_css, attr_name, default=None):
+        try:
+            elements = self.xpath(xpath_or_css) if self._is_xpath(xpath_or_css) else self.css(xpath_or_css)
+            if not elements:
+                return default
+            if hasattr(elements, 'attrib'):
+                return elements.attrib.get(attr_name, default)
+            elif len(elements) > 0 and hasattr(elements[0], 'attrib'):
+                return elements[0].attrib.get(attr_name, default)
+            return default
+        except Exception:
+            return default
+
+    def extract_attrs(self, xpath_or_css, attr_name, default=None):
+        if default is None:
+            default = []
+            
+        try:
+            elements = self.xpath(xpath_or_css) if self._is_xpath(xpath_or_css) else self.css(xpath_or_css)
+            if not elements:
+                return default
+                
+            result = []
+            for element in elements:
+                if hasattr(element, 'attrib'):
+                    attr_value = element.attrib.get(attr_name)
+                    if attr_value is not None:
+                        result.append(attr_value)
+                        
+            return result if result else default
+        except Exception:
+            return default
+
+
+def test_selector_methods():
+    """测试选择器方法"""
+    print("测试选择器方法...")
+    print("=" * 50)
+    
+    # 创建测试HTML
+    html_content = """
+    <html>
+    <head>
+        <title>测试页面</title>
+    </head>
+    <body>
+        <div class="content">
+            <h1>主标题</h1>
+            <p class="intro">介绍段落</p>
+            <ul class="list">
+                <li>项目1</li>
+                <li>项目2</li>
+                <li>项目3</li>
+            </ul>
+            <a href="https://example.com" class="link">链接文本</a>
+            <img src="image.jpg" alt="图片描述" class="image">
+        </div>
+    </body>
+    </html>
+    """
+    
+    response = MockResponse(html_content)
+    
+    # 测试 extract_text
+    print("1. 测试 extract_text:")
+    title = response.extract_text('title')
+    print(f"   标题: {title}")
+    
+    h1_text = response.extract_text('.content h1')
+    print(f"   H1文本: {h1_text}")
+    
+    # 测试XPath
+    title_xpath = response.extract_text('//title')
+    print(f"   XPath标题: {title_xpath}")
+    
+    print()
+    
+    # 测试 extract_texts
+    print("2. 测试 extract_texts:")
+    list_items = response.extract_texts('.list li')
+    print(f"   列表项: {list_items}")
+    
+    # 测试XPath
+    list_items_xpath = response.extract_texts('//ul[@class="list"]/li')
+    print(f"   XPath列表项: {list_items_xpath}")
+    
+    print()
+    
+    # 测试 extract_attr
+    print("3. 测试 extract_attr:")
+    link_href = response.extract_attr('.link', 'href')
+    print(f"   链接href: {link_href}")
+    
+    img_alt = response.extract_attr('.image', 'alt')
+    print(f"   图片alt: {img_alt}")
+    
+    # 测试XPath
+    link_href_xpath = response.extract_attr('//a[@class="link"]', 'href')
+    print(f"   XPath链接href: {link_href_xpath}")
+    
+    print()
+    
+    # 测试 extract_attrs
+    print("4. 测试 extract_attrs:")
+    all_links = response.extract_attrs('a', 'href')
+    print(f"   所有链接: {all_links}")
+    
+    print()
+    
+    # 测试边界情况
+    print("5. 测试边界情况:")
+    non_exist = response.extract_text('.non-exist', default='默认文本')
+    print(f"   不存在元素的默认值: {non_exist}")
+    
+    non_exist_attr = response.extract_attr('.non-exist', 'href', default='默认链接')
+    print(f"   不存在属性的默认值: {non_exist_attr}")
+    
+    print()
+    print("所有测试完成！")
+
+
+if __name__ == '__main__':
+    test_selector_methods()

tests/simple_url_test.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response URL 处理方法简单测试
+"""
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+# 直接导入需要的模块
+from urllib.parse import urlparse, urlsplit, parse_qs, urlencode, quote, unquote, urldefrag
+
+
+def test_url_methods():
+    """测试 URL 处理方法"""
+    print("测试 Response URL 处理方法")
+    
+    # 测试数据
+    test_url = "https://example.com/test?param1=value1&param2=value2#section1"
+    print(f"测试URL: {test_url}")
+    
+    # 1. 测试 urlparse
+    print("\n1. 测试 urlparse:")
+    parsed = urlparse(test_url)
+    print(f"  scheme: {parsed.scheme}")
+    print(f"  netloc: {parsed.netloc}")
+    print(f"  path: {parsed.path}")
+    print(f"  query: {parsed.query}")
+    print(f"  fragment: {parsed.fragment}")
+    
+    # 2. 测试 urlsplit
+    print("\n2. 测试 urlsplit:")
+    split_result = urlsplit(test_url)
+    print(f"  scheme: {split_result.scheme}")
+    print(f"  netloc: {split_result.netloc}")
+    print(f"  path: {split_result.path}")
+    print(f"  query: {split_result.query}")
+    print(f"  fragment: {split_result.fragment}")
+    
+    # 3. 测试 parse_qs
+    print("\n3. 测试 parse_qs:")
+    query_dict = parse_qs(parsed.query)
+    print(f"  解析结果: {query_dict}")
+    
+    # 4. 测试 urlencode
+    print("\n4. 测试 urlencode:")
+    test_dict = {"name": "张三", "age": 25, "city": "北京"}
+    encoded = urlencode(test_dict)
+    print(f"  编码结果: {encoded}")
+    
+    # 5. 测试 quote/unquote
+    print("\n5. 测试 quote/unquote:")
+    original = "hello world 你好"
+    quoted = quote(original)
+    print(f"  原始字符串: {original}")
+    print(f"  URL编码: {quoted}")
+    
+    unquoted = unquote(quoted)
+    print(f"  URL解码: {unquoted}")
+    print(f"  编码解码是否一致: {original == unquoted}")
+    
+    # 6. 测试 urldefrag
+    print("\n6. 测试 urldefrag:")
+    url_without_frag, fragment = urldefrag(test_url)
+    print(f"  去除片段的URL: {url_without_frag}")
+    print(f"  片段: {fragment}")
+    
+    print("\n所有测试完成！")
+
+
+if __name__ == '__main__':
+    test_url_methods()

tests/simulate_mysql_update_test.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+"""
+模拟 MySQL ON DUPLICATE KEY UPDATE 行为测试
+演示不同情况下的影响行数
+"""
+import sys
+import os
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.db_helper import SQLBuilder
+
+
+def simulate_mysql_scenarios():
+    """模拟不同的 MySQL 场景"""
+    print("=== MySQL 场景模拟测试 ===\n")
+    
+    table = "news_items"
+    
+    # 场景1: 新记录插入
+    print("场景1: 插入新记录")
+    new_data = {
+        'title': '新文章标题',
+        'publish_time': '2025-10-09 10:00',
+        'url': 'https://example.com/new-article',
+        'source': '新来源',
+        'content': '新文章内容'
+    }
+    
+    sql1 = SQLBuilder.make_insert(
+        table=table,
+        data=new_data,
+        auto_update=False,
+        update_columns=('title', 'publish_time'),
+        insert_ignore=False
+    )
+    
+    print(f"SQL: {sql1[:100]}...")
+    print("预期行为: 正常插入，影响行数 = 1")
+    print()
+    
+    # 场景2: 冲突但字段值相同
+    print("场景2: 主键冲突，更新字段值相同")
+    duplicate_data = {
+        'title': '已有文章标题',  # 假设数据库中已存在相同标题的记录
+        'publish_time': '2025-10-09 09:00',  # 与数据库中记录相同的发布时间
+        'url': 'https://example.com/existing-article',
+        'source': '来源',
+        'content': '文章内容'
+    }
+    
+    sql2 = SQLBuilder.make_insert(
+        table=table,
+        data=duplicate_data,
+        auto_update=False,
+        update_columns=('title', 'publish_time'),
+        insert_ignore=False
+    )
+    
+    print(f"SQL: {sql2[:100]}...")
+    print("预期行为: 触发 ON DUPLICATE KEY UPDATE，但字段值未变化，影响行数 = 0")
+    print()
+    
+    # 场景3: 冲突且字段值不同
+    print("场景3: 主键冲突，更新字段值不同")
+    updated_data = {
+        'title': '已有文章标题',  # 与数据库中记录相同
+        'publish_time': '2025-10-09 11:00',  # 与数据库中记录不同的发布时间
+        'url': 'https://example.com/existing-article',
+        'source': '来源',
+        'content': '文章内容'
+    }
+    
+    sql3 = SQLBuilder.make_insert(
+        table=table,
+        data=updated_data,
+        auto_update=False,
+        update_columns=('title', 'publish_time'),
+        insert_ignore=False
+    )
+    
+    print(f"SQL: {sql3[:100]}...")
+    print("预期行为: 触发 ON DUPLICATE KEY UPDATE，字段值变化，影响行数 = 2")
+    print("(MySQL 5.7+ 版本中，更新操作返回的影响行数为 2)")
+    print()
+    
+    # 场景4: 使用 INSERT IGNORE
+    print("场景4: 使用 INSERT IGNORE")
+    ignore_data = {
+        'title': '忽略重复标题',  # 假设数据库中已存在相同标题的记录
+        'publish_time': '2025-10-09 12:00',
+        'url': 'https://example.com/ignore-article',
+        'source': '忽略来源',
+        'content': '忽略内容'
+    }
+    
+    sql4 = SQLBuilder.make_insert(
+        table=table,
+        data=ignore_data,
+        auto_update=False,
+        update_columns=(),
+        insert_ignore=True
+    )
+    
+    print(f"SQL: {sql4[:100]}...")
+    print("预期行为: 遇到重复记录时忽略插入，影响行数 = 0")
+    print()
+    
+    # 场景5: 使用 REPLACE INTO
+    print("场景5: 使用 REPLACE INTO")
+    replace_data = {
+        'title': '替换文章标题',  # 假设数据库中已存在相同标题的记录
+        'publish_time': '2025-10-09 13:00',
+        'url': 'https://example.com/replace-article',
+        'source': '替换来源',
+        'content': '替换内容'
+    }
+    
+    sql5 = SQLBuilder.make_insert(
+        table=table,
+        data=replace_data,
+        auto_update=True,  # 使用 REPLACE INTO
+        update_columns=(),
+        insert_ignore=False
+    )
+    
+    print(f"SQL: {sql5[:100]}...")
+    print("预期行为: 删除旧记录并插入新记录，影响行数 = 2")
+    print()
+    
+    print("=== 总结 ===")
+    print("1. 当使用 MYSQL_UPDATE_COLUMNS 时，影响行数为 0 并不表示错误")
+    print("2. 这可能意味着更新的字段值与现有记录相同")
+    print("3. 如果需要确保更新，可以在 update_columns 中包含更多字段")
+    print("4. 如果需要完全替换记录，使用 MYSQL_AUTO_UPDATE = True")
+
+
+if __name__ == "__main__":
+    simulate_mysql_scenarios()

tests/test_asyncmy_usage.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+import asyncio
+from asyncmy import create_pool
+
+async def test_asyncmy_usage():
+    """测试asyncmy库的正确使用方式"""
+    try:
+        # 创建连接池
+        pool = await create_pool(
+            host='127.0.0.1',
+            port=3306,
+            user='root',
+            password='123456',
+            db='test',
+            minsize=1,
+            maxsize=5
+        )
+        
+        # 获取连接
+        conn = await pool.acquire()
+        try:
+            # 获取游标
+            cursor = await conn.cursor()
+            try:
+                # 执行SQL
+                result = cursor.execute("SELECT 1")
+                print(f"execute返回类型: {type(result)}")
+                print(f"execute返回值: {result}")
+                
+                # 检查是否需要await
+                if hasattr(result, '__await__'):
+                    print("execute返回的是协程对象，需要await")
+                    result = await result
+                else:
+                    print("execute返回的不是协程对象，不需要await")
+                    
+                # 提交事务
+                await conn.commit()
+                
+            finally:
+                await cursor.close()
+        finally:
+            pool.release(conn)
+            
+        # 关闭连接池
+        pool.close()
+        await pool.wait_closed()
+        
+        print("测试完成")
+        
+    except Exception as e:
+        print(f"测试出错: {e}")
+        import traceback
+        traceback.print_exc()
+
+if __name__ == "__main__":
+    asyncio.run(test_asyncmy_usage())

tests/test_crawler_process_import.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+测试CrawlerProcess导入功能
+"""
+
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+
+def test_crawler_process_import():
+    """测试CrawlerProcess导入功能"""
+    print("测试CrawlerProcess导入功能...")
+    
+    try:
+        # 测试直接从crawlo导入CrawlerProcess
+        from crawlo import CrawlerProcess
+        print(f"  成功从crawlo导入CrawlerProcess: {CrawlerProcess}")
+        
+        # 测试创建实例
+        process = CrawlerProcess()
+        print(f"  成功创建CrawlerProcess实例: {process}")
+        
+        print("CrawlerProcess导入测试通过!")
+        
+    except ImportError as e:
+        print(f"  导入失败: {e}")
+        # 如果直接导入失败，尝试从crawler模块导入
+        try:
+            from crawlo.crawler import CrawlerProcess
+            print(f"  成功从crawlo.crawler导入CrawlerProcess: {CrawlerProcess}")
+        except ImportError as e2:
+            print(f"  从crawler模块导入也失败: {e2}")
+
+
+if __name__ == '__main__':
+    test_crawler_process_import()

tests/test_crawler_process_spider_modules.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+测试CrawlerProcess与SPIDER_MODULES的集成
+"""
+
+import sys
+import os
+import asyncio
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.settings.setting_manager import SettingManager
+
+
+def test_crawler_process_spider_modules():
+    """测试CrawlerProcess与SPIDER_MODULES的集成"""
+    print("测试CrawlerProcess与SPIDER_MODULES的集成...")
+    
+    # 创建一个包含SPIDER_MODULES的设置
+    settings = SettingManager({
+        'SPIDER_MODULES': ['tests.test_spiders'],
+        'SPIDER_LOADER_WARN_ONLY': True,
+        'CONCURRENCY': 1,
+        'LOG_LEVEL': 'INFO'
+    })
+    
+    # 创建CrawlerProcess实例
+    process = CrawlerProcess(settings=settings)
+    
+    # 测试获取爬虫名称
+    spider_names = process.get_spider_names()
+    print(f"发现的爬虫: {spider_names}")
+    
+    # 测试检查爬虫是否已注册
+    is_registered = process.is_spider_registered('test_spider')
+    print(f"爬虫'test_spider'是否已注册: {is_registered}")
+    
+    # 测试获取爬虫类
+    spider_class = process.get_spider_class('test_spider')
+    print(f"爬虫'test_spider'的类: {spider_class}")
+    
+    print("测试完成!")
+
+
+if __name__ == '__main__':
+    test_crawler_process_spider_modules()

tests/test_crawlo_proxy_integration.py

@@ -61,8 +61,14 @@ async def test_proxy_integration():
     config = CrawloConfig.standalone(
         concurrency=1,
         download_delay=0.1,
-        PROXY_ENABLED=True,
-        PROXY_API_URL="https://api.proxyprovider.com/get",  # 模拟代理API
+        # 代理配置
+        # 高级代理配置（适用于ProxyMiddleware）
+        # 只要配置了代理API URL，中间件就会自动启用
+        PROXY_API_URL="https://proxy-api.example.com/get",  # 模拟代理API
+        
+        # 代理配置（适用于ProxyMiddleware）
+        # 只要配置了代理列表，中间件就会自动启用
+        # PROXY_LIST=["http://proxy1:8080", "http://proxy2:8080"],
         LOG_LEVEL='WARNING'  # 减少日志输出
     )