crawlo 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/simple_crawlo_test.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简单的Crawlo性能测试
+"""
+import asyncio
+import time
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo import Spider, Request
+from crawlo.crawler import CrawlerProcess
+from crawlo.items import Item, Field
+
+class NewsItem(Item):
+    title = Field()
+    publish_time = Field()
+    url = Field()
+    source = Field()
+    content = Field()
+
+class OfweekSimpleSpider(Spider):
+    name = "ofweek_simple"
+    
+    def start_requests(self):
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "https://ee.ofweek.com/CATList-2800-8100-ee-2.html",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+        }
+        cookies = {
+            "__utmz": "57425525.1730117117.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
+            "Hm_lvt_abe9900db162c6d089cdbfd107db0f03": "1739244841",
+            "Hm_lvt_af50e2fc51af73da7720fb324b88a975": "1740100727",
+            "JSESSIONID": "FEA96D3B5FC31350B2285E711BF2A541",
+            "Hm_lvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477622",
+            "HMACCOUNT": "08DF0D235A291EAA",
+            "__utma": "57425525.2080994505.1730117117.1747970718.1757477622.50",
+            "__utmc": "57425525",
+            "__utmt": "1",
+            "__utmb": "57425525.2.10.1757477622",
+            "Hm_lpvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477628",
+            "index_burying_point": "c64d6c31e69d560efe319cc9f8be279f"
+        }
+
+        # 使用较少的页数进行测试
+        max_page = 5
+        for page in range(1, max_page + 1):
+            url = f"https://ee.ofweek.com/CATList-2800-8100-ee-{page}.html"
+            yield Request(url=url, callback=self.parse, headers=headers, cookies=cookies)
+
+    def parse(self, response):
+        from urllib.parse import urljoin
+        rows = response.xpath("//div[@class=\"main_left\"]/div[@class=\"list_model\"]/div[@class=\"model_right model_right2\"]")
+        
+        for row in rows:
+            try:
+                url = row.xpath("./h3/a/@href").extract_first()
+                title = row.xpath("./h3/a/text()").extract_first()
+
+                if not url or not title:
+                    continue
+
+                absolute_url = urljoin(response.url, url)
+                if not absolute_url.startswith(("http://", "https://")):
+                    continue
+
+                yield Request(
+                    url=absolute_url,
+                    meta={"title": title.strip() if title else "", "parent_url": response.url},
+                    callback=self.parse_detail
+                )
+            except Exception:
+                continue
+
+    def parse_detail(self, response):
+        title = response.meta.get("title", "")
+        content_elements = response.xpath("//div[@class=\"TRS_Editor\"]|//*[@id=\"articleC\"]")
+        if content_elements:
+            content = content_elements.xpath(".//text()").extract()
+            content = "\n".join([text.strip() for text in content if text.strip()])
+        else:
+            content = ""
+
+        publish_time = response.xpath("//div[@class=\"time fl\"]/text()").extract_first()
+        if publish_time:
+            publish_time = publish_time.strip()
+
+        source = response.xpath("//div[@class=\"source-name\"]/text()").extract_first()
+
+        item = NewsItem()
+        item["title"] = title.strip() if title else ""
+        item["publish_time"] = publish_time if publish_time else ""
+        item["url"] = response.url
+        item["source"] = source if source else ""
+        item["content"] = content
+
+        yield item
+
+async def main():
+    start_time = time.time()
+    
+    process = CrawlerProcess(settings={
+        "CONCURRENCY": 8,
+        "DOWNLOAD_DELAY": 0.1,
+        "LOG_LEVEL": "ERROR",  # 减少日志输出以提高性能
+    })
+    await process.crawl(OfweekSimpleSpider)
+    
+    end_time = time.time()
+    execution_time = end_time - start_time
+    
+    print(f"Crawlo执行时间: {execution_time:.2f}秒")
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/simple_log_test.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+简单的日志系统测试
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+# 确保日志目录存在
+os.makedirs('/examples/ofweek_standalone/logs', exist_ok=True)
+
+# 测试日志系统
+from crawlo.utils.log import LoggerManager, get_logger
+
+print("=== 简单日志系统测试 ===")
+
+# 1. 直接配置日志系统
+print("1. 配置日志系统...")
+LoggerManager.configure(
+    LOG_LEVEL='INFO',
+    LOG_FILE='/Users/oscar/projects/Crawlo/examples/ofweek_standalone/logs/simple_test.log'
+)
+
+# 2. 创建logger
+print("2. 创建logger...")
+logger = get_logger('test.logger')
+print(f"   Logger: {logger}")
+print(f"   Handlers: {len(logger.handlers)}")
+
+for i, handler in enumerate(logger.handlers):
+    handler_type = type(handler).__name__
+    print(f"     Handler {i}: {handler_type}")
+    if hasattr(handler, 'baseFilename'):
+        print(f"       File: {handler.baseFilename}")
+
+# 3. 测试日志输出
+print("3. 测试日志输出...")
+logger.info("这是一条测试INFO日志")
+logger.debug("这是一条测试DEBUG日志")
+logger.warning("这是一条测试WARNING日志")
+
+print("4. 检查日志文件...")
+log_file = '/Users/oscar/projects/Crawlo/examples/ofweek_standalone/logs/simple_test.log'
+if os.path.exists(log_file):
+    print(f"   日志文件存在: {log_file}")
+    with open(log_file, 'r', encoding='utf-8') as f:
+        content = f.read()
+        print(f"   文件内容长度: {len(content)} 字符")
+        if content:
+            print("   文件内容:")
+            print(content)
+        else:
+            print("   文件为空")
+else:
+    print(f"   日志文件不存在: {log_file}")
+
+print("=== 测试完成 ===")

tests/simple_optimization_test.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简单的Crawlo框架优化验证测试
+用于快速验证优化效果
+"""
+
+import asyncio
+import time
+import sys
+import os
+from pathlib import Path
+
+# 添加项目根目录到 Python 路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import Spider
+from crawlo import Request, Item
+
+
+class SimpleTestSpider(Spider):
+    """简单的测试爬虫"""
+    name = 'simple_test'
+    
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # 使用较少的测试页面以加快测试速度
+        self.start_urls = ['https://httpbin.org/delay/0' for _ in range(10)]
+    
+    def parse(self, response):
+        """简单解析响应"""
+        yield {'url': response.url, 'status': response.status_code}
+
+
+async def test_concurrent_performance():
+    """测试并发性能"""
+    print("开始并发性能测试...")
+    
+    # 配置设置
+    settings = {
+        'CONCURRENT_REQUESTS': 10,
+        'DOWNLOAD_DELAY': 0.1,
+        'RANDOMIZE_DOWNLOAD_DELAY': False,
+        'SCHEDULER_MAX_QUEUE_SIZE': 5000,
+        'BACKPRESSURE_RATIO': 0.9,
+    }
+    
+    # 创建爬虫进程
+    process = CrawlerProcess(settings=settings)
+    
+    # 记录开始时间
+    start_time = time.time()
+    
+    # 添加测试爬虫
+    crawler = await process.crawl(SimpleTestSpider)
+    
+    # 计算性能指标
+    metrics = crawler.metrics
+    duration = metrics.get_total_duration()
+    pages = 10
+    rps = pages / duration if duration > 0 else 0
+    
+    print(f"完成时间: {duration:.2f} 秒")
+    print(f"每秒请求数: {rps:.2f} RPS")
+    
+    return duration, rps
+
+
+async def test_sequential_performance():
+    """测试顺序执行性能"""
+    print("\n开始顺序执行性能测试...")
+    
+    # 配置设置
+    settings = {
+        'CONCURRENT_REQUESTS': 1,  # 顺序执行
+        'DOWNLOAD_DELAY': 0.1,
+        'RANDOMIZE_DOWNLOAD_DELAY': False,
+        'SCHEDULER_MAX_QUEUE_SIZE': 5000,
+        'BACKPRESSURE_RATIO': 0.9,
+    }
+    
+    # 创建爬虫进程
+    process = CrawlerProcess(settings=settings)
+    
+    # 记录开始时间
+    start_time = time.time()
+    
+    # 添加测试爬虫
+    crawler = await process.crawl(SimpleTestSpider)
+    
+    # 计算性能指标
+    metrics = crawler.metrics
+    duration = metrics.get_total_duration()
+    pages = 10
+    rps = pages / duration if duration > 0 else 0
+    
+    print(f"完成时间: {duration:.2f} 秒")
+    print(f"每秒请求数: {rps:.2f} RPS")
+    
+    return duration, rps
+
+
+async def main():
+    """主函数"""
+    print("Crawlo 框架优化效果验证测试")
+    print("=" * 40)
+    
+    # 测试并发性能
+    concurrent_duration, concurrent_rps = await test_concurrent_performance()
+    
+    # 测试顺序执行性能
+    sequential_duration, sequential_rps = await test_sequential_performance()
+    
+    # 输出比较结果
+    print("\n=== 性能对比 ===")
+    print(f"并发执行: {concurrent_duration:.2f}s, {concurrent_rps:.2f} RPS")
+    print(f"顺序执行: {sequential_duration:.2f}s, {sequential_rps:.2f} RPS")
+    
+    if concurrent_duration < sequential_duration:
+        improvement = (sequential_duration - concurrent_duration) / sequential_duration * 100
+        print(f"并发执行比顺序执行快 {improvement:.1f}%")
+    else:
+        print("并发执行性能未达到预期，请检查优化效果")
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

tests/simple_spider_test.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简单的爬虫测试脚本
+"""
+
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class TestSpider(Spider):
+    """测试爬虫"""
+    name = 'test_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        yield Request('https://httpbin.org/get', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        return []
+
+
+def main():
+    """主函数"""
+    print("开始测试爬虫功能...")
+    
+    # 初始化框架
+    from crawlo.initialization import initialize_framework
+    settings = initialize_framework()
+    
+    # 创建爬虫进程
+    from crawlo.crawler import CrawlerProcess
+    process = CrawlerProcess(settings=settings)
+    
+    # 运行爬虫
+    import asyncio
+    asyncio.run(process.crawl(TestSpider))
+    
+    print("爬虫测试完成！")
+
+
+if __name__ == "__main__":
+    main()

tests/simple_test.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简化的框架测试
+"""
+import os
+import sys
+sys.path.insert(0, '/')
+
+# 设置基本配置
+test_log_file = '/Users/oscar/projects/Crawlo/simple_test.log'
+if os.path.exists(test_log_file):
+    os.remove(test_log_file)
+
+# 最简单的测试
+try:
+    from crawlo.utils.log import LoggerManager
+    
+    print("配置日志系统...")
+    LoggerManager.configure(
+        LOG_LEVEL='INFO',
+        LOG_FILE=test_log_file
+    )
+    
+    from crawlo.utils.log import get_logger
+    logger = get_logger('test.simple')
+    
+    print("测试日志输出...")
+    logger.info("这是一条测试信息")
+    logger.info("Crawlo框架初始化完成")
+    logger.info("Crawlo Framework Started 1.3.3")
+    
+    print("检查日志文件...")
+    if os.path.exists(test_log_file):
+        with open(test_log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"日志文件内容: {len(content)} 字符")
+            print("内容:")
+            print(content)
+    else:
+        print("日志文件未创建")
+        
+except Exception as e:
+    print(f"错误: {e}")
+    import traceback
+    traceback.print_exc()
+
+print("测试完成")

tests/test_all_commands.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试所有 crawlo 命令
+"""
+
+import sys
+import os
+import subprocess
+import tempfile
+import shutil
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+def run_command(cmd, cwd=None, capture_output=True):
+    """运行命令并返回结果"""
+    try:
+        result = subprocess.run(
+            cmd, 
+            shell=True, 
+            cwd=cwd, 
+            capture_output=capture_output, 
+            text=True, 
+            timeout=30
+        )
+        return result.returncode, result.stdout, result.stderr
+    except subprocess.TimeoutExpired:
+        return -1, "", "Command timed out"
+    except Exception as e:
+        return -1, "", str(e)
+
+def test_help_command():
+    """测试 help 命令"""
+    print("测试 help 命令...")
+    
+    # 测试 -h 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli -h")
+    assert code == 0, f"help 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "help 输出不包含框架名称"
+    
+    # 测试 --help 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli --help")
+    assert code == 0, f"help 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "help 输出不包含框架名称"
+    
+    print("✅ help 命令测试通过")
+
+def test_version_command():
+    """测试 version 命令"""
+    print("测试 version 命令...")
+    
+    # 测试 -v 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli -v")
+    assert code == 0, f"version 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "version 输出不包含框架名称"
+    
+    # 测试 --version 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli --version")
+    assert code == 0, f"version 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "version 输出不包含框架名称"
+    
+    print("✅ version 命令测试通过")
+
+def test_startproject_command():
+    """测试 startproject 命令"""
+    print("测试 startproject 命令...")
+    
+    # 创建临时目录进行测试
+    with tempfile.TemporaryDirectory() as temp_dir:
+        project_name = "test_project"
+        project_path = Path(temp_dir) / project_name
+        
+        # 测试创建项目
+        code, stdout, stderr = run_command(
+            f"python -m crawlo.cli startproject {project_name}", 
+            cwd=temp_dir
+        )
+        
+        # 检查项目是否创建成功
+        assert code == 0, f"startproject 命令失败: {stderr}"
+        assert project_path.exists(), f"项目目录未创建: {project_path}"
+        
+        # 检查必要的文件是否存在
+        required_files = [
+            "crawlo.cfg",
+            "settings.py",
+            "spiders/__init__.py",
+            "items.py",
+            "middlewares.py"
+        ]
+        
+        for file_path in required_files:
+            full_path = project_path / file_path
+            assert full_path.exists(), f"必要文件不存在: {full_path}"
+        
+        print("✅ startproject 命令测试通过")
+
+def test_list_command():
+    """测试 list 命令"""
+    print("测试 list 命令...")
+    
+    # 在示例项目目录中测试 list 命令
+    example_dir = Path(__file__).parent.parent / "examples" / "ofweek_standalone"
+    
+    # 测试普通 list 命令
+    code, stdout, stderr = run_command("python -m crawlo.cli list", cwd=example_dir)
+    # list 命令可能会因为环境问题失败，但我们检查是否有输出
+    assert code == 0 or len(stdout) > 0, f"list 命令失败: {stderr}"
+    
+    # 测试 --json 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli list --json", cwd=example_dir)
+    # list 命令可能会因为环境问题失败，但我们检查是否有输出
+    assert code == 0 or len(stdout) > 0, f"list --json 命令失败: {stderr}"
+    
+    print("✅ list 命令测试通过")
+
+def test_genspider_command():
+    """测试 genspider 命令"""
+    print("测试 genspider 命令...")
+    
+    # 创建临时项目进行测试
+    with tempfile.TemporaryDirectory() as temp_dir:
+        project_name = "test_project"
+        project_path = Path(temp_dir) / project_name
+        
+        # 先创建项目
+        code, stdout, stderr = run_command(
+            f"python -m crawlo.cli startproject {project_name}", 
+            cwd=temp_dir
+        )
+        assert code == 0, f"创建项目失败: {stderr}"
+        
+        # 测试生成爬虫
+        spider_name = "test_spider"
+        domain = "example.com"
+        code, stdout, stderr = run_command(
+            f"python -m crawlo.cli genspider {spider_name} {domain}", 
+            cwd=project_path
+        )
+        assert code == 0, f"genspider 命令失败: {stderr}"
+        
+        # 检查爬虫文件是否创建
+        spider_file = project_path / project_name / "spiders" / f"{spider_name}.py"
+        assert spider_file.exists(), f"爬虫文件未创建: {spider_file}"
+        
+        # 检查文件内容
+        with open(spider_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            assert spider_name in content, "爬虫文件不包含爬虫名称"
+            assert domain in content, "爬虫文件不包含域名"
+        
+        print("✅ genspider 命令测试通过")
+
+def test_check_command():
+    """测试 check 命令"""
+    print("测试 check 命令...")
+    
+    # 在示例项目目录中测试 check 命令
+    example_dir = Path(__file__).parent.parent / "examples" / "ofweek_standalone"
+    
+    # 测试普通 check 命令
+    code, stdout, stderr = run_command("python -m crawlo.cli check", cwd=example_dir)
+    # check 命令可能会因为环境问题失败，但我们检查是否有输出
+    assert code == 0 or len(stdout) > 0, f"check 命令失败: {stderr}"
+    
+    print("✅ check 命令测试通过")
+
+def test_stats_command():
+    """测试 stats 命令"""
+    print("测试 stats 命令...")
+    
+    # 在示例项目目录中测试 stats 命令
+    example_dir = Path(__file__).parent.parent / "examples" / "ofweek_standalone"
+    
+    # 测试普通 stats 命令
+    code, stdout, stderr = run_command("python -m crawlo.cli stats", cwd=example_dir)
+    # stats 命令可能会因为没有统计数据而返回非0，但我们检查是否有输出
+    assert code == 0 or len(stdout) > 0, f"stats 命令失败: {stderr}"
+    
+    print("✅ stats 命令测试通过")
+
+def main():
+    """主函数"""
+    print("开始测试所有 crawlo 命令...")
+    print("=" * 50)
+    
+    try:
+        # 测试 help 命令
+        test_help_command()
+        print()
+        
+        # 测试 version 命令
+        test_version_command()
+        print()
+        
+        # 测试 startproject 命令
+        test_startproject_command()
+        print()
+        
+        # 测试 genspider 命令
+        test_genspider_command()
+        print()
+        
+        # 测试 list 命令
+        test_list_command()
+        print()
+        
+        # 测试 check 命令
+        test_check_command()
+        print()
+        
+        # 测试 stats 命令
+        test_stats_command()
+        print()
+        
+        print("=" * 50)
+        print("所有命令测试通过！")
+        
+    except Exception as e:
+        print("=" * 50)
+        print(f"测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())