crawlo 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/queue_blocking_test.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+队列阻塞行为测试脚本
+验证队列满时的阻塞行为
+"""
+
+import sys
+import os
+import asyncio
+import time
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.network.request import Request
+from crawlo.queue.queue_manager import QueueConfig, QueueManager
+
+
+async def test_queue_blocking_behavior():
+    """测试队列阻塞行为"""
+    print("开始测试队列阻塞行为...")
+    
+    # 初始化框架
+    from crawlo.initialization import initialize_framework
+    settings = initialize_framework()
+    
+    # 创建一个小型队列配置进行测试
+    settings.set('SCHEDULER_MAX_QUEUE_SIZE', 5)  # 设置非常小的队列大小
+    
+    # 创建队列配置
+    queue_config = QueueConfig.from_settings(settings)
+    queue_config.max_queue_size = 5  # 设置小队列大小进行测试
+    
+    # 创建队列管理器
+    queue_manager = QueueManager(queue_config)
+    
+    # 初始化队列
+    await queue_manager.initialize()
+    
+    print(f"队列类型: {queue_manager._queue_type}")
+    print(f"队列最大大小: {queue_config.max_queue_size}")
+    
+    # 创建生产者任务
+    async def producer(queue_manager, name, count):
+        """生产者：向队列添加请求"""
+        print(f"生产者 {name} 开始工作，将添加 {count} 个请求")
+        start_time = time.time()
+        
+        for i in range(count):
+            request = Request(url=f'https://example.com/page{name}_{i}.html')
+            try:
+                # 这里应该会阻塞当队列满时
+                await queue_manager.put(request)
+                print(f"生产者 {name} 成功添加请求 {i}")
+            except Exception as e:
+                print(f"生产者 {name} 添加请求 {i} 失败: {e}")
+        
+        end_time = time.time()
+        print(f"生产者 {name} 完成工作，耗时 {end_time - start_time:.2f} 秒")
+    
+    # 创建消费者任务
+    async def consumer(queue_manager, name, count):
+        """消费者：从队列获取请求"""
+        print(f"消费者 {name} 开始工作，将获取 {count} 个请求")
+        start_time = time.time()
+        
+        retrieved_count = 0
+        while retrieved_count < count:
+            try:
+                request = await queue_manager.get(timeout=2.0)
+                if request:
+                    print(f"消费者 {name} 成功获取请求: {request.url}")
+                    retrieved_count += 1
+                    # 模拟处理时间
+                    await asyncio.sleep(0.1)
+                else:
+                    print(f"消费者 {name} 超时，没有获取到请求")
+                    break
+            except Exception as e:
+                print(f"消费者 {name} 获取请求失败: {e}")
+                break
+        
+        end_time = time.time()
+        print(f"消费者 {name} 完成工作，获取了 {retrieved_count} 个请求，耗时 {end_time - start_time:.2f} 秒")
+    
+    # 同时运行生产者和消费者
+    print("开始并发测试...")
+    
+    # 生产者尝试添加10个请求到大小为5的队列
+    # 消费者会逐渐消费，生产者应该会被阻塞直到有空间
+    tasks = [
+        asyncio.create_task(producer(queue_manager, "P1", 10)),
+        asyncio.create_task(consumer(queue_manager, "C1", 10))
+    ]
+    
+    # 等待所有任务完成
+    await asyncio.gather(*tasks, return_exceptions=True)
+    
+    final_queue_size = await queue_manager.size()
+    print(f"队列最终大小: {final_queue_size}")
+    
+    # 关闭队列
+    await queue_manager.close()
+    
+    print("队列阻塞行为测试完成！")
+
+
+def main():
+    """主函数"""
+    print("开始队列阻塞行为测试...")
+    asyncio.run(test_queue_blocking_behavior())
+
+
+if __name__ == "__main__":
+    main()

tests/queue_test.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+队列系统测试脚本
+"""
+
+import sys
+import os
+import asyncio
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.network.request import Request
+from crawlo.queue.queue_manager import QueueConfig, QueueManager
+
+
+async def test_queue_system():
+    """测试队列系统"""
+    print("开始测试队列系统...")
+    
+    # 初始化框架
+    from crawlo.initialization import initialize_framework
+    settings = initialize_framework()
+    
+    # 创建一个小型队列配置进行测试
+    settings.set('SCHEDULER_MAX_QUEUE_SIZE', 10)
+    
+    # 创建队列配置
+    queue_config = QueueConfig.from_settings(settings)
+    queue_config.max_queue_size = 10  # 设置小队列大小进行测试
+    
+    # 创建队列管理器
+    queue_manager = QueueManager(queue_config)
+    
+    # 初始化队列
+    await queue_manager.initialize()
+    
+    print(f"队列类型: {queue_manager._queue_type}")
+    print(f"队列最大大小: {queue_config.max_queue_size}")
+    
+    # 测试添加请求到队列
+    print("测试添加请求到队列...")
+    tasks = []
+    for i in range(15):  # 尝试添加15个请求，超过队列大小
+        request = Request(url=f'https://example.com/page{i}.html')
+        task = asyncio.create_task(queue_manager.put(request))
+        tasks.append(task)
+    
+    # 等待所有任务完成
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+    
+    success_count = sum(1 for result in results if result is True)
+    print(f"成功添加 {success_count} 个请求到队列")
+    
+    queue_size = await queue_manager.size()
+    print(f"队列当前大小: {queue_size}")
+    
+    # 测试从队列获取请求
+    print("测试从队列获取请求...")
+    retrieved_count = 0
+    while retrieved_count < 15:
+        try:
+            request = await queue_manager.get(timeout=1.0)
+            if request:
+                print(f"成功获取请求: {request.url}")
+                retrieved_count += 1
+            else:
+                break
+        except Exception as e:
+            print(f"获取请求失败: {e}")
+            break
+    
+    print(f"总共获取了 {retrieved_count} 个请求")
+    
+    final_queue_size = await queue_manager.size()
+    print(f"队列最终大小: {final_queue_size}")
+    
+    # 关闭队列
+    await queue_manager.close()
+    
+    print("队列系统测试完成！")
+
+
+def main():
+    """主函数"""
+    print("开始队列系统测试...")
+    asyncio.run(test_queue_system())
+
+
+if __name__ == "__main__":
+    main()

tests/scrapy_comparison/ofweek_scrapy.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+import scrapy
+from urllib.parse import urljoin
+
+
+class NewsItem(scrapy.Item):
+    title = scrapy.Field()
+    publish_time = scrapy.Field()
+    url = scrapy.Field()
+    source = scrapy.Field()
+    content = scrapy.Field()
+
+
+class OfweekScrapySpider(scrapy.Spider):
+    name = 'ofweek_scrapy'
+    allowed_domains = ['ee.ofweek.com']
+    
+    def start_requests(self):
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "https://ee.ofweek.com/CATList-2800-8100-ee-2.html",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+            "sec-ch-ua": "\"Not;A=Brand\";v=\"99\", \"Google Chrome\";v=\"139\", \"Chromium\";v=\"139\"",
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": "\"Windows\""
+        }
+        cookies = {
+            "__utmz": "57425525.1730117117.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
+            "Hm_lvt_abe9900db162c6d089cdbfd107db0f03": "1739244841",
+            "Hm_lvt_af50e2fc51af73da7720fb324b88a975": "1740100727",
+            "JSESSIONID": "FEA96D3B5FC31350B2285E711BF2A541",
+            "Hm_lvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477622",
+            "HMACCOUNT": "08DF0D235A291EAA",
+            "__utma": "57425525.2080994505.1730117117.1747970718.1757477622.50",
+            "__utmc": "57425525",
+            "__utmt": "1",
+            "__utmb": "57425525.2.10.1757477622",
+            "Hm_lpvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477628",
+            "index_burying_point": "c64d6c31e69d560efe319cc9f8be279f"
+        }
+
+        # 使用较少的页数进行测试
+        max_page = 50
+        for page in range(1, max_page + 1):
+            url = f'https://ee.ofweek.com/CATList-2800-8100-ee-{page}.html'
+            yield scrapy.Request(
+                url=url,
+                callback=self.parse,
+                headers=headers,
+                cookies=cookies
+            )
+
+    def parse(self, response):
+        self.logger.info(f'正在解析页面: {response.url}')
+
+        rows = response.xpath(
+            '//div[@class="main_left"]/div[@class="list_model"]/div[@class="model_right model_right2"]')
+        self.logger.info(f"在页面 {response.url} 中找到 {len(rows)} 个条目")
+
+        for row in rows:
+            try:
+                # 提取URL和标题
+                url = row.xpath('./h3/a/@href').extract_first()
+                title = row.xpath('./h3/a/text()').extract_first()
+
+                # 容错处理
+                if not url:
+                    self.logger.warning(f"条目缺少URL，跳过")
+                    continue
+
+                if not title:
+                    self.logger.warning(f"条目缺少标题，跳过")
+                    continue
+
+                # 确保 URL 是绝对路径
+                absolute_url = urljoin(response.url, url)
+
+                # 验证URL格式
+                if not absolute_url.startswith(('http://', 'https://')):
+                    self.logger.warning(f"无效的URL格式，跳过: {absolute_url}")
+                    continue
+
+                self.logger.info(f"提取到详情页链接: {absolute_url}, 标题: {title}")
+                yield scrapy.Request(
+                    url=absolute_url,
+                    meta={
+                        "title": title.strip() if title else '',
+                        "parent_url": response.url
+                    },
+                    callback=self.parse_detail
+                )
+            except Exception as e:
+                self.logger.error(f"处理条目时出错: {e}")
+                continue
+
+    def parse_detail(self, response):
+        self.logger.info(f'正在解析详情页: {response.url}')
+
+        try:
+            title = response.meta.get('title', '')
+
+            # 提取内容，增加容错处理
+            content_elements = response.xpath('//div[@class="TRS_Editor"]|//*[@id="articleC"]')
+            if content_elements:
+                content = content_elements.xpath('.//text()').extract()
+                content = '\n'.join([text.strip() for text in content if text.strip()])
+            else:
+                content = ''
+                self.logger.warning(f"未找到内容区域: {response.url}")
+
+            # 提取发布时间
+            publish_time = response.xpath('//div[@class="time fl"]/text()').extract_first()
+            if publish_time:
+                publish_time = publish_time.strip()
+
+            source = response.xpath('//div[@class="source-name"]/text()').extract_first()
+
+            # 创建数据项
+            item = NewsItem()
+            item['title'] = title.strip() if title else ''
+            item['publish_time'] = publish_time if publish_time else ''
+            item['url'] = response.url
+            item['source'] = source if source else ''
+            item['content'] = content
+
+            self.logger.info(f"成功提取详情页数据: {item['title']}")
+            yield item
+
+        except Exception as e:
+            self.logger.error(f"解析详情页 {response.url} 时出错: {e}")

tests/scrapy_comparison/scrapy_test.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+import scrapy
+from urllib.parse import urljoin
+
+class NewsItem(scrapy.Item):
+    title = scrapy.Field()
+    publish_time = scrapy.Field()
+    url = scrapy.Field()
+    source = scrapy.Field()
+    content = scrapy.Field()
+
+class OfweekScrapyTestSpider(scrapy.Spider):
+    name = 'ofweek_scrapy_test'
+    allowed_domains = ['ee.ofweek.com']
+    
+    def start_requests(self):
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "https://ee.ofweek.com/CATList-2800-8100-ee-2.html",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+        }
+        cookies = {
+            "__utmz": "57425525.1730117117.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
+            "Hm_lvt_abe9900db162c6d089cdbfd107db0f03": "1739244841",
+            "Hm_lvt_af50e2fc51af73da7720fb324b88a975": "1740100727",
+            "JSESSIONID": "FEA96D3B5FC31350B2285E711BF2A541",
+            "Hm_lvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477622",
+            "HMACCOUNT": "08DF0D235A291EAA",
+            "__utma": "57425525.2080994505.1730117117.1747970718.1757477622.50",
+            "__utmc": "57425525",
+            "__utmt": "1",
+            "__utmb": "57425525.2.10.1757477622",
+            "Hm_lpvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477628",
+            "index_burying_point": "c64d6c31e69d560efe319cc9f8be279f"
+        }
+
+        # 使用较少的页数进行测试
+        max_page = 5
+        for page in range(1, max_page + 1):
+            url = f'https://ee.ofweek.com/CATList-2800-8100-ee-{page}.html'
+            yield scrapy.Request(
+                url=url,
+                callback=self.parse,
+                headers=headers,
+                cookies=cookies
+            )
+
+    def parse(self, response):
+        self.logger.info(f'正在解析页面: {response.url}')
+
+        rows = response.xpath(
+            '//div[@class="main_left"]/div[@class="list_model"]/div[@class="model_right model_right2"]')
+        self.logger.info(f"在页面 {response.url} 中找到 {len(rows)} 个条目")
+
+        for row in rows:
+            try:
+                # 提取URL和标题
+                url = row.xpath('./h3/a/@href').extract_first()
+                title = row.xpath('./h3/a/text()').extract_first()
+
+                # 容错处理
+                if not url:
+                    self.logger.warning(f"条目缺少URL，跳过")
+                    continue
+
+                if not title:
+                    self.logger.warning(f"条目缺少标题，跳过")
+                    continue
+
+                # 确保 URL 是绝对路径
+                absolute_url = urljoin(response.url, url)
+
+                # 验证URL格式
+                if not absolute_url.startswith(('http://', 'https://')):
+                    self.logger.warning(f"无效的URL格式，跳过: {absolute_url}")
+                    continue
+
+                self.logger.info(f"提取到详情页链接: {absolute_url}, 标题: {title}")
+                yield scrapy.Request(
+                    url=absolute_url,
+                    meta={
+                        "title": title.strip() if title else '',
+                        "parent_url": response.url
+                    },
+                    callback=self.parse_detail
+                )
+            except Exception as e:
+                self.logger.error(f"处理条目时出错: {e}")
+                continue
+
+    def parse_detail(self, response):
+        self.logger.info(f'正在解析详情页: {response.url}')
+
+        try:
+            title = response.meta.get('title', '')
+
+            # 提取内容，增加容错处理
+            content_elements = response.xpath('//div[@class="TRS_Editor"]|//*[@id="articleC"]')
+            if content_elements:
+                content = content_elements.xpath('.//text()').extract()
+                content = '\n'.join([text.strip() for text in content if text.strip()])
+            else:
+                content = ''
+                self.logger.warning(f"未找到内容区域: {response.url}")
+
+            # 提取发布时间
+            publish_time = response.xpath('//div[@class="time fl"]/text()').extract_first()
+            if publish_time:
+                publish_time = publish_time.strip()
+
+            source = response.xpath('//div[@class="source-name"]/text()').extract_first()
+
+            # 创建数据项
+            item = NewsItem()
+            item['title'] = title.strip() if title else ''
+            item['publish_time'] = publish_time if publish_time else ''
+            item['url'] = response.url
+            item['source'] = source if source else ''
+            item['content'] = content
+
+            self.logger.info(f"成功提取详情页数据: {item['title']}")
+            yield item
+
+        except Exception as e:
+            self.logger.error(f"解析详情页 {response.url} 时出错: {e}")

tests/simple_command_test.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+简单测试所有 crawlo 命令
+"""
+
+import sys
+import os
+import subprocess
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+def run_command(cmd, cwd=None):
+    """运行命令并返回结果"""
+    try:
+        result = subprocess.run(
+            cmd, 
+            shell=True, 
+            cwd=cwd, 
+            capture_output=True, 
+            text=True, 
+            timeout=30
+        )
+        return result.returncode, result.stdout, result.stderr
+    except subprocess.TimeoutExpired:
+        return -1, "", "Command timed out"
+    except Exception as e:
+        return -1, "", str(e)
+
+def test_help_command():
+    """测试 help 命令"""
+    print("测试 help 命令...")
+    
+    # 测试 -h 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli -h")
+    assert code == 0, f"help 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "help 输出不包含框架名称"
+    
+    # 测试 --help 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli --help")
+    assert code == 0, f"help 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "help 输出不包含框架名称"
+    
+    print("✅ help 命令测试通过")
+
+def test_version_command():
+    """测试 version 命令"""
+    print("测试 version 命令...")
+    
+    # 测试 -v 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli -v")
+    assert code == 0, f"version 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "version 输出不包含框架名称"
+    
+    # 测试 --version 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli --version")
+    assert code == 0, f"version 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "version 输出不包含框架名称"
+    
+    print("✅ version 命令测试通过")
+
+def test_command_help():
+    """测试各命令的帮助信息"""
+    print("测试各命令的帮助信息...")
+    
+    commands = ["startproject", "genspider", "run", "check", "list", "stats"]
+    
+    for command in commands:
+        code, stdout, stderr = run_command(f"python -m crawlo.cli {command} --help")
+        # 命令帮助通常返回非0状态码，但我们检查输出
+        assert len(stdout) > 0 or len(stderr) > 0, f"{command} 命令帮助无输出"
+        print(f"✅ {command} 命令帮助测试通过")
+
+def test_invalid_command():
+    """测试无效命令"""
+    print("测试无效命令...")
+    
+    code, stdout, stderr = run_command("python -m crawlo.cli invalid_command")
+    assert code != 0, "无效命令应该返回非0状态码"
+    assert "Unknown command" in stderr or "Unknown command" in stdout, "应该提示未知命令"
+    
+    print("✅ 无效命令测试通过")
+
+def main():
+    """主函数"""
+    print("开始简单测试所有 crawlo 命令...")
+    print("=" * 50)
+    
+    try:
+        # 测试 help 命令
+        test_help_command()
+        print()
+        
+        # 测试 version 命令
+        test_version_command()
+        print()
+        
+        # 测试各命令的帮助信息
+        test_command_help()
+        print()
+        
+        # 测试无效命令
+        test_invalid_command()
+        print()
+        
+        print("=" * 50)
+        print("所有命令简单测试通过！")
+        
+    except Exception as e:
+        print("=" * 50)
+        print(f"测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())