crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/simple_spider_test.py

@@ -1,50 +1,50 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-简单的爬虫测试脚本
-"""
-
-import sys
-import os
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.spider import Spider
-from crawlo import Request
-
-
-class TestSpider(Spider):
-    """测试爬虫"""
-    name = 'test_spider'
-    
-    def start_requests(self):
-        """发起测试请求"""
-        yield Request('https://httpbin.org/get', callback=self.parse)
-    
-    def parse(self, response):
-        """解析响应"""
-        print(f"成功获取响应: {response.url}")
-        print(f"状态码: {response.status_code}")
-        return []
-
-
-def main():
-    """主函数"""
-    print("开始测试爬虫功能...")
-    
-    # 初始化框架
-    from crawlo.initialization import initialize_framework
-    settings = initialize_framework()
-    
-    # 创建爬虫进程
-    from crawlo.crawler import CrawlerProcess
-    process = CrawlerProcess(settings=settings)
-    
-    # 运行爬虫
-    import asyncio
-    asyncio.run(process.crawl(TestSpider))
-    
-    print("爬虫测试完成！")
-
-
-if __name__ == "__main__":
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简单的爬虫测试脚本
+"""
+
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class TestSpider(Spider):
+    """测试爬虫"""
+    name = 'test_spider'
+    
+    def start_requests(self):
+        """发起测试请求"""
+        yield Request('https://httpbin.org/get', callback=self.parse)
+    
+    def parse(self, response):
+        """解析响应"""
+        print(f"成功获取响应: {response.url}")
+        print(f"状态码: {response.status_code}")
+        return []
+
+
+def main():
+    """主函数"""
+    print("开始测试爬虫功能...")
+    
+    # 初始化框架
+    from crawlo.initialization import initialize_framework
+    settings = initialize_framework()
+    
+    # 创建爬虫进程
+    from crawlo.crawler import CrawlerProcess
+    process = CrawlerProcess(settings=settings)
+    
+    # 运行爬虫
+    import asyncio
+    asyncio.run(process.crawl(TestSpider))
+    
+    print("爬虫测试完成！")
+
+
+if __name__ == "__main__":
     main()

tests/simple_test.py

@@ -1,48 +1,48 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-简化的框架测试
-"""
-import os
-import sys
-sys.path.insert(0, '/')
-
-# 设置基本配置
-test_log_file = '/Users/oscar/projects/Crawlo/simple_test.log'
-if os.path.exists(test_log_file):
-    os.remove(test_log_file)
-
-# 最简单的测试
-try:
-    from crawlo.utils.log import LoggerManager
-    
-    print("配置日志系统...")
-    LoggerManager.configure(
-        LOG_LEVEL='INFO',
-        LOG_FILE=test_log_file
-    )
-    
-    from crawlo.utils.log import get_logger
-    logger = get_logger('test.simple')
-    
-    print("测试日志输出...")
-    logger.info("这是一条测试信息")
-    logger.info("Crawlo框架初始化完成")
-    logger.info("Crawlo Framework Started 1.3.3")
-    
-    print("检查日志文件...")
-    if os.path.exists(test_log_file):
-        with open(test_log_file, 'r', encoding='utf-8') as f:
-            content = f.read()
-            print(f"日志文件内容: {len(content)} 字符")
-            print("内容:")
-            print(content)
-    else:
-        print("日志文件未创建")
-        
-except Exception as e:
-    print(f"错误: {e}")
-    import traceback
-    traceback.print_exc()
-
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简化的框架测试
+"""
+import os
+import sys
+sys.path.insert(0, '/')
+
+# 设置基本配置
+test_log_file = '/Users/oscar/projects/Crawlo/simple_test.log'
+if os.path.exists(test_log_file):
+    os.remove(test_log_file)
+
+# 最简单的测试
+try:
+    from crawlo.utils.log import LoggerManager
+    
+    print("配置日志系统...")
+    LoggerManager.configure(
+        LOG_LEVEL='INFO',
+        LOG_FILE=test_log_file
+    )
+    
+    from crawlo.utils.log import get_logger
+    logger = get_logger('test.simple')
+    
+    print("测试日志输出...")
+    logger.info("这是一条测试信息")
+    logger.info("Crawlo框架初始化完成")
+    logger.info("Crawlo Framework Started 1.3.3")
+    
+    print("检查日志文件...")
+    if os.path.exists(test_log_file):
+        with open(test_log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"日志文件内容: {len(content)} 字符")
+            print("内容:")
+            print(content)
+    else:
+        print("日志文件未创建")
+        
+except Exception as e:
+    print(f"错误: {e}")
+    import traceback
+    traceback.print_exc()
+
 print("测试完成")

tests/spider_log_timing_test.py

@@ -1,178 +1,178 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-"""
-模拟爬虫场景测试日志文件生成时机
-"""
-
-import os
-import sys
-import time
-import asyncio
-from pathlib import Path
-
-# 添加项目根目录到Python路径
-project_root = Path(__file__).parent.parent
-sys.path.insert(0, str(project_root))
-
-from crawlo.logging import configure_logging as configure, get_logger
-
-
-class MockSpider:
-    """模拟爬虫类"""
-    
-    def __init__(self, name):
-        self.name = name
-        self.logger = get_logger(f'spider.{name}')
-    
-    async def start_requests(self):
-        """开始请求"""
-        self.logger.info(f"Spider {self.name} 开始请求")
-        # 模拟一些网络请求
-        for i in range(3):
-            self.logger.info(f"Spider {self.name} 发送请求 {i+1}")
-            await asyncio.sleep(0.1)
-    
-    async def parse(self, response):
-        """解析响应"""
-        self.logger.info(f"Spider {self.name} 解析响应")
-        # 模拟解析过程
-        await asyncio.sleep(0.05)
-        self.logger.info(f"Spider {self.name} 提取数据")
-        return {"data": f"item from {self.name}"}
-    
-    async def crawl(self):
-        """执行爬取"""
-        self.logger.info(f"Spider {self.name} 开始爬取")
-        await self.start_requests()
-        await self.parse("mock_response")
-        self.logger.info(f"Spider {self.name} 爬取完成")
-
-
-async def test_spider_logging_timing():
-    """测试爬虫日志记录时机"""
-    print("=== 测试爬虫日志记录时机 ===")
-    
-    # 设置日志文件路径
-    log_file = "logs/spider_timing_test.log"
-    
-    # 删除可能存在的旧日志文件
-    if os.path.exists(log_file):
-        os.remove(log_file)
-    
-    # 配置日志系统
-    print("1. 配置日志系统...")
-    configure(
-        level='INFO',
-        file_path=log_file,
-        console_enabled=True,
-        file_enabled=True
-    )
-    
-    print(f"   配置后日志文件是否存在: {os.path.exists(log_file)}")
-    
-    # 创建爬虫实例
-    print("\n2. 创建爬虫实例...")
-    spider = MockSpider("test_spider")
-    print(f"   创建爬虫后日志文件是否存在: {os.path.exists(log_file)}")
-    
-    # 检查文件内容
-    if os.path.exists(log_file):
-        with open(log_file, 'r', encoding='utf-8') as f:
-            content = f.read()
-            print(f"   日志文件行数: {len(content.splitlines())}")
-    
-    # 执行爬取
-    print("\n3. 执行爬取...")
-    await spider.crawl()
-    
-    print(f"   爬取完成后日志文件是否存在: {os.path.exists(log_file)}")
-    
-    # 检查最终文件内容
-    if os.path.exists(log_file):
-        with open(log_file, 'r', encoding='utf-8') as f:
-            lines = f.readlines()
-            print(f"\n最终日志文件内容:")
-            print(f"  总行数: {len(lines)}")
-            print(f"  文件大小: {os.path.getsize(log_file)} 字节")
-            if lines:
-                print(f"  第一行: {lines[0].strip()}")
-                print(f"  最后一行: {lines[-1].strip()}")
-    
-    # 等待一段时间后再次检查
-    print("\n4. 等待2秒后再次检查...")
-    time.sleep(2)
-    
-    if os.path.exists(log_file):
-        with open(log_file, 'r', encoding='utf-8') as f:
-            lines = f.readlines()
-            print(f"  等待后总行数: {len(lines)}")
-
-
-async def test_multiple_spiders():
-    """测试多个爬虫的日志记录"""
-    print("\n=== 测试多个爬虫的日志记录 ===")
-    
-    # 设置日志文件路径
-    log_file = "logs/multi_spider_test.log"
-    
-    # 删除可能存在的旧日志文件
-    if os.path.exists(log_file):
-        os.remove(log_file)
-    
-    # 配置日志系统
-    configure(
-        level='INFO',
-        file_path=log_file,
-        console_enabled=True,
-        file_enabled=True
-    )
-    
-    # 创建多个爬虫实例
-    spiders = [MockSpider(f"spider_{i}") for i in range(3)]
-    
-    print("1. 顺序执行多个爬虫...")
-    for i, spider in enumerate(spiders):
-        print(f"   执行爬虫 {i+1}...")
-        await spider.crawl()
-        
-        # 检查当前日志文件状态
-        if os.path.exists(log_file):
-            with open(log_file, 'r', encoding='utf-8') as f:
-                lines = f.readlines()
-                print(f"     当前日志行数: {len(lines)}")
-    
-    print(f"\n   所有爬虫执行完成后日志文件是否存在: {os.path.exists(log_file)}")
-    
-    # 检查最终文件内容
-    if os.path.exists(log_file):
-        with open(log_file, 'r', encoding='utf-8') as f:
-            lines = f.readlines()
-            print(f"\n最终日志文件内容:")
-            print(f"  总行数: {len(lines)}")
-            print(f"  文件大小: {os.path.getsize(log_file)} 字节")
-            if lines:
-                print(f"  第一行: {lines[0].strip()}")
-                print(f"  最后一行: {lines[-1].strip()}")
-
-
-async def main():
-    """主函数"""
-    print("开始测试爬虫日志记录时机...")
-    
-    try:
-        await test_spider_logging_timing()
-        await test_multiple_spiders()
-        
-        print("\n=== 所有测试完成 ===")
-        
-    except Exception as e:
-        print(f"\n测试过程中出现错误: {e}")
-        import traceback
-        traceback.print_exc()
-        return 1
-    
-    return 0
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+模拟爬虫场景测试日志文件生成时机
+"""
+
+import os
+import sys
+import time
+import asyncio
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.logging import configure_logging as configure, get_logger
+
+
+class MockSpider:
+    """模拟爬虫类"""
+    
+    def __init__(self, name):
+        self.name = name
+        self.logger = get_logger(f'spider.{name}')
+    
+    async def start_requests(self):
+        """开始请求"""
+        self.logger.info(f"Spider {self.name} 开始请求")
+        # 模拟一些网络请求
+        for i in range(3):
+            self.logger.info(f"Spider {self.name} 发送请求 {i+1}")
+            await asyncio.sleep(0.1)
+    
+    async def parse(self, response):
+        """解析响应"""
+        self.logger.info(f"Spider {self.name} 解析响应")
+        # 模拟解析过程
+        await asyncio.sleep(0.05)
+        self.logger.info(f"Spider {self.name} 提取数据")
+        return {"data": f"item from {self.name}"}
+    
+    async def crawl(self):
+        """执行爬取"""
+        self.logger.info(f"Spider {self.name} 开始爬取")
+        await self.start_requests()
+        await self.parse("mock_response")
+        self.logger.info(f"Spider {self.name} 爬取完成")
+
+
+async def test_spider_logging_timing():
+    """测试爬虫日志记录时机"""
+    print("=== 测试爬虫日志记录时机 ===")
+    
+    # 设置日志文件路径
+    log_file = "logs/spider_timing_test.log"
+    
+    # 删除可能存在的旧日志文件
+    if os.path.exists(log_file):
+        os.remove(log_file)
+    
+    # 配置日志系统
+    print("1. 配置日志系统...")
+    configure(
+        level='INFO',
+        file_path=log_file,
+        console_enabled=True,
+        file_enabled=True
+    )
+    
+    print(f"   配置后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 创建爬虫实例
+    print("\n2. 创建爬虫实例...")
+    spider = MockSpider("test_spider")
+    print(f"   创建爬虫后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 检查文件内容
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"   日志文件行数: {len(content.splitlines())}")
+    
+    # 执行爬取
+    print("\n3. 执行爬取...")
+    await spider.crawl()
+    
+    print(f"   爬取完成后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 检查最终文件内容
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+            print(f"\n最终日志文件内容:")
+            print(f"  总行数: {len(lines)}")
+            print(f"  文件大小: {os.path.getsize(log_file)} 字节")
+            if lines:
+                print(f"  第一行: {lines[0].strip()}")
+                print(f"  最后一行: {lines[-1].strip()}")
+    
+    # 等待一段时间后再次检查
+    print("\n4. 等待2秒后再次检查...")
+    time.sleep(2)
+    
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+            print(f"  等待后总行数: {len(lines)}")
+
+
+async def test_multiple_spiders():
+    """测试多个爬虫的日志记录"""
+    print("\n=== 测试多个爬虫的日志记录 ===")
+    
+    # 设置日志文件路径
+    log_file = "logs/multi_spider_test.log"
+    
+    # 删除可能存在的旧日志文件
+    if os.path.exists(log_file):
+        os.remove(log_file)
+    
+    # 配置日志系统
+    configure(
+        level='INFO',
+        file_path=log_file,
+        console_enabled=True,
+        file_enabled=True
+    )
+    
+    # 创建多个爬虫实例
+    spiders = [MockSpider(f"spider_{i}") for i in range(3)]
+    
+    print("1. 顺序执行多个爬虫...")
+    for i, spider in enumerate(spiders):
+        print(f"   执行爬虫 {i+1}...")
+        await spider.crawl()
+        
+        # 检查当前日志文件状态
+        if os.path.exists(log_file):
+            with open(log_file, 'r', encoding='utf-8') as f:
+                lines = f.readlines()
+                print(f"     当前日志行数: {len(lines)}")
+    
+    print(f"\n   所有爬虫执行完成后日志文件是否存在: {os.path.exists(log_file)}")
+    
+    # 检查最终文件内容
+    if os.path.exists(log_file):
+        with open(log_file, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+            print(f"\n最终日志文件内容:")
+            print(f"  总行数: {len(lines)}")
+            print(f"  文件大小: {os.path.getsize(log_file)} 字节")
+            if lines:
+                print(f"  第一行: {lines[0].strip()}")
+                print(f"  最后一行: {lines[-1].strip()}")
+
+
+async def main():
+    """主函数"""
+    print("开始测试爬虫日志记录时机...")
+    
+    try:
+        await test_spider_logging_timing()
+        await test_multiple_spiders()
+        
+        print("\n=== 所有测试完成 ===")
+        
+    except Exception as e:
+        print(f"\n测试过程中出现错误: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == '__main__':
     asyncio.run(main())