crawlo 1.4.5__py3-none-any.whl → 1.4.7__py3-none-any.whl

tests/ofweek_scrapy/ofweek_scrapy/settings.py

@@ -1,85 +1,85 @@
-# Scrapy settings for ofweek_scrapy project
-
-BOT_NAME = 'ofweek_scrapy'
-
-SPIDER_MODULES = ['ofweek_scrapy.spiders']
-NEWSPIDER_MODULE = 'ofweek_scrapy.spiders'
-
-# Obey robots.txt rules
-ROBOTSTXT_OBEY = False
-
-# Configure maximum concurrent requests performed by Scrapy (default: 16)
-# 使用与Crawlo相同的并发数
-CONCURRENT_REQUESTS = 8
-
-# Configure a delay for requests for the same website (default: 0)
-# 使用与Crawlo相同的下载延迟
-DOWNLOAD_DELAY = 1.0
-# The download delay setting will honor only one of:
-#CONCURRENT_REQUESTS_PER_DOMAIN = 16
-#CONCURRENT_REQUESTS_PER_IP = 16
-
-# Disable cookies (enabled by default)
-#COOKIES_ENABLED = False
-
-# Disable Telnet Console (enabled by default)
-#TELNETCONSOLE_ENABLED = False
-
-# Override the default request headers:
-#DEFAULT_REQUEST_HEADERS = {
-#    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-#    'Accept-Language': 'en',
-#}
-
-# Enable or disable spider middlewares
-# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-#SPIDER_MIDDLEWARES = {
-#    'ofweek_scrapy.middlewares.OfweekScrapySpiderMiddleware': 543,
-#}
-
-# Enable or disable downloader middlewares
-# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
-#DOWNLOADER_MIDDLEWARES = {
-#    'ofweek_scrapy.middlewares.OfweekScrapyDownloaderMiddleware': 543,
-#}
-
-# Enable or disable extensions
-# See https://docs.scrapy.org/en/latest/topics/extensions.html
-#EXTENSIONS = {
-#    'scrapy.extensions.telnet.TelnetConsole': None,
-#}
-
-# Configure item pipelines
-# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-#ITEM_PIPELINES = {
-#    'ofweek_scrapy.pipelines.OfweekScrapyPipeline': 300,
-#}
-
-# Enable and configure the AutoThrottle extension (disabled by default)
-# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
-#AUTOTHROTTLE_ENABLED = True
-# The initial download delay
-#AUTOTHROTTLE_START_DELAY = 5
-# The maximum download delay to be set in case of high latencies
-#AUTOTHROTTLE_MAX_DELAY = 60
-# The average number of requests Scrapy should be sending in parallel to
-# each remote server
-#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
-# Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG = False
-
-# Enable and configure HTTP caching (disabled by default)
-# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
-#HTTPCACHE_ENABLED = True
-#HTTPCACHE_EXPIRATION_SECS = 0
-#HTTPCACHE_DIR = 'httpcache'
-#HTTPCACHE_IGNORE_HTTP_CODES = []
-#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
-
-# Set settings whose default value is deprecated to a future-proof value
-REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
-TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
-FEED_EXPORT_ENCODING = 'utf-8'
-
-# 日志配置
+# Scrapy settings for ofweek_scrapy project
+
+BOT_NAME = 'ofweek_scrapy'
+
+SPIDER_MODULES = ['ofweek_scrapy.spiders']
+NEWSPIDER_MODULE = 'ofweek_scrapy.spiders'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+# 使用与Crawlo相同的并发数
+CONCURRENT_REQUESTS = 8
+
+# Configure a delay for requests for the same website (default: 0)
+# 使用与Crawlo相同的下载延迟
+DOWNLOAD_DELAY = 1.0
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#    'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'ofweek_scrapy.middlewares.OfweekScrapySpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'ofweek_scrapy.middlewares.OfweekScrapyDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    'ofweek_scrapy.pipelines.OfweekScrapyPipeline': 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
+TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+FEED_EXPORT_ENCODING = 'utf-8'
+
+# 日志配置
 LOG_LEVEL = 'INFO'

tests/ofweek_scrapy/scrapy.cfg

@@ -1,11 +1,11 @@
-# Automatically created by: scrapy startproject
-#
-# For more information about the [deploy] section see:
-# https://scrapyd.readthedocs.io/en/latest/deploy.html
-
-[settings]
-default = ofweek_scrapy.settings
-
-[deploy]
-#url = http://localhost:6800/
-project = ofweek_scrapy
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = ofweek_scrapy.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = ofweek_scrapy

tests/optimized_performance_test.py

@@ -1,212 +1,212 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-优化后的Crawlo框架性能测试脚本
-用于评估优化后的框架性能表现
-"""
-
-import asyncio
-import time
-import sys
-import os
-from pathlib import Path
-
-# 添加项目根目录到 Python 路径
-project_root = Path(__file__).parent.parent
-sys.path.insert(0, str(project_root))
-
-from crawlo.crawler import CrawlerProcess
-from crawlo.spider import Spider
-from crawlo import Request, Item
-
-
-class TestItem(Item):
-    """测试用的Item类"""
-    def __init__(self):
-        super().__init__()
-        self.url = ''
-        self.status = 0
-        self.page_id = ''
-
-
-def create_test_spider_class(spider_name, page_count):
-    """动态创建测试爬虫类"""
-    class TestSpider(Spider):
-        # 显式设置name属性
-        name = spider_name
-        
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-            # 使用较少的测试页面以加快测试速度
-            self.start_urls = [f'https://httpbin.org/delay/0?page={i}' for i in range(page_count)]
-        
-        def parse(self, response):
-            """简单解析响应"""
-            # 正确返回Item对象
-            item = TestItem()
-            item['url'] = response.url
-            item['status'] = response.status_code
-            item['page_id'] = response.url.split('page=')[-1] if 'page=' in response.url else 'unknown'
-            yield item
-    
-    return TestSpider
-
-
-class OptimizedPerformanceTester:
-    """优化后的性能测试器"""
-    
-    def __init__(self):
-        self.results = {}
-    
-    def test_initialization_performance(self):
-        """测试初始化性能"""
-        print("测试初始化性能...")
-        
-        start_time = time.time()
-        settings = {
-            'CONCURRENT_REQUESTS': 10,
-        }
-        process = CrawlerProcess(settings=settings)
-        end_time = time.time()
-        
-        init_time = end_time - start_time
-        print(f"初始化时间: {init_time:.4f} 秒")
-        return init_time
-    
-    async def run_crawler_test(self, test_pages=20, concurrent_requests=10, test_name="performance_test"):
-        """运行爬虫性能测试"""
-        # 配置设置
-        settings = {
-            'CONCURRENT_REQUESTS': concurrent_requests,
-            'DOWNLOAD_DELAY': 0.1,  # 减少延迟以提高性能
-            'RANDOMIZE_DOWNLOAD_DELAY': False,
-            'SCHEDULER_MAX_QUEUE_SIZE': 5000,  # 使用优化后的队列大小
-            'BACKPRESSURE_RATIO': 0.9,  # 使用优化后的背压比率
-        }
-        
-        # 创建测试爬虫类
-        TestSpiderClass = create_test_spider_class(test_name, test_pages)
-        
-        # 注册爬虫类
-        from crawlo.spider import get_global_spider_registry
-        registry = get_global_spider_registry()
-        registry[TestSpiderClass.name] = TestSpiderClass
-        
-        # 创建爬虫进程
-        process = CrawlerProcess(settings=settings)
-        
-        # 添加测试爬虫
-        crawler = await process.crawl(TestSpiderClass.name)
-        
-        # 计算性能指标
-        metrics = crawler.metrics
-        duration = metrics.get_total_duration()
-        rps = test_pages / duration if duration > 0 else 0
-        
-        return {
-            'duration': duration,
-            'rps': rps,
-            'pages': test_pages,
-            'concurrent': concurrent_requests
-        }
-    
-    async def run_scale_tests(self):
-        """运行不同规模的测试"""
-        print("\n=== 运行规模测试 ===")
-        scales = [10, 20, 50]  # 减少测试规模以加快测试速度
-        results = []
-        
-        for i, scale in enumerate(scales):
-            test_name = f"scale_test_{i}_{scale}"
-            print(f"测试规模: {scale} 个页面")
-            try:
-                result = await self.run_crawler_test(test_pages=scale, test_name=test_name)
-                results.append(result)
-                print(f"  完成时间: {result['duration']:.2f} 秒")
-                print(f"  每秒请求数: {result['rps']:.2f} RPS")
-            except Exception as e:
-                print(f"  测试失败: {e}")
-                import traceback
-                traceback.print_exc()
-            print()
-        
-        return results
-    
-    async def run_concurrency_tests(self):
-        """运行不同并发数的测试"""
-        print("\n=== 运行并发测试 ===")
-        concurrencies = [1, 5, 10, 20]  # 增加更高并发数的测试
-        results = []
-        
-        for i, concurrency in enumerate(concurrencies):
-            test_name = f"concurrency_test_{i}_{concurrency}"
-            print(f"测试并发数: {concurrency}")
-            try:
-                result = await self.run_crawler_test(
-                    test_pages=50,  # 增加页面数以更好地测试并发效果
-                    concurrent_requests=concurrency,
-                    test_name=test_name
-                )
-                results.append(result)
-                print(f"  完成时间: {result['duration']:.2f} 秒")
-                print(f"  每秒请求数: {result['rps']:.2f} RPS")
-            except Exception as e:
-                print(f"  测试失败: {e}")
-                import traceback
-                traceback.print_exc()
-            print()
-        
-        return results
-    
-    async def run_performance_suite(self):
-        """运行完整的性能测试套件"""
-        print("开始 Crawlo 框架优化后性能测试")
-        print("=" * 50)
-        
-        # 测试初始化性能
-        init_time = self.test_initialization_performance()
-        
-        # 运行规模测试
-        scale_results = await self.run_scale_tests()
-        
-        # 运行并发测试
-        concurrency_results = await self.run_concurrency_tests()
-        
-        # 汇总结果
-        print("\n=== 性能测试汇总 ===")
-        print(f"初始化时间: {init_time:.4f} 秒")
-        
-        print("\n规模测试结果:")
-        for result in scale_results:
-            if 'duration' in result:
-                print(f"  {result['pages']} 页面: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
-        
-        print("\n并发测试结果:")
-        for result in concurrency_results:
-            if 'duration' in result:
-                print(f"  {result['concurrent']} 并发: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
-        
-        return {
-            'initialization': init_time,
-            'scale_tests': scale_results,
-            'concurrency_tests': concurrency_results
-        }
-
-
-async def main():
-    """主函数"""
-    tester = OptimizedPerformanceTester()
-    results = await tester.run_performance_suite()
-    
-    print("\n=== 测试完成 ===")
-    print("优化后性能测试已完成，结果如上所示。")
-    
-    # 保存结果到文件
-    import json
-    with open('optimized_performance_test_results.json', 'w', encoding='utf-8') as f:
-        json.dump(results, f, ensure_ascii=False, indent=2)
-    print("结果已保存到 optimized_performance_test_results.json")
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+优化后的Crawlo框架性能测试脚本
+用于评估优化后的框架性能表现
+"""
+
+import asyncio
+import time
+import sys
+import os
+from pathlib import Path
+
+# 添加项目根目录到 Python 路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import Spider
+from crawlo import Request, Item
+
+
+class TestItem(Item):
+    """测试用的Item类"""
+    def __init__(self):
+        super().__init__()
+        self.url = ''
+        self.status = 0
+        self.page_id = ''
+
+
+def create_test_spider_class(spider_name, page_count):
+    """动态创建测试爬虫类"""
+    class TestSpider(Spider):
+        # 显式设置name属性
+        name = spider_name
+        
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            # 使用较少的测试页面以加快测试速度
+            self.start_urls = [f'https://httpbin.org/delay/0?page={i}' for i in range(page_count)]
+        
+        def parse(self, response):
+            """简单解析响应"""
+            # 正确返回Item对象
+            item = TestItem()
+            item['url'] = response.url
+            item['status'] = response.status_code
+            item['page_id'] = response.url.split('page=')[-1] if 'page=' in response.url else 'unknown'
+            yield item
+    
+    return TestSpider
+
+
+class OptimizedPerformanceTester:
+    """优化后的性能测试器"""
+    
+    def __init__(self):
+        self.results = {}
+    
+    def test_initialization_performance(self):
+        """测试初始化性能"""
+        print("测试初始化性能...")
+        
+        start_time = time.time()
+        settings = {
+            'CONCURRENT_REQUESTS': 10,
+        }
+        process = CrawlerProcess(settings=settings)
+        end_time = time.time()
+        
+        init_time = end_time - start_time
+        print(f"初始化时间: {init_time:.4f} 秒")
+        return init_time
+    
+    async def run_crawler_test(self, test_pages=20, concurrent_requests=10, test_name="performance_test"):
+        """运行爬虫性能测试"""
+        # 配置设置
+        settings = {
+            'CONCURRENT_REQUESTS': concurrent_requests,
+            'DOWNLOAD_DELAY': 0.1,  # 减少延迟以提高性能
+            'RANDOMIZE_DOWNLOAD_DELAY': False,
+            'SCHEDULER_MAX_QUEUE_SIZE': 5000,  # 使用优化后的队列大小
+            'BACKPRESSURE_RATIO': 0.9,  # 使用优化后的背压比率
+        }
+        
+        # 创建测试爬虫类
+        TestSpiderClass = create_test_spider_class(test_name, test_pages)
+        
+        # 注册爬虫类
+        from crawlo.spider import get_global_spider_registry
+        registry = get_global_spider_registry()
+        registry[TestSpiderClass.name] = TestSpiderClass
+        
+        # 创建爬虫进程
+        process = CrawlerProcess(settings=settings)
+        
+        # 添加测试爬虫
+        crawler = await process.crawl(TestSpiderClass.name)
+        
+        # 计算性能指标
+        metrics = crawler.metrics
+        duration = metrics.get_total_duration()
+        rps = test_pages / duration if duration > 0 else 0
+        
+        return {
+            'duration': duration,
+            'rps': rps,
+            'pages': test_pages,
+            'concurrent': concurrent_requests
+        }
+    
+    async def run_scale_tests(self):
+        """运行不同规模的测试"""
+        print("\n=== 运行规模测试 ===")
+        scales = [10, 20, 50]  # 减少测试规模以加快测试速度
+        results = []
+        
+        for i, scale in enumerate(scales):
+            test_name = f"scale_test_{i}_{scale}"
+            print(f"测试规模: {scale} 个页面")
+            try:
+                result = await self.run_crawler_test(test_pages=scale, test_name=test_name)
+                results.append(result)
+                print(f"  完成时间: {result['duration']:.2f} 秒")
+                print(f"  每秒请求数: {result['rps']:.2f} RPS")
+            except Exception as e:
+                print(f"  测试失败: {e}")
+                import traceback
+                traceback.print_exc()
+            print()
+        
+        return results
+    
+    async def run_concurrency_tests(self):
+        """运行不同并发数的测试"""
+        print("\n=== 运行并发测试 ===")
+        concurrencies = [1, 5, 10, 20]  # 增加更高并发数的测试
+        results = []
+        
+        for i, concurrency in enumerate(concurrencies):
+            test_name = f"concurrency_test_{i}_{concurrency}"
+            print(f"测试并发数: {concurrency}")
+            try:
+                result = await self.run_crawler_test(
+                    test_pages=50,  # 增加页面数以更好地测试并发效果
+                    concurrent_requests=concurrency,
+                    test_name=test_name
+                )
+                results.append(result)
+                print(f"  完成时间: {result['duration']:.2f} 秒")
+                print(f"  每秒请求数: {result['rps']:.2f} RPS")
+            except Exception as e:
+                print(f"  测试失败: {e}")
+                import traceback
+                traceback.print_exc()
+            print()
+        
+        return results
+    
+    async def run_performance_suite(self):
+        """运行完整的性能测试套件"""
+        print("开始 Crawlo 框架优化后性能测试")
+        print("=" * 50)
+        
+        # 测试初始化性能
+        init_time = self.test_initialization_performance()
+        
+        # 运行规模测试
+        scale_results = await self.run_scale_tests()
+        
+        # 运行并发测试
+        concurrency_results = await self.run_concurrency_tests()
+        
+        # 汇总结果
+        print("\n=== 性能测试汇总 ===")
+        print(f"初始化时间: {init_time:.4f} 秒")
+        
+        print("\n规模测试结果:")
+        for result in scale_results:
+            if 'duration' in result:
+                print(f"  {result['pages']} 页面: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
+        
+        print("\n并发测试结果:")
+        for result in concurrency_results:
+            if 'duration' in result:
+                print(f"  {result['concurrent']} 并发: {result['duration']:.2f}s, {result['rps']:.2f} RPS")
+        
+        return {
+            'initialization': init_time,
+            'scale_tests': scale_results,
+            'concurrency_tests': concurrency_results
+        }
+
+
+async def main():
+    """主函数"""
+    tester = OptimizedPerformanceTester()
+    results = await tester.run_performance_suite()
+    
+    print("\n=== 测试完成 ===")
+    print("优化后性能测试已完成，结果如上所示。")
+    
+    # 保存结果到文件
+    import json
+    with open('optimized_performance_test_results.json', 'w', encoding='utf-8') as f:
+        json.dump(results, f, ensure_ascii=False, indent=2)
+    print("结果已保存到 optimized_performance_test_results.json")
+
+
+if __name__ == '__main__':
     asyncio.run(main())