PyPI - crawlo - Versions diffs - 1.1.4__py3-none-any.whl → 1.1.5__py3-none-any.whl - Mend

crawlo 1.1.4py3-none-any.whl → 1.1.5py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of crawlo might be problematic. Click here for more details.

Files changed (186) hide show

crawlo/__init__.py +61 -34
crawlo/__version__.py +1 -1
crawlo/cleaners/__init__.py +61 -0
crawlo/cleaners/data_formatter.py +226 -0
crawlo/cleaners/encoding_converter.py +126 -0
crawlo/cleaners/text_cleaner.py +233 -0
crawlo/cli.py +40 -40
crawlo/commands/__init__.py +13 -13
crawlo/commands/check.py +594 -594
crawlo/commands/genspider.py +151 -151
crawlo/commands/list.py +155 -155
crawlo/commands/run.py +285 -285
crawlo/commands/startproject.py +300 -196
crawlo/commands/stats.py +188 -188
crawlo/commands/utils.py +186 -186
crawlo/config.py +309 -279
crawlo/config_validator.py +253 -0
crawlo/core/__init__.py +2 -2
crawlo/core/engine.py +346 -172
crawlo/core/processor.py +40 -40
crawlo/core/scheduler.py +137 -166
crawlo/crawler.py +1027 -1027
crawlo/downloader/__init__.py +266 -242
crawlo/downloader/aiohttp_downloader.py +220 -212
crawlo/downloader/cffi_downloader.py +256 -251
crawlo/downloader/httpx_downloader.py +259 -259
crawlo/downloader/hybrid_downloader.py +214 -0
crawlo/downloader/playwright_downloader.py +403 -0
crawlo/downloader/selenium_downloader.py +473 -0
crawlo/event.py +11 -11
crawlo/exceptions.py +81 -81
crawlo/extension/__init__.py +37 -37
crawlo/extension/health_check.py +141 -141
crawlo/extension/log_interval.py +57 -57
crawlo/extension/log_stats.py +81 -81
crawlo/extension/logging_extension.py +43 -43
crawlo/extension/memory_monitor.py +104 -88
crawlo/extension/performance_profiler.py +133 -117
crawlo/extension/request_recorder.py +107 -107
crawlo/filters/__init__.py +154 -154
crawlo/filters/aioredis_filter.py +280 -242
crawlo/filters/memory_filter.py +269 -269
crawlo/items/__init__.py +23 -23
crawlo/items/base.py +21 -21
crawlo/items/fields.py +53 -53
crawlo/items/items.py +104 -104
crawlo/middleware/__init__.py +21 -21
crawlo/middleware/default_header.py +32 -32
crawlo/middleware/download_delay.py +28 -28
crawlo/middleware/middleware_manager.py +135 -135
crawlo/middleware/proxy.py +272 -248
crawlo/middleware/request_ignore.py +30 -30
crawlo/middleware/response_code.py +18 -18
crawlo/middleware/response_filter.py +26 -26
crawlo/middleware/retry.py +124 -124
crawlo/mode_manager.py +206 -201
crawlo/network/__init__.py +21 -21
crawlo/network/request.py +338 -311
crawlo/network/response.py +360 -271
crawlo/pipelines/__init__.py +21 -21
crawlo/pipelines/bloom_dedup_pipeline.py +156 -156
crawlo/pipelines/console_pipeline.py +39 -39
crawlo/pipelines/csv_pipeline.py +316 -316
crawlo/pipelines/database_dedup_pipeline.py +224 -224
crawlo/pipelines/json_pipeline.py +218 -218
crawlo/pipelines/memory_dedup_pipeline.py +115 -115
crawlo/pipelines/mongo_pipeline.py +131 -131
crawlo/pipelines/mysql_pipeline.py +316 -316
crawlo/pipelines/pipeline_manager.py +56 -56
crawlo/pipelines/redis_dedup_pipeline.py +166 -162
crawlo/project.py +153 -153
crawlo/queue/pqueue.py +37 -37
crawlo/queue/queue_manager.py +320 -307
crawlo/queue/redis_priority_queue.py +277 -209
crawlo/settings/__init__.py +7 -7
crawlo/settings/default_settings.py +216 -278
crawlo/settings/setting_manager.py +99 -99
crawlo/spider/__init__.py +639 -639
crawlo/stats_collector.py +59 -59
crawlo/subscriber.py +130 -130
crawlo/task_manager.py +30 -30
crawlo/templates/crawlo.cfg.tmpl +10 -10
crawlo/templates/project/__init__.py.tmpl +3 -3
crawlo/templates/project/items.py.tmpl +17 -17
crawlo/templates/project/middlewares.py.tmpl +110 -110
crawlo/templates/project/pipelines.py.tmpl +97 -97
crawlo/templates/project/run.py.tmpl +251 -251
crawlo/templates/project/settings.py.tmpl +326 -279
crawlo/templates/project/settings_distributed.py.tmpl +120 -0
crawlo/templates/project/settings_gentle.py.tmpl +95 -0
crawlo/templates/project/settings_high_performance.py.tmpl +152 -0
crawlo/templates/project/settings_simple.py.tmpl +69 -0
crawlo/templates/project/spiders/__init__.py.tmpl +5 -5
crawlo/templates/spider/spider.py.tmpl +141 -141
crawlo/tools/__init__.py +183 -0
crawlo/tools/anti_crawler.py +269 -0
crawlo/tools/authenticated_proxy.py +241 -0
crawlo/tools/data_validator.py +181 -0
crawlo/tools/date_tools.py +36 -0
crawlo/tools/distributed_coordinator.py +387 -0
crawlo/tools/retry_mechanism.py +221 -0
crawlo/tools/scenario_adapter.py +263 -0
crawlo/utils/__init__.py +35 -7
crawlo/utils/batch_processor.py +261 -0
crawlo/utils/controlled_spider_mixin.py +439 -439
crawlo/utils/date_tools.py +290 -233
crawlo/utils/db_helper.py +343 -343
crawlo/utils/enhanced_error_handler.py +360 -0
crawlo/utils/env_config.py +106 -0
crawlo/utils/error_handler.py +126 -0
crawlo/utils/func_tools.py +82 -82
crawlo/utils/large_scale_config.py +286 -286
crawlo/utils/large_scale_helper.py +343 -343
crawlo/utils/log.py +128 -128
crawlo/utils/performance_monitor.py +285 -0
crawlo/utils/queue_helper.py +175 -175
crawlo/utils/redis_connection_pool.py +335 -0
crawlo/utils/redis_key_validator.py +200 -0
crawlo/utils/request.py +267 -267
crawlo/utils/request_serializer.py +219 -219
crawlo/utils/spider_loader.py +62 -62
crawlo/utils/system.py +11 -11
crawlo/utils/tools.py +4 -4
crawlo/utils/url.py +39 -39
{crawlo-1.1.4.dist-info → crawlo-1.1.5.dist-info}/METADATA +401 -403
crawlo-1.1.5.dist-info/RECORD +185 -0
examples/__init__.py +7 -7
tests/__init__.py +7 -7
tests/advanced_tools_example.py +276 -0
tests/authenticated_proxy_example.py +237 -0
tests/cleaners_example.py +161 -0
tests/config_validation_demo.py +103 -0
{examples → tests}/controlled_spider_example.py +205 -205
tests/date_tools_example.py +181 -0
tests/dynamic_loading_example.py +524 -0
tests/dynamic_loading_test.py +105 -0
tests/env_config_example.py +134 -0
tests/error_handling_example.py +172 -0
tests/redis_key_validation_demo.py +131 -0
tests/response_improvements_example.py +145 -0
tests/test_advanced_tools.py +149 -0
tests/test_all_redis_key_configs.py +146 -0
tests/test_authenticated_proxy.py +142 -0
tests/test_cleaners.py +55 -0
tests/test_comprehensive.py +147 -0
tests/test_config_validator.py +194 -0
tests/test_date_tools.py +124 -0
tests/test_dynamic_downloaders_proxy.py +125 -0
tests/test_dynamic_proxy.py +93 -0
tests/test_dynamic_proxy_config.py +147 -0
tests/test_dynamic_proxy_real.py +110 -0
tests/test_edge_cases.py +304 -0
tests/test_enhanced_error_handler.py +271 -0
tests/test_env_config.py +122 -0
tests/test_error_handler_compatibility.py +113 -0
tests/test_final_validation.py +153 -153
tests/test_framework_env_usage.py +104 -0
tests/test_integration.py +357 -0
tests/test_item_dedup_redis_key.py +123 -0
tests/test_parsel.py +30 -0
tests/test_performance.py +328 -0
tests/test_proxy_health_check.py +32 -32
tests/test_proxy_middleware_integration.py +136 -136
tests/test_proxy_providers.py +56 -56
tests/test_proxy_stats.py +19 -19
tests/test_proxy_strategies.py +59 -59
tests/test_queue_manager_redis_key.py +177 -0
tests/test_redis_config.py +28 -28
tests/test_redis_connection_pool.py +295 -0
tests/test_redis_key_naming.py +182 -0
tests/test_redis_key_validator.py +124 -0
tests/test_redis_queue.py +224 -224
tests/test_request_serialization.py +70 -70
tests/test_response_improvements.py +153 -0
tests/test_scheduler.py +241 -241
tests/test_simple_response.py +62 -0
tests/test_telecom_spider_redis_key.py +206 -0
tests/test_template_content.py +88 -0
tests/test_template_redis_key.py +135 -0
tests/test_tools.py +154 -0
tests/tools_example.py +258 -0
crawlo/core/enhanced_engine.py +0 -190
crawlo-1.1.4.dist-info/RECORD +0 -117
{crawlo-1.1.4.dist-info → crawlo-1.1.5.dist-info}/WHEEL +0 -0
{crawlo-1.1.4.dist-info → crawlo-1.1.5.dist-info}/entry_points.txt +0 -0
{crawlo-1.1.4.dist-info → crawlo-1.1.5.dist-info}/top_level.txt +0 -0

tests/test_response_improvements.py ADDED Viewed

@@ -0,0 +1,153 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response 改进功能测试
+"""
+import unittest
+from crawlo.network.response import Response
+class TestResponseImprovements(unittest.TestCase):
+    """Response 改进功能测试类"""
+    def setUp(self):
+        """测试前准备"""
+        # 创建一个模拟的HTML响应
+        html_content = """
+        <html>
+        <head>
+            <title>测试页面</title>
+        </head>
+        <body>
+            <div class="content">
+                <h1>主标题</h1>
+                <p class="intro">这是介绍段落</p>
+                <ul class="list">
+                    <li>项目1</li>
+                    <li>项目2</li>
+                    <li>项目3</li>
+                </ul>
+                <a href="https://example.com" class="link">链接文本</a>
+                <img src="image.jpg" alt="图片描述" class="image">
+            </div>
+        </body>
+        </html>
+        """
+        self.response = Response(
+            url="https://example.com/test",
+            body=html_content.encode('utf-8'),
+            headers={"content-type": "text/html; charset=utf-8"}
+        )
+    def test_extract_text_with_css(self):
+        """测试使用CSS选择器提取文本"""
+        # 测试提取单个元素文本
+        title = self.response.extract_text('title')
+        self.assertEqual(title, "测试页面")
+        # 测试提取class元素文本
+        h1_text = self.response.extract_text('.content h1')
+        self.assertEqual(h1_text, "主标题")
+        # 测试提取带有默认值的情况
+        non_exist = self.response.extract_text('.non-exist', default='默认值')
+        self.assertEqual(non_exist, '默认值')
+    def test_extract_text_with_xpath(self):
+        """测试使用XPath选择器提取文本"""
+        # 测试提取单个元素文本
+        title = self.response.extract_text('//title')
+        self.assertEqual(title, "测试页面")
+        # 测试提取class元素文本
+        h1_text = self.response.extract_text('//div[@class="content"]/h1')
+        self.assertEqual(h1_text, "主标题")
+    def test_extract_texts_with_css(self):
+        """测试使用CSS选择器提取多个文本"""
+        # 测试提取多个li元素的文本
+        list_items = self.response.extract_texts('.list li')
+        expected = ["项目1", "项目2", "项目3"]
+        self.assertEqual(list_items, expected)
+        # 测试提取不存在元素的默认值
+        non_exist = self.response.extract_texts('.non-exist', default=['默认值'])
+        self.assertEqual(non_exist, ['默认值'])
+    def test_extract_texts_with_xpath(self):
+        """测试使用XPath选择器提取多个文本"""
+        # 测试提取多个li元素的文本
+        list_items = self.response.extract_texts('//ul[@class="list"]/li')
+        expected = ["项目1", "项目2", "项目3"]
+        self.assertEqual(list_items, expected)
+    def test_extract_attr(self):
+        """测试提取元素属性"""
+        # 测试提取链接的href属性
+        link_href = self.response.extract_attr('.link', 'href')
+        self.assertEqual(link_href, "https://example.com")
+        # 测试提取图片的alt属性
+        img_alt = self.response.extract_attr('.image', 'alt')
+        self.assertEqual(img_alt, "图片描述")
+        # 测试提取不存在属性的默认值
+        non_exist = self.response.extract_attr('.link', 'non-exist', default='默认值')
+        self.assertEqual(non_exist, '默认值')
+    def test_extract_attrs(self):
+        """测试提取多个元素的属性"""
+        # 测试提取所有li元素的属性（这里我们测试class属性）
+        list_classes = self.response.extract_attrs('.list li', 'class')
+        # 注意：在当前HTML中li元素没有class属性，所以应该返回空列表
+        self.assertEqual(list_classes, [])
+        # 测试提取所有图片元素的alt属性
+        img_alts = self.response.extract_attrs('.image', 'alt')
+        self.assertEqual(img_alts, ['图片描述'])
+        # 测试提取不存在元素时的默认值
+        non_exist = self.response.extract_attrs('.non-exist-elements', 'alt', default=['默认值'])
+        self.assertEqual(non_exist, ['默认值'])
+    def test_extract_text_from_elements(self):
+        """测试从复杂元素中提取文本"""
+        # 创建包含嵌套标签的HTML
+        complex_html = """
+        <div class="complex">
+            <p>段落文本 <strong>粗体文本</strong> 普通文本</p>
+            <p>第二段落 <em>斜体文本</em></p>
+        </div>
+        """
+        complex_response = Response(
+            url="https://example.com/complex",
+            body=complex_html.encode('utf-8')
+        )
+        # 测试提取复杂元素的文本
+        complex_text = complex_response.extract_text('.complex p', join_str=' ')
+        self.assertIn("段落文本", complex_text)
+        self.assertIn("粗体文本", complex_text)
+        self.assertIn("普通文本", complex_text)
+    def test_edge_cases(self):
+        """测试边界情况"""
+        # 测试空响应
+        empty_response = Response(url="https://example.com/empty", body=b"")
+        empty_text = empty_response.extract_text('title', default='默认标题')
+        self.assertEqual(empty_text, '默认标题')
+        # 测试只包含空白字符的元素
+        whitespace_html = "<div class='whitespace'>   </div>"
+        whitespace_response = Response(
+            url="https://example.com/whitespace",
+            body=whitespace_html.encode('utf-8')
+        )
+        whitespace_text = whitespace_response.extract_text('.whitespace')
+        self.assertEqual(whitespace_text, '')
+if __name__ == '__main__':
+    unittest.main()

tests/test_scheduler.py CHANGED Viewed

@@ -1,242 +1,242 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-测试修复后的 Scheduler 分布式队列功能
-"""
-import asyncio
-import sys
-from unittest.mock import Mock
-from crawlo.core.scheduler import Scheduler
-from crawlo.network.request import Request
-from crawlo.utils.log import get_logger
-class MockCrawler:
-    """模拟 Crawler 对象"""
-    def __init__(self, use_redis=True):
-        self.settings = MockSettings(use_redis)
-        self.stats = Mock()
-class MockSettings:
-    """模拟 Settings 对象"""
-    def __init__(self, use_redis=True):
-        self.use_redis = use_redis
-    def get(self, key, default=None):
-        config = {
-            'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',
-            'LOG_LEVEL': 'INFO',
-            'DEPTH_PRIORITY': 1,
-            'SCHEDULER_MAX_QUEUE_SIZE': 100,
-            'SCHEDULER_QUEUE_NAME': 'test:crawlo:requests',
-            'FILTER_DEBUG': False,
-            'PROJECT_NAME': 'test',
-        }
-        if self.use_redis:
-            config['REDIS_URL'] = 'redis://localhost:6379/0'
-        return config.get(key, default)
-    def get_int(self, key, default=0):
-        value = self.get(key, default)
-        return int(value) if value is not None else default
-    def get_bool(self, key, default=False):
-        value = self.get(key, default)
-        if isinstance(value, bool):
-            return value
-        if isinstance(value, str):
-            return value.lower() in ('true', '1', 'yes')
-        return bool(value) if value is not None else default
-class MockFilter:
-    """模拟去重过滤器"""
-    def __init__(self):
-        self.seen = set()
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls()
-    async def requested(self, request):
-        if request.url in self.seen:
-            return True
-        self.seen.add(request.url)
-        return False
-    def log_stats(self, request):
-        pass
-async def test_memory_scheduler():
-    """测试内存调度器"""
-    print("🔍 测试内存调度器...")
-    crawler = MockCrawler(use_redis=False)
-    scheduler = Scheduler.create_instance(crawler)
-    # 模拟去重过滤器
-    scheduler.dupe_filter = MockFilter()
-    scheduler.open()
-    # 测试入队
-    request1 = Request(url="https://example1.com")
-    request2 = Request(url="https://example2.com")
-    success1 = await scheduler.enqueue_request(request1)
-    success2 = await scheduler.enqueue_request(request2)
-    print(f"   📤 入队结果: {success1}, {success2}")
-    print(f"   📊 队列大小: {len(scheduler)}")
-    # 测试出队
-    req1 = await scheduler.next_request()
-    req2 = await scheduler.next_request()
-    print(f"   📥 出队结果: {req1.url if req1 else None}, {req2.url if req2 else None}")
-    print(f"   📊 剩余大小: {len(scheduler)}")
-    await scheduler.close()
-    print("   ✅ 内存调度器测试完成")
-async def test_redis_scheduler():
-    """测试 Redis 调度器"""
-    print("🔍 测试 Redis 调度器...")
-    try:
-        crawler = MockCrawler(use_redis=True)
-        scheduler = Scheduler.create_instance(crawler)
-        # 模拟去重过滤器
-        scheduler.dupe_filter = MockFilter()
-        scheduler.open()
-        # 测试入队
-        request1 = Request(url="https://redis-test1.com", priority=5)
-        request2 = Request(url="https://redis-test2.com", priority=3)
-        request3 = Request(url="https://redis-test3.com", priority=8)
-        success1 = await scheduler.enqueue_request(request1)
-        success2 = await scheduler.enqueue_request(request2)
-        success3 = await scheduler.enqueue_request(request3)
-        print(f"   📤 入队结果: {success1}, {success2}, {success3}")
-        print(f"   📊 队列大小: {len(scheduler)}")
-        # 等待一小段时间让 Redis 操作完成
-        await asyncio.sleep(0.5)
-        # 测试出队（应该按优先级排序）
-        req1 = await scheduler.next_request()
-        req2 = await scheduler.next_request()
-        req3 = await scheduler.next_request()
-        print("   📥 出队结果（按优先级）:")
-        if req1:
-            print(f"      {req1.url} (优先级: {getattr(req1, 'priority', 0)})")
-        if req2:
-            print(f"      {req2.url} (优先级: {getattr(req2, 'priority', 0)})")
-        if req3:
-            print(f"      {req3.url} (优先级: {getattr(req3, 'priority', 0)})")
-        print(f"   📊 剩余大小: {len(scheduler)}")
-        await scheduler.close()
-        print("   ✅ Redis 调度器测试完成")
-    except Exception as e:
-        print(f"   ❌ Redis 调度器测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-async def test_concurrent_redis():
-    """测试并发 Redis 操作"""
-    print("🔍 测试并发 Redis 操作...")
-    async def producer(scheduler, name, count):
-        """生产者"""
-        for i in range(count):
-            request = Request(url=f"https://{name}-{i}.com", priority=i % 10)
-            await scheduler.enqueue_request(request)
-            await asyncio.sleep(0.01)
-        print(f"   ✅ 生产者 {name} 完成 ({count} 个请求)")
-    async def consumer(scheduler, name, count):
-        """消费者"""
-        consumed = 0
-        for _ in range(count):
-            request = await scheduler.next_request()
-            if request:
-                consumed += 1
-                await asyncio.sleep(0.005)
-            else:
-                break
-        print(f"   ✅ 消费者 {name} 处理了 {consumed} 个请求")
-    try:
-        crawler = MockCrawler(use_redis=True)
-        scheduler = Scheduler.create_instance(crawler)
-        scheduler.dupe_filter = MockFilter()
-        scheduler.open()
-        # 并发运行生产者和消费者
-        tasks = [
-            producer(scheduler, "producer-1", 5),
-            producer(scheduler, "producer-2", 5),
-            consumer(scheduler, "consumer-1", 3),
-            consumer(scheduler, "consumer-2", 3),
-            consumer(scheduler, "consumer-3", 4),
-        ]
-        await asyncio.gather(*tasks, return_exceptions=True)
-        print(f"   📊 最终队列大小: {len(scheduler)}")
-        await scheduler.close()
-        print("   ✅ 并发测试完成")
-    except Exception as e:
-        print(f"   ❌ 并发测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-async def main():
-    """主测试函数"""
-    print("🚀 开始测试修复后的 Scheduler...")
-    print("=" * 50)
-    try:
-        # 1. 测试内存调度器
-        await test_memory_scheduler()
-        print()
-        # 2. 测试 Redis 调度器
-        await test_redis_scheduler()
-        print()
-        # 3. 测试并发操作
-        await test_concurrent_redis()
-        print("=" * 50)
-        print("🎉 所有 Scheduler 测试完成！")
-    except Exception as e:
-        print("=" * 50)
-        print(f"❌ 测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-if __name__ == "__main__":
-    # 设置日志级别避免过多输出
-    import logging
-    logging.getLogger('crawlo').setLevel(logging.WARNING)
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试修复后的 Scheduler 分布式队列功能
+"""
+import asyncio
+import sys
+from unittest.mock import Mock
+from crawlo.core.scheduler import Scheduler
+from crawlo.network.request import Request
+from crawlo.utils.log import get_logger
+class MockCrawler:
+    """模拟 Crawler 对象"""
+    def __init__(self, use_redis=True):
+        self.settings = MockSettings(use_redis)
+        self.stats = Mock()
+class MockSettings:
+    """模拟 Settings 对象"""
+    def __init__(self, use_redis=True):
+        self.use_redis = use_redis
+    def get(self, key, default=None):
+        config = {
+            'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',
+            'LOG_LEVEL': 'INFO',
+            'DEPTH_PRIORITY': 1,
+            'SCHEDULER_MAX_QUEUE_SIZE': 100,
+            'SCHEDULER_QUEUE_NAME': 'test:crawlo:requests',
+            'FILTER_DEBUG': False,
+            'PROJECT_NAME': 'test',
+        }
+        if self.use_redis:
+            config['REDIS_URL'] = 'redis://localhost:6379/0'
+        return config.get(key, default)
+    def get_int(self, key, default=0):
+        value = self.get(key, default)
+        return int(value) if value is not None else default
+    def get_bool(self, key, default=False):
+        value = self.get(key, default)
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            return value.lower() in ('true', '1', 'yes')
+        return bool(value) if value is not None else default
+class MockFilter:
+    """模拟去重过滤器"""
+    def __init__(self):
+        self.seen = set()
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls()
+    async def requested(self, request):
+        if request.url in self.seen:
+            return True
+        self.seen.add(request.url)
+        return False
+    def log_stats(self, request):
+        pass
+async def test_memory_scheduler():
+    """测试内存调度器"""
+    print("🔍 测试内存调度器...")
+    crawler = MockCrawler(use_redis=False)
+    scheduler = Scheduler.create_instance(crawler)
+    # 模拟去重过滤器
+    scheduler.dupe_filter = MockFilter()
+    scheduler.open()
+    # 测试入队
+    request1 = Request(url="https://example1.com")
+    request2 = Request(url="https://example2.com")
+    success1 = await scheduler.enqueue_request(request1)
+    success2 = await scheduler.enqueue_request(request2)
+    print(f"   📤 入队结果: {success1}, {success2}")
+    print(f"   📊 队列大小: {len(scheduler)}")
+    # 测试出队
+    req1 = await scheduler.next_request()
+    req2 = await scheduler.next_request()
+    print(f"   📥 出队结果: {req1.url if req1 else None}, {req2.url if req2 else None}")
+    print(f"   📊 剩余大小: {len(scheduler)}")
+    await scheduler.close()
+    print("   ✅ 内存调度器测试完成")
+async def test_redis_scheduler():
+    """测试 Redis 调度器"""
+    print("🔍 测试 Redis 调度器...")
+    try:
+        crawler = MockCrawler(use_redis=True)
+        scheduler = Scheduler.create_instance(crawler)
+        # 模拟去重过滤器
+        scheduler.dupe_filter = MockFilter()
+        scheduler.open()
+        # 测试入队
+        request1 = Request(url="https://redis-test1.com", priority=5)
+        request2 = Request(url="https://redis-test2.com", priority=3)
+        request3 = Request(url="https://redis-test3.com", priority=8)
+        success1 = await scheduler.enqueue_request(request1)
+        success2 = await scheduler.enqueue_request(request2)
+        success3 = await scheduler.enqueue_request(request3)
+        print(f"   📤 入队结果: {success1}, {success2}, {success3}")
+        print(f"   📊 队列大小: {len(scheduler)}")
+        # 等待一小段时间让 Redis 操作完成
+        await asyncio.sleep(0.5)
+        # 测试出队（应该按优先级排序）
+        req1 = await scheduler.next_request()
+        req2 = await scheduler.next_request()
+        req3 = await scheduler.next_request()
+        print("   📥 出队结果（按优先级）:")
+        if req1:
+            print(f"      {req1.url} (优先级: {getattr(req1, 'priority', 0)})")
+        if req2:
+            print(f"      {req2.url} (优先级: {getattr(req2, 'priority', 0)})")
+        if req3:
+            print(f"      {req3.url} (优先级: {getattr(req3, 'priority', 0)})")
+        print(f"   📊 剩余大小: {len(scheduler)}")
+        await scheduler.close()
+        print("   ✅ Redis 调度器测试完成")
+    except Exception as e:
+        print(f"   ❌ Redis 调度器测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+async def test_concurrent_redis():
+    """测试并发 Redis 操作"""
+    print("🔍 测试并发 Redis 操作...")
+    async def producer(scheduler, name, count):
+        """生产者"""
+        for i in range(count):
+            request = Request(url=f"https://{name}-{i}.com", priority=i % 10)
+            await scheduler.enqueue_request(request)
+            await asyncio.sleep(0.01)
+        print(f"   ✅ 生产者 {name} 完成 ({count} 个请求)")
+    async def consumer(scheduler, name, count):
+        """消费者"""
+        consumed = 0
+        for _ in range(count):
+            request = await scheduler.next_request()
+            if request:
+                consumed += 1
+                await asyncio.sleep(0.005)
+            else:
+                break
+        print(f"   ✅ 消费者 {name} 处理了 {consumed} 个请求")
+    try:
+        crawler = MockCrawler(use_redis=True)
+        scheduler = Scheduler.create_instance(crawler)
+        scheduler.dupe_filter = MockFilter()
+        scheduler.open()
+        # 并发运行生产者和消费者
+        tasks = [
+            producer(scheduler, "producer-1", 5),
+            producer(scheduler, "producer-2", 5),
+            consumer(scheduler, "consumer-1", 3),
+            consumer(scheduler, "consumer-2", 3),
+            consumer(scheduler, "consumer-3", 4),
+        ]
+        await asyncio.gather(*tasks, return_exceptions=True)
+        print(f"   📊 最终队列大小: {len(scheduler)}")
+        await scheduler.close()
+        print("   ✅ 并发测试完成")
+    except Exception as e:
+        print(f"   ❌ 并发测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+async def main():
+    """主测试函数"""
+    print("🚀 开始测试修复后的 Scheduler...")
+    print("=" * 50)
+    try:
+        # 1. 测试内存调度器
+        await test_memory_scheduler()
+        print()
+        # 2. 测试 Redis 调度器
+        await test_redis_scheduler()
+        print()
+        # 3. 测试并发操作
+        await test_concurrent_redis()
+        print("=" * 50)
+        print("🎉 所有 Scheduler 测试完成！")
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+if __name__ == "__main__":
+    # 设置日志级别避免过多输出
+    import logging
+    logging.getLogger('crawlo').setLevel(logging.WARNING)
     asyncio.run(main())

crawlo 1.1.4__py3-none-any.whl → 1.1.5__py3-none-any.whl

Potentially problematic release.

crawlo 1.1.4py3-none-any.whl → 1.1.5py3-none-any.whl