crawlo 1.1.2__py3-none-any.whl → 1.1.3__py3-none-any.whl

tests/test_scheduler.py

@@ -1,242 +1,242 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-测试修复后的 Scheduler 分布式队列功能
-"""
-import asyncio
-import sys
-from unittest.mock import Mock
-from crawlo.core.scheduler import Scheduler
-from crawlo.network.request import Request
-from crawlo.utils.log import get_logger
-
-
-class MockCrawler:
-    """模拟 Crawler 对象"""
-    def __init__(self, use_redis=True):
-        self.settings = MockSettings(use_redis)
-        self.stats = Mock()
-
-
-class MockSettings:
-    """模拟 Settings 对象"""
-    def __init__(self, use_redis=True):
-        self.use_redis = use_redis
-        
-    def get(self, key, default=None):
-        config = {
-            'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',
-            'LOG_LEVEL': 'INFO',
-            'DEPTH_PRIORITY': 1,
-            'SCHEDULER_MAX_QUEUE_SIZE': 100,
-            'SCHEDULER_QUEUE_NAME': 'test:crawlo:requests',
-            'FILTER_DEBUG': False,
-            'PROJECT_NAME': 'test',
-        }
-        if self.use_redis:
-            config['REDIS_URL'] = 'redis://localhost:6379/0'
-        
-        return config.get(key, default)
-    
-    def get_int(self, key, default=0):
-        value = self.get(key, default)
-        return int(value) if value is not None else default
-        
-    def get_bool(self, key, default=False):
-        value = self.get(key, default)
-        if isinstance(value, bool):
-            return value
-        if isinstance(value, str):
-            return value.lower() in ('true', '1', 'yes')
-        return bool(value) if value is not None else default
-
-
-class MockFilter:
-    """模拟去重过滤器"""
-    def __init__(self):
-        self.seen = set()
-        
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls()
-    
-    async def requested(self, request):
-        if request.url in self.seen:
-            return True
-        self.seen.add(request.url)
-        return False
-    
-    def log_stats(self, request):
-        pass
-
-
-async def test_memory_scheduler():
-    """测试内存调度器"""
-    print("🔍 测试内存调度器...")
-    
-    crawler = MockCrawler(use_redis=False)
-    scheduler = Scheduler.create_instance(crawler)
-    
-    # 模拟去重过滤器
-    scheduler.dupe_filter = MockFilter()
-    
-    scheduler.open()
-    
-    # 测试入队
-    request1 = Request(url="https://example1.com")
-    request2 = Request(url="https://example2.com")
-    
-    success1 = await scheduler.enqueue_request(request1)
-    success2 = await scheduler.enqueue_request(request2)
-    
-    print(f"   📤 入队结果: {success1}, {success2}")
-    print(f"   📊 队列大小: {len(scheduler)}")
-    
-    # 测试出队
-    req1 = await scheduler.next_request()
-    req2 = await scheduler.next_request()
-    
-    print(f"   📥 出队结果: {req1.url if req1 else None}, {req2.url if req2 else None}")
-    print(f"   📊 剩余大小: {len(scheduler)}")
-    
-    await scheduler.close()
-    print("   ✅ 内存调度器测试完成")
-
-
-async def test_redis_scheduler():
-    """测试 Redis 调度器"""
-    print("🔍 测试 Redis 调度器...")
-    
-    try:
-        crawler = MockCrawler(use_redis=True)
-        scheduler = Scheduler.create_instance(crawler)
-        
-        # 模拟去重过滤器
-        scheduler.dupe_filter = MockFilter()
-        
-        scheduler.open()
-        
-        # 测试入队
-        request1 = Request(url="https://redis-test1.com", priority=5)
-        request2 = Request(url="https://redis-test2.com", priority=3)
-        request3 = Request(url="https://redis-test3.com", priority=8)
-        
-        success1 = await scheduler.enqueue_request(request1)
-        success2 = await scheduler.enqueue_request(request2)
-        success3 = await scheduler.enqueue_request(request3)
-        
-        print(f"   📤 入队结果: {success1}, {success2}, {success3}")
-        print(f"   📊 队列大小: {len(scheduler)}")
-        
-        # 等待一小段时间让 Redis 操作完成
-        await asyncio.sleep(0.5)
-        
-        # 测试出队（应该按优先级排序）
-        req1 = await scheduler.next_request()
-        req2 = await scheduler.next_request()
-        req3 = await scheduler.next_request()
-        
-        print("   📥 出队结果（按优先级）:")
-        if req1:
-            print(f"      {req1.url} (优先级: {getattr(req1, 'priority', 0)})")
-        if req2:
-            print(f"      {req2.url} (优先级: {getattr(req2, 'priority', 0)})")
-        if req3:
-            print(f"      {req3.url} (优先级: {getattr(req3, 'priority', 0)})")
-            
-        print(f"   📊 剩余大小: {len(scheduler)}")
-        
-        await scheduler.close()
-        print("   ✅ Redis 调度器测试完成")
-        
-    except Exception as e:
-        print(f"   ❌ Redis 调度器测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-
-
-async def test_concurrent_redis():
-    """测试并发 Redis 操作"""
-    print("🔍 测试并发 Redis 操作...")
-    
-    async def producer(scheduler, name, count):
-        """生产者"""
-        for i in range(count):
-            request = Request(url=f"https://{name}-{i}.com", priority=i % 10)
-            await scheduler.enqueue_request(request)
-            await asyncio.sleep(0.01)
-        print(f"   ✅ 生产者 {name} 完成 ({count} 个请求)")
-    
-    async def consumer(scheduler, name, count):
-        """消费者"""
-        consumed = 0
-        for _ in range(count):
-            request = await scheduler.next_request()
-            if request:
-                consumed += 1
-                await asyncio.sleep(0.005)
-            else:
-                break
-        print(f"   ✅ 消费者 {name} 处理了 {consumed} 个请求")
-    
-    try:
-        crawler = MockCrawler(use_redis=True)
-        scheduler = Scheduler.create_instance(crawler)
-        scheduler.dupe_filter = MockFilter()
-        scheduler.open()
-        
-        # 并发运行生产者和消费者
-        tasks = [
-            producer(scheduler, "producer-1", 5),
-            producer(scheduler, "producer-2", 5),
-            consumer(scheduler, "consumer-1", 3),
-            consumer(scheduler, "consumer-2", 3),
-            consumer(scheduler, "consumer-3", 4),
-        ]
-        
-        await asyncio.gather(*tasks, return_exceptions=True)
-        
-        print(f"   📊 最终队列大小: {len(scheduler)}")
-        
-        await scheduler.close()
-        print("   ✅ 并发测试完成")
-        
-    except Exception as e:
-        print(f"   ❌ 并发测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-
-
-async def main():
-    """主测试函数"""
-    print("🚀 开始测试修复后的 Scheduler...")
-    print("=" * 50)
-    
-    try:
-        # 1. 测试内存调度器
-        await test_memory_scheduler()
-        print()
-        
-        # 2. 测试 Redis 调度器
-        await test_redis_scheduler()
-        print()
-        
-        # 3. 测试并发操作
-        await test_concurrent_redis()
-        
-        print("=" * 50)
-        print("🎉 所有 Scheduler 测试完成！")
-        
-    except Exception as e:
-        print("=" * 50)
-        print(f"❌ 测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-
-
-if __name__ == "__main__":
-    # 设置日志级别避免过多输出
-    import logging
-    logging.getLogger('crawlo').setLevel(logging.WARNING)
-    
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试修复后的 Scheduler 分布式队列功能
+"""
+import asyncio
+import sys
+from unittest.mock import Mock
+from crawlo.core.scheduler import Scheduler
+from crawlo.network.request import Request
+from crawlo.utils.log import get_logger
+
+
+class MockCrawler:
+    """模拟 Crawler 对象"""
+    def __init__(self, use_redis=True):
+        self.settings = MockSettings(use_redis)
+        self.stats = Mock()
+
+
+class MockSettings:
+    """模拟 Settings 对象"""
+    def __init__(self, use_redis=True):
+        self.use_redis = use_redis
+        
+    def get(self, key, default=None):
+        config = {
+            'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',
+            'LOG_LEVEL': 'INFO',
+            'DEPTH_PRIORITY': 1,
+            'SCHEDULER_MAX_QUEUE_SIZE': 100,
+            'SCHEDULER_QUEUE_NAME': 'test:crawlo:requests',
+            'FILTER_DEBUG': False,
+            'PROJECT_NAME': 'test',
+        }
+        if self.use_redis:
+            config['REDIS_URL'] = 'redis://localhost:6379/0'
+        
+        return config.get(key, default)
+    
+    def get_int(self, key, default=0):
+        value = self.get(key, default)
+        return int(value) if value is not None else default
+        
+    def get_bool(self, key, default=False):
+        value = self.get(key, default)
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            return value.lower() in ('true', '1', 'yes')
+        return bool(value) if value is not None else default
+
+
+class MockFilter:
+    """模拟去重过滤器"""
+    def __init__(self):
+        self.seen = set()
+        
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls()
+    
+    async def requested(self, request):
+        if request.url in self.seen:
+            return True
+        self.seen.add(request.url)
+        return False
+    
+    def log_stats(self, request):
+        pass
+
+
+async def test_memory_scheduler():
+    """测试内存调度器"""
+    print("🔍 测试内存调度器...")
+    
+    crawler = MockCrawler(use_redis=False)
+    scheduler = Scheduler.create_instance(crawler)
+    
+    # 模拟去重过滤器
+    scheduler.dupe_filter = MockFilter()
+    
+    scheduler.open()
+    
+    # 测试入队
+    request1 = Request(url="https://example1.com")
+    request2 = Request(url="https://example2.com")
+    
+    success1 = await scheduler.enqueue_request(request1)
+    success2 = await scheduler.enqueue_request(request2)
+    
+    print(f"   📤 入队结果: {success1}, {success2}")
+    print(f"   📊 队列大小: {len(scheduler)}")
+    
+    # 测试出队
+    req1 = await scheduler.next_request()
+    req2 = await scheduler.next_request()
+    
+    print(f"   📥 出队结果: {req1.url if req1 else None}, {req2.url if req2 else None}")
+    print(f"   📊 剩余大小: {len(scheduler)}")
+    
+    await scheduler.close()
+    print("   ✅ 内存调度器测试完成")
+
+
+async def test_redis_scheduler():
+    """测试 Redis 调度器"""
+    print("🔍 测试 Redis 调度器...")
+    
+    try:
+        crawler = MockCrawler(use_redis=True)
+        scheduler = Scheduler.create_instance(crawler)
+        
+        # 模拟去重过滤器
+        scheduler.dupe_filter = MockFilter()
+        
+        scheduler.open()
+        
+        # 测试入队
+        request1 = Request(url="https://redis-test1.com", priority=5)
+        request2 = Request(url="https://redis-test2.com", priority=3)
+        request3 = Request(url="https://redis-test3.com", priority=8)
+        
+        success1 = await scheduler.enqueue_request(request1)
+        success2 = await scheduler.enqueue_request(request2)
+        success3 = await scheduler.enqueue_request(request3)
+        
+        print(f"   📤 入队结果: {success1}, {success2}, {success3}")
+        print(f"   📊 队列大小: {len(scheduler)}")
+        
+        # 等待一小段时间让 Redis 操作完成
+        await asyncio.sleep(0.5)
+        
+        # 测试出队（应该按优先级排序）
+        req1 = await scheduler.next_request()
+        req2 = await scheduler.next_request()
+        req3 = await scheduler.next_request()
+        
+        print("   📥 出队结果（按优先级）:")
+        if req1:
+            print(f"      {req1.url} (优先级: {getattr(req1, 'priority', 0)})")
+        if req2:
+            print(f"      {req2.url} (优先级: {getattr(req2, 'priority', 0)})")
+        if req3:
+            print(f"      {req3.url} (优先级: {getattr(req3, 'priority', 0)})")
+            
+        print(f"   📊 剩余大小: {len(scheduler)}")
+        
+        await scheduler.close()
+        print("   ✅ Redis 调度器测试完成")
+        
+    except Exception as e:
+        print(f"   ❌ Redis 调度器测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+async def test_concurrent_redis():
+    """测试并发 Redis 操作"""
+    print("🔍 测试并发 Redis 操作...")
+    
+    async def producer(scheduler, name, count):
+        """生产者"""
+        for i in range(count):
+            request = Request(url=f"https://{name}-{i}.com", priority=i % 10)
+            await scheduler.enqueue_request(request)
+            await asyncio.sleep(0.01)
+        print(f"   ✅ 生产者 {name} 完成 ({count} 个请求)")
+    
+    async def consumer(scheduler, name, count):
+        """消费者"""
+        consumed = 0
+        for _ in range(count):
+            request = await scheduler.next_request()
+            if request:
+                consumed += 1
+                await asyncio.sleep(0.005)
+            else:
+                break
+        print(f"   ✅ 消费者 {name} 处理了 {consumed} 个请求")
+    
+    try:
+        crawler = MockCrawler(use_redis=True)
+        scheduler = Scheduler.create_instance(crawler)
+        scheduler.dupe_filter = MockFilter()
+        scheduler.open()
+        
+        # 并发运行生产者和消费者
+        tasks = [
+            producer(scheduler, "producer-1", 5),
+            producer(scheduler, "producer-2", 5),
+            consumer(scheduler, "consumer-1", 3),
+            consumer(scheduler, "consumer-2", 3),
+            consumer(scheduler, "consumer-3", 4),
+        ]
+        
+        await asyncio.gather(*tasks, return_exceptions=True)
+        
+        print(f"   📊 最终队列大小: {len(scheduler)}")
+        
+        await scheduler.close()
+        print("   ✅ 并发测试完成")
+        
+    except Exception as e:
+        print(f"   ❌ 并发测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+async def main():
+    """主测试函数"""
+    print("🚀 开始测试修复后的 Scheduler...")
+    print("=" * 50)
+    
+    try:
+        # 1. 测试内存调度器
+        await test_memory_scheduler()
+        print()
+        
+        # 2. 测试 Redis 调度器
+        await test_redis_scheduler()
+        print()
+        
+        # 3. 测试并发操作
+        await test_concurrent_redis()
+        
+        print("=" * 50)
+        print("🎉 所有 Scheduler 测试完成！")
+        
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    # 设置日志级别避免过多输出
+    import logging
+    logging.getLogger('crawlo').setLevel(logging.WARNING)
+    
     asyncio.run(main())

crawlo-1.1.2.dist-info/RECORD

@@ -1,108 +0,0 @@
-crawlo/__init__.py,sha256=esOolburYDjtF43D5N9Kh6TSQW2yKcz888ilhBSinBc,825
-crawlo/__version__.py,sha256=5SgGjThsHu_ITn8V83BvCziqCwxdXxTQqcC3KQMHPfM,22
-crawlo/cli.py,sha256=CtR2Pfa7SyRxEKPaXqt-6E6K5Vq5z3rfdAI95UO4cbU,1166
-crawlo/config.py,sha256=i0Amz6wNPgv_aVcdCBRRlcwuZLSa87cH9OEmTQvB97Q,8329
-crawlo/crawler.py,sha256=v6i5tjgSOtbMoqiw1qdgKx1cY4kcVcd5l5bUTWtJNNU,36461
-crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
-crawlo/exceptions.py,sha256=pthF1lJlJHyRZm-mE6NAo5WzK3GYJqmRqIuIlK1Odx8,1129
-crawlo/mode_manager.py,sha256=WIxrq9S3EAH0D71LH1AxvcqXomeABqoXgtUN4A--DKY,6702
-crawlo/project.py,sha256=xWN2eTAjf_Pza-wWvvV4JjScQRWxe9hXlztX81ccUMc,5182
-crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
-crawlo/subscriber.py,sha256=3d4eYtkSgPj-18-mTZM6RQLSil-ux13FUcmfFxr3sMk,3730
-crawlo/task_manager.py,sha256=19InAxS9oJ9EMj20Aw2urN-v6BeC22dkgPuW-B9-4UI,819
-crawlo/commands/__init__.py,sha256=AMYjXG7ulE8dPVmgWVo0uqXsaCYUUZYmmu2-7kFzH1M,342
-crawlo/commands/check.py,sha256=172OiAxnX5wwSlszUsyPgMZwAoIbGDTdfhtRz309ilc,22843
-crawlo/commands/genspider.py,sha256=-jGJdfXLsefX_H1ydQ2wirdu6p6wmhClzVXY_0L-1aE,5050
-crawlo/commands/list.py,sha256=yByqQeZBgvjewOKxpnOobpeJ7Hnbs-CWsoyITqZu2ZY,5781
-crawlo/commands/run.py,sha256=8Qngjsl8Q4RBdO39a__wKGsheY2PFuPit2hds_jwEbM,10524
-crawlo/commands/startproject.py,sha256=bzNgpkKzUEggY2m7Iw810mSPe8wOPFBqSCO0jZX3z_g,7138
-crawlo/commands/stats.py,sha256=6pAgkEi8MBnCer2rWmKpaTYr1jaM6HeMG9owAvEzJyY,6064
-crawlo/commands/utils.py,sha256=nohMvUU2zLvX0XzXk6KeCNxP0EvSWj9DiVLxM_7tD5o,5106
-crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
-crawlo/core/engine.py,sha256=8Dcew1XyxChW5Fz1wFEWpJlPrQb2hKDWKul8e61S-Q0,6662
-crawlo/core/enhanced_engine.py,sha256=9I9Uxdy2oAz8zDGTzEiytuKu__VDVmIN8zwZKfrD8bw,6254
-crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
-crawlo/core/scheduler.py,sha256=iNZl47PvurnLLXRZN8Jfj1un6YP2q9m5n56X3Eb7_A8,5615
-crawlo/downloader/__init__.py,sha256=tl0mE54reR-PuJYSsXsKP2VY5uzvq4lITxZwKKjNzPs,7663
-crawlo/downloader/aiohttp_downloader.py,sha256=UKupGYPOWrscAVsjhFgKYElTa9tbEeltqV7nuWqjIeE,8005
-crawlo/downloader/cffi_downloader.py,sha256=-GVfSIhi1Ip56suSiGf8jnUE2EBF1P56vw0uxLh_T6I,10440
-crawlo/downloader/httpx_downloader.py,sha256=FJcdE3BoxUYFQDPm3quaveSjweAKfAQRaeJm1hQViLg,12003
-crawlo/extension/__init__.py,sha256=PEBbMxi6ULBZ9ivEJ4T7IH_R376BQJ6Rz_D9Ce-Cqbs,1133
-crawlo/extension/log_interval.py,sha256=S-hSoiz9GdmgHrac4vDQ52fleoBcH-kzdPUD8YRAons,1922
-crawlo/extension/log_stats.py,sha256=VinABjzuFa-JmNXSX86gQ_3Oyx1y3vUhpWTEu9HRRqg,1677
-crawlo/extension/logging_extension.py,sha256=Jce2HHc9ejsIWQitkHZuhnLbT_1MIfcvywfzYkkJ5eY,1202
-crawlo/filters/__init__.py,sha256=XC_Q4ykZtSNYizYlAcehVwBBNO3lZ2zuWwafzXiuWyQ,4241
-crawlo/filters/aioredis_filter.py,sha256=3vbPOY3L_H7TUgL8K6AtaP6QDtoIg5xpE70Kxpil6dM,8376
-crawlo/filters/memory_filter.py,sha256=VJO0UFRYGxmV8dj4G1subsQ-FtvPcGLbvd7IVtqXnOs,9260
-crawlo/items/__init__.py,sha256=bqekZrRlDhxfWie0UbCs656TptYseoe9QJ67I4E7Elk,386
-crawlo/items/base.py,sha256=tAYrPJgblp3ZEihDXvappdYc6pGdim6x2_9QSmMKI2o,577
-crawlo/items/fields.py,sha256=wMlakQTsEwyrlLzMt1gI4pScLQZMqd3E1xcfH4dbSqk,1801
-crawlo/items/items.py,sha256=e-3nXI9ckD64vcDxxQiAU6ufbtJMs09gbZQcYjxgwHY,3374
-crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0,549
-crawlo/middleware/default_header.py,sha256=i_Uj07JObyeZFxL7ZAZmvZsHvA1HGtkNab1sA0d-nWI,1067
-crawlo/middleware/download_delay.py,sha256=2M-TchDA7MwyTfYy0Hzh_bW9wlHlpiP-oQlys7crTj0,966
-crawlo/middleware/middleware_manager.py,sha256=j1hkWRFB5rnC5SnB7oXWE5eUNv8blS9krDIDM5fIDs8,6213
-crawlo/middleware/proxy.py,sha256=m2ZZ50En9hUtgrqSqA6hItGT74xMqccHFPhZshutIco,9811
-crawlo/middleware/request_ignore.py,sha256=QI2z4fUnJ-4xvPTZAmsL-GqR4RFHS1xq9iDr5KFrMco,997
-crawlo/middleware/response_code.py,sha256=tmef2QVl3JCiTMii6VQkASlOY2OyqmOPoOfNxIK1eF8,659
-crawlo/middleware/response_filter.py,sha256=ep8ZxDlfIefi9YqK8dPASEp5TTDRo9QEY_jMceC411s,837
-crawlo/middleware/retry.py,sha256=pH4fdW71DwQzMY7APutX7yqtvSTAoznYeM3ux2EOT7E,4146
-crawlo/network/__init__.py,sha256=BLPERYPo22g1BXrW--wUnlolrdFUmOPjgOB8XQQJlck,397
-crawlo/network/request.py,sha256=5bzXJJWcAgGngNos-k4eIh9QaJdnGw0_SLgdrNt_0-4,11103
-crawlo/network/response.py,sha256=M0iG8ggcEpFzN7aAinKwVbxHAhANLYj8Zp4-FwaTpYo,9812
-crawlo/pipelines/__init__.py,sha256=Tvs9FQ_2qz8GcmoEqLrz_YFZrRpL3gGIssiI5Ce9b5I,254
-crawlo/pipelines/console_pipeline.py,sha256=KABkR3J-rqO0Awox7lizxKR2XuHfVhWPiVRgIybwwu4,1248
-crawlo/pipelines/csv_pipeline.py,sha256=6FBT2AoU6iNU-5NfgWRq7-JpF9dK2nBokjxx-y4jIas,12174
-crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
-crawlo/pipelines/mongo_pipeline.py,sha256=lv-Zn_mWdE_jVy7Nh30Lzqm3YhtLRV5rMy-m4rBWYe0,4442
-crawlo/pipelines/mysql_pipeline.py,sha256=6g6PGTwAyzH5VStlPAg2SdG5t1lPw1Zu-cY7x6Mz16s,7861
-crawlo/pipelines/pipeline_manager.py,sha256=VrbebOYiqrobtKhp5II18w-odCICdWkmRg5WPK0Emz4,2112
-crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/queue/pqueue.py,sha256=yzF900ko2RReVNZtfk_VV3PzSXEUALI6SHf97geuu6s,1058
-crawlo/queue/queue_manager.py,sha256=OQ9YBbc9Y4uE_5WbQhe233FFdhaeiOhDVbqWF7ev42U,11303
-crawlo/queue/redis_priority_queue.py,sha256=hf7hiIiRS-2DIq1bIJbs1yLEZjuMnHwc-n2_I7PlE9A,7294
-crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
-crawlo/settings/default_settings.py,sha256=jRM_Cqt3tQ7V4mgqBTAIQY3BcfGsLkKdvW1TcbWHv48,9627
-crawlo/settings/setting_manager.py,sha256=SxKB1aCWh4OySM_bH9cYng9I3PAmrSP-Q8XOZEWEwbI,2899
-crawlo/spider/__init__.py,sha256=Z_rK23l5yt-DuwJPg8bcqodM_FIs4-iHLaKOimGumcE,20452
-crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
-crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
-crawlo/templates/project/items.py.tmpl,sha256=3h-4nuneUoCAGUzSLyLNsdgeAAUXzz4rV0QctzCkIJA,301
-crawlo/templates/project/middlewares.py.tmpl,sha256=Ua2vG3WXliMBSmwsbYaSwzx2lZPw9vLrG2dLqtZWGRg,3157
-crawlo/templates/project/pipelines.py.tmpl,sha256=-dxK7T396AVTbh0FhivLDZq4lKxw3p1-5Y6Ij4jgImE,12041
-crawlo/templates/project/run.py.tmpl,sha256=RQ_k_B-Pp1R853-N_58yUGjIouXa6ipBtUPQenLjswE,7519
-crawlo/templates/project/settings.py.tmpl,sha256=NxPvXGnomTV5NkczPIC7Tscn2vjmHUE4CYmbQB2SmDQ,7884
-crawlo/templates/project/spiders/__init__.py.tmpl,sha256=j_YKsw6HQMJyqlk3WUouP3bsr-XVxshRoSNakHBc00g,106
-crawlo/templates/spider/spider.py.tmpl,sha256=Wx1yMBsqDQvJGb9HoEVecX6XZcVrJiBj9blXYrernTo,6167
-crawlo/utils/__init__.py,sha256=BDORpyjMN7VGPKImnCDKSkprS-petgD7ezc9rMlBvb0,123
-crawlo/utils/controlled_spider_mixin.py,sha256=BpH76XxBJtYQ0rmCX6vrG-8gQXT9-LP65ibetzfqncU,12500
-crawlo/utils/date_tools.py,sha256=0yG0tzGb1VFgWDJJ_cow2LJfz3kj_w2MqSjmfKKESl8,6961
-crawlo/utils/db_helper.py,sha256=3ib5-agrlwf2t5S_QtLRYH75wvJDlYbRqRmDEbpH5Bo,10559
-crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
-crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
-crawlo/utils/large_scale_helper.py,sha256=JJqcGSI6VaVe3MSL6IWjmCp8XQIu6T4U-BvBLSttr_s,12157
-crawlo/utils/log.py,sha256=A3lPyhD8kD88cV23KOL-_eT8g69xGQ5L1toDB2AO0mc,4005
-crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
-crawlo/utils/request.py,sha256=yoLB2rY8d78vgPjIWpdhY5SalIKjyLIvTG_UH6EMdVI,8798
-crawlo/utils/request_serializer.py,sha256=bPoSQqE2ksiMyP3WiPB3w3UqZs4f_LgkAw4Pj0qyBDo,8565
-crawlo/utils/spider_loader.py,sha256=pEDUsYOTGjszA6KgjiMlYN4GS5fP4uakkhcp3JTFFQY,2187
-crawlo/utils/system.py,sha256=HvWV1acxou0Rn0L7pNq4CnV_GWFeU0Tmjy3_nLD8M64,237
-crawlo/utils/tools.py,sha256=5Uv25Wy4m_ndZY0-n-eX-t3PxvaZ6wR3-Wvx-o7_Vrs,271
-crawlo/utils/url.py,sha256=rlgX2VlJv6JvLmCDTsbxzMSXE6R5ZL_0dLEqprsA-JU,1482
-examples/__init__.py,sha256=6i631BPnS_TR_BWUjtjB5CBO-zv9kRkwQTQvSya2wHE,123
-tests/__init__.py,sha256=scL1IPVT1iucth7v8ffrjRdeW7QrC_Y7AMmFVMdTY1Y,129
-tests/test_final_validation.py,sha256=fBxf_6YcAEa_HyV_oGAXmmVHY4i6FdA4J6klCmc36hQ,4925
-tests/test_proxy_health_check.py,sha256=xo3QMP1YNw9hu7JDHZOYCUZmFFKLJpHSh4SbxXhCRPQ,1091
-tests/test_proxy_middleware_integration.py,sha256=zcl7fR9Toc-I-stSUTzKZPwcfh3kgrpjI5SbkZ6AVmE,4305
-tests/test_proxy_providers.py,sha256=XwWZCywTYguSsUxSm6fsbaoH1p9dKjqSIx9-sqKZehA,1693
-tests/test_proxy_stats.py,sha256=Til_yksrRz2yBVw-yJi5-36LhNW3vTwpXTm4BdR9PUM,507
-tests/test_proxy_strategies.py,sha256=ZkziozkvZd3KWOQnpHQ8Upd3WpyoX7gN0qFGluNm348,1809
-tests/test_redis_config.py,sha256=TqzFRojc6esGXjGhUCvSLYQDUTAgEJsty9vRVuNraMU,893
-tests/test_redis_queue.py,sha256=o6xViXxJcdx-1eMcG3vhAQEIm8h346HnZb7JXs7ZjwM,6622
-tests/test_request_serialization.py,sha256=8sVdppAsohJ5u-m1WvablCndwL-M_36YPLdGKwgeznM,2289
-tests/test_scheduler.py,sha256=-FOkTWzaMdr6yfO1Msu74hI_GgSfD7iRxO-cFA-9Iyk,7442
-crawlo-1.1.2.dist-info/METADATA,sha256=fIXPyz2xk3GN82z5693BePRT1OWvul312NOh41CBRXA,17061
-crawlo-1.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.1.2.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.1.2.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.1.2.dist-info/RECORD,,