crawlo 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

crawlo-1.1.3.dist-info/RECORD

@@ -0,0 +1,113 @@
+crawlo/__init__.py,sha256=-9NQRRYzQvX-sVT-phr3qiJRePbs1G2Gd6h76y8UpZU,859
+crawlo/__version__.py,sha256=MNg8Ih17kliJwlnU9Nip6ZLjE9MDf-ibrAgQJVctAy4,23
+crawlo/cli.py,sha256=hjAJKx9pba375sATvvcy-dtZyBIgXj8fRBq9RFIZHA4,1206
+crawlo/config.py,sha256=ShwUnG3K3JmYeIg3oFzHHRYDs1vJQ-H71CPaq0rNNZ0,8608
+crawlo/crawler.py,sha256=mZknc51PvB_2tGFhxe4DRMvoMH9qO8CydlIRysvLZC4,37488
+crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
+crawlo/exceptions.py,sha256=YVIDnC1bKSMv3fXH_6tinWMuD9HmKHIaUfO4_fkX5sY,1247
+crawlo/mode_manager.py,sha256=n9-NPR7KK7SkoCoI8JdQkVMwL5_9T4MzdSfhTg-BfV4,6902
+crawlo/project.py,sha256=DOf_zzdA_A_nilff6Dp5KJXA6KphHYMalAYv336-cO8,5335
+crawlo/stats_collector.py,sha256=v4jC9BAe-23w93hWzbeMCCgQ9VuFPyxw5JV9ItbGH8w,1636
+crawlo/subscriber.py,sha256=udlHeTR0ymGQhCDxVUGwUzeeeR4TYCEJrJwFnkgr0cU,3836
+crawlo/task_manager.py,sha256=PScfEB03306Txa0l38AeQ_0WVhKzeWOFyT3bnrkbHW0,849
+crawlo/commands/__init__.py,sha256=kZ3qATqDPmMUCNUQSFfBfIA8fp_1dgBwIAWbmFN3_To,355
+crawlo/commands/check.py,sha256=jW8SgfkOS35j4VS7nRZBZdFCBX9CVFez5LR2sfP_H1U,23437
+crawlo/commands/genspider.py,sha256=_3GwFMYK79BuKk__5L0ljuwWwOzN80MeuhRkL4Ql11A,5201
+crawlo/commands/list.py,sha256=octTk0QZhapiyM7WgCPersP2v3MesthbJeG9vMqVFOs,5936
+crawlo/commands/run.py,sha256=m7SFTxmw4mZJ_eS1a9fHG-c6FvQcRHXfW71xenYBYYc,10809
+crawlo/commands/startproject.py,sha256=G88oa5OdK1wCFBCMpOm3P7ZM1Ahv5Lq_HUq-XysxZwU,7334
+crawlo/commands/stats.py,sha256=iEKdxHoqsJuTkn8zAF9ekBVO1--8__BeD7xohYG5NwE,6252
+crawlo/commands/utils.py,sha256=b7yW6UlOLprR3gN9oOdhcl3fsCwWRE3-_gDxWz5xhMo,5292
+crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
+crawlo/core/engine.py,sha256=vQPuk6gMcmJnPfMSg8-RrRpRSXg_i7ZraJfgpvz-NwE,6833
+crawlo/core/enhanced_engine.py,sha256=5cxv8EVszB6yrpW32m4CF73UMuv4tOzVjejVB7pK1u0,6443
+crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
+crawlo/core/scheduler.py,sha256=hnMcHN8nq904ibfGNZpcFbIXAr0shUKmTYgY5eE2-Pc,5969
+crawlo/downloader/__init__.py,sha256=O7T7MjCfiwfqXQYSF59CPsFYZWmfI21weO41J-SxcRM,7905
+crawlo/downloader/aiohttp_downloader.py,sha256=wzJsOlMx_sPSrQ-OF71EcB1j6p7zTdmYevYdBsPN-34,8217
+crawlo/downloader/cffi_downloader.py,sha256=swia_ZqJbbBCzhkqaG7EEVdvj6S123d0kfA6QscqF5I,10691
+crawlo/downloader/httpx_downloader.py,sha256=MpgDeIdGqNsiSKLOEDBnr5Z0eUbhHnqVEmAuoIfJmFU,12296
+crawlo/extension/__init__.py,sha256=wCyNLL-IdjoITKA_mWjr4PjIsUtWKogj-TFfFhDavf4,1164
+crawlo/extension/log_interval.py,sha256=FOWeTOuWtOpCz2UPV5F_--QIa8yomltSpjxbw3F7bkU,1971
+crawlo/extension/log_stats.py,sha256=JFJsdK7UWkhP4TEAF-H-S7SpQbDpBryS0AT6e6jZCBo,1721
+crawlo/extension/logging_extension.py,sha256=rty2_up53KV05nCazuBuz2ZapHKq0ti7mGVBzMTr0ak,1236
+crawlo/filters/__init__.py,sha256=lX-QOCDTiTRFoiK1qrZ5HABo7LgZfcxScx_lELYEvJk,4395
+crawlo/filters/aioredis_filter.py,sha256=LbUXtfJXyOF9L-Fxztka8djTtEaFDsNNCydaE9aDqUM,8611
+crawlo/filters/memory_filter.py,sha256=FzGJPhVKfZ8P23kP6de-VSfE8oVMjjpfWzKJIdiMtZU,9529
+crawlo/items/__init__.py,sha256=rFpx1qFBo0Ik7bSdnXC8EVTJUOQdoJYGVdhYjaH00nk,409
+crawlo/items/base.py,sha256=hwGJEdFWOdaZfalFX8umRkh_HUWLEbCjvq4j70fplMQ,598
+crawlo/items/fields.py,sha256=fpS0vlRPpZYjTaMDgI9Q8z_YQqruwf6fi4Dgm6R2oEk,1854
+crawlo/items/items.py,sha256=OmVEvMmgofMU95GkaiWkfNQ2fjsH2fY9sw3SKcmUhLs,3478
+crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
+crawlo/middleware/default_header.py,sha256=OVW4vpRPp3Y6qYXtiEYlGqVjCYcbuv1Iecc7zEgwCsI,1099
+crawlo/middleware/download_delay.py,sha256=P2eyAJXwdLdC4yYuLhvKZVa1b5YQvQD0GpsR8aDW8-8,994
+crawlo/middleware/middleware_manager.py,sha256=G9R9SnDVey_zS2k1zm_358EPHDxRWfP953I4QavL7P0,6348
+crawlo/middleware/proxy.py,sha256=1kdKjKhSkjilJqkBYV8khTK964XbBLxJ4ssh8NPcQSE,10059
+crawlo/middleware/request_ignore.py,sha256=jdybWFVXuA5YsAPfZJFzLTWkYhEAewNgxuhFqczPW9M,1027
+crawlo/middleware/response_code.py,sha256=vgXWv3mMu_v9URvhKA9myIFH4u6L4EwNme80wL4DCGc,677
+crawlo/middleware/response_filter.py,sha256=O2gkV_Yjart8kmmXTGzrtZnb_Uuefap4uL2Cu01iRs4,863
+crawlo/middleware/retry.py,sha256=D_v4-8grofGsLkGJIhLKklMbdHO0gtij7DIS4G52NJc,4248
+crawlo/network/__init__.py,sha256=bvEnpEUBZJ79URfNZbsHhsBKna54hM2-x_BV8eotTA4,418
+crawlo/network/request.py,sha256=qaKIlOg8ETC8vcKQ__M7aD0own4DLi9vqYBg3Getmkg,11386
+crawlo/network/response.py,sha256=3IAmNzV8b-wn4Z-N2mchgn5U7Fcpudl1D-zcY-lH_mw,10009
+crawlo/pipelines/__init__.py,sha256=FDe2Pr5tiHtV8hFlheElRO_O1aVKvSWlkTcAl9BXAKA,637
+crawlo/pipelines/bloom_dedup_pipeline.py,sha256=n0Ay7MtIEJ8L4Otiha4zRvI9toFUSNFugTNubi-Q3aw,5798
+crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
+crawlo/pipelines/csv_pipeline.py,sha256=qbXZoqq4FIR9QkUGpC0ryWzmqGJSrM2bxmWLM4I1nXM,12490
+crawlo/pipelines/database_dedup_pipeline.py,sha256=6_zKtCNgFBPJyTI3Mk4l75fMQ2UQQGKFfNTpZqGs_zI,8224
+crawlo/pipelines/json_pipeline.py,sha256=wrCsh8YInmcPLAkhPrHObMx89VZfhf-c7qRrYsTixPE,8585
+crawlo/pipelines/memory_dedup_pipeline.py,sha256=oQcBODO-I2p6B7Nm_klXvuhzSMIHP-JWwC4_o6Gkgcc,3954
+crawlo/pipelines/mongo_pipeline.py,sha256=Yr48D0T61-_Y-EpgWXf7BUn9w8e-Pj5P07QDSPZ0pYU,4558
+crawlo/pipelines/mysql_pipeline.py,sha256=bsAFqpxrCijzvX-IusxOtvTvQEUCt5uHNTyYMo_pIq4,8056
+crawlo/pipelines/pipeline_manager.py,sha256=Kw37RC2GESWDnDJ6qIN1MA0qc27Uyhu77ebm1r-FgeU,2168
+crawlo/pipelines/redis_dedup_pipeline.py,sha256=4Mpsscucq2_6UAcKlaa6FIj_KhsX0IeD22cperUcUIo,6165
+crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/queue/pqueue.py,sha256=qTFOuvEXsYEZbm0ULjsOeZo0XtSsZ-SHpx7nFEtmluE,1095
+crawlo/queue/queue_manager.py,sha256=yejs-wasl_gkRZF8qNXV0LX2qIdi0yJrNZa-fkLKnes,11774
+crawlo/queue/redis_priority_queue.py,sha256=N0lD83hmmYi2bRa-e-rnHA8DB5GbdTDavBlHZGcepoQ,8324
+crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
+crawlo/settings/default_settings.py,sha256=Zc1lJwMlPKI6jiG5nX-QORSgg3zqt9jPcCD6HEHnrp8,10104
+crawlo/settings/setting_manager.py,sha256=4xXOzKwZCgAp8ybwvVcs2R--CsOD7c6dBIkj6DJHB3c,2998
+crawlo/spider/__init__.py,sha256=xAH6NfE_6K2aY_VSL9DoGjcmMHJDd5Nxr7TG1Y8vQAE,21091
+crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
+crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
+crawlo/templates/project/items.py.tmpl,sha256=bXx-oCldMr2EgBKUAH9LH5gMnbyLiWX-EySAaMzcu2g,318
+crawlo/templates/project/middlewares.py.tmpl,sha256=OgvIfsWyTNbfBwsJqxQiEAWCNjXt8iQVj7Hr26RCEIU,3243
+crawlo/templates/project/pipelines.py.tmpl,sha256=076j3H3dKpmie9VIiIiQiVVlB_tic_hDaN4bJ3VnULc,12775
+crawlo/templates/project/run.py.tmpl,sha256=IBD0F0XEgBR6gR34PhYAjKiZDdvLufZJkABHapTsoYo,8428
+crawlo/templates/project/settings.py.tmpl,sha256=HfsKW1lHG7KFw3lqhHSi3KkhnJ4gO8C9p8zfE5pC30w,8307
+crawlo/templates/project/spiders/__init__.py.tmpl,sha256=zMbePipgLsctQUEnda4WkHz8rDLUX--rc8ruI6zkpWc,111
+crawlo/templates/spider/spider.py.tmpl,sha256=9DEEp8m1zUY022VoNGkauxNXB0UtZlSps1aw3_ywLRs,6344
+crawlo/utils/__init__.py,sha256=XCYumI8wJ1jU_Myn_K0LT-LVygPDUCdETCbXM3EWvlo,130
+crawlo/utils/controlled_spider_mixin.py,sha256=os5v8eFX08Nbqm48W-xWdY04LboqDgOgOL-93IX9zvg,16914
+crawlo/utils/date_tools.py,sha256=lcEFP2Z5b-6pUTHczrzCCuqiHP_4_2zamomMGPZrExo,7194
+crawlo/utils/db_helper.py,sha256=ZqOt1d3mErVv4TOvoWlov0niUxORB9aHByTmMoNFIDw,10902
+crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
+crawlo/utils/large_scale_config.py,sha256=lsraHTAQx3sMPjTnCBY_SVIpkuIBUb3zD9eFvmccOOM,8440
+crawlo/utils/large_scale_helper.py,sha256=ZazAI7KV3V-3hzc4a3BWxTXfEO2XIEBMzxTLM9S1l_Q,12500
+crawlo/utils/log.py,sha256=YD2FfXuuE2MC9ZdQQZ0H7KysE7l_LHZqQepaTPlcApo,4133
+crawlo/utils/queue_helper.py,sha256=gFmkh1jKlIcN1rmo2Jl6vYcLP5ByUWlfHO9eNlZPBLs,4918
+crawlo/utils/request.py,sha256=ejdKpTwc-HE04HQybafhOVywzz57IV3pY0YMkSLyGUo,9065
+crawlo/utils/request_serializer.py,sha256=bZhuonZV2AdB_X0aje7sDljqWAIrEzUYwEaxXytaWsg,8784
+crawlo/utils/spider_loader.py,sha256=V0CBTicJBYBZafhwLfDEfuEc_hJ2mSoiptT6qKufI9U,2249
+crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
+crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
+crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
+examples/__init__.py,sha256=NkRbV8_S1tb8S2AW6BE2U6P2-eGOPwMR1k0YQAwQpSE,130
+examples/controlled_spider_example.py,sha256=2SAQKoREGHe-OzVaSkGpopCcrou6QXmeW7rLdmsyopw,7981
+tests/__init__.py,sha256=409aRX8hsPffiZCVjOogtxwhACzBp8G2UTJyUQSxhK0,136
+tests/test_final_validation.py,sha256=UNHMOkcOBx9jPdnYuYCF4Cx5GlXakBeHybOP27lpbAg,5078
+tests/test_proxy_health_check.py,sha256=_tDlxa_6TdL3M5RLkHF82roXJ8WIuG5hELBp2GADyKQ,1123
+tests/test_proxy_middleware_integration.py,sha256=mTPK_XvbmLCV_QoVZzA3ybWOOX61493Ew78WfTp-bYQ,4441
+tests/test_proxy_providers.py,sha256=u_R2fhab90vqvQEaOAztpAOe9tJXvUMIdoDxmStmXJ4,1749
+tests/test_proxy_stats.py,sha256=ES00CEoDITYPFBGPk8pecFzD3ItYIv6NSpcqNd8-kvo,526
+tests/test_proxy_strategies.py,sha256=9Z1pXmTNyw-eIhGXlf2abZbJx6igLohYq-_3hldQ5uE,1868
+tests/test_redis_config.py,sha256=Kbl3PURGNM1BUIspakEOA-ZOl2xxTHb_8KbftwjYOsg,921
+tests/test_redis_queue.py,sha256=5LTc86A5qqF5VbmkvkF2OnLAxlJ7ClfJPw0dODxekFk,6846
+tests/test_request_serialization.py,sha256=Jf7Kr7edL0ENwxh8ABa1W_O3dWyqNlvoSfQM1Mykpys,2359
+tests/test_scheduler.py,sha256=elAPFh-Ph49bbJQlTBEsRwzhoX82EdryqQbpc_wsobU,7683
+crawlo-1.1.3.dist-info/METADATA,sha256=o02XZ1pmSz28mIePbTgfDN67y3POps84PFa9b1ImIzY,20601
+crawlo-1.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.1.3.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.1.3.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.1.3.dist-info/RECORD,,

examples/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 12:36
-# @Author  :   oscar
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 12:36
+# @Author  :   oscar
+# @Desc    :   None
+"""

examples/controlled_spider_example.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+受控爬虫混入类使用示例
+演示如何使用 ControlledRequestMixin 和 AsyncControlledRequestMixin 来控制大规模请求生成
+"""
+
+import asyncio
+from crawlo.spider import Spider
+from crawlo.network import Request
+from crawlo.utils.controlled_spider_mixin import ControlledRequestMixin, AsyncControlledRequestMixin
+
+
+class LargeScaleSpider(Spider, ControlledRequestMixin):
+    """
+    同步版本的受控爬虫示例
+    适用于需要生成大量请求但要控制并发的场景
+    """
+    name = 'large_scale_spider'
+    
+    def __init__(self):
+        Spider.__init__(self)
+        ControlledRequestMixin.__init__(self)
+        
+        # 配置受控生成参数
+        self.max_pending_requests = 150     # 最大待处理请求数
+        self.batch_size = 75               # 每批生成请求数
+        self.generation_interval = 0.02     # 生成间隔（秒）
+        self.backpressure_threshold = 300   # 背压阈值
+    
+    def _original_start_requests(self):
+        """
+        提供原始的大量请求
+        这里模拟爬取一个电商网站的商品页面
+        """
+        # 模拟爬取 10,000 个商品页面
+        base_urls = [
+            "https://example-shop.com/products",
+            "https://example-shop.com/electronics", 
+            "https://example-shop.com/clothing",
+            "https://example-shop.com/books",
+            "https://example-shop.com/home"
+        ]
+        
+        for category in base_urls:
+            # 每个分类爬取 2000 页
+            for page in range(1, 2001):
+                yield Request(
+                    url=f"{category}?page={page}",
+                    callback=self.parse_product_list,
+                    meta={'category': category.split('/')[-1], 'page': page}
+                )
+    
+    def _process_request_before_yield(self, request):
+        """
+        在 yield 请求前进行处理
+        可以添加去重、优先级设置、请求头设置等逻辑
+        """
+        # 设置请求头
+        request.headers.setdefault('User-Agent', 
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36')
+        
+        # 根据分类设置优先级
+        category = request.meta.get('category', '')
+        if category == 'electronics':
+            request.priority = 10  # 电子产品优先级最高
+        elif category == 'clothing':
+            request.priority = 8
+        else:
+            request.priority = 5
+        
+        # 可以在这里添加去重逻辑
+        # if self.is_duplicate_request(request):
+        #     return None  # 跳过重复请求
+        
+        return request
+    
+    async def parse_product_list(self, response):
+        """解析商品列表页面"""
+        # 提取商品链接
+        product_links = response.css('a.product-link::attr(href)').getall()
+        
+        for link in product_links:
+            # 生成商品详情页请求
+            yield Request(
+                url=response.urljoin(link),
+                callback=self.parse_product_detail,
+                meta={'category': response.meta['category']}
+            )
+        
+        # 提取分页信息
+        next_page = response.css('a.next-page::attr(href)').get()
+        if next_page:
+            yield Request(
+                url=response.urljoin(next_page),
+                callback=self.parse_product_list,
+                meta=response.meta
+            )
+    
+    async def parse_product_detail(self, response):
+        """解析商品详情页面"""
+        yield {
+            'title': response.css('h1.product-title::text').get(),
+            'price': response.css('.price::text').re_first(r'\d+\.?\d*'),
+            'description': response.css('.product-description::text').get(),
+            'category': response.meta['category'],
+            'url': response.url,
+            'in_stock': 'in-stock' in response.css('.availability::attr(class)').get(''),
+            'rating': response.css('.rating::attr(data-rating)').get(),
+            'reviews_count': response.css('.reviews-count::text').re_first(r'\d+')
+        }
+
+
+class AsyncLargeScaleSpider(Spider, AsyncControlledRequestMixin):
+    """
+    异步版本的受控爬虫示例
+    使用异步控制来实现更精确的并发管理
+    """
+    name = 'async_large_scale_spider'
+    
+    def __init__(self):
+        Spider.__init__(self)
+        AsyncControlledRequestMixin.__init__(self)
+        
+        # 配置异步控制参数
+        self.max_concurrent_generations = 15    # 最大同时生成数
+        self.queue_monitor_interval = 0.5       # 队列监控间隔
+    
+    def _original_start_requests(self):
+        """
+        提供原始的大量请求
+        这里模拟爬取新闻网站的文章
+        """
+        # 模拟爬取 50,000 篇新闻文章
+        news_sites = [
+            "https://news-site-1.com",
+            "https://news-site-2.com", 
+            "https://news-site-3.com",
+            "https://tech-news.com",
+            "https://finance-news.com"
+        ]
+        
+        categories = ['tech', 'finance', 'sports', 'politics', 'entertainment']
+        
+        for site in news_sites:
+            for category in categories:
+                # 每个分类爬取 2000 页
+                for page in range(1, 2001):
+                    yield Request(
+                        url=f"{site}/{category}?page={page}",
+                        callback=self.parse_article_list,
+                        meta={'site': site, 'category': category, 'page': page}
+                    )
+    
+    def _process_request_before_yield(self, request):
+        """异步版本的请求预处理"""
+        # 设置请求头
+        request.headers.setdefault('User-Agent', 
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
+        
+        # 根据新闻类型设置优先级
+        category = request.meta.get('category', '')
+        if category in ['tech', 'finance']:
+            request.priority = 10  # 科技和财经新闻优先级最高
+        elif category in ['sports', 'politics']:
+            request.priority = 8
+        else:
+            request.priority = 5
+        
+        # 设置延迟（避免对服务器造成过大压力）
+        site = request.meta.get('site', '')
+        if 'tech-news.com' in site:
+            request.meta['download_delay'] = 0.5  # 科技新闻站点较敏感，增加延迟
+        
+        return request
+    
+    async def parse_article_list(self, response):
+        """解析文章列表页面"""
+        # 提取文章链接
+        article_links = response.css('a.article-link::attr(href)').getall()
+        
+        for link in article_links:
+            yield Request(
+                url=response.urljoin(link),
+                callback=self.parse_article_detail,
+                meta={
+                    'site': response.meta['site'],
+                    'category': response.meta['category']
+                }
+            )
+    
+    async def parse_article_detail(self, response):
+        """解析文章详情页面"""
+        yield {
+            'title': response.css('h1.article-title::text').get(),
+            'content': ' '.join(response.css('.article-content p::text').getall()),
+            'author': response.css('.author::text').get(),
+            'publish_date': response.css('.publish-date::attr(datetime)').get(),
+            'category': response.meta['category'],
+            'site': response.meta['site'],
+            'url': response.url,
+            'tags': response.css('.tags a::text').getall(),
+            'views': response.css('.views-count::text').re_first(r'\d+'),
+            'comments': response.css('.comments-count::text').re_first(r'\d+')
+        }

tests/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-08-24 12:36
-# @Author  :   crawl-coder
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-08-24 12:36
+# @Author  :   crawl-coder
+# @Desc    :   None
+"""

tests/test_final_validation.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+最终验证测试：确认分布式队列的 logger 序列化问题已完全解决
+"""
+import asyncio
+import pickle
+import sys
+sys.path.insert(0, "..")
+
+from crawlo.network.request import Request
+from crawlo.spider import Spider
+from crawlo.core.scheduler import Scheduler
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.utils.log import get_logger
+from unittest.mock import Mock
+
+
+class TestSpider(Spider):
+    """测试爬虫"""
+    name = "validation_spider"
+    
+    def __init__(self):
+        super().__init__()
+        # 故意添加多个 logger 来测试清理
+        self.custom_logger = get_logger("custom")
+        self.debug_logger = get_logger("debug")
+        self.nested_data = {
+            'logger': get_logger("nested"),
+            'sub': {
+                'logger_ref': get_logger("sub_logger")
+            }
+        }
+    
+    def parse(self, response):
+        # 验证主 logger 还在
+        self.logger.info(f"✅ 主 logger 工作正常: {response.url}")
+        return {"url": response.url, "status": "success"}
+
+
+def test_scheduler_cleaning():
+    """测试调度器的 logger 清理"""
+    print("🔍 测试调度器 logger 清理...")
+    
+    spider = TestSpider()
+    request = Request(
+        url="https://scheduler-test.com",
+        callback=spider.parse,
+        meta={"logger": get_logger("meta_logger")}
+    )
+    
+    # Mock crawler 和 scheduler
+    class MockCrawler:
+        def __init__(self):
+            self.spider = spider
+    
+    class MockScheduler(Scheduler):
+        def __init__(self):
+            self.crawler = MockCrawler()
+            self.logger = get_logger("MockScheduler")
+    
+    scheduler = MockScheduler()
+    
+    # 清理前检查
+    print(f"   🔧 清理前 - spider.logger: {spider.logger is not None}")
+    print(f"   🔧 清理前 - spider.custom_logger: {spider.custom_logger is not None}")
+    print(f"   🔧 清理前 - request.callback: {request.callback is not None}")
+    
+    # 执行清理
+    cleaned_request = scheduler._deep_clean_loggers(request)
+    
+    # 清理后检查
+    print(f"   ✅ 清理后 - spider.logger: {spider.logger is not None}")
+    print(f"   ✅ 清理后 - spider.custom_logger: {spider.custom_logger is None}")
+    print(f"   ✅ 清理后 - request.callback: {cleaned_request.callback is None}")
+    
+    # 序列化测试
+    try:
+        serialized = pickle.dumps(cleaned_request)
+        print(f"   ✅ 调度器清理后序列化成功，大小: {len(serialized)} bytes")
+        return True
+    except Exception as e:
+        print(f"   ❌ 调度器清理后序列化失败: {e}")
+        return False
+
+
+async def test_redis_queue_cleaning():
+    """测试 Redis 队列的 logger 清理"""
+    print("\\n🔍 测试 Redis 队列 logger 清理...")
+    
+    spider = TestSpider()
+    request = Request(
+        url="https://redis-test.com",
+        callback=spider.parse,
+        meta={"logger": get_logger("meta_logger")}
+    )
+    
+    try:
+        queue = RedisPriorityQueue(redis_url="redis://127.0.0.1:6379/0")
+        await queue.connect()
+        
+        # 入队测试
+        success = await queue.put(request, priority=0)
+        print(f"   ✅ Redis 队列入队成功: {success}")
+        
+        if success:
+            # 出队测试
+            retrieved = await queue.get(timeout=2.0)
+            if retrieved:
+                print(f"   ✅ Redis 队列出队成功: {retrieved.url}")
+                print(f"   ✅ callback 信息保存: {'_callback_info' in retrieved.meta}")
+                await queue.close()
+                return True
+            else:
+                print("   ❌ 出队失败")
+                await queue.close()
+                return False
+        else:
+            await queue.close()
+            return False
+            
+    except Exception as e:
+        print(f"   ❌ Redis 队列测试失败: {e}")
+        return False
+
+
+async def main():
+    """主测试函数"""
+    print("🚀 开始最终验证测试...")
+    print("=" * 60)
+    
+    # 测试 1: 调度器清理
+    scheduler_ok = test_scheduler_cleaning()
+    
+    # 测试 2: Redis 队列清理
+    redis_ok = await test_redis_queue_cleaning()
+    
+    print("\\n" + "=" * 60)
+    print("📊 测试结果汇总:")
+    print(f"   调度器 logger 清理: {'✅ 通过' if scheduler_ok else '❌ 失败'}")
+    print(f"   Redis 队列清理: {'✅ 通过' if redis_ok else '❌ 失败'}")
+    
+    if scheduler_ok and redis_ok:
+        print("\\n🎉 所有测试通过！")
+        print("✅ 分布式队列的 logger 序列化问题已完全修复！")
+        print("✅ Crawlo 现在可以正常使用 Redis 分布式队列了！")
+        return True
+    else:
+        print("\\n❌ 部分测试失败，需要进一步修复")
+        return False
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/test_proxy_health_check.py

@@ -1,33 +1,33 @@
-# tests/test_proxy_health_check.py
-import pytest
-from unittest.mock import AsyncMock, patch
-from crawlo.proxy.health_check import check_single_proxy
-import httpx
-
-
-@pytest.mark.asyncio
-@patch('httpx.AsyncClient')
-async def test_health_check_success(mock_client_class):
-    """测试健康检查：成功"""
-    mock_resp = AsyncMock()
-    mock_resp.status_code = 200
-    mock_client_class.return_value.__aenter__.return_value.get.return_value = mock_resp
-
-    proxy_info = {'url': 'http://good:8080', 'healthy': False}
-    await check_single_proxy(proxy_info)
-
-    assert proxy_info['healthy'] is True
-    assert proxy_info['failures'] == 0
-
-
-@pytest.mark.asyncio
-@patch('httpx.AsyncClient')
-async def test_health_check_failure(mock_client_class):
-    """测试健康检查：失败"""
-    mock_client_class.return_value.__aenter__.return_value.get.side_effect = httpx.ConnectError("Failed")
-
-    proxy_info = {'url': 'http://bad:8080', 'healthy': True, 'failures': 0}
-    await check_single_proxy(proxy_info)
-
-    assert proxy_info['healthy'] is False
+# tests/test_proxy_health_check.py
+import pytest
+from unittest.mock import AsyncMock, patch
+from crawlo.proxy.health_check import check_single_proxy
+import httpx
+
+
+@pytest.mark.asyncio
+@patch('httpx.AsyncClient')
+async def test_health_check_success(mock_client_class):
+    """测试健康检查：成功"""
+    mock_resp = AsyncMock()
+    mock_resp.status_code = 200
+    mock_client_class.return_value.__aenter__.return_value.get.return_value = mock_resp
+
+    proxy_info = {'url': 'http://good:8080', 'healthy': False}
+    await check_single_proxy(proxy_info)
+
+    assert proxy_info['healthy'] is True
+    assert proxy_info['failures'] == 0
+
+
+@pytest.mark.asyncio
+@patch('httpx.AsyncClient')
+async def test_health_check_failure(mock_client_class):
+    """测试健康检查：失败"""
+    mock_client_class.return_value.__aenter__.return_value.get.side_effect = httpx.ConnectError("Failed")
+
+    proxy_info = {'url': 'http://bad:8080', 'healthy': True, 'failures': 0}
+    await check_single_proxy(proxy_info)
+
+    assert proxy_info['healthy'] is False
     assert proxy_info['failures'] == 1