crawlo 1.1.2__py3-none-any.whl → 1.1.4__py3-none-any.whl

{crawlo-1.1.2.dist-info → crawlo-1.1.4.dist-info}/RECORD

@@ -1,14 +1,14 @@
 crawlo/__init__.py,sha256=esOolburYDjtF43D5N9Kh6TSQW2yKcz888ilhBSinBc,825
-crawlo/__version__.py,sha256=5SgGjThsHu_ITn8V83BvCziqCwxdXxTQqcC3KQMHPfM,22
+crawlo/__version__.py,sha256=XxXhu8-QnuD9hA8Ah0WX5rgpt_DwOQmAwcK-FtpngyQ,22
 crawlo/cli.py,sha256=CtR2Pfa7SyRxEKPaXqt-6E6K5Vq5z3rfdAI95UO4cbU,1166
 crawlo/config.py,sha256=i0Amz6wNPgv_aVcdCBRRlcwuZLSa87cH9OEmTQvB97Q,8329
 crawlo/crawler.py,sha256=v6i5tjgSOtbMoqiw1qdgKx1cY4kcVcd5l5bUTWtJNNU,36461
 crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
-crawlo/exceptions.py,sha256=pthF1lJlJHyRZm-mE6NAo5WzK3GYJqmRqIuIlK1Odx8,1129
+crawlo/exceptions.py,sha256=sMay0wnWLfc_FXWslqxm60qz6b66LXs3EdN_w8ygE9k,1166
 crawlo/mode_manager.py,sha256=WIxrq9S3EAH0D71LH1AxvcqXomeABqoXgtUN4A--DKY,6702
 crawlo/project.py,sha256=xWN2eTAjf_Pza-wWvvV4JjScQRWxe9hXlztX81ccUMc,5182
 crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
-crawlo/subscriber.py,sha256=3d4eYtkSgPj-18-mTZM6RQLSil-ux13FUcmfFxr3sMk,3730
+crawlo/subscriber.py,sha256=gioTIqRdEwVG-bwIiQonbk1vWWAqTh9hzVkrqZ1AfP0,5006
 crawlo/task_manager.py,sha256=19InAxS9oJ9EMj20Aw2urN-v6BeC22dkgPuW-B9-4UI,819
 crawlo/commands/__init__.py,sha256=AMYjXG7ulE8dPVmgWVo0uqXsaCYUUZYmmu2-7kFzH1M,342
 crawlo/commands/check.py,sha256=172OiAxnX5wwSlszUsyPgMZwAoIbGDTdfhtRz309ilc,22843
@@ -22,17 +22,21 @@ crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
 crawlo/core/engine.py,sha256=8Dcew1XyxChW5Fz1wFEWpJlPrQb2hKDWKul8e61S-Q0,6662
 crawlo/core/enhanced_engine.py,sha256=9I9Uxdy2oAz8zDGTzEiytuKu__VDVmIN8zwZKfrD8bw,6254
 crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
-crawlo/core/scheduler.py,sha256=iNZl47PvurnLLXRZN8Jfj1un6YP2q9m5n56X3Eb7_A8,5615
+crawlo/core/scheduler.py,sha256=fiU-Q-lzyC3B6ih8NSWqjP1Xw_ryNVb_4dLUARtWRBE,5804
 crawlo/downloader/__init__.py,sha256=tl0mE54reR-PuJYSsXsKP2VY5uzvq4lITxZwKKjNzPs,7663
 crawlo/downloader/aiohttp_downloader.py,sha256=UKupGYPOWrscAVsjhFgKYElTa9tbEeltqV7nuWqjIeE,8005
 crawlo/downloader/cffi_downloader.py,sha256=-GVfSIhi1Ip56suSiGf8jnUE2EBF1P56vw0uxLh_T6I,10440
-crawlo/downloader/httpx_downloader.py,sha256=FJcdE3BoxUYFQDPm3quaveSjweAKfAQRaeJm1hQViLg,12003
-crawlo/extension/__init__.py,sha256=PEBbMxi6ULBZ9ivEJ4T7IH_R376BQJ6Rz_D9Ce-Cqbs,1133
-crawlo/extension/log_interval.py,sha256=S-hSoiz9GdmgHrac4vDQ52fleoBcH-kzdPUD8YRAons,1922
-crawlo/extension/log_stats.py,sha256=VinABjzuFa-JmNXSX86gQ_3Oyx1y3vUhpWTEu9HRRqg,1677
-crawlo/extension/logging_extension.py,sha256=Jce2HHc9ejsIWQitkHZuhnLbT_1MIfcvywfzYkkJ5eY,1202
+crawlo/downloader/httpx_downloader.py,sha256=7jfQfvAtfk8yD_mvwUbWLhYOxMM7r1nudiU7m_Jl9wc,12037
+crawlo/extension/__init__.py,sha256=Sg588p6UhyrwFNTiD2wqGW-i3xgLX6HlLuQPKT7mayE,1526
+crawlo/extension/health_check.py,sha256=IVaaVo_0CcZtf1LoCAYXIBvs3wZ7hdmT6U4-NYWAgP0,5527
+crawlo/extension/log_interval.py,sha256=VCIeNqXcWDnxj4m6l77cjqgRzV8LfsPMb22X0Xc1Vwc,2417
+crawlo/extension/log_stats.py,sha256=Ssxz6R1YpWIj5WJvQ2cJ9F5oR7FUFdj-ITc9lV92SSU,2908
+crawlo/extension/logging_extension.py,sha256=ET6VAu1J2qNMz4NnG1G3zQLRhbsvV7l6xRIuQLE6DaE,1626
+crawlo/extension/memory_monitor.py,sha256=gg-GK5RD9XhnrN_zp3KTmPKyWDmKLMv_OTY-HxSxBNI,3664
+crawlo/extension/performance_profiler.py,sha256=NvQuuvE83dXJ-1URpN8OF9za9J1l7xhVbV22JynPQpA,4235
+crawlo/extension/request_recorder.py,sha256=RC23yzXClnVv9j2ljQvjBkUfWznfnDHsrQejKhE9y5E,4074
 crawlo/filters/__init__.py,sha256=XC_Q4ykZtSNYizYlAcehVwBBNO3lZ2zuWwafzXiuWyQ,4241
-crawlo/filters/aioredis_filter.py,sha256=3vbPOY3L_H7TUgL8K6AtaP6QDtoIg5xpE70Kxpil6dM,8376
+crawlo/filters/aioredis_filter.py,sha256=WhkFZcVAym9wLSUa8WTVctYfEibjxG42umtmacO1IY0,8370
 crawlo/filters/memory_filter.py,sha256=VJO0UFRYGxmV8dj4G1subsQ-FtvPcGLbvd7IVtqXnOs,9260
 crawlo/items/__init__.py,sha256=bqekZrRlDhxfWie0UbCs656TptYseoe9QJ67I4E7Elk,386
 crawlo/items/base.py,sha256=tAYrPJgblp3ZEihDXvappdYc6pGdim6x2_9QSmMKI2o,577
@@ -46,36 +50,40 @@ crawlo/middleware/proxy.py,sha256=m2ZZ50En9hUtgrqSqA6hItGT74xMqccHFPhZshutIco,98
 crawlo/middleware/request_ignore.py,sha256=QI2z4fUnJ-4xvPTZAmsL-GqR4RFHS1xq9iDr5KFrMco,997
 crawlo/middleware/response_code.py,sha256=tmef2QVl3JCiTMii6VQkASlOY2OyqmOPoOfNxIK1eF8,659
 crawlo/middleware/response_filter.py,sha256=ep8ZxDlfIefi9YqK8dPASEp5TTDRo9QEY_jMceC411s,837
-crawlo/middleware/retry.py,sha256=pH4fdW71DwQzMY7APutX7yqtvSTAoznYeM3ux2EOT7E,4146
+crawlo/middleware/retry.py,sha256=-7zpRURugiTTm4QYUSUlbnURD5mcT2Ji0yHvCgY1wGc,4124
 crawlo/network/__init__.py,sha256=BLPERYPo22g1BXrW--wUnlolrdFUmOPjgOB8XQQJlck,397
-crawlo/network/request.py,sha256=5bzXJJWcAgGngNos-k4eIh9QaJdnGw0_SLgdrNt_0-4,11103
-crawlo/network/response.py,sha256=M0iG8ggcEpFzN7aAinKwVbxHAhANLYj8Zp4-FwaTpYo,9812
-crawlo/pipelines/__init__.py,sha256=Tvs9FQ_2qz8GcmoEqLrz_YFZrRpL3gGIssiI5Ce9b5I,254
+crawlo/network/request.py,sha256=tPAiOVJyF3Kk-midqydTGXgv5M5tsYJRtwUUJTrUsrE,11075
+crawlo/network/response.py,sha256=cUvdjsB2cQ-qWEKHNGIkwWGgCg-EnQ81xTrjrUOVno0,9738
+crawlo/pipelines/__init__.py,sha256=lrdVDjeHLNkA4_MAwI1auk_I9xfeU1SlBWXiammb6lc,616
+crawlo/pipelines/bloom_dedup_pipeline.py,sha256=QQxGFGEoMHN4Vx2kq7G_i1o9pmuXp8clZebilOar3fk,5642
 crawlo/pipelines/console_pipeline.py,sha256=KABkR3J-rqO0Awox7lizxKR2XuHfVhWPiVRgIybwwu4,1248
 crawlo/pipelines/csv_pipeline.py,sha256=6FBT2AoU6iNU-5NfgWRq7-JpF9dK2nBokjxx-y4jIas,12174
+crawlo/pipelines/database_dedup_pipeline.py,sha256=wVBXEGArFR3uxoN7yfJSOarBmtGrJpOqowAqa7OUs98,8000
 crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
-crawlo/pipelines/mongo_pipeline.py,sha256=lv-Zn_mWdE_jVy7Nh30Lzqm3YhtLRV5rMy-m4rBWYe0,4442
-crawlo/pipelines/mysql_pipeline.py,sha256=6g6PGTwAyzH5VStlPAg2SdG5t1lPw1Zu-cY7x6Mz16s,7861
+crawlo/pipelines/memory_dedup_pipeline.py,sha256=5jeL2jEq7sioYmXlzfkx-LNSbWyChrXeWx8d15YEZOA,3839
+crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
+crawlo/pipelines/mysql_pipeline.py,sha256=cwgJvRORTRea_Eep2coBaMf3G8PQVTQA1qrnIlDZApc,13480
 crawlo/pipelines/pipeline_manager.py,sha256=VrbebOYiqrobtKhp5II18w-odCICdWkmRg5WPK0Emz4,2112
+crawlo/pipelines/redis_dedup_pipeline.py,sha256=TaokJ4wP5-Cxf-ueFJdh4SX58hchT0QzZ5RBDXHDN64,6003
 crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 crawlo/queue/pqueue.py,sha256=yzF900ko2RReVNZtfk_VV3PzSXEUALI6SHf97geuu6s,1058
-crawlo/queue/queue_manager.py,sha256=OQ9YBbc9Y4uE_5WbQhe233FFdhaeiOhDVbqWF7ev42U,11303
-crawlo/queue/redis_priority_queue.py,sha256=hf7hiIiRS-2DIq1bIJbs1yLEZjuMnHwc-n2_I7PlE9A,7294
+crawlo/queue/queue_manager.py,sha256=NMR0Fo8XFBg6_g7htq4D80cS6Ilo0EKt5QtyF-KxNuM,11467
+crawlo/queue/redis_priority_queue.py,sha256=boJCKqcKxRw9XCCzaHy5qmrX9DvzPiQBzOkBHI5odfE,8116
 crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
-crawlo/settings/default_settings.py,sha256=jRM_Cqt3tQ7V4mgqBTAIQY3BcfGsLkKdvW1TcbWHv48,9627
+crawlo/settings/default_settings.py,sha256=B4_61tNJvqzVvyqt9AtRV7Iq5q8G4pJOExcN0ve7S_A,11559
 crawlo/settings/setting_manager.py,sha256=SxKB1aCWh4OySM_bH9cYng9I3PAmrSP-Q8XOZEWEwbI,2899
 crawlo/spider/__init__.py,sha256=Z_rK23l5yt-DuwJPg8bcqodM_FIs4-iHLaKOimGumcE,20452
 crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
 crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
-crawlo/templates/project/items.py.tmpl,sha256=3h-4nuneUoCAGUzSLyLNsdgeAAUXzz4rV0QctzCkIJA,301
-crawlo/templates/project/middlewares.py.tmpl,sha256=Ua2vG3WXliMBSmwsbYaSwzx2lZPw9vLrG2dLqtZWGRg,3157
-crawlo/templates/project/pipelines.py.tmpl,sha256=-dxK7T396AVTbh0FhivLDZq4lKxw3p1-5Y6Ij4jgImE,12041
-crawlo/templates/project/run.py.tmpl,sha256=RQ_k_B-Pp1R853-N_58yUGjIouXa6ipBtUPQenLjswE,7519
-crawlo/templates/project/settings.py.tmpl,sha256=NxPvXGnomTV5NkczPIC7Tscn2vjmHUE4CYmbQB2SmDQ,7884
+crawlo/templates/project/items.py.tmpl,sha256=mt1Mm--H2Ouos3r7JPkYh0r33rgYJf1YOMz0OZy8TYs,297
+crawlo/templates/project/middlewares.py.tmpl,sha256=jpmj7b7Zb7d3nVyxcaVNdp4KqSts6l2cPSqn_oJUSrM,3775
+crawlo/templates/project/pipelines.py.tmpl,sha256=k_4MJnwZ6GPqVwJSEDURUlTxWybmts4vHrF0de2vgAk,2620
+crawlo/templates/project/run.py.tmpl,sha256=ktkYOslcCh9mpklg6yE5VqfATx3Frj_jNT5z5gHjQ4o,8177
+crawlo/templates/project/settings.py.tmpl,sha256=O_teIARjzRD3aMvPnuIgjaDHdjwW-3beyzfo1QH-Hr8,9580
 crawlo/templates/project/spiders/__init__.py.tmpl,sha256=j_YKsw6HQMJyqlk3WUouP3bsr-XVxshRoSNakHBc00g,106
-crawlo/templates/spider/spider.py.tmpl,sha256=Wx1yMBsqDQvJGb9HoEVecX6XZcVrJiBj9blXYrernTo,6167
+crawlo/templates/spider/spider.py.tmpl,sha256=a8S9j43z5gE4auMhf_OnnuVHSZN3JbMDu8Bczu8zIZY,4944
 crawlo/utils/__init__.py,sha256=BDORpyjMN7VGPKImnCDKSkprS-petgD7ezc9rMlBvb0,123
-crawlo/utils/controlled_spider_mixin.py,sha256=BpH76XxBJtYQ0rmCX6vrG-8gQXT9-LP65ibetzfqncU,12500
+crawlo/utils/controlled_spider_mixin.py,sha256=VjT30pNW_YIgmTD0nb7DDl2D3HvpnAYFzgSgV3fxFN0,16475
 crawlo/utils/date_tools.py,sha256=0yG0tzGb1VFgWDJJ_cow2LJfz3kj_w2MqSjmfKKESl8,6961
 crawlo/utils/db_helper.py,sha256=3ib5-agrlwf2t5S_QtLRYH75wvJDlYbRqRmDEbpH5Bo,10559
 crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
@@ -90,6 +98,7 @@ crawlo/utils/system.py,sha256=HvWV1acxou0Rn0L7pNq4CnV_GWFeU0Tmjy3_nLD8M64,237
 crawlo/utils/tools.py,sha256=5Uv25Wy4m_ndZY0-n-eX-t3PxvaZ6wR3-Wvx-o7_Vrs,271
 crawlo/utils/url.py,sha256=rlgX2VlJv6JvLmCDTsbxzMSXE6R5ZL_0dLEqprsA-JU,1482
 examples/__init__.py,sha256=6i631BPnS_TR_BWUjtjB5CBO-zv9kRkwQTQvSya2wHE,123
+examples/controlled_spider_example.py,sha256=SP_k4mdKPvD1JCPs9UCm68jcy2Frg84vvXv9-14RC6I,7776
 tests/__init__.py,sha256=scL1IPVT1iucth7v8ffrjRdeW7QrC_Y7AMmFVMdTY1Y,129
 tests/test_final_validation.py,sha256=fBxf_6YcAEa_HyV_oGAXmmVHY4i6FdA4J6klCmc36hQ,4925
 tests/test_proxy_health_check.py,sha256=xo3QMP1YNw9hu7JDHZOYCUZmFFKLJpHSh4SbxXhCRPQ,1091
@@ -101,8 +110,8 @@ tests/test_redis_config.py,sha256=TqzFRojc6esGXjGhUCvSLYQDUTAgEJsty9vRVuNraMU,89
 tests/test_redis_queue.py,sha256=o6xViXxJcdx-1eMcG3vhAQEIm8h346HnZb7JXs7ZjwM,6622
 tests/test_request_serialization.py,sha256=8sVdppAsohJ5u-m1WvablCndwL-M_36YPLdGKwgeznM,2289
 tests/test_scheduler.py,sha256=-FOkTWzaMdr6yfO1Msu74hI_GgSfD7iRxO-cFA-9Iyk,7442
-crawlo-1.1.2.dist-info/METADATA,sha256=fIXPyz2xk3GN82z5693BePRT1OWvul312NOh41CBRXA,17061
-crawlo-1.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.1.2.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.1.2.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.1.2.dist-info/RECORD,,
+crawlo-1.1.4.dist-info/METADATA,sha256=2I2NA0BR-MWoPZmRUkWrUQYMjuPiUi9mrogIYPWpASU,19781
+crawlo-1.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.1.4.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.1.4.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.1.4.dist-info/RECORD,,

examples/controlled_spider_example.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+受控爬虫混入类使用示例
+演示如何使用 ControlledRequestMixin 和 AsyncControlledRequestMixin 来控制大规模请求生成
+"""
+
+import asyncio
+from crawlo.spider import Spider
+from crawlo.network import Request
+from crawlo.utils.controlled_spider_mixin import ControlledRequestMixin, AsyncControlledRequestMixin
+
+
+class LargeScaleSpider(Spider, ControlledRequestMixin):
+    """
+    同步版本的受控爬虫示例
+    适用于需要生成大量请求但要控制并发的场景
+    """
+    name = 'large_scale_spider'
+    
+    def __init__(self):
+        Spider.__init__(self)
+        ControlledRequestMixin.__init__(self)
+        
+        # 配置受控生成参数
+        self.max_pending_requests = 150     # 最大待处理请求数
+        self.batch_size = 75               # 每批生成请求数
+        self.generation_interval = 0.02     # 生成间隔（秒）
+        self.backpressure_threshold = 300   # 背压阈值
+    
+    def _original_start_requests(self):
+        """
+        提供原始的大量请求
+        这里模拟爬取一个电商网站的商品页面
+        """
+        # 模拟爬取 10,000 个商品页面
+        base_urls = [
+            "https://example-shop.com/products",
+            "https://example-shop.com/electronics", 
+            "https://example-shop.com/clothing",
+            "https://example-shop.com/books",
+            "https://example-shop.com/home"
+        ]
+        
+        for category in base_urls:
+            # 每个分类爬取 2000 页
+            for page in range(1, 2001):
+                yield Request(
+                    url=f"{category}?page={page}",
+                    callback=self.parse_product_list,
+                    meta={'category': category.split('/')[-1], 'page': page}
+                )
+    
+    def _process_request_before_yield(self, request):
+        """
+        在 yield 请求前进行处理
+        可以添加去重、优先级设置、请求头设置等逻辑
+        """
+        # 设置请求头
+        request.headers.setdefault('User-Agent', 
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36')
+        
+        # 根据分类设置优先级
+        category = request.meta.get('category', '')
+        if category == 'electronics':
+            request.priority = 10  # 电子产品优先级最高
+        elif category == 'clothing':
+            request.priority = 8
+        else:
+            request.priority = 5
+        
+        # 可以在这里添加去重逻辑
+        # if self.is_duplicate_request(request):
+        #     return None  # 跳过重复请求
+        
+        return request
+    
+    async def parse_product_list(self, response):
+        """解析商品列表页面"""
+        # 提取商品链接
+        product_links = response.css('a.product-link::attr(href)').getall()
+        
+        for link in product_links:
+            # 生成商品详情页请求
+            yield Request(
+                url=response.urljoin(link),
+                callback=self.parse_product_detail,
+                meta={'category': response.meta['category']}
+            )
+        
+        # 提取分页信息
+        next_page = response.css('a.next-page::attr(href)').get()
+        if next_page:
+            yield Request(
+                url=response.urljoin(next_page),
+                callback=self.parse_product_list,
+                meta=response.meta
+            )
+    
+    async def parse_product_detail(self, response):
+        """解析商品详情页面"""
+        yield {
+            'title': response.css('h1.product-title::text').get(),
+            'price': response.css('.price::text').re_first(r'\d+\.?\d*'),
+            'description': response.css('.product-description::text').get(),
+            'category': response.meta['category'],
+            'url': response.url,
+            'in_stock': 'in-stock' in response.css('.availability::attr(class)').get(''),
+            'rating': response.css('.rating::attr(data-rating)').get(),
+            'reviews_count': response.css('.reviews-count::text').re_first(r'\d+')
+        }
+
+
+class AsyncLargeScaleSpider(Spider, AsyncControlledRequestMixin):
+    """
+    异步版本的受控爬虫示例
+    使用异步控制来实现更精确的并发管理
+    """
+    name = 'async_large_scale_spider'
+    
+    def __init__(self):
+        Spider.__init__(self)
+        AsyncControlledRequestMixin.__init__(self)
+        
+        # 配置异步控制参数
+        self.max_concurrent_generations = 15    # 最大同时生成数
+        self.queue_monitor_interval = 0.5       # 队列监控间隔
+    
+    def _original_start_requests(self):
+        """
+        提供原始的大量请求
+        这里模拟爬取新闻网站的文章
+        """
+        # 模拟爬取 50,000 篇新闻文章
+        news_sites = [
+            "https://news-site-1.com",
+            "https://news-site-2.com", 
+            "https://news-site-3.com",
+            "https://tech-news.com",
+            "https://finance-news.com"
+        ]
+        
+        categories = ['tech', 'finance', 'sports', 'politics', 'entertainment']
+        
+        for site in news_sites:
+            for category in categories:
+                # 每个分类爬取 2000 页
+                for page in range(1, 2001):
+                    yield Request(
+                        url=f"{site}/{category}?page={page}",
+                        callback=self.parse_article_list,
+                        meta={'site': site, 'category': category, 'page': page}
+                    )
+    
+    def _process_request_before_yield(self, request):
+        """异步版本的请求预处理"""
+        # 设置请求头
+        request.headers.setdefault('User-Agent', 
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
+        
+        # 根据新闻类型设置优先级
+        category = request.meta.get('category', '')
+        if category in ['tech', 'finance']:
+            request.priority = 10  # 科技和财经新闻优先级最高
+        elif category in ['sports', 'politics']:
+            request.priority = 8
+        else:
+            request.priority = 5
+        
+        # 设置延迟（避免对服务器造成过大压力）
+        site = request.meta.get('site', '')
+        if 'tech-news.com' in site:
+            request.meta['download_delay'] = 0.5  # 科技新闻站点较敏感，增加延迟
+        
+        return request
+    
+    async def parse_article_list(self, response):
+        """解析文章列表页面"""
+        # 提取文章链接
+        article_links = response.css('a.article-link::attr(href)').getall()
+        
+        for link in article_links:
+            yield Request(
+                url=response.urljoin(link),
+                callback=self.parse_article_detail,
+                meta={
+                    'site': response.meta['site'],
+                    'category': response.meta['category']
+                }
+            )
+    
+    async def parse_article_detail(self, response):
+        """解析文章详情页面"""
+        yield {
+            'title': response.css('h1.article-title::text').get(),
+            'content': ' '.join(response.css('.article-content p::text').getall()),
+            'author': response.css('.author::text').get(),
+            'publish_date': response.css('.publish-date::attr(datetime)').get(),
+            'category': response.meta['category'],
+            'site': response.meta['site'],
+            'url': response.url,
+            'tags': response.css('.tags a::text').getall(),
+            'views': response.css('.views-count::text').re_first(r'\d+'),
+            'comments': response.css('.comments-count::text').re_first(r'\d+')
+        }