crawlo 1.1.9__py3-none-any.whl → 1.2.1__py3-none-any.whl

crawlo-1.2.1.dist-info/RECORD

@@ -0,0 +1,220 @@
+crawlo/__init__.py,sha256=jSOsZbDJ_Q5wZV8onSXx5LgNM7Z1q3zCROGdImBDr2I,1373
+crawlo/__version__.py,sha256=1y7BBR2l-3BuWrDXe70lefKZxLd53JVfWAfn4y7Mn_0,23
+crawlo/cli.py,sha256=ZrrOKAvqgGJgqoyakzItt-Jroa9JBF9vQG-1lz4wBPM,2094
+crawlo/config.py,sha256=_pORwcVEgjEhrqVaApu51X0Il3TBK6w9aezGnnqYu8Y,9847
+crawlo/config_validator.py,sha256=M118EATR-tITzRSe2oSinV5oh2QsooMCkEJ5WS8ma_0,10155
+crawlo/crawler.py,sha256=24EE7zFPByeYLJnf1K_R9fhJMqaFUjBSa6TuUhlY4TI,37398
+crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
+crawlo/exceptions.py,sha256=YVIDnC1bKSMv3fXH_6tinWMuD9HmKHIaUfO4_fkX5sY,1247
+crawlo/mode_manager.py,sha256=q34nX-bMCLCKFfeYy_34VrBT86sL_K030LlLX2H3m0I,7728
+crawlo/project.py,sha256=MH2OYXIZdc9EJolUpUBXTcAgS7qvPP366LkOy62iP_8,6831
+crawlo/stats_collector.py,sha256=v4jC9BAe-23w93hWzbeMCCgQ9VuFPyxw5JV9ItbGH8w,1636
+crawlo/subscriber.py,sha256=Aj0kPpbBYlzOb1uViDFraoaThsQEVlqOSYUaFT3jSDs,5136
+crawlo/task_manager.py,sha256=PScfEB03306Txa0l38AeQ_0WVhKzeWOFyT3bnrkbHW0,849
+crawlo/cleaners/__init__.py,sha256=lxL-ZWDKW-DdobdgKUQ27wNmBiUhGnD0CVG6HWkX3_o,1261
+crawlo/cleaners/data_formatter.py,sha256=iBDHpZBZvn9O7pLkTQilE1TzYJQEc3z3f6HXoVus0f0,7808
+crawlo/cleaners/encoding_converter.py,sha256=G3khLlk0uBeTwIutsWxVUeSuyc1GMC1BDNJDwsU9ryg,4238
+crawlo/cleaners/text_cleaner.py,sha256=16e6WqIIb9qANMiK-vCEl4TvgkId19Aa2W1NMLU-jFQ,6707
+crawlo/commands/__init__.py,sha256=orvY6wLOBwGUEJKeF3h_T1fxj8AaQLjngBDd-3xKOE4,392
+crawlo/commands/check.py,sha256=jW8SgfkOS35j4VS7nRZBZdFCBX9CVFez5LR2sfP_H1U,23437
+crawlo/commands/genspider.py,sha256=_3GwFMYK79BuKk__5L0ljuwWwOzN80MeuhRkL4Ql11A,5201
+crawlo/commands/help.py,sha256=reSUkeqLzl13tfjSFoYYVrs8y-gYFHjiznn8vrK8P3M,5314
+crawlo/commands/list.py,sha256=octTk0QZhapiyM7WgCPersP2v3MesthbJeG9vMqVFOs,5936
+crawlo/commands/run.py,sha256=Go9hAEUMuG3GphBgemG5S5W4MF39XOxp7-E06rX-pTU,11043
+crawlo/commands/startproject.py,sha256=UYGelGY4dM6Zu3U4G5m8snKqbsfgszhvfpAJLl5b5tM,15772
+crawlo/commands/stats.py,sha256=iEKdxHoqsJuTkn8zAF9ekBVO1--8__BeD7xohYG5NwE,6252
+crawlo/commands/utils.py,sha256=b7yW6UlOLprR3gN9oOdhcl3fsCwWRE3-_gDxWz5xhMo,5292
+crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
+crawlo/core/engine.py,sha256=kkNHQt1Dycb1hrEmG1m3seyX9C5qQp4GhOF-0H97t-A,14078
+crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
+crawlo/core/scheduler.py,sha256=LGeytsc04itdX7ulsl46MXBTyRFhBUu7UCSQzbtKZsk,5362
+crawlo/downloader/__init__.py,sha256=8-r4_Wc_X64FJtKzNQamwsZsc428creKeFo19VxF33o,8565
+crawlo/downloader/aiohttp_downloader.py,sha256=Ck4ybwpox3sC7v4IGc8RIcvmIBf9v6jAdyNGHByY1m4,8620
+crawlo/downloader/cffi_downloader.py,sha256=U2CMYQ8ZyKMQNNPvyI2USqI-Dclq2d7ZRgp3LxLjTFs,10966
+crawlo/downloader/httpx_downloader.py,sha256=MpgDeIdGqNsiSKLOEDBnr5Z0eUbhHnqVEmAuoIfJmFU,12296
+crawlo/downloader/hybrid_downloader.py,sha256=nn_cD4SOKj0_TE2zuLfXfJhQppeftZDL5u8DZuITwVQ,8256
+crawlo/downloader/playwright_downloader.py,sha256=L-TVzG7cYfuBlqW0XSZuz5C_r9fpJrmYNcoQ-cDEna4,16663
+crawlo/downloader/selenium_downloader.py,sha256=ho0ovIVU7e99xMT2_Clgj6SIH3GjxPx7yPN85C0_w9o,21508
+crawlo/extension/__init__.py,sha256=g7Nj1vm8L8n9ZmIbTzOXMYTkMSAJj4pw8TefaDyd52k,1563
+crawlo/extension/health_check.py,sha256=Yiu50Dkn_rIS6NnzurIoQmwDM2d-SmEetO1d5wor2Xc,5668
+crawlo/extension/log_interval.py,sha256=2R3XVdM1grDN8wh9TTHRB_WmQypCr5YSGvESNDnS16s,2474
+crawlo/extension/log_stats.py,sha256=5CpcTHj0lCtFvhBTI3pG41gkiL_hPtv5FWY87L6EVjM,2989
+crawlo/extension/logging_extension.py,sha256=euecjvj71aK6dElMzYCuNXiNb9jO8qQmL5QspQ1xoz8,1669
+crawlo/extension/memory_monitor.py,sha256=fClPchpCkVjcIiU0AJHCKDd7HEiz5B4KqNqKTRZ2hcU,4394
+crawlo/extension/performance_profiler.py,sha256=BjWD3LOb4VwjQJQvQtWNg7GluEwFquI1CztNfgMzy3c,5032
+crawlo/extension/request_recorder.py,sha256=KA_RmcfscDxP5wPdolO76yKfRj-1jmHhG3jkVGO1pbc,4181
+crawlo/filters/__init__.py,sha256=lX-QOCDTiTRFoiK1qrZ5HABo7LgZfcxScx_lELYEvJk,4395
+crawlo/filters/aioredis_filter.py,sha256=oveE9qFRtK8C9FZGRSZusRakXcAF-B2vkwbpsc0fX1E,10308
+crawlo/filters/memory_filter.py,sha256=FzGJPhVKfZ8P23kP6de-VSfE8oVMjjpfWzKJIdiMtZU,9529
+crawlo/items/__init__.py,sha256=rFpx1qFBo0Ik7bSdnXC8EVTJUOQdoJYGVdhYjaH00nk,409
+crawlo/items/base.py,sha256=hwGJEdFWOdaZfalFX8umRkh_HUWLEbCjvq4j70fplMQ,598
+crawlo/items/fields.py,sha256=fpS0vlRPpZYjTaMDgI9Q8z_YQqruwf6fi4Dgm6R2oEk,1854
+crawlo/items/items.py,sha256=OmVEvMmgofMU95GkaiWkfNQ2fjsH2fY9sw3SKcmUhLs,3478
+crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
+crawlo/middleware/default_header.py,sha256=PHz_VWRZga1gqkpTGY_lDoG2c_ktt03dDacVKUzNu_I,5348
+crawlo/middleware/download_delay.py,sha256=2iWnJFtWDlqDy5MsAob8TPiJQoiz9v21yatkBI0eptg,3542
+crawlo/middleware/middleware_manager.py,sha256=G9R9SnDVey_zS2k1zm_358EPHDxRWfP953I4QavL7P0,6348
+crawlo/middleware/offsite.py,sha256=Y1SQaNjZOmWPNCHy884X0hfwVR31sIdJ1ofv83n0NDU,4142
+crawlo/middleware/proxy.py,sha256=eHeEDaCm72cpipXM7b-KO3i8ScYMsPuu5byLZUFuHhY,15604
+crawlo/middleware/request_ignore.py,sha256=cjjwtACdbH4Me0NF7RwYn6cJQNlrJktO_ZaRE5OjiF8,2665
+crawlo/middleware/response_code.py,sha256=Nk5fQtgyzZVOfFvFcgF442rk9hXBm9ixzTh4nH8BhSM,5099
+crawlo/middleware/response_filter.py,sha256=tUe5Hc2N8p5_4kO8pdDa6sChUBkP2Mk7XAo762ScZOk,4386
+crawlo/middleware/retry.py,sha256=D_v4-8grofGsLkGJIhLKklMbdHO0gtij7DIS4G52NJc,4248
+crawlo/network/__init__.py,sha256=bvEnpEUBZJ79URfNZbsHhsBKna54hM2-x_BV8eotTA4,418
+crawlo/network/request.py,sha256=amZdRvDJvFpx82thtWwv6uo5uv9vtCwbvw3jAc3sCrY,12633
+crawlo/network/response.py,sha256=QwJhL3xJfPVy_gwtGrg61oAgaqCoCmjyj1Ug7Zju7Pg,13060
+crawlo/pipelines/__init__.py,sha256=FDe2Pr5tiHtV8hFlheElRO_O1aVKvSWlkTcAl9BXAKA,637
+crawlo/pipelines/bloom_dedup_pipeline.py,sha256=n0Ay7MtIEJ8L4Otiha4zRvI9toFUSNFugTNubi-Q3aw,5798
+crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
+crawlo/pipelines/csv_pipeline.py,sha256=qbXZoqq4FIR9QkUGpC0ryWzmqGJSrM2bxmWLM4I1nXM,12490
+crawlo/pipelines/database_dedup_pipeline.py,sha256=6_zKtCNgFBPJyTI3Mk4l75fMQ2UQQGKFfNTpZqGs_zI,8224
+crawlo/pipelines/json_pipeline.py,sha256=wrCsh8YInmcPLAkhPrHObMx89VZfhf-c7qRrYsTixPE,8585
+crawlo/pipelines/memory_dedup_pipeline.py,sha256=oQcBODO-I2p6B7Nm_klXvuhzSMIHP-JWwC4_o6Gkgcc,3954
+crawlo/pipelines/mongo_pipeline.py,sha256=PohTKTGw3QRvuP-T6SrquwW3FAHSno8jQ2D2cH_d75U,5837
+crawlo/pipelines/mysql_pipeline.py,sha256=RRKMuO-7BTomyRFYCmDpfvjBTcU4SGdjGDV4wBeKWck,13796
+crawlo/pipelines/pipeline_manager.py,sha256=g7lQBllYpKLymDWShjsR4SsXG41XnzD5mUR0UDBt0ZQ,2419
+crawlo/pipelines/redis_dedup_pipeline.py,sha256=U2mms0XBkTgC8_DywIigGboChjUH_l0-cfo_ioYQNsI,6333
+crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/queue/pqueue.py,sha256=qTFOuvEXsYEZbm0ULjsOeZo0XtSsZ-SHpx7nFEtmluE,1095
+crawlo/queue/queue_manager.py,sha256=wdQq0UYJcmyw9JPXiro2GP3q58vtjeIV3ouWyFmuhBc,13211
+crawlo/queue/redis_priority_queue.py,sha256=oVvGjkcMpRqJEEBbRlAL16TKcp38Gg9YMdZCUQa7wFI,13081
+crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
+crawlo/settings/default_settings.py,sha256=_s5mXXYFhZ6Yx_8ATALDmrOOb-4e8iAiaGyi7LNYJss,9385
+crawlo/settings/setting_manager.py,sha256=uqgm2uZQp7Wx1hMSf2FnA0dh6f4jzZ13NQbRvIfYwsQ,3829
+crawlo/spider/__init__.py,sha256=xAH6NfE_6K2aY_VSL9DoGjcmMHJDd5Nxr7TG1Y8vQAE,21091
+crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
+crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
+crawlo/templates/project/items.py.tmpl,sha256=8_3DBA8HrS2XbfHzsMZNJiZbFY6fDJUUMFoFti_obJk,314
+crawlo/templates/project/middlewares.py.tmpl,sha256=fxHqi-Sjec5GiHJciprOU-6SAUTzM728NlZckIqf9hM,4278
+crawlo/templates/project/pipelines.py.tmpl,sha256=j9oqEhCezmmHlBhMWgYtlgup4jhWnMlv6AEiAOHODkg,2704
+crawlo/templates/project/run.py.tmpl,sha256=yOxpPkyffXLwa7NDx2Y96c8U-QN81_3mqZSuB526DNs,1271
+crawlo/templates/project/settings.py.tmpl,sha256=1481CdB6VdDzNXyYe0a5Qn5z-UJ0ELIELuSVbExnrOg,12167
+crawlo/templates/project/settings_distributed.py.tmpl,sha256=Nli-qR-UB4TXAq4mXjx17y-yAv46NwwpcVidjGeM00A,4321
+crawlo/templates/project/settings_gentle.py.tmpl,sha256=ljXf9vKV2c-cN8yvf5U4UWsPWrMVmJUfbXladIdS2mg,3320
+crawlo/templates/project/settings_high_performance.py.tmpl,sha256=3wf8fFYZ5EVE2742JlcwwrPF794vEIEmbxFSbqyGnJQ,5434
+crawlo/templates/project/settings_simple.py.tmpl,sha256=UKfSEZFAXDywswwws9SvR5vBMng2maJ37DJMfK0nAmI,2365
+crawlo/templates/project/spiders/__init__.py.tmpl,sha256=zMbePipgLsctQUEnda4WkHz8rDLUX--rc8ruI6zkpWc,111
+crawlo/templates/spider/spider.py.tmpl,sha256=KvU-9YpN6MifDE7XzejjyyQS7RUjLDLZ8zqJcLwSsu0,5198
+crawlo/tools/__init__.py,sha256=6BcoEE7t119VzdwqnULrWnYDNpXu86FHkWgx_WL1-Sk,3853
+crawlo/tools/anti_crawler.py,sha256=LwLC6BkxDSkxc5H1hQ6kY9j7O0PZGAMPZECr7gbqw2M,9431
+crawlo/tools/authenticated_proxy.py,sha256=ULCK0Cc9F2rGhRqu6kzKBdxzK9v2n1CsatSQ_PMxpAg,7272
+crawlo/tools/data_validator.py,sha256=bLWnkpFdclJuqjtSAgMI5nznN4vAuPwE1YaiFWKWenM,5490
+crawlo/tools/date_tools.py,sha256=dNm06OwJYQ7KeVkZoFcfFhMwJPsHlB3JJfSplOA0XfY,635
+crawlo/tools/distributed_coordinator.py,sha256=QM4ECyhTba9jtV7-AkOuT_lDJObeKVeBsRLZZFS_A7Y,12863
+crawlo/tools/retry_mechanism.py,sha256=VbHo9-pT8ADbwyPuKcRfeUo4BtVZQy6yvrSvugoiZRA,8021
+crawlo/tools/scenario_adapter.py,sha256=pzysL1B2uQ1ZSEncVHd9Hv2viHNgaxP44YAUcDcppfw,9660
+crawlo/utils/__init__.py,sha256=5unISyKR7pZdj7EwP5tadGYKvyb_W1uoC52YzqSbxis,592
+crawlo/utils/batch_processor.py,sha256=wnpWL41Su6gSvbzNgRWv7sRxKzBFV0J27V9gC3WsLJc,9155
+crawlo/utils/controlled_spider_mixin.py,sha256=os5v8eFX08Nbqm48W-xWdY04LboqDgOgOL-93IX9zvg,16914
+crawlo/utils/date_tools.py,sha256=gsgHweZFOzeW-K91wRlBEg_XLKBCXl7UZJfnjI_W-J0,9197
+crawlo/utils/db_helper.py,sha256=ZqOt1d3mErVv4TOvoWlov0niUxORB9aHByTmMoNFIDw,10902
+crawlo/utils/enhanced_error_handler.py,sha256=o6oxCtWFcAGslON3FZNRgxvHutd9z0tqbc0R5kmF_lk,14265
+crawlo/utils/env_config.py,sha256=9hJ4T-ufAzWPZugytgLMZStZxD17JaE900S8xpGoyn0,2974
+crawlo/utils/error_handler.py,sha256=7AHwZHJD-CTq0DNOwbDjUvzdbushDI_xWJiwe4KzBeQ,4411
+crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
+crawlo/utils/large_scale_config.py,sha256=lsraHTAQx3sMPjTnCBY_SVIpkuIBUb3zD9eFvmccOOM,8440
+crawlo/utils/large_scale_helper.py,sha256=ZazAI7KV3V-3hzc4a3BWxTXfEO2XIEBMzxTLM9S1l_Q,12500
+crawlo/utils/log.py,sha256=YD2FfXuuE2MC9ZdQQZ0H7KysE7l_LHZqQepaTPlcApo,4133
+crawlo/utils/performance_monitor.py,sha256=ku_QAx3MX8U-Dfd5fqMr-YV5j1srYJbtF_5jdVJx6tY,9879
+crawlo/utils/queue_helper.py,sha256=gFmkh1jKlIcN1rmo2Jl6vYcLP5ByUWlfHO9eNlZPBLs,4918
+crawlo/utils/redis_connection_pool.py,sha256=rGaKcn2YVeP_bA38svkl3A-lU3DpfX6x0TzdaxWvIlo,10665
+crawlo/utils/redis_key_validator.py,sha256=5lft8SPbHkBf9NdrQaW13R-fXLJoMBkJlc1w38wuPmU,5836
+crawlo/utils/request.py,sha256=ejdKpTwc-HE04HQybafhOVywzz57IV3pY0YMkSLyGUo,9065
+crawlo/utils/request_serializer.py,sha256=bZhuonZV2AdB_X0aje7sDljqWAIrEzUYwEaxXytaWsg,8784
+crawlo/utils/spider_loader.py,sha256=V0CBTicJBYBZafhwLfDEfuEc_hJ2mSoiptT6qKufI9U,2249
+crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
+crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
+crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
+examples/__init__.py,sha256=NkRbV8_S1tb8S2AW6BE2U6P2-eGOPwMR1k0YQAwQpSE,130
+examples/aiohttp_settings.py,sha256=xEagH-6lGmUaWBiGJ1gqtHTDw60PzR9q1qzkpMrd_oc,1058
+examples/curl_cffi_settings.py,sha256=0oLNQ2MI-hf8t7ibYmbXw68v3yd_Xu9j9j4_3RNsvGQ,1027
+examples/default_header_middleware_example.py,sha256=1hUPu-Zw4okWLox2pD8FTV7MQqYgRjDus5mVsGODinQ,4599
+examples/default_header_spider_example.py,sha256=JIkG81aqXhGi-TKeFGl8s4ewMhdhwss-QBODj5il3VI,4723
+examples/download_delay_middleware_example.py,sha256=utIsjO6in9vqiDZ59fhV8It62rx9_9DMyJXbdbh8kSE,4855
+examples/httpx_settings.py,sha256=cOYrH4PlsrfSD6r8z9ErjU_2cPLWifXIvu8YfmVSRxg,1036
+examples/multi_downloader_proxy_example.py,sha256=mje7W5GsCbLvXClQ724n9AFgZbJMgQ80hMh-N5fzQv0,2576
+examples/offsite_middleware_example.py,sha256=X6GqLG6J1iDjs04CVwh-XvTv7sW0r2o6OyBX3xxBV7U,2069
+examples/offsite_spider_example.py,sha256=01LPOkNZEAlMwXuK2jI8fqUNkgm_CDmrNQRzL8RCE5A,3172
+examples/proxy_spider_example.py,sha256=CvreU4UhLXnFFDffKthJAgO6ebzOvwHQbXBNQfw8rOY,5495
+examples/request_ignore_middleware_example.py,sha256=O9YdM6-ivsBMyXmMAhw2kI9j3VmSXf0cftBnPsJGEQA,2023
+examples/request_ignore_spider_example.py,sha256=S6Y50w7wrL8l5vemq4lm13_0SONHBIs3pN3_6LfR0Hw,3169
+examples/response_code_middleware_example.py,sha256=AvR4RuCK4wL7yUd0qoOB5ScBbdYefUa95SmBr_Pv9pQ,2082
+examples/response_filter_middleware_example.py,sha256=9QDqzDkcfSF8WVyUNgu-ybj6x43OXI_Xl1v2gVHqWhE,2651
+examples/tong_hua_shun_settings.py,sha256=3YYojtVvxm3JqRkR-hNL9WDR3AeoIZW3iikRnM4C6OU,1926
+examples/tong_hua_shun_spider.py,sha256=5ToQOv01LS0GlmMVknwIY2CpUZ0vTq-sBUP6t8aOfqg,6690
+tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md,sha256=4W6HlT9Uc3cyu77T9pfbkrMxpAZ-xq_L9MU-GbukLV0,3427
+tests/__init__.py,sha256=409aRX8hsPffiZCVjOogtxwhACzBp8G2UTJyUQSxhK0,136
+tests/advanced_tools_example.py,sha256=1_iitECKCuWUYMNNGo61l3lmwMRrWdA8F_Xw56UaGZY,9340
+tests/authenticated_proxy_example.py,sha256=b307_RybOtxXVQK0ITboLvHmOTwIN4yTF9aup4dYF7Q,8477
+tests/cleaners_example.py,sha256=tEGf-x8CNlXpKurgmNxTmmPjQa45dRUSBjnOJVGGSeU,5330
+tests/config_validation_demo.py,sha256=ah4cLZnozMdwKzACOoa5R52dWfya282WyulCGjadioM,3218
+tests/controlled_spider_example.py,sha256=2SAQKoREGHe-OzVaSkGpopCcrou6QXmeW7rLdmsyopw,7981
+tests/date_tools_example.py,sha256=XI3iFEzeo7Nb5YepK8WHytIaBegtxWVSISpqQgpV6M8,5042
+tests/dynamic_loading_example.py,sha256=7LdeQZFevrb-U1_dgr4oX3aYo2Da4HvE_0KIf1fw4Ew,18786
+tests/dynamic_loading_test.py,sha256=dzDW7b66HeDsIYsYgvNRihE3V6b6gEbUGQpp-eJbcIM,3413
+tests/env_config_example.py,sha256=_ZRDh_LR23ZKpy9E--y_KM0QIOiZF5vRT98QTn52TY8,4951
+tests/error_handling_example.py,sha256=sqvSoogDefyDoEBz3AVPG4pe49gRqBxT5BDFkijmYaA,5332
+tests/redis_key_validation_demo.py,sha256=R0ZTJJSeMB-ptVfzwegw7bw83FDDcR2XUNnUZqstSPE,4475
+tests/response_improvements_example.py,sha256=t1cbG3nesp82bqog4_ku1GvQzNbhRyWa5EaKTmOPrSk,5402
+tests/test_advanced_tools.py,sha256=HT_TcwfFzli-CavIJSqQqnCxnBn5FDMX09zL7AJ5tNY,5398
+tests/test_all_redis_key_configs.py,sha256=Uk91dnGwGaGsdukpvbTiXd4NnrsOa-THWD2cwSZkqag,5805
+tests/test_authenticated_proxy.py,sha256=lnvmQwuf0zaZP_E05EzcNFR2VJbwTkLjOmZGNoJKaC4,4339
+tests/test_cleaners.py,sha256=qyd20RNuBHIVHz7X5JjLwlIZedn7yHZ4uB3X78BpaF4,1819
+tests/test_comprehensive.py,sha256=dvRJeeVYc1cgXK9Y171hH9Y847zZpWSAFFH-EI3UepQ,5182
+tests/test_config_validator.py,sha256=Ec1h8Mw-fVz1d9JoATIWZb0nTc8pYjhTCSjPm3tvkTQ,6825
+tests/test_crawlo_proxy_integration.py,sha256=O0wK1u73qOuahRHWBXeDuMk5y1qMBMHLVVBYVP3U5SM,5576
+tests/test_date_tools.py,sha256=pcLDyhLrZ_jh-PhPm4CvLZEgNeH9kLMPKN5zacHwuWM,4053
+tests/test_default_header_middleware.py,sha256=v-ei_1EY7cvFSsySrQPXF5-DmyGsq2yzjYhhrwFMOXs,6003
+tests/test_double_crawlo_fix.py,sha256=qkJugMiEM2KQFHZ_Iza7Y0cER8p-Mecr57zjwHdsaSE,8033
+tests/test_double_crawlo_fix_simple.py,sha256=aPQQRex7ShxC_QJyjRkgelbn0Lnl3fFTwsPH5OglpwM,4807
+tests/test_download_delay_middleware.py,sha256=Idc6KzhL3hY3aDKgn1j_v5-mLIHz7dTnV5c4tJVZh5Q,9107
+tests/test_downloader_proxy_compatibility.py,sha256=DU19UnjhS3jdP1lfoF098bptKF4jpaowc4G274SgV8c,8915
+tests/test_dynamic_downloaders_proxy.py,sha256=t_aWpxOHi4h3_fg2ImtIq7IIJ0r3PTHtnXiopPe2ZlM,4450
+tests/test_dynamic_proxy.py,sha256=zi7Ocbhc9GL1zCs0XhmG2NvBBeIZ2d2hPJVh18lH4Y0,3172
+tests/test_dynamic_proxy_config.py,sha256=C_9CEjCJtrr0SxIXCyLDhSIi88ujF7UAT1F-FAphd0w,5853
+tests/test_dynamic_proxy_real.py,sha256=krWnbFIH26mWNPhOfPMmx3ZxJfOreZxMZFGwVb_8-K8,3511
+tests/test_edge_cases.py,sha256=kd5irijfGB0MkL-eukmc-WFm_xfbzQ30tPq-FMJaxsg,10856
+tests/test_enhanced_error_handler.py,sha256=cROcikmLvL11NgjCj-fGfVlKV3sLq7VN4DY_9U54IBU,8705
+tests/test_env_config.py,sha256=Qu1sDeADs69dSr1x0QmEe8nJrMHneE_4JClt-N901e8,4867
+tests/test_error_handler_compatibility.py,sha256=xJ43cmCwfBGh-qBwCGiMDPPlfNDLw4ZrmlrHN9IojkU,4241
+tests/test_final_validation.py,sha256=UNHMOkcOBx9jPdnYuYCF4Cx5GlXakBeHybOP27lpbAg,5078
+tests/test_framework_env_usage.py,sha256=bFb_ptdLeX2obdJWEqEHPWweiWR-wR2BpvEaJMQK7h4,4201
+tests/test_integration.py,sha256=l-G_z_btCtK0W4VvP1yNJFaZIZtKXDTeD1Ns2C-v6Nw,11677
+tests/test_item_dedup_redis_key.py,sha256=GIdyFBm56EbFzTvgnYcimgQFoMgvEZW8z1KFVUu3HJ4,3877
+tests/test_offsite_middleware.py,sha256=1DYktO_D-hiLEB6dBnc0iOvnWimqOdE6kimnS8aof_s,7764
+tests/test_parsel.py,sha256=wuZqRFIm9xx1tt6o3Xi_OjvwhT_MPmHiUEj2ax06zlo,701
+tests/test_performance.py,sha256=cjmWwIXBsHibgVyjyIt30iiXSDVse2HD6lFb0YlZsAQ,11613
+tests/test_proxy_api.py,sha256=SOBedDX8JTl3iPhluaCyybvA6B6k1poYkN7hY9gNpho,11029
+tests/test_proxy_health_check.py,sha256=_tDlxa_6TdL3M5RLkHF82roXJ8WIuG5hELBp2GADyKQ,1123
+tests/test_proxy_middleware.py,sha256=EdQAfwwAJIBxw9JmUFTDEu_pdxapaTlcJr7KcrY6-AY,4021
+tests/test_proxy_middleware_enhanced.py,sha256=QR-p26F63N7MxNjZ2QJUeerh_xdnCDejkrGPIh7Fh4U,7035
+tests/test_proxy_middleware_integration.py,sha256=mTPK_XvbmLCV_QoVZzA3ybWOOX61493Ew78WfTp-bYQ,4441
+tests/test_proxy_providers.py,sha256=u_R2fhab90vqvQEaOAztpAOe9tJXvUMIdoDxmStmXJ4,1749
+tests/test_proxy_stats.py,sha256=ES00CEoDITYPFBGPk8pecFzD3ItYIv6NSpcqNd8-kvo,526
+tests/test_proxy_strategies.py,sha256=9Z1pXmTNyw-eIhGXlf2abZbJx6igLohYq-_3hldQ5uE,1868
+tests/test_queue_manager_double_crawlo.py,sha256=5oPn3xnAXcZ2ZJweLTYEKDM-sQXQ0MZJdtjqD_Zmlrc,6995
+tests/test_queue_manager_redis_key.py,sha256=tvy3qkmB6XNpnJ4SOgjKvxE83hltCdL5Z32CupQ2VZ0,6454
+tests/test_real_scenario_proxy.py,sha256=dhlQcbiGd-zp2L9EJ12q7HfDxuEAyi_SxBSoXH6YyZM,8490
+tests/test_redis_config.py,sha256=Kbl3PURGNM1BUIspakEOA-ZOl2xxTHb_8KbftwjYOsg,921
+tests/test_redis_connection_pool.py,sha256=wLPmJ94jUajpShNrnnl4pTbX9ZIGqCEZgzzecasAC4s,9471
+tests/test_redis_key_naming.py,sha256=44A174UkKdj2ipjJa4N0RWIN8CYun6w08yylg8Yu9ac,6957
+tests/test_redis_key_validator.py,sha256=Yw0EsDQuI3Gzt_s86BubBP2VU_s7fyI-TthEX5J5C-o,4427
+tests/test_redis_queue.py,sha256=5LTc86A5qqF5VbmkvkF2OnLAxlJ7ClfJPw0dODxekFk,6846
+tests/test_request_ignore_middleware.py,sha256=QN81wgG_W_XfXCF9LvJNxCNwbOH6_tZnLIwLDTK2K5Q,6229
+tests/test_request_serialization.py,sha256=Jf7Kr7edL0ENwxh8ABa1W_O3dWyqNlvoSfQM1Mykpys,2359
+tests/test_response_code_middleware.py,sha256=wSe525bm-bk_iWMjPDzUu1LfOQrwJY8_MLKAspq2dzk,12193
+tests/test_response_filter_middleware.py,sha256=YWrGzJ7wmftTjJXcNTtJl3b3EdJsO4oR22ZLWwgErhg,16327
+tests/test_response_improvements.py,sha256=vNqHKyoEoYeEGAUiRzdsff2V6yvJ9QnDwGg7gmN38Ow,6028
+tests/test_retry_middleware.py,sha256=RmSYSf0GagGPGAVi5TXJWc0bZlmAI_hwFr2FYhvuKrk,8097
+tests/test_scheduler.py,sha256=elAPFh-Ph49bbJQlTBEsRwzhoX82EdryqQbpc_wsobU,7683
+tests/test_simple_response.py,sha256=_ui2PuVZvJcAuLY7HZ8xcsy_tDBimgBqX0ukj3kE5J0,1549
+tests/test_telecom_spider_redis_key.py,sha256=SUCUK_CgHgl1wj59o2a1g4qJpLUyW4reXv-5Bjuk6Ko,7675
+tests/test_template_content.py,sha256=URwjlAzMCdUN0sW_OupUcuSNMxp1OKgW79JOpkLPXnw,2965
+tests/test_template_redis_key.py,sha256=dOFutic8CL3tOzGbYhWbMrYiXZ8R3fhNoF5VKax5Iy0,4946
+tests/test_tools.py,sha256=fgzXL2L7eBV_nGjeMxH8IMhfc0dviQ80XgzZkJp_4dA,5266
+tests/tools_example.py,sha256=uXNS4xXJ-OD_xInAn2zjKLG_nlbgVGXZLoJtfhaG9lI,7926
+crawlo-1.2.1.dist-info/METADATA,sha256=5D7qv6sQyaTf7K2nE2ii4lc97YZQQmmKEJqxAifFHoE,20484
+crawlo-1.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.2.1.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.2.1.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.2.1.dist-info/RECORD,,

examples/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 12:36
-# @Author  :   oscar
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 12:36
+# @Author  :   oscar
+# @Desc    :   None
+"""

examples/aiohttp_settings.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+aiohttp下载器配置示例
+"""
+
+# 基础配置
+SETTINGS = {
+    # 下载器配置
+    'DOWNLOADER': 'crawlo.downloader.aiohttp_downloader.AioHttpDownloader',
+    'DOWNLOADER_TYPE': 'aiohttp',
+    
+    # aiohttp特定配置
+    'AIOHTTP_AUTO_DECOMPRESS': True,
+    'AIOHTTP_FORCE_CLOSE': False,
+    
+    # 代理配置
+    'PROXY_ENABLED': True,
+    'PROXY_API_URL': 'http://test.proxy.api:8080/proxy/getitem/',
+    'PROXY_EXTRACTOR': 'proxy',
+    'PROXY_REFRESH_INTERVAL': 60,
+    'PROXY_POOL_SIZE': 5,
+    
+    # 通用下载配置
+    'DOWNLOAD_TIMEOUT': 30,
+    'CONNECTION_POOL_LIMIT': 100,
+    'CONNECTION_POOL_LIMIT_PER_HOST': 20,
+    'DOWNLOAD_MAXSIZE': 10 * 1024 * 1024,  # 10MB
+    'VERIFY_SSL': True,
+    
+    # 日志配置
+    'LOG_LEVEL': 'INFO',
+}
+
+def get_settings():
+    """获取配置"""
+    return SETTINGS
+
+if __name__ == "__main__":
+    print("aiohttp下载器配置:")
+    for key, value in SETTINGS.items():
+        print(f"  {key}: {value}")

examples/curl_cffi_settings.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+curl-cffi下载器配置示例
+"""
+
+# 基础配置
+SETTINGS = {
+    # 下载器配置
+    'DOWNLOADER': 'crawlo.downloader.cffi_downloader.CurlCffiDownloader',
+    'DOWNLOADER_TYPE': 'curl_cffi',
+    
+    # curl-cffi特定配置
+    'CURL_BROWSER_TYPE': 'chrome',
+    
+    # 代理配置
+    'PROXY_ENABLED': True,
+    'PROXY_API_URL': 'http://test.proxy.api:8080/proxy/getitem/',
+    'PROXY_EXTRACTOR': 'proxy',
+    'PROXY_REFRESH_INTERVAL': 60,
+    'PROXY_POOL_SIZE': 5,
+    
+    # 通用下载配置
+    'DOWNLOAD_TIMEOUT': 30,
+    'CONNECTION_POOL_LIMIT': 100,
+    'CONNECTION_POOL_LIMIT_PER_HOST': 20,
+    'DOWNLOAD_MAXSIZE': 10 * 1024 * 1024,  # 10MB
+    'VERIFY_SSL': True,
+    
+    # 日志配置
+    'LOG_LEVEL': 'INFO',
+}
+
+def get_settings():
+    """获取配置"""
+    return SETTINGS
+
+if __name__ == "__main__":
+    print("curl-cffi下载器配置:")
+    for key, value in SETTINGS.items():
+        print(f"  {key}: {value}")

examples/default_header_middleware_example.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+DefaultHeaderMiddleware 使用示例
+展示如何配置和使用DefaultHeaderMiddleware来添加默认请求头，支持随机更换功能
+"""
+
+# 基础配置
+SETTINGS = {
+    # 默认请求头配置
+    'DEFAULT_REQUEST_HEADERS': {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+    },
+    
+    # 固定User-Agent（优先级高于随机User-Agent）
+    # 'USER_AGENT': 'Custom User Agent String',
+    
+    # 自定义User-Agent列表（可选，如果不配置将使用内置列表）
+    # 'USER_AGENTS': [
+    #     'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+    #     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+    # ],
+    
+    # 启用随机User-Agent功能
+    'RANDOM_USER_AGENT_ENABLED': True,
+    
+    # 指定User-Agent设备类型（可选值: "desktop", "mobile", "all"）
+    'USER_AGENT_DEVICE_TYPE': 'all',
+    
+    # 随机请求头配置（启用RANDOMNESS时使用）
+    'RANDOM_HEADERS': {
+        'Accept': [
+            'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
+        ],
+        'Accept-Language': [
+            'zh-CN,zh;q=0.9,en;q=0.8',
+            'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+            'zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7'
+        ],
+        'Cache-Control': 'no-cache'  # 固定值
+    },
+    
+    # 启用随机性功能（用于随机请求头）
+    'RANDOMNESS': True,
+    
+    # 中间件配置（DefaultHeaderMiddleware已默认启用）
+    'MIDDLEWARES': [
+        # === 请求预处理阶段 ===
+        'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',  # 1. 忽略无效请求
+        'crawlo.middleware.download_delay.DownloadDelayMiddleware',  # 2. 控制请求频率
+        'crawlo.middleware.default_header.DefaultHeaderMiddleware',  # 3. 添加默认请求头
+        'crawlo.middleware.proxy.ProxyMiddleware',  # 4. 设置代理
+        'crawlo.middleware.offsite.OffsiteMiddleware',  # 5. 站外请求过滤
+        
+        # === 响应处理阶段 ===
+        'crawlo.middleware.retry.RetryMiddleware',  # 6. 失败请求重试
+        'crawlo.middleware.response_code.ResponseCodeMiddleware',  # 7. 处理特殊状态码
+        'crawlo.middleware.response_filter.ResponseFilterMiddleware',  # 8. 响应内容过滤
+    ],
+    
+    # 其他常用配置
+    'DOWNLOAD_DELAY': 1,
+    'CONCURRENCY': 8,
+    'LOG_LEVEL': 'INFO',
+}
+
+def get_settings():
+    """获取配置"""
+    return SETTINGS
+
+if __name__ == "__main__":
+    print("DefaultHeaderMiddleware配置示例:")
+    print("=" * 40)
+    print("默认请求头配置:")
+    for key, value in SETTINGS['DEFAULT_REQUEST_HEADERS'].items():
+        print(f"  {key}: {value}")
+    
+    print(f"\n随机User-Agent功能: {'启用' if SETTINGS['RANDOM_USER_AGENT_ENABLED'] else '禁用'}")
+    print(f"User-Agent设备类型: {SETTINGS['USER_AGENT_DEVICE_TYPE']}")
+    
+    print(f"\n随机请求头功能: {'启用' if SETTINGS['RANDOMNESS'] else '禁用'}")
+    print("随机请求头配置:")
+    for key, value in SETTINGS['RANDOM_HEADERS'].items():
+        if isinstance(value, list):
+            print(f"  {key}: {len(value)}个可选值")
+        else:
+            print(f"  {key}: {value}")
+    
+    print("\n中间件列表:")
+    for i, middleware in enumerate(SETTINGS['MIDDLEWARES'], 1):
+        print(f"  {i}. {middleware}")
+    
+    print("\n" + "=" * 40)
+    print("DefaultHeaderMiddleware功能说明:")
+    print("✓ 自动为所有请求添加默认请求头")
+    print("✓ 不会覆盖请求中已存在的同名头部")
+    print("✓ 支持随机User-Agent更换（降低被识别风险）")
+    print("✓ 支持按设备类型选择User-Agent（桌面/移动/全部）")
+    print("✓ 支持随机请求头更换（增加请求多样性）")
+    print("✓ 可通过设置相关配置项来启用/禁用随机功能")
+    print("✓ 在调试模式下会记录添加的请求头信息")

examples/default_header_spider_example.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+使用DefaultHeaderMiddleware的爬虫示例
+展示如何在实际爬虫中使用DefaultHeaderMiddleware添加默认请求头，支持随机更换功能
+"""
+
+from crawlo.spider import Spider
+from crawlo.network.request import Request
+
+
+class HeaderExampleSpider(Spider):
+    """
+    示例爬虫，演示DefaultHeaderMiddleware的使用，包括随机更换功能
+    """
+    
+    # 爬虫名称
+    name = "header_example_spider"
+    
+    # 自定义设置
+    custom_settings = {
+        # 默认请求头配置
+        'DEFAULT_REQUEST_HEADERS': {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Connection': 'keep-alive',
+        },
+        
+        # 启用随机User-Agent功能
+        'RANDOM_USER_AGENT_ENABLED': True,
+        
+        # 指定User-Agent设备类型（可选值: "desktop", "mobile", "all"）
+        'USER_AGENT_DEVICE_TYPE': 'all',
+        
+        # 或者自定义User-Agent列表（可选，如果不配置将使用内置列表）
+        # 'USER_AGENTS': [
+        #     'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+        #     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+        # ],
+        
+        # 随机请求头配置
+        'RANDOM_HEADERS': {
+            'Accept': [
+                'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+                'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
+            ],
+            'Accept-Language': [
+                'zh-CN,zh;q=0.9,en;q=0.8',
+                'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+                'zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7'
+            ],
+            'Cache-Control': 'no-cache'
+        },
+        
+        # 启用随机性功能
+        'RANDOMNESS': True,
+        
+        # 请求延迟（秒）
+        'DOWNLOAD_DELAY': 1,
+        
+        # 并发数
+        'CONCURRENCY': 4,
+        
+        # 日志级别
+        'LOG_LEVEL': 'INFO',
+    }
+    
+    def start_requests(self):
+        """
+        开始请求
+        """
+        urls = [
+            'https://httpbin.org/headers',  # 可以查看发送的请求头
+            'https://httpbin.org/user-agent',  # 可以查看User-Agent
+        ]
+        
+        for url in urls:
+            yield Request(url=url, callback=self.parse_headers)
+    
+    async def parse_headers(self, response):
+        """
+        处理响应，查看发送的请求头
+        """
+        self.logger.info(f"请求URL: {response.url}")
+        self.logger.info(f"状态码: {response.status_code}")
+        
+        # 输出响应内容（包含请求头信息）
+        try:
+            import json
+            data = json.loads(response.body.decode('utf-8'))
+            self.logger.info("发送的请求头:")
+            headers = data.get('headers', {})
+            for key, value in headers.items():
+                self.logger.info(f"  {key}: {value}")
+        except Exception as e:
+            self.logger.warning(f"解析响应失败: {e}")
+            self.logger.info(f"响应内容: {response.body[:200]}...")
+
+
+# 运行爬虫的示例代码
+if __name__ == "__main__":
+    """
+    运行说明:
+    
+    1. 确保已在项目根目录下安装了crawlo:
+       pip install -e .
+       
+    2. 运行爬虫:
+       crawlo run header_example_spider
+       
+    3. 观察日志输出:
+       - 可以看到发送的请求头信息
+       - 验证随机User-Agent是否正确更换
+       - 确认随机请求头是否按预期设置
+       
+    DefaultHeaderMiddleware的随机功能优势:
+    ✓ 自动为每个请求随机更换User-Agent，降低被识别为爬虫的风险
+    ✓ 支持按设备类型选择User-Agent（桌面/移动设备）
+    ✓ 支持随机更换其他请求头，增加请求的多样性
+    ✓ 可配置的随机策略，灵活适应不同网站的要求
+    ✓ 保持默认请求头的一致性，同时增加随机性
+    """
+    print("DefaultHeaderSpider示例")
+    print("=" * 30)
+    print("此爬虫演示了DefaultHeaderMiddleware的随机更换功能")
+    print("请使用以下命令运行:")
+    print("  crawlo run header_example_spider")