PyPI - crawlo - Versions diffs - 1.4.5__py3-none-any.whl → 1.4.7__py3-none-any.whl - Mend

crawlo 1.4.5py3-none-any.whl → 1.4.7py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of crawlo might be problematic. Click here for more details.

Files changed (375) hide show

crawlo/__init__.py +90 -89
crawlo/__version__.py +1 -1
crawlo/cli.py +75 -75
crawlo/commands/__init__.py +14 -14
crawlo/commands/check.py +594 -594
crawlo/commands/genspider.py +186 -186
crawlo/commands/help.py +140 -138
crawlo/commands/list.py +155 -155
crawlo/commands/run.py +379 -341
crawlo/commands/startproject.py +460 -460
crawlo/commands/stats.py +187 -187
crawlo/commands/utils.py +196 -196
crawlo/config.py +320 -312
crawlo/config_validator.py +277 -277
crawlo/core/__init__.py +52 -52
crawlo/core/engine.py +451 -438
crawlo/core/processor.py +47 -47
crawlo/core/scheduler.py +290 -291
crawlo/crawler.py +698 -657
crawlo/data/__init__.py +5 -5
crawlo/data/user_agents.py +194 -194
crawlo/downloader/__init__.py +280 -276
crawlo/downloader/aiohttp_downloader.py +233 -233
crawlo/downloader/cffi_downloader.py +250 -245
crawlo/downloader/httpx_downloader.py +265 -259
crawlo/downloader/hybrid_downloader.py +212 -212
crawlo/downloader/playwright_downloader.py +425 -402
crawlo/downloader/selenium_downloader.py +486 -472
crawlo/event.py +45 -11
crawlo/exceptions.py +215 -82
crawlo/extension/__init__.py +65 -64
crawlo/extension/health_check.py +141 -141
crawlo/extension/log_interval.py +94 -94
crawlo/extension/log_stats.py +70 -70
crawlo/extension/logging_extension.py +53 -61
crawlo/extension/memory_monitor.py +104 -104
crawlo/extension/performance_profiler.py +133 -133
crawlo/extension/request_recorder.py +107 -107
crawlo/factories/__init__.py +27 -27
crawlo/factories/base.py +68 -68
crawlo/factories/crawler.py +104 -103
crawlo/factories/registry.py +84 -84
crawlo/factories/utils.py +135 -0
crawlo/filters/__init__.py +170 -153
crawlo/filters/aioredis_filter.py +348 -264
crawlo/filters/memory_filter.py +261 -276
crawlo/framework.py +306 -292
crawlo/initialization/__init__.py +44 -44
crawlo/initialization/built_in.py +391 -434
crawlo/initialization/context.py +141 -141
crawlo/initialization/core.py +240 -194
crawlo/initialization/phases.py +230 -149
crawlo/initialization/registry.py +143 -145
crawlo/initialization/utils.py +49 -0
crawlo/interfaces.py +23 -23
crawlo/items/__init__.py +23 -23
crawlo/items/base.py +23 -23
crawlo/items/fields.py +52 -52
crawlo/items/items.py +104 -104
crawlo/logging/__init__.py +42 -46
crawlo/logging/config.py +277 -197
crawlo/logging/factory.py +175 -171
crawlo/logging/manager.py +104 -112
crawlo/middleware/__init__.py +87 -24
crawlo/middleware/default_header.py +132 -132
crawlo/middleware/download_delay.py +104 -104
crawlo/middleware/middleware_manager.py +142 -142
crawlo/middleware/offsite.py +123 -123
crawlo/middleware/proxy.py +209 -386
crawlo/middleware/request_ignore.py +86 -86
crawlo/middleware/response_code.py +150 -150
crawlo/middleware/response_filter.py +136 -136
crawlo/middleware/retry.py +124 -124
crawlo/mode_manager.py +287 -253
crawlo/network/__init__.py +21 -21
crawlo/network/request.py +375 -379
crawlo/network/response.py +569 -664
crawlo/pipelines/__init__.py +53 -22
crawlo/pipelines/base_pipeline.py +452 -0
crawlo/pipelines/bloom_dedup_pipeline.py +146 -146
crawlo/pipelines/console_pipeline.py +39 -39
crawlo/pipelines/csv_pipeline.py +316 -316
crawlo/pipelines/database_dedup_pipeline.py +197 -197
crawlo/pipelines/json_pipeline.py +218 -218
crawlo/pipelines/memory_dedup_pipeline.py +105 -105
crawlo/pipelines/mongo_pipeline.py +140 -132
crawlo/pipelines/mysql_pipeline.py +470 -326
crawlo/pipelines/pipeline_manager.py +100 -100
crawlo/pipelines/redis_dedup_pipeline.py +155 -156
crawlo/project.py +347 -347
crawlo/queue/__init__.py +10 -0
crawlo/queue/pqueue.py +38 -38
crawlo/queue/queue_manager.py +591 -525
crawlo/queue/redis_priority_queue.py +519 -370
crawlo/settings/__init__.py +7 -7
crawlo/settings/default_settings.py +285 -270
crawlo/settings/setting_manager.py +219 -219
crawlo/spider/__init__.py +657 -657
crawlo/stats_collector.py +82 -73
crawlo/subscriber.py +129 -129
crawlo/task_manager.py +138 -138
crawlo/templates/crawlo.cfg.tmpl +10 -10
crawlo/templates/project/__init__.py.tmpl +2 -4
crawlo/templates/project/items.py.tmpl +13 -17
crawlo/templates/project/middlewares.py.tmpl +38 -38
crawlo/templates/project/pipelines.py.tmpl +35 -36
crawlo/templates/project/settings.py.tmpl +110 -157
crawlo/templates/project/settings_distributed.py.tmpl +156 -161
crawlo/templates/project/settings_gentle.py.tmpl +170 -171
crawlo/templates/project/settings_high_performance.py.tmpl +171 -172
crawlo/templates/project/settings_minimal.py.tmpl +99 -77
crawlo/templates/project/settings_simple.py.tmpl +168 -169
crawlo/templates/project/spiders/__init__.py.tmpl +9 -9
crawlo/templates/run.py.tmpl +23 -30
crawlo/templates/spider/spider.py.tmpl +33 -144
crawlo/templates/spiders_init.py.tmpl +5 -10
crawlo/tools/__init__.py +86 -189
crawlo/tools/date_tools.py +289 -289
crawlo/tools/distributed_coordinator.py +384 -384
crawlo/tools/scenario_adapter.py +262 -262
crawlo/tools/text_cleaner.py +232 -232
crawlo/utils/__init__.py +50 -50
crawlo/utils/batch_processor.py +276 -259
crawlo/utils/config_manager.py +442 -0
crawlo/utils/controlled_spider_mixin.py +439 -439
crawlo/utils/db_helper.py +250 -244
crawlo/utils/error_handler.py +410 -410
crawlo/utils/fingerprint.py +121 -121
crawlo/utils/func_tools.py +82 -82
crawlo/utils/large_scale_helper.py +344 -344
crawlo/utils/leak_detector.py +335 -0
crawlo/utils/log.py +79 -79
crawlo/utils/misc.py +81 -81
crawlo/utils/mongo_connection_pool.py +157 -0
crawlo/utils/mysql_connection_pool.py +197 -0
crawlo/utils/performance_monitor.py +285 -285
crawlo/utils/queue_helper.py +175 -175
crawlo/utils/redis_checker.py +91 -0
crawlo/utils/redis_connection_pool.py +578 -388
crawlo/utils/redis_key_validator.py +198 -198
crawlo/utils/request.py +278 -256
crawlo/utils/request_serializer.py +225 -225
crawlo/utils/resource_manager.py +337 -0
crawlo/utils/selector_helper.py +137 -137
crawlo/utils/singleton.py +70 -0
crawlo/utils/spider_loader.py +201 -201
crawlo/utils/text_helper.py +94 -94
crawlo/utils/{url.py → url_utils.py} +39 -39
crawlo-1.4.7.dist-info/METADATA +689 -0
crawlo-1.4.7.dist-info/RECORD +347 -0
examples/__init__.py +7 -7
tests/__init__.py +7 -7
tests/advanced_tools_example.py +217 -275
tests/authenticated_proxy_example.py +110 -106
tests/baidu_performance_test.py +108 -108
tests/baidu_test.py +59 -59
tests/bug_check_test.py +250 -250
tests/cleaners_example.py +160 -160
tests/comprehensive_framework_test.py +212 -212
tests/comprehensive_test.py +81 -81
tests/comprehensive_testing_summary.md +186 -186
tests/config_validation_demo.py +142 -142
tests/controlled_spider_example.py +205 -205
tests/date_tools_example.py +180 -180
tests/debug_configure.py +69 -69
tests/debug_framework_logger.py +84 -84
tests/debug_log_config.py +126 -126
tests/debug_log_levels.py +63 -63
tests/debug_pipelines.py +66 -66
tests/detailed_log_test.py +233 -233
tests/direct_selector_helper_test.py +96 -96
tests/distributed_dedup_test.py +467 -0
tests/distributed_test.py +66 -66
tests/distributed_test_debug.py +76 -76
tests/dynamic_loading_example.py +523 -523
tests/dynamic_loading_test.py +104 -104
tests/error_handling_example.py +171 -171
tests/explain_mysql_update_behavior.py +77 -0
tests/final_comprehensive_test.py +151 -151
tests/final_log_test.py +260 -260
tests/final_validation_test.py +182 -182
tests/fix_log_test.py +142 -142
tests/framework_performance_test.py +202 -202
tests/log_buffering_test.py +111 -111
tests/log_generation_timing_test.py +153 -153
tests/monitor_redis_dedup.sh +72 -0
tests/ofweek_scrapy/ofweek_scrapy/items.py +12 -12
tests/ofweek_scrapy/ofweek_scrapy/middlewares.py +100 -100
tests/ofweek_scrapy/ofweek_scrapy/pipelines.py +13 -13
tests/ofweek_scrapy/ofweek_scrapy/settings.py +84 -84
tests/ofweek_scrapy/scrapy.cfg +11 -11
tests/optimized_performance_test.py +211 -211
tests/performance_comparison.py +244 -244
tests/queue_blocking_test.py +113 -113
tests/queue_test.py +89 -89
tests/redis_key_validation_demo.py +130 -130
tests/request_params_example.py +150 -150
tests/response_improvements_example.py +144 -144
tests/scrapy_comparison/ofweek_scrapy.py +138 -138
tests/scrapy_comparison/scrapy_test.py +133 -133
tests/simple_cli_test.py +55 -0
tests/simple_command_test.py +119 -119
tests/simple_crawlo_test.py +126 -126
tests/simple_follow_test.py +38 -38
tests/simple_log_test2.py +137 -137
tests/simple_optimization_test.py +128 -128
tests/simple_queue_type_test.py +41 -41
tests/simple_response_selector_test.py +94 -94
tests/simple_selector_helper_test.py +154 -154
tests/simple_selector_test.py +207 -207
tests/simple_spider_test.py +49 -49
tests/simple_url_test.py +73 -73
tests/simulate_mysql_update_test.py +140 -0
tests/spider_log_timing_test.py +177 -177
tests/test_advanced_tools.py +148 -148
tests/test_all_commands.py +230 -230
tests/test_all_pipeline_fingerprints.py +133 -133
tests/test_all_redis_key_configs.py +145 -145
tests/test_asyncmy_usage.py +57 -0
tests/test_batch_processor.py +178 -178
tests/test_cleaners.py +54 -54
tests/test_cli_arguments.py +119 -0
tests/test_component_factory.py +174 -174
tests/test_config_consistency.py +80 -80
tests/test_config_merge.py +152 -152
tests/test_config_validator.py +182 -182
tests/test_controlled_spider_mixin.py +79 -79
tests/test_crawler_process_import.py +38 -38
tests/test_crawler_process_spider_modules.py +47 -47
tests/test_crawlo_proxy_integration.py +114 -108
tests/test_date_tools.py +123 -123
tests/test_dedup_fix.py +220 -220
tests/test_dedup_pipeline_consistency.py +124 -124
tests/test_default_header_middleware.py +313 -313
tests/test_distributed.py +65 -65
tests/test_double_crawlo_fix.py +204 -204
tests/test_double_crawlo_fix_simple.py +124 -124
tests/test_download_delay_middleware.py +221 -221
tests/test_downloader_proxy_compatibility.py +272 -268
tests/test_edge_cases.py +305 -305
tests/test_encoding_core.py +56 -56
tests/test_encoding_detection.py +126 -126
tests/test_enhanced_error_handler.py +270 -270
tests/test_enhanced_error_handler_comprehensive.py +245 -245
tests/test_error_handler_compatibility.py +112 -112
tests/test_factories.py +252 -252
tests/test_factory_compatibility.py +196 -196
tests/test_final_validation.py +153 -153
tests/test_fingerprint_consistency.py +135 -135
tests/test_fingerprint_simple.py +51 -51
tests/test_get_component_logger.py +83 -83
tests/test_hash_performance.py +99 -99
tests/test_integration.py +169 -169
tests/test_item_dedup_redis_key.py +122 -122
tests/test_large_scale_helper.py +235 -235
tests/test_logging_enhancements.py +374 -374
tests/test_logging_final.py +184 -184
tests/test_logging_integration.py +312 -312
tests/test_logging_system.py +282 -282
tests/test_middleware_debug.py +141 -141
tests/test_mode_consistency.py +51 -51
tests/test_multi_directory.py +67 -67
tests/test_multiple_spider_modules.py +80 -80
tests/test_mysql_pipeline_config.py +165 -0
tests/test_mysql_pipeline_error.py +99 -0
tests/test_mysql_pipeline_init_log.py +83 -0
tests/test_mysql_pipeline_integration.py +133 -0
tests/test_mysql_pipeline_refactor.py +144 -0
tests/test_mysql_pipeline_refactor_simple.py +86 -0
tests/test_mysql_pipeline_robustness.py +196 -0
tests/test_mysql_pipeline_types.py +89 -0
tests/test_mysql_update_columns.py +94 -0
tests/test_offsite_middleware.py +244 -244
tests/test_offsite_middleware_simple.py +203 -203
tests/test_optimized_selector_naming.py +100 -100
tests/test_parsel.py +29 -29
tests/test_performance.py +327 -327
tests/test_performance_monitor.py +115 -115
tests/test_pipeline_fingerprint_consistency.py +86 -86
tests/test_priority_behavior.py +211 -211
tests/test_priority_consistency.py +151 -151
tests/test_priority_consistency_fixed.py +249 -249
tests/test_proxy_health_check.py +32 -32
tests/test_proxy_middleware.py +217 -121
tests/test_proxy_middleware_enhanced.py +212 -216
tests/test_proxy_middleware_integration.py +142 -137
tests/test_proxy_middleware_refactored.py +207 -184
tests/test_proxy_only.py +84 -0
tests/test_proxy_providers.py +56 -56
tests/test_proxy_stats.py +19 -19
tests/test_proxy_strategies.py +59 -59
tests/test_proxy_with_downloader.py +153 -0
tests/test_queue_empty_check.py +41 -41
tests/test_queue_manager_double_crawlo.py +173 -173
tests/test_queue_manager_redis_key.py +179 -179
tests/test_queue_naming.py +154 -154
tests/test_queue_type.py +106 -106
tests/test_queue_type_redis_config_consistency.py +130 -130
tests/test_random_headers_default.py +322 -322
tests/test_random_headers_necessity.py +308 -308
tests/test_random_user_agent.py +72 -72
tests/test_redis_config.py +28 -28
tests/test_redis_connection_pool.py +294 -294
tests/test_redis_key_naming.py +181 -181
tests/test_redis_key_validator.py +123 -123
tests/test_redis_queue.py +224 -224
tests/test_redis_queue_name_fix.py +175 -175
tests/test_redis_queue_type_fallback.py +129 -129
tests/test_request_ignore_middleware.py +182 -182
tests/test_request_params.py +111 -111
tests/test_request_serialization.py +70 -70
tests/test_response_code_middleware.py +349 -349
tests/test_response_filter_middleware.py +427 -427
tests/test_response_follow.py +104 -104
tests/test_response_improvements.py +152 -152
tests/test_response_selector_methods.py +92 -92
tests/test_response_url_methods.py +70 -70
tests/test_response_urljoin.py +86 -86
tests/test_retry_middleware.py +333 -333
tests/test_retry_middleware_realistic.py +273 -273
tests/test_scheduler.py +252 -252
tests/test_scheduler_config_update.py +133 -133
tests/test_scrapy_style_encoding.py +112 -112
tests/test_selector_helper.py +100 -100
tests/test_selector_optimizations.py +146 -146
tests/test_simple_response.py +61 -61
tests/test_spider_loader.py +49 -49
tests/test_spider_loader_comprehensive.py +69 -69
tests/test_spider_modules.py +84 -84
tests/test_spiders/test_spider.py +9 -9
tests/test_telecom_spider_redis_key.py +205 -205
tests/test_template_content.py +87 -87
tests/test_template_redis_key.py +134 -134
tests/test_tools.py +159 -159
tests/test_user_agent_randomness.py +176 -176
tests/test_user_agents.py +96 -96
tests/untested_features_report.md +138 -138
tests/verify_debug.py +51 -51
tests/verify_distributed.py +117 -117
tests/verify_log_fix.py +111 -111
tests/verify_mysql_warnings.py +110 -0
crawlo/logging/async_handler.py +0 -181
crawlo/logging/monitor.py +0 -153
crawlo/logging/sampler.py +0 -167
crawlo/middleware/simple_proxy.py +0 -65
crawlo/tools/authenticated_proxy.py +0 -241
crawlo/tools/data_formatter.py +0 -226
crawlo/tools/data_validator.py +0 -181
crawlo/tools/encoding_converter.py +0 -127
crawlo/tools/network_diagnostic.py +0 -365
crawlo/tools/request_tools.py +0 -83
crawlo/tools/retry_mechanism.py +0 -224
crawlo/utils/env_config.py +0 -143
crawlo/utils/large_scale_config.py +0 -287
crawlo/utils/system.py +0 -11
crawlo/utils/tools.py +0 -5
crawlo-1.4.5.dist-info/METADATA +0 -329
crawlo-1.4.5.dist-info/RECORD +0 -347
tests/env_config_example.py +0 -134
tests/ofweek_scrapy/ofweek_scrapy/spiders/ofweek_spider.py +0 -162
tests/test_authenticated_proxy.py +0 -142
tests/test_comprehensive.py +0 -147
tests/test_dynamic_downloaders_proxy.py +0 -125
tests/test_dynamic_proxy.py +0 -93
tests/test_dynamic_proxy_config.py +0 -147
tests/test_dynamic_proxy_real.py +0 -110
tests/test_env_config.py +0 -122
tests/test_framework_env_usage.py +0 -104
tests/test_large_scale_config.py +0 -113
tests/test_proxy_api.py +0 -265
tests/test_real_scenario_proxy.py +0 -196
tests/tools_example.py +0 -261
{crawlo-1.4.5.dist-info → crawlo-1.4.7.dist-info}/WHEEL +0 -0
{crawlo-1.4.5.dist-info → crawlo-1.4.7.dist-info}/entry_points.txt +0 -0
{crawlo-1.4.5.dist-info → crawlo-1.4.7.dist-info}/top_level.txt +0 -0

crawlo-1.4.5.dist-info/RECORD DELETED Viewed

@@ -1,347 +0,0 @@
-crawlo/__init__.py,sha256=n5vFwi0iuYrpAIyoNJZzWHV1gvF-vh-Yze3jiuwEXqM,2180
-crawlo/__version__.py,sha256=47Hd5fKyrYgSfmOfBF7ibw9EyAE1ctXOQOLg_x_Ld9w,23
-crawlo/cli.py,sha256=AQnAB5NMI-Ic1VPw_Jjng8L4AI4-wMozOwzE6CfXkZU,2402
-crawlo/config.py,sha256=EQIT7WpkXAlr2ocd5SYJYOKTSWUlQx2AkTHX7ErEWxw,9798
-crawlo/config_validator.py,sha256=oY4-2bwXUlwHAnGgkI-EznviDfML_dcxbWSGXNSxC2k,11516
-crawlo/crawler.py,sha256=6f9eDeUEZVfnUywaZ6CnL5R3bHO4sG82z-Syl3zZKvE,27360
-crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
-crawlo/exceptions.py,sha256=YVIDnC1bKSMv3fXH_6tinWMuD9HmKHIaUfO4_fkX5sY,1247
-crawlo/framework.py,sha256=9gP6VN4MHqutGXaxnwpNMSULfVYbNp906UdZiJGywlQ,9458
-crawlo/interfaces.py,sha256=q1vwMSiZLfLpPhFa9Y0hAcjYEKvLkW2fZ2fmoAZ-5TE,653
-crawlo/mode_manager.py,sha256=e8QmwsnndFx_hGME_7w-hazKo0GOYjUr-7FBf7dWxgc,8903
-crawlo/project.py,sha256=9wnlHd-rYAC3TT1Fc1ftyUBx7mbDT6TQCqoaIP6N3iA,13998
-crawlo/stats_collector.py,sha256=hIjlnX750jU4Oncyand1jBccfaX4Tu7egd2DBYu2N7A,2379
-crawlo/subscriber.py,sha256=h8fx69NJZeWem0ZkCmfHAi2kgfDGFObHpwN0aGNUM6Y,5115
-crawlo/task_manager.py,sha256=Ic6PFUqZOhLXuZ_UEk_8Neb9FmqYv8I2RzV3vLzFNSU,5966
-crawlo/commands/__init__.py,sha256=orvY6wLOBwGUEJKeF3h_T1fxj8AaQLjngBDd-3xKOE4,392
-crawlo/commands/check.py,sha256=TKDhI_sj7kErgiJpt2vCZ9QL-g6yWjrrPWKbgh8pgEU,23199
-crawlo/commands/genspider.py,sha256=JB4ZuFpKsYwtjx3DSsxugH7e3kqxhDWPG5ZKfvM0isI,6041
-crawlo/commands/help.py,sha256=8xPC0iNCg1rRBoK2bb6noAEANc1JwrdM35eF-j6yeZM,5111
-crawlo/commands/list.py,sha256=trzcd3kG6DhkOqYZADcl3yR7M8iJBgRw5fE-g9e0gVM,5877
-crawlo/commands/run.py,sha256=EjpIilgCTkXGVSV4rEISbJubdhqrok9nNe5-xDfDK5E,13169
-crawlo/commands/startproject.py,sha256=boZrMyn6TgCi1jt3D3DQfui6hJitjwNO8mqlWKNOBns,17366
-crawlo/commands/stats.py,sha256=vlGJLyiXZtY0ASdzCK59JNereSsAel4W9JCGaOzCr-8,6201
-crawlo/commands/utils.py,sha256=YVNEEzlm_qNY3SVvU8h6o2lQMkVgypvoB4ZFrP4gln0,5578
-crawlo/core/__init__.py,sha256=BWkj3AqZwp2Bk73UzUlC_qsqv_MH_HNrzy4DY1hosj4,1330
-crawlo/core/engine.py,sha256=znJ0VDFBImYi6KkTD8GHNo-V9BDnPSv9iYfTYLPsVSc,19379
-crawlo/core/processor.py,sha256=hR5MrbeZvDUx0ShKntr4qwkeVZzjlPJ8EAKgIFkNVts,1555
-crawlo/core/scheduler.py,sha256=G9xtrvE1wsTSOTOFUKDEphJvy6Xk5icuCGXTScYy7nQ,14084
-crawlo/data/__init__.py,sha256=UPqgioMdu3imSUmpLWzVlpvoBnEfaPSAT-crCcWd7iw,121
-crawlo/data/user_agents.py,sha256=zjjFkldQkqtrn45j0WZplaZLannPxZDeAU0JofxQcBc,9891
-crawlo/downloader/__init__.py,sha256=P5pl-BGYCkdKWgoIewcYPz7ocVLixVfYuCDFmYyuqIw,8966
-crawlo/downloader/aiohttp_downloader.py,sha256=-dIFucMOQhiiEmtgEpG2Lqh1vF-PvDddbIrZ8Hge0Ig,9556
-crawlo/downloader/cffi_downloader.py,sha256=QxoeocCE2DsQCnhZla6-BjhplaTZDWMbEJmNrghWSDA,10488
-crawlo/downloader/httpx_downloader.py,sha256=MpgDeIdGqNsiSKLOEDBnr5Z0eUbhHnqVEmAuoIfJmFU,12296
-crawlo/downloader/hybrid_downloader.py,sha256=dNnFeegRnyLaOxTWI6XrWKqqVPx80AZBZNgmrcKRVBM,8240
-crawlo/downloader/playwright_downloader.py,sha256=L-TVzG7cYfuBlqW0XSZuz5C_r9fpJrmYNcoQ-cDEna4,16663
-crawlo/downloader/selenium_downloader.py,sha256=P8GuhEw6OYVeN3oeksuBLpUJCELXiu0mAR23X6IIOAA,21508
-crawlo/extension/__init__.py,sha256=wwaTTWYUzbg5b84sQn2JvBlyuhVGkw-REkhVlR2vymA,2980
-crawlo/extension/health_check.py,sha256=stDpyP4gOzAdbBlPbSf0rge0QounAhF8CtrGq5fa_7c,5657
-crawlo/extension/log_interval.py,sha256=N25aNjFkjh9br6g3ViFqRrz06C2geAdfGas-OT2oZh8,4497
-crawlo/extension/log_stats.py,sha256=CWjMb_V1o8j8uwGFvh9SZ7EYLl_OYzmuIsOT5V-_BE4,2452
-crawlo/extension/logging_extension.py,sha256=WnHVoC4aLHYLapAN0ylt3k5aanP_T1GOyJrAVy-6ePE,2415
-crawlo/extension/memory_monitor.py,sha256=fClPchpCkVjcIiU0AJHCKDd7HEiz5B4KqNqKTRZ2hcU,4394
-crawlo/extension/performance_profiler.py,sha256=BjWD3LOb4VwjQJQvQtWNg7GluEwFquI1CztNfgMzy3c,5032
-crawlo/extension/request_recorder.py,sha256=KA_RmcfscDxP5wPdolO76yKfRj-1jmHhG3jkVGO1pbc,4181
-crawlo/factories/__init__.py,sha256=24dH70p05pZerO9-9gaKpTawRGeGvQYw7j5brvq8GUg,714
-crawlo/factories/base.py,sha256=loB_vyc0CsQK0BgwRoSOFS8gPcmv-b9irtjC9UaBGA4,1832
-crawlo/factories/crawler.py,sha256=e9zl4qytByzsYbz66klY3cZTvQei0-9GjdFK4XCyXcg,3198
-crawlo/factories/registry.py,sha256=YU87CdsntOz609M0aQbGcCG9glPinUJxOn-_CaM4f-M,2595
-crawlo/filters/__init__.py,sha256=noSe07tp2Ip_zXwAbS021BojrqNRaObDO-2YV6DOQfc,4381
-crawlo/filters/aioredis_filter.py,sha256=WglGW-XLjsy8r_NDrNsXk_nzwaIq081MBnooHqCCQZA,9841
-crawlo/filters/memory_filter.py,sha256=gIPXCw650v81XRiz0MhWXH-zcn24ERzDTzBQZRoy1YU,9890
-crawlo/initialization/__init__.py,sha256=uNRMm9GccMYZi51scpvo-CPx_3ayp3Y81psBHlUoDfw,1132
-crawlo/initialization/built_in.py,sha256=DlZf4k9FlU52tnwlFtKqWHqlFZpo-VHB0qP61rVqJzo,16259
-crawlo/initialization/context.py,sha256=wG9t-M-Qttj7TN6gDumPX5Q5GHaPDUpLTZZDne2r3TE,4863
-crawlo/initialization/core.py,sha256=GWc9QNSp2JmHlCAhgq1aqGDXHcO6QlxFAVfePKC1xeo,6872
-crawlo/initialization/phases.py,sha256=iWhGITh9eudfSmzf2G3DLPAIJkCDrv9TVBtnAoS1_3c,4176
-crawlo/initialization/registry.py,sha256=kKVegqWxtPCaZ1mTyVHN4yFecAGDOPFJfebkP-SoobE,4919
-crawlo/items/__init__.py,sha256=rFpx1qFBo0Ik7bSdnXC8EVTJUOQdoJYGVdhYjaH00nk,409
-crawlo/items/base.py,sha256=q0YTJlqUtizsqXwfWlk0ndcINV9dDyUckwMx8_JrkeY,602
-crawlo/items/fields.py,sha256=l-DIwK6CCpdzNvf6ELz7Ckc7YCghZD9UCXA8vhNn2UE,1852
-crawlo/items/items.py,sha256=OmVEvMmgofMU95GkaiWkfNQ2fjsH2fY9sw3SKcmUhLs,3478
-crawlo/logging/__init__.py,sha256=NlvL0sc9NZqy-Poplwpd2wsUleGyg33MyRb1wxyG-zs,1184
-crawlo/logging/async_handler.py,sha256=cMBVi9Ue1y5yZ7r0Uzwr7j_4nyxwmcGXwSE1J7_yIOw,5259
-crawlo/logging/config.py,sha256=0N7w542vlKyE5tSLAbDJYm9U4_1lQro_TQKVIJE_pA4,7167
-crawlo/logging/factory.py,sha256=b4Z0fBmP00GpvpJ7k4QxqYP32n_EqG5KD3ouUWU7L4U,6656
-crawlo/logging/manager.py,sha256=aem7yla0q83rf2CpwQEyg5YMbey4TzkquBVWiKtcqdQ,3182
-crawlo/logging/monitor.py,sha256=mzZWm3rQ2mGUoAmkEJPUkBmR0VWK66l14aqqhQ0zwE8,4935
-crawlo/logging/sampler.py,sha256=1BoRMpusP3wbXRnet5xl9_Yb_3_-AUq9WJhK9gYg7v4,5292
-crawlo/middleware/__init__.py,sha256=khNCstVcYlL14SbLZ8ys9ub1-C8k4FIiMQ99Vw9wA-0,635
-crawlo/middleware/default_header.py,sha256=Pw-ev8ffi16GeCh84R5L3hAZgp3G1QXS-H5kV3JEp4Q,5164
-crawlo/middleware/download_delay.py,sha256=2iWnJFtWDlqDy5MsAob8TPiJQoiz9v21yatkBI0eptg,3542
-crawlo/middleware/middleware_manager.py,sha256=H_o0nwo_xQ8aSRnnvEs2Ho3fS-3WNi_5AjChhqvRYnk,6645
-crawlo/middleware/offsite.py,sha256=4tUkPqXMMXsi1WwYnJ_e7wMd6sRgK19QHRCYq8-w8jk,4682
-crawlo/middleware/proxy.py,sha256=uKk5OSLIs7jv9bBgkZwsi1rIpthooxhMrGBC2BPRDCc,16022
-crawlo/middleware/request_ignore.py,sha256=7qdX4zAimjSGwdod_aWUbOTfzLBWZ5KzLVFchGMCxCI,2663
-crawlo/middleware/response_code.py,sha256=d5t0hmP8QliuvvtFOqW-ogCBtZxg2eyjsOtlQAEUxM8,4533
-crawlo/middleware/response_filter.py,sha256=tVGr06bfJBR3xAHI2G5c3WimFsGHu8qoJtDcsVuCATU,4384
-crawlo/middleware/retry.py,sha256=Acfo95B9wF8fQTCQIqluZOS2hHdnknQu_FOHvpGKJp0,4248
-crawlo/middleware/simple_proxy.py,sha256=rQ4RkqewGvDRCw021nGrg8ngkBzg3wqrEVqvSmBgQ6M,2256
-crawlo/network/__init__.py,sha256=bvEnpEUBZJ79URfNZbsHhsBKna54hM2-x_BV8eotTA4,418
-crawlo/network/request.py,sha256=e6-YLgK7SU8D19n21mQwqt_b_aeRVJFOgWPIBPal2ys,14178
-crawlo/network/response.py,sha256=-URnNc_J7qBSG19uJbfuF6A_14MHLOtY78FvcZDzbsI,23418
-crawlo/pipelines/__init__.py,sha256=FDe2Pr5tiHtV8hFlheElRO_O1aVKvSWlkTcAl9BXAKA,637
-crawlo/pipelines/bloom_dedup_pipeline.py,sha256=vIF_6noJAdpotrJpnCmrVXCi59gRmHHn28mYW6VukbM,5465
-crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
-crawlo/pipelines/csv_pipeline.py,sha256=qbXZoqq4FIR9QkUGpC0ryWzmqGJSrM2bxmWLM4I1nXM,12490
-crawlo/pipelines/database_dedup_pipeline.py,sha256=IxahtD_mhni-Y21_idOMX58_Htf46A7n52enG9VR2PI,7296
-crawlo/pipelines/json_pipeline.py,sha256=wrCsh8YInmcPLAkhPrHObMx89VZfhf-c7qRrYsTixPE,8585
-crawlo/pipelines/memory_dedup_pipeline.py,sha256=lKkYPu6vkpPjfQ1-xOLvPFT4VdTI8QVx0yjqtVR0ZB0,3598
-crawlo/pipelines/mongo_pipeline.py,sha256=PohTKTGw3QRvuP-T6SrquwW3FAHSno8jQ2D2cH_d75U,5837
-crawlo/pipelines/mysql_pipeline.py,sha256=pLJQJUKqzWrrOxuO-eHXNq5xLza0DHeuGnpwX2Pc4NI,14186
-crawlo/pipelines/pipeline_manager.py,sha256=_DtWfxcTinIf5ApzUOVjZksd2tPbc7qeKi92IVd_kbs,4387
-crawlo/pipelines/redis_dedup_pipeline.py,sha256=RB1kXLr8ZuWNrgZKYwt--tlmnWsQTbuwTsSt3pafol8,6077
-crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/queue/pqueue.py,sha256=bbgd3l1VfqYXfz-4VFaiWLmJit1LdB3qHalCtNqyrqI,1210
-crawlo/queue/queue_manager.py,sha256=8rKygMxr6DgSjnGsKFmvlTI5XAARvQIN_ENkAruHGXs,21532
-crawlo/queue/redis_priority_queue.py,sha256=vLvg2toKaRrXD1QyEdu1ZjTmANv7clFaBF7mCtstBmI,15995
-crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
-crawlo/settings/default_settings.py,sha256=kBcE5PF-sfB12cjIxHeNPEvzSWSHYDu6saEgrTGXn5o,11970
-crawlo/settings/setting_manager.py,sha256=yI1tGaludevxKGGZO3Pn4aYofrg2cwYwvMZCFC5PPZw,8595
-crawlo/spider/__init__.py,sha256=QGhe_yNsnfnCF3G9nSoWEw23b8SkP5oSFU5W79C5DzI,21881
-crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
-crawlo/templates/run.py.tmpl,sha256=g8yst2hkqhKGNotR33fDxwmEsX6aEvhrXY_cfYos_vc,788
-crawlo/templates/spiders_init.py.tmpl,sha256=p6UK8KWr8FDydNxiAh6Iz29MY5WmgXIkf2z-buOGhOM,354
-crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
-crawlo/templates/project/items.py.tmpl,sha256=hpQ2AfUmhddnzMuKM5LF6t44dOfFXwJRAZlWFKUFOZw,343
-crawlo/templates/project/middlewares.py.tmpl,sha256=eEobZl8g_0DtiwLYbirQULqOacH-yUrrs4PUrGcJ2UE,1098
-crawlo/templates/project/pipelines.py.tmpl,sha256=7BeaQDMHbIjhKzRtzlCMiFlU8xgMzDs2PIHq3EVUAlQ,887
-crawlo/templates/project/settings.py.tmpl,sha256=mL9_JAyz8R35r-ywRHi4T-dtal7oczU5kodEWxldw40,5265
-crawlo/templates/project/settings_distributed.py.tmpl,sha256=RHzfWZITv-0ErCR9OYEswAZHpA5d9fYil0ZoGCtFt8g,5459
-crawlo/templates/project/settings_gentle.py.tmpl,sha256=pmjrBLjnpGcR90RkcJrM5O8PsTrRhUB92QR3R4TJyko,5733
-crawlo/templates/project/settings_high_performance.py.tmpl,sha256=9QhXSzfxIsMPyq0kZY9h2YBllyXGpGE37bMEbSrs_Ag,5823
-crawlo/templates/project/settings_minimal.py.tmpl,sha256=1qUPhSdHtvLSHTpytUJ8K63sMROhTwkz8e4tVg1fYoM,2222
-crawlo/templates/project/settings_simple.py.tmpl,sha256=sIyrCIVXsHSKl8Yjj8HkGs-ppMFH26a5yp6egVNlT2Q,5585
-crawlo/templates/project/spiders/__init__.py.tmpl,sha256=llhcIItXpm0TlEeumeLwp4fcYv2NHl8Iru7tLhDhxiE,216
-crawlo/templates/spider/spider.py.tmpl,sha256=KvU-9YpN6MifDE7XzejjyyQS7RUjLDLZ8zqJcLwSsu0,5198
-crawlo/tools/__init__.py,sha256=sXDMZNP6EwZIFivGcRthxqD1DFMMS8UOJvULAzHD-w4,3927
-crawlo/tools/authenticated_proxy.py,sha256=ULCK0Cc9F2rGhRqu6kzKBdxzK9v2n1CsatSQ_PMxpAg,7272
-crawlo/tools/data_formatter.py,sha256=iBDHpZBZvn9O7pLkTQilE1TzYJQEc3z3f6HXoVus0f0,7808
-crawlo/tools/data_validator.py,sha256=bLWnkpFdclJuqjtSAgMI5nznN4vAuPwE1YaiFWKWenM,5490
-crawlo/tools/date_tools.py,sha256=QOT3W5MqcEQhVM3cTZYxu1MRfgX-TI4aF1RI9s0QbdE,9195
-crawlo/tools/distributed_coordinator.py,sha256=kkRbRoxz7iXKI3AQElyTptDpYl352ErbSkM3wjSHVwU,12574
-crawlo/tools/encoding_converter.py,sha256=CqHAsR2rwxuzsyR-TeQNb79HX5mH4KAUixEY-sX7204,4170
-crawlo/tools/network_diagnostic.py,sha256=X1hSbUthIVbMHCU7ti43Zpu8XTaDJd5Oxr2zAkEuSB0,13013
-crawlo/tools/request_tools.py,sha256=oXrk4yWMACVa65fDQCQgzsg6a94FH4_lS7qNR53FHYU,2420
-crawlo/tools/retry_mechanism.py,sha256=4AQ_HLuYt4hYMI9XHoKFk2GQKEiDJB5pAnsMCfjc6Bk,7777
-crawlo/tools/scenario_adapter.py,sha256=pzysL1B2uQ1ZSEncVHd9Hv2viHNgaxP44YAUcDcppfw,9660
-crawlo/tools/text_cleaner.py,sha256=UrMGcgRnJaufjmDKIDsRYKMA8znCAArHDgouttWPygk,6690
-crawlo/utils/__init__.py,sha256=nxLnfqcEGLnsfSEagoKNyu-pm2ByU9BwE5tLxcS71Qo,1003
-crawlo/utils/batch_processor.py,sha256=8LNy-K2SrQVUxmGEWxQyYw_j9M-erN4Ie7O4d3zpBvM,9142
-crawlo/utils/controlled_spider_mixin.py,sha256=8CuM3Cr2wQLHbaO_ohbCsPcImJnyfZHpERbSeMgQ-AQ,16936
-crawlo/utils/db_helper.py,sha256=xTgBTXSWTNXM19rLsypPtnsswO0HdDV1K7zn_wYk4s0,8137
-crawlo/utils/env_config.py,sha256=W-VD_WF63DHxsyJysvp1eJwRh3L_pBRl_PitQAY3nQY,4079
-crawlo/utils/error_handler.py,sha256=e2LeUGT_OMcNKcjiX9Pp-NuQh5spsHBqIPBd7VxA2IQ,16247
-crawlo/utils/fingerprint.py,sha256=3IbctH3zwyBjN_12SH7-vrFt-akA2WSo3iAzHc6u--s,3689
-crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
-crawlo/utils/large_scale_config.py,sha256=NZMsDj4qbVx06Fu0aHqNKX1yzo6WFT7CgrhVnvw1ZFs,8372
-crawlo/utils/large_scale_helper.py,sha256=4ORkZcIrwJ0SlKOUh7l7WIuERORuRhNBgHCM71Rz0n0,12452
-crawlo/utils/log.py,sha256=KmUWVYq8t6fSGOC88nnYCDxwBUdoPWvaBmpOSHn2oWI,2914
-crawlo/utils/misc.py,sha256=m_TbfMf4Aoe70zmkv7XWyFg8Rz0qOYPXepwB6EcYr7Y,2519
-crawlo/utils/performance_monitor.py,sha256=32KspSo7RWvCX_fl0ZFn4ScWWOqbVVwEhPRd921Ez6I,9832
-crawlo/utils/queue_helper.py,sha256=gFmkh1jKlIcN1rmo2Jl6vYcLP5ByUWlfHO9eNlZPBLs,4918
-crawlo/utils/redis_connection_pool.py,sha256=EsPZkmQctWkoYU2wcrqkgwnIWnE6nG4XCXECKn216JA,12575
-crawlo/utils/redis_key_validator.py,sha256=-UTTx0Ul184pzwSply8hVdH0lp-gkXXOc_gEHR_7VlU,5809
-crawlo/utils/request.py,sha256=RcINrLvShfZ5VHu1T_hJJRXp-viKWSo35C2JOgWyl2k,8641
-crawlo/utils/request_serializer.py,sha256=b5abcgjJk4IU6Wfg46AmOAU2wmzu_WqcpEbuAncRMGQ,8931
-crawlo/utils/selector_helper.py,sha256=BVczzsSzPL5zF5KHXK3hyuqEl9o0ADYEuCH7Aw8aj98,4332
-crawlo/utils/spider_loader.py,sha256=oxifl0p4SOFhvvnD38Em4zGtC7sRr_pw4dki01MoAq0,7677
-crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
-crawlo/utils/text_helper.py,sha256=TTZgQPayMFUOYj8syt47Gwa4AQVY15W1b56STJetAKE,2920
-crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
-crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
-examples/__init__.py,sha256=NkRbV8_S1tb8S2AW6BE2U6P2-eGOPwMR1k0YQAwQpSE,130
-tests/__init__.py,sha256=409aRX8hsPffiZCVjOogtxwhACzBp8G2UTJyUQSxhK0,136
-tests/advanced_tools_example.py,sha256=1_iitECKCuWUYMNNGo61l3lmwMRrWdA8F_Xw56UaGZY,9340
-tests/authenticated_proxy_example.py,sha256=fKmHXXxIxCJXjEplttCWRh7PZhbxkBSxJF91Bx-qOME,3019
-tests/baidu_performance_test.py,sha256=wxdaI7UwKboMYH_qcaqZLxAStvndH60bvKGzD8F-jaI,3974
-tests/baidu_test.py,sha256=NKYnwDbPJX3tmKtRn7uQ_QWzUXiLTQC-Gdr1cQkJzEo,1874
-tests/bug_check_test.py,sha256=EIDOUk_QgtBOWKuBLm_WHbgJ0fsDuJACJ-nuxnBIdkQ,8056
-tests/cleaners_example.py,sha256=blVqSJ7SeWUNd17JjHZJgVTzWH65XKevLyaMB_Wg8qA,5324
-tests/comprehensive_framework_test.py,sha256=_1N-OGbKvBTNachNvIjkL_izr4uv6OUybUkhxxz5MAk,5977
-tests/comprehensive_test.py,sha256=wypCaB56IV8w8nd5VA5LSXUQ3IgLf0AKKUiCci6yEJQ,2969
-tests/comprehensive_testing_summary.md,sha256=S01gclEas5oDhSLjxT_mOoeMl8FE4NMA-gveQOrIS4g,6513
-tests/config_validation_demo.py,sha256=jbZ7h-HGsJmuqBb1euB_AhmKjllkvPmItRF1K0MQrVM,4171
-tests/controlled_spider_example.py,sha256=2SAQKoREGHe-OzVaSkGpopCcrou6QXmeW7rLdmsyopw,7981
-tests/date_tools_example.py,sha256=XI3iFEzeo7Nb5YepK8WHytIaBegtxWVSISpqQgpV6M8,5042
-tests/debug_configure.py,sha256=9VZRJtA0u_SbCC_ZMdsiT9raseRE-C2spZpZuXZNysI,2213
-tests/debug_framework_logger.py,sha256=p-dfOjEipjXQd6yk75S-SMoepzwujiwuv3rNwNSyjow,3413
-tests/debug_log_config.py,sha256=cPS6qOLnynYTFOxpjcy9OUgIqrkasWb9f2c_PASc2_E,3714
-tests/debug_log_levels.py,sha256=CZWG3KGDq-hYJ5TPhoZTyjKFKkkM-AoK3oP1w-JC1sc,2168
-tests/debug_pipelines.py,sha256=FMb36bH9lQxBLb-nM579hBRK1S16Vxu1t_BC3Dj8O2w,2164
-tests/detailed_log_test.py,sha256=oTCFF_Un7Jq2gV4rpRDFOxlHJSthnQhvEf0CSItfB7I,7501
-tests/direct_selector_helper_test.py,sha256=p7_x3x87JUnpKplmwYO4zN5ympcPJSPdHsviso-LmpI,2862
-tests/distributed_test.py,sha256=u6cEiymZzCItaTClKTxwVjNmOj9_PZii4_eGNAVMDW8,1825
-tests/distributed_test_debug.py,sha256=pUv6ZKEJ5pK9xOA7lgVk6WW3cBAtnb1bsuZzJ8oGLvY,2181
-tests/dynamic_loading_example.py,sha256=7LdeQZFevrb-U1_dgr4oX3aYo2Da4HvE_0KIf1fw4Ew,18786
-tests/dynamic_loading_test.py,sha256=dzDW7b66HeDsIYsYgvNRihE3V6b6gEbUGQpp-eJbcIM,3413
-tests/env_config_example.py,sha256=_ZRDh_LR23ZKpy9E--y_KM0QIOiZF5vRT98QTn52TY8,4951
-tests/error_handling_example.py,sha256=grTeo1X17rFz4lhgASb0g5yu4NWbmNz5neyuonnNR40,5294
-tests/final_comprehensive_test.py,sha256=szTNbtwKfYNmE0kzDPCsE_kvnTG7FNKl2JERakGhKIk,4314
-tests/final_log_test.py,sha256=CpZ4ZvvuvFiBvz1a50qN599XIU086ett_I0bSX42BLU,9367
-tests/final_validation_test.py,sha256=4cuTr58i46JI6M4Tz54e7vrVFrOr3R7HSWgyQPKmM9M,5244
-tests/fix_log_test.py,sha256=hcRy0j3j0CT0oLN7KNA0VL-_o4M-uE1amR6GziBflfU,4440
-tests/framework_performance_test.py,sha256=Qp47VrsCK0ylEhDkFOm7lnD8rVkaJ7u1MopsEhAomrE,6985
-tests/log_buffering_test.py,sha256=0B5UY1yQuxnBU1pEyz3IBYweN__4fOkPXly-kYfOpNU,3226
-tests/log_generation_timing_test.py,sha256=zHb_m2FqlpRCYw-wqFWFn8cbVH8UR3VvXKSM6nNnbgo,4681
-tests/optimized_performance_test.py,sha256=bA0dN4j7ViyTSSiCJEjlkJ9Y7jspTFKs2xX7UXHE8Gs,7379
-tests/performance_comparison.py,sha256=UevHOM_9z2ILedf_xZ_8F8QiPjb_M8WTfGQrxzKtgco,9266
-tests/queue_blocking_test.py,sha256=hp-6hmTOO64oOAWVtlN8cFJ95GjbK3t9fj-4q_TKowk,3955
-tests/queue_test.py,sha256=HeBiBXqAgIAbUkLVQ3McS6NdRselA30m3lnuxNBvZbk,2689
-tests/redis_key_validation_demo.py,sha256=WD2jvuBwHhLYIb3lVFtvYSSnmXWn1EW4EPCEwFhfi6M,4467
-tests/request_params_example.py,sha256=J50NdsnK1sDrqG-5m3oA-mu1_wHwVwHIfsWxGeQpz7o,4250
-tests/response_improvements_example.py,sha256=t1cbG3nesp82bqog4_ku1GvQzNbhRyWa5EaKTmOPrSk,5402
-tests/simple_command_test.py,sha256=8TowzW45ukKTPeaNC5uij3RR7rqPULiBr2PguSSMdP8,3688
-tests/simple_crawlo_test.py,sha256=FYDn5cgAxHN81QSYa_wcJcxJit7aLnIopnkHKKr83dE,4801
-tests/simple_follow_test.py,sha256=3vNT5Eqwza6fxAY9Xl_9xtFGdfrPwm6NnVHdRmJsH8A,1053
-tests/simple_log_test2.py,sha256=Rn3XerVlkT0M-vbQmrQL7bVIZG3REnJNmMvUvKr6C20,3944
-tests/simple_optimization_test.py,sha256=hflvaC81ra1ZrPOp-Z7rQrH95OnSADvAjy95BLulD6o,3678
-tests/simple_queue_type_test.py,sha256=wAf4XLKl9oS5BlfrRJ1SLY-kYmNq4YY0LdIC7HmW-yg,1193
-tests/simple_response_selector_test.py,sha256=0naeRUX1n-oAW6VRj-12c6nre2D0RjJ0dD3Nx7BBTjY,2844
-tests/simple_selector_helper_test.py,sha256=l9FsVhQ-z-ICqqetLIyeSaI8Dn6bXNCD8sLdr0tpvms,4438
-tests/simple_selector_test.py,sha256=XzOYzpEzr0yaioLV6v-4XC60VZMd5jRthlyp7Ud02o4,6630
-tests/simple_spider_test.py,sha256=RzziJg-fbIVJ6_CgbismfkwrLwpJp4WWp2RLgG7Tpws,1168
-tests/simple_url_test.py,sha256=g9RBn46V7fHZTU0BrB5pl5AGCbw6QuKOXClVACb-MEQ,2297
-tests/spider_log_timing_test.py,sha256=pvYpKZemClr4mCR76xywhsiWbT5sPdzD_taZKFjlgvM,5573
-tests/test_advanced_tools.py,sha256=HT_TcwfFzli-CavIJSqQqnCxnBn5FDMX09zL7AJ5tNY,5398
-tests/test_all_commands.py,sha256=VgVa9SzU5Irvn5igHpC2W4E_6ZDWDt7jc-T4UPK_PFE,7718
-tests/test_all_pipeline_fingerprints.py,sha256=NDrBYr0f9CAhjmSezTS4NUrAdcotrSX3ElJTWqjXXbU,5308
-tests/test_all_redis_key_configs.py,sha256=dWc4Dsr07_vuSpb4hwkMpyy6XO8SI7vglVjGuGvXoa4,5710
-tests/test_authenticated_proxy.py,sha256=lnvmQwuf0zaZP_E05EzcNFR2VJbwTkLjOmZGNoJKaC4,4339
-tests/test_batch_processor.py,sha256=4_nYlu9R1JkDCFHq0bYc9LUNqsg41r7sQ879hkrhEts,7212
-tests/test_cleaners.py,sha256=HDK8_YU7GUj_3hGU415cxEeUR74mnDSk0yroLlgDI0I,1816
-tests/test_component_factory.py,sha256=V3hO5pJHSDtViLAykXSUqkeH4g-GB4GczwutrTatS2U,5809
-tests/test_comprehensive.py,sha256=dvRJeeVYc1cgXK9Y171hH9Y847zZpWSAFFH-EI3UepQ,5182
-tests/test_config_consistency.py,sha256=RgSxyaypMpysltsGSh1vFMeOShiZZG0rmUKzEhNLpYw,2001
-tests/test_config_merge.py,sha256=ts1j-TIKkFS0EO5q1I4O7f4YUKR5MLTmRSqOpOlv094,5606
-tests/test_config_validator.py,sha256=Z4gBHkI0_fEx-xgiiG4T33F4BAuePuF81obpNTXfseY,6202
-tests/test_controlled_spider_mixin.py,sha256=AQ493ic6AxZAKd7QCgnUES92BBWCMNteTd5DjoQlhwo,2864
-tests/test_crawler_process_import.py,sha256=iIPqSCpv2VRb_hWTu5euLME4PDFf7NwixeBypRuv39Y,1175
-tests/test_crawler_process_spider_modules.py,sha256=uMr4esj6ascVBzt0WrPd3ZOQfKD00O6tJrNhuWOdvV0,1395
-tests/test_crawlo_proxy_integration.py,sha256=81DVwosMoiSMxj4V_jLzcL7aqvSv_8ucggkQyXsvzT0,2733
-tests/test_date_tools.py,sha256=pcLDyhLrZ_jh-PhPm4CvLZEgNeH9kLMPKN5zacHwuWM,4053
-tests/test_dedup_fix.py,sha256=UFdm8lIi0ZIdp40W8ruxRD69bxzijuFUfNyJmB4Fwl0,8788
-tests/test_dedup_pipeline_consistency.py,sha256=dn5EAZSU5gQOV5EQwreHp76i5aQZ9tEdltSGO7dif5M,5176
-tests/test_default_header_middleware.py,sha256=UDjEPIUCre1M6ndjV_uXLVCfY7WJwyN-1Xn15hzbKMo,13126
-tests/test_distributed.py,sha256=78Pn4HPLIaO8t1IiaSkckBmuEVTcnC8IDw7znf9_Zcw,1790
-tests/test_double_crawlo_fix.py,sha256=lZwrT5ij6Jbh0EzZswhw05FXwgKaEZsSHekLTrJJajg,7856
-tests/test_double_crawlo_fix_simple.py,sha256=NDmCEeyvpf_D1tGQMA66iLPPKlAnSZcEg71e7GHYcjg,4768
-tests/test_download_delay_middleware.py,sha256=Idc6KzhL3hY3aDKgn1j_v5-mLIHz7dTnV5c4tJVZh5Q,9107
-tests/test_downloader_proxy_compatibility.py,sha256=0hgIzWXIqd92YXEB5sNneyp4Sk7PaG76up2cd6N9QQY,8903
-tests/test_dynamic_downloaders_proxy.py,sha256=t_aWpxOHi4h3_fg2ImtIq7IIJ0r3PTHtnXiopPe2ZlM,4450
-tests/test_dynamic_proxy.py,sha256=zi7Ocbhc9GL1zCs0XhmG2NvBBeIZ2d2hPJVh18lH4Y0,3172
-tests/test_dynamic_proxy_config.py,sha256=C_9CEjCJtrr0SxIXCyLDhSIi88ujF7UAT1F-FAphd0w,5853
-tests/test_dynamic_proxy_real.py,sha256=krWnbFIH26mWNPhOfPMmx3ZxJfOreZxMZFGwVb_8-K8,3511
-tests/test_edge_cases.py,sha256=460JtYR6yuTo8J4wqJScMzDkrrDUE2Q8R425AaUycIQ,11127
-tests/test_encoding_core.py,sha256=k5fZET0R1KInhAlbbHEJv4m9d6NuibOxxfIcR43TS7Y,1681
-tests/test_encoding_detection.py,sha256=Zb1KkF2CR57qa0Hr_Iv8msompGJZT2EIL_2mGp0zX9Q,4245
-tests/test_enhanced_error_handler.py,sha256=Ku_86jv7iDe25v8ZxalcXxJJjIiIvQXWH8ZldbwdVm8,8581
-tests/test_enhanced_error_handler_comprehensive.py,sha256=j_cxyIPGks9A3untKhAdj5HU0hrLbbzOLu0uAtGUlJo,9369
-tests/test_env_config.py,sha256=Qu1sDeADs69dSr1x0QmEe8nJrMHneE_4JClt-N901e8,4867
-tests/test_error_handler_compatibility.py,sha256=xJ43cmCwfBGh-qBwCGiMDPPlfNDLw4ZrmlrHN9IojkU,4241
-tests/test_factories.py,sha256=wKFfr8YBXPs-AQ8YOFgDhINn5uivKqPBZQPUe5GL9Ig,8865
-tests/test_factory_compatibility.py,sha256=zzTXd3ku3iedgxgB1DxTt3zfetiIl6wCjL9yXIUCpic,6260
-tests/test_final_validation.py,sha256=OuZI01O0E68Pao--bD-BFDTRZFPc_Mt4W-OXUzlt6ZA,4966
-tests/test_fingerprint_consistency.py,sha256=68V5u_2hNABI5pNWzXUrA1PJ08Xh9x3-JsMSNNjORMo,4956
-tests/test_fingerprint_simple.py,sha256=qiSba8gF3Zl91QO_ijJO7KstLdjATs30V_GZCNHShig,1626
-tests/test_framework_env_usage.py,sha256=bFb_ptdLeX2obdJWEqEHPWweiWR-wR2BpvEaJMQK7h4,4201
-tests/test_get_component_logger.py,sha256=UKj5uT1F3L3atoJFmpk7QSDO2fZHgw-7Y84vMFbHRkM,2285
-tests/test_hash_performance.py,sha256=4eVPwbu66Oun0LVyTTNd9d2cj2V1xq0YZkRg8Z0TO-Q,3211
-tests/test_integration.py,sha256=lVEzKNAjFzFZHRNZAyJmXxa_5Ogf_qqL4APqs620o58,4839
-tests/test_item_dedup_redis_key.py,sha256=dp_H59exJLaZHh5oMtmMEOWh-DNZwbnwIFYDjOpHgd0,3842
-tests/test_large_scale_config.py,sha256=Ik32ilAOQXsyw2sHR53gDPNNjY0AXybQ9ya2JY-EeqM,4296
-tests/test_large_scale_helper.py,sha256=0L6EKHcKgh7XHvoW4wRSkxmw8GolUwSOCgZ_-ZmCyDo,8371
-tests/test_logging_enhancements.py,sha256=YHcYWC8PG_AP5wZnmOr6H7QuU7m-3xzxEhppM0Jubvg,12731
-tests/test_logging_final.py,sha256=K9vxyODslXza05hElVEcvzbXgzthYKK5CRj4UJTftIw,6376
-tests/test_logging_integration.py,sha256=5WpExyt6BmYBZwrjqtQIGOw1Id64opJBAIahDk70Mlc,11131
-tests/test_logging_system.py,sha256=LGfK14ZEWzRtl3_VkBGz-AaVa_dDtuC5zu40m8FvmMo,9206
-tests/test_middleware_debug.py,sha256=gtiaWCxBSTcaNkdqXirM7CsThr_HfiCueBdQCpp7rqg,4572
-tests/test_mode_consistency.py,sha256=t72WX0etC_AayaL2AT6e2lIgbfP-zxTgYAiTARSN2Jk,1276
-tests/test_multi_directory.py,sha256=sH9Y3B-fuESlc7J1aICa-AlBcCW8HFR-Q5j2anUr8l0,2196
-tests/test_multiple_spider_modules.py,sha256=M0wPyQW7HMasbMIgn_R78wjZEj4A_DgqaGHp0qF9Y0c,2567
-tests/test_offsite_middleware.py,sha256=njpXTdngOqBs60Wj6xgo5EEXlJnMHd7vtYGi9dVauW0,10602
-tests/test_offsite_middleware_simple.py,sha256=4MfDKSXGHcoFLYnnxCH2rmnzztWyN0xByYLoHtepyiA,7918
-tests/test_optimized_selector_naming.py,sha256=fbmlB5S2kBwtQWpWoQ4lQ7rUQm2_DeWK-t6KqvIRTUQ,2787
-tests/test_parsel.py,sha256=wuZqRFIm9xx1tt6o3Xi_OjvwhT_MPmHiUEj2ax06zlo,701
-tests/test_performance.py,sha256=Lqs2iu3dmWipZkBPARcwIjDLXsqe42ntz1M4RzqqXKo,11457
-tests/test_performance_monitor.py,sha256=paW3HGg6ReHb9lwnOivGCrI8STwbwp_mbuhgfds1h3I,4187
-tests/test_pipeline_fingerprint_consistency.py,sha256=LL55oGSDGy0K8LxoyKa6ogNHXhJlZHe509vCFbibLkk,2847
-tests/test_priority_behavior.py,sha256=JQ5uv80cAUKV9Eh3S8j5zxYSSL-dmzhwhuKOINM26zU,9325
-tests/test_priority_consistency.py,sha256=rVX7nku5N_QpB_ffDu3xqREkCWPX5aNNiXy112o9wpA,5756
-tests/test_priority_consistency_fixed.py,sha256=MlYi5PIr5wxunC3Ku4ilnxOatKyRu2qIvhV7pjadkjg,10765
-tests/test_proxy_api.py,sha256=XnmklS-xU4ke_560gV6AIlBsRmG8YLQTGFAZrTUZuhc,11013
-tests/test_proxy_health_check.py,sha256=_tDlxa_6TdL3M5RLkHF82roXJ8WIuG5hELBp2GADyKQ,1123
-tests/test_proxy_middleware.py,sha256=EdQAfwwAJIBxw9JmUFTDEu_pdxapaTlcJr7KcrY6-AY,4021
-tests/test_proxy_middleware_enhanced.py,sha256=QR-p26F63N7MxNjZ2QJUeerh_xdnCDejkrGPIh7Fh4U,7035
-tests/test_proxy_middleware_integration.py,sha256=mTPK_XvbmLCV_QoVZzA3ybWOOX61493Ew78WfTp-bYQ,4441
-tests/test_proxy_middleware_refactored.py,sha256=VbkTWkmmomcyswobA_gf3p_bERl_eexY2e6ohJQS_A8,6960
-tests/test_proxy_providers.py,sha256=u_R2fhab90vqvQEaOAztpAOe9tJXvUMIdoDxmStmXJ4,1749
-tests/test_proxy_stats.py,sha256=ES00CEoDITYPFBGPk8pecFzD3ItYIv6NSpcqNd8-kvo,526
-tests/test_proxy_strategies.py,sha256=9Z1pXmTNyw-eIhGXlf2abZbJx6igLohYq-_3hldQ5uE,1868
-tests/test_queue_empty_check.py,sha256=ZJC6jOgZq0Wb0-ubrB1ZNcCaUiWeCxoNZmjkd6PY6t0,1182
-tests/test_queue_manager_double_crawlo.py,sha256=MijZ3JuyHMuqGbRC-8kclFr-4O7m_T8CqezP4qiWk-E,6957
-tests/test_queue_manager_redis_key.py,sha256=txHLq5XUZZN7h9HUlqlUCEVCTe2IXdf9r7F_P2zNVdY,7117
-tests/test_queue_naming.py,sha256=GzPqxvATF9A5iQMgn2wQM68qTHTbsQ9dPhktAkyzDG4,4786
-tests/test_queue_type.py,sha256=N8y3s0Cwco-XphRcIiR4bQuNAEOgmxvlRoR1jzghTtY,3304
-tests/test_queue_type_redis_config_consistency.py,sha256=1ew7Zp9CxH1DQ0RUmsZMV-n7s8B5dCly6u3FkJbR1qE,5259
-tests/test_random_headers_default.py,sha256=ulDb3_kRpnTCN1-TO3m6wVM-eMkZS_ezsSbd1ur8Xpg,12772
-tests/test_random_headers_necessity.py,sha256=SSbNQIE347oCQvuG6yaAambFU-3MyQzTV5jN1kArRGY,11741
-tests/test_random_user_agent.py,sha256=6HjU4iUcMk-J6bR2N5FhIkWDfnaFKAPNVyRzxmQQ14k,2302
-tests/test_real_scenario_proxy.py,sha256=clmLvBfap5OpsaCE08MAWap-78jhVrxYfVfDNyoa4Hg,8454
-tests/test_redis_config.py,sha256=51_Fy1PqIhS0MMO2nR4q6oQjBFxfqcUPK_4NNf5s83g,903
-tests/test_redis_connection_pool.py,sha256=pKfXdE3Cm_L_fNqI9zqFmqiidCwR0t7hiM_Fu_V1cNI,9328
-tests/test_redis_key_naming.py,sha256=MTFk656JhiGVTsMctBDhBNOMFcBDZrsQA3UfPZ-Dgj4,6911
-tests/test_redis_key_validator.py,sha256=GszSzGADgk3uN6Bye1d8pS-AtMVgB8jwqW-22gPNM6M,4418
-tests/test_redis_queue.py,sha256=WQV3MtGg8rJzHgC2kRfXM6lSMXpwXJVQZfqn2dVrhg0,6758
-tests/test_redis_queue_name_fix.py,sha256=9V-qDmUz97wOdPv8pKcVnQRxeZ126WSdB5JBnow4ho8,5770
-tests/test_redis_queue_type_fallback.py,sha256=Bg6b5bqt-TKotE_wWdkaZNQr2WXT8gaDfolv8785NyE,4692
-tests/test_request_ignore_middleware.py,sha256=QN81wgG_W_XfXCF9LvJNxCNwbOH6_tZnLIwLDTK2K5Q,6229
-tests/test_request_params.py,sha256=l2etiDebqylPBym1e9DSDn4wxwTHv8DQHKq9AzlzlG0,4287
-tests/test_request_serialization.py,sha256=Ikgec8tt_sPCK6jcZyK8vRw84zRNE6nxQy9rba1WKmE,2332
-tests/test_response_code_middleware.py,sha256=wSe525bm-bk_iWMjPDzUu1LfOQrwJY8_MLKAspq2dzk,12193
-tests/test_response_filter_middleware.py,sha256=YWrGzJ7wmftTjJXcNTtJl3b3EdJsO4oR22ZLWwgErhg,16327
-tests/test_response_follow.py,sha256=gjVZ_knsuHUaCDOjRPk-qG9HRCwReXlVrIx_KpveRHM,3738
-tests/test_response_improvements.py,sha256=vNqHKyoEoYeEGAUiRzdsff2V6yvJ9QnDwGg7gmN38Ow,6028
-tests/test_response_selector_methods.py,sha256=6aS7q_PBx601MnXbCze-ZWNO-uCKFVjhxcCg9NJqKrI,2738
-tests/test_response_url_methods.py,sha256=plOpSN3JLRI8-lbj4cva8-_jRFdDwmax9Gkv6O2Ac-s,2759
-tests/test_response_urljoin.py,sha256=uXTWhFx8-XBb-Vaghn9YKJz5ThkwRuNykBWW4S7f3go,3379
-tests/test_retry_middleware.py,sha256=mi7s4HDAqmmd9nvyxs3ZgxdEKOYkCgDu3rDvU_9o8vQ,11133
-tests/test_retry_middleware_realistic.py,sha256=Sam5y4jCN8oeElU4xxeS5zjAyzS-P8siPV7OaifgsyU,9679
-tests/test_scheduler.py,sha256=1fCu35QgK5gzgrhD0aUZj5lxL0QbokzPav-yEJxz9Ig,8182
-tests/test_scheduler_config_update.py,sha256=LuxjEbt20QrPyVkjSFxvTnFtUxwMaHB6TcqjFyo8bow,4261
-tests/test_scrapy_style_encoding.py,sha256=2K_0lHsYqop4qb5lO1U8g7hbae4nkMPrbEvVTl5TT9Y,3408
-tests/test_selector_helper.py,sha256=-fw8p-uJixTKso7OLUBTVJ2oOjL8LIJA1WDetzthGO0,2818
-tests/test_selector_optimizations.py,sha256=5t5RrDkcy0YtK2Es9DBfi3Cejfv6yV4dagulIQhmEho,4665
-tests/test_simple_response.py,sha256=_ui2PuVZvJcAuLY7HZ8xcsy_tDBimgBqX0ukj3kE5J0,1549
-tests/test_spider_loader.py,sha256=-myi78LztwABeaCpJj-DzO2CxNEYW8lavtVuUreoHcI,1314
-tests/test_spider_loader_comprehensive.py,sha256=gp6SWrDQcrg4RFNkLJQWDQ16NDfpdOlg0rCyJ86-F-8,2591
-tests/test_spider_modules.py,sha256=wxPs28FtpGnQTemMY6r7WxVrwYo3bHnAd5dq94qj1K4,2797
-tests/test_telecom_spider_redis_key.py,sha256=c-gfixPul2VlYMQJGf0H5ZgYJ461fQgSKbCPrbAU45M,7625
-tests/test_template_content.py,sha256=2RgCdOA3pMUSOqC_JbTGeW7KonbTqJ0ySYJNWegU-v0,2903
-tests/test_template_redis_key.py,sha256=99-s0_-8MFJbIvGG_X__sH0qkXWTtJv8fdTdlftsq4I,4876
-tests/test_tools.py,sha256=z50Bvq_q8FwpyxNkmh00_A3sXkSv2l1Q_EbK02FDYgk,5504
-tests/test_user_agent_randomness.py,sha256=tE8_zh-BjMAQ9CTgScxZze6JarNher6COkdoLU68YfA,5681
-tests/test_user_agents.py,sha256=e4haX-o8Janl-PawGJ9MemZyMqTX33_tBF_WnYSVoUw,3327
-tests/tools_example.py,sha256=Rxu5vVKnj3CZ3mCx-EEotBWPtZs2S7ktyqq-SYeclxU,7999
-tests/untested_features_report.md,sha256=31aUlsw_1OKe0_ijAjeH85kJ7HJ8qzKLJdOHDjWtYdY,4169
-tests/verify_debug.py,sha256=iQ4Efwg9bQTHscr73VYAAZ8rBIe1u6mQfeaEK5YgneY,1564
-tests/verify_distributed.py,sha256=0IolM4ymuPOz_uTfHSWFO3Vxzp7Lo6i0zhSbzJhHFtI,4045
-tests/verify_log_fix.py,sha256=7reyVl3MXTDASyChgU5BAYuzuxvFjSLG9HywAHso0qg,4336
-tests/ofweek_scrapy/scrapy.cfg,sha256=D_8rsW65iTbH7nG1kI25jYTCpoQKBVa2shajrsC6fBw,280
-tests/ofweek_scrapy/ofweek_scrapy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/ofweek_scrapy/ofweek_scrapy/items.py,sha256=Y_TwwHPAgOXTuCTdnhRxil7vYPk1_rzj1ZatTq4AX-I,280
-tests/ofweek_scrapy/ofweek_scrapy/middlewares.py,sha256=O4jVSXZgxtsRzU9O_O3YdkS7_QLndzv3uYP-Op8g254,3654
-tests/ofweek_scrapy/ofweek_scrapy/pipelines.py,sha256=ZO6WqTqPpTwLvnwO7YL0E35OPp4zSfJ_GhMeshNRSow,379
-tests/ofweek_scrapy/ofweek_scrapy/settings.py,sha256=X3Y6goZluAz0n2bepWAKEhZX0URFfe9_lBRBCPgtLPk,2933
-tests/ofweek_scrapy/ofweek_scrapy/spiders/__init__.py,sha256=ULwecZkx3_NTphkz7y_qiazBeUoHFnCCWnKSjoDCZj0,161
-tests/ofweek_scrapy/ofweek_scrapy/spiders/ofweek_spider.py,sha256=gcfKze-ipzP7JTDGCL3TgtjwIwfgI7dPL6GmdXVT0fs,6880
-tests/scrapy_comparison/ofweek_scrapy.py,sha256=rhVds_WjYum1bLuWWe90HtXE51fZXEqhhPSc822ZasQ,5790
-tests/scrapy_comparison/scrapy_test.py,sha256=-IsGUHPBgEL0TmXjeLZl-TUA01B7Dsc2nRo4JZbFwZA,5599
-tests/test_spiders/__init__.py,sha256=Ws2DhfUA0Xh5Cxr9M46td7B6hyNoLTyAhZ60FnIh6D0,20
-tests/test_spiders/test_spider.py,sha256=kNGEg80HMMFgzVseI1jJjljZEBy3QYKt_3SXGASffFM,168
-crawlo-1.4.5.dist-info/METADATA,sha256=o0MSsONyv_KU7dMNANtCZlkLpVdDUz8zGJKd5i2DM1g,9355
-crawlo-1.4.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.4.5.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.4.5.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.4.5.dist-info/RECORD,,

tests/env_config_example.py DELETED Viewed

@@ -1,134 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-环境变量配置工具使用示例
-展示如何在 Crawlo 项目中正确使用环境变量配置工具
-"""
-import os
-import sys
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-from crawlo.utils.env_config import get_env_var, get_redis_config, get_runtime_config
-from crawlo.settings.setting_manager import SettingManager
-from crawlo.settings import default_settings
-def example_basic_usage():
-    """基本使用示例"""
-    print("=== 基本环境变量使用示例 ===")
-    # 获取字符串环境变量
-    project_name = get_env_var('PROJECT_NAME', 'my_crawler', str)
-    print(f"项目名称: {project_name}")
-    # 获取整数环境变量
-    concurrency = get_env_var('CONCURRENCY', 8, int)
-    print(f"并发数: {concurrency}")
-    # 获取布尔环境变量
-    debug_mode = get_env_var('DEBUG_MODE', False, bool)
-    print(f"调试模式: {debug_mode}")
-def example_redis_config():
-    """Redis配置示例"""
-    print("\n=== Redis配置示例 ===")
-    # 获取Redis配置
-    redis_config = get_redis_config()
-    print(f"Redis主机: {redis_config['REDIS_HOST']}")
-    print(f"Redis端口: {redis_config['REDIS_PORT']}")
-    print(f"Redis密码: {'*' * len(redis_config['REDIS_PASSWORD']) if redis_config['REDIS_PASSWORD'] else '无'}")
-    print(f"Redis数据库: {redis_config['REDIS_DB']}")
-    # 生成Redis URL
-    if redis_config['REDIS_PASSWORD']:
-        redis_url = f"redis://:{redis_config['REDIS_PASSWORD']}@{redis_config['REDIS_HOST']}:{redis_config['REDIS_PORT']}/{redis_config['REDIS_DB']}"
-    else:
-        redis_url = f"redis://{redis_config['REDIS_HOST']}:{redis_config['REDIS_PORT']}/{redis_config['REDIS_DB']}"
-    print(f"Redis URL: {redis_url}")
-def example_runtime_config():
-    """运行时配置示例"""
-    print("\n=== 运行时配置示例 ===")
-    # 获取运行时配置
-    runtime_config = get_runtime_config()
-    print(f"运行模式: {runtime_config['CRAWLO_MODE']}")
-    print(f"项目名称: {runtime_config['PROJECT_NAME']}")
-    print(f"并发数: {runtime_config['CONCURRENCY']}")
-def example_settings_integration():
-    """与Settings集成示例"""
-    print("\n=== 与Settings集成示例 ===")
-    # 创建设置管理器
-    settings = SettingManager()
-    # 更新Redis相关设置
-    redis_config = get_redis_config()
-    settings.set('REDIS_HOST', redis_config['REDIS_HOST'])
-    settings.set('REDIS_PORT', redis_config['REDIS_PORT'])
-    settings.set('REDIS_PASSWORD', redis_config['REDIS_PASSWORD'])
-    settings.set('REDIS_DB', redis_config['REDIS_DB'])
-    # 更新运行时设置
-    runtime_config = get_runtime_config()
-    settings.set('PROJECT_NAME', runtime_config['PROJECT_NAME'])
-    settings.set('RUN_MODE', runtime_config['CRAWLO_MODE'])
-    settings.set('CONCURRENCY', runtime_config['CONCURRENCY'])
-    # 显示一些关键设置
-    print(f"项目名称: {settings.get('PROJECT_NAME')}")
-    print(f"运行模式: {settings.get('RUN_MODE')}")
-    print(f"并发数: {settings.get_int('CONCURRENCY')}")
-    print(f"Redis主机: {settings.get('REDIS_HOST')}")
-    print(f"Redis端口: {settings.get_int('REDIS_PORT')}")
-def example_env_setup():
-    """环境变量设置示例"""
-    print("\n=== 环境变量设置示例 ===")
-    print("在命令行中设置环境变量的示例:")
-    print("  Windows (PowerShell):")
-    print("    $env:PROJECT_NAME = \"my_distributed_crawler\"")
-    print("    $env:REDIS_HOST = \"redis.example.com\"")
-    print("    $env:REDIS_PORT = \"6380\"")
-    print("    $env:CONCURRENCY = \"16\"")
-    print("    $env:CRAWLO_MODE = \"distributed\"")
-    print()
-    print("  Linux/macOS:")
-    print("    export PROJECT_NAME=\"my_distributed_crawler\"")
-    print("    export REDIS_HOST=\"redis.example.com\"")
-    print("    export REDIS_PORT=\"6380\"")
-    print("    export CONCURRENCY=\"16\"")
-    print("    export CRAWLO_MODE=\"distributed\"")
-if __name__ == '__main__':
-    # 设置一些测试环境变量
-    os.environ['PROJECT_NAME'] = 'test_crawler'
-    os.environ['CONCURRENCY'] = '12'
-    os.environ['DEBUG_MODE'] = 'true'
-    os.environ['REDIS_HOST'] = 'redis.test.com'
-    os.environ['REDIS_PORT'] = '6380'
-    os.environ['REDIS_PASSWORD'] = 'test_password'
-    os.environ['CRAWLO_MODE'] = 'distributed'
-    # 运行示例
-    example_basic_usage()
-    example_redis_config()
-    example_runtime_config()
-    example_settings_integration()
-    example_env_setup()
-    # 清理测试环境变量
-    for var in ['PROJECT_NAME', 'CONCURRENCY', 'DEBUG_MODE', 'REDIS_HOST',
-                'REDIS_PORT', 'REDIS_PASSWORD', 'CRAWLO_MODE']:
-        if var in os.environ:
-            del os.environ[var]

tests/ofweek_scrapy/ofweek_scrapy/spiders/ofweek_spider.py DELETED Viewed

@@ -1,162 +0,0 @@
-import scrapy
-from urllib.parse import urljoin
-class OfweekSpider(scrapy.Spider):
-    name = 'ofweek'
-    allowed_domains = ['ee.ofweek.com']
-    def start_requests(self):
-        """生成初始请求"""
-        headers = {
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
-            "Cache-Control": "no-cache",
-            "Connection": "keep-alive",
-            "Pragma": "no-cache",
-            "Referer": "https://ee.ofweek.com/CATList-2800-8100-ee-2.html",
-            "Sec-Fetch-Dest": "document",
-            "Sec-Fetch-Mode": "navigate",
-            "Sec-Fetch-Site": "same-origin",
-            "Sec-Fetch-User": "?1",
-            "Upgrade-Insecure-Requests": "1",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
-        }
-        cookies = {
-            "__utmz": "57425525.1730117117.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
-            "Hm_lvt_abe9900db162c6d089cdbfd107db0f03": "1739244841",
-            "Hm_lvt_af50e2fc51af73da7720fb324b88a975": "1740100727",
-            "JSESSIONID": "FEA96D3B5FC31350B2285E711BF2A541",
-            "Hm_lvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477622",
-            "HMACCOUNT": "08DF0D235A291EAA",
-            "__utma": "57425525.2080994505.1730117117.1747970718.1757477622.50",
-            "__utmc": "57425525",
-            "__utmt": "1",
-            "__utmb": "57425525.2.10.1757477622",
-            "Hm_lpvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477628",
-            "index_burying_point": "c64d6c31e69d560efe319cc9f8be279f"
-        }
-        # 使用与Crawlo相同的页数进行公平测试
-        max_page = 50  # 原来是1851，现在改为50页进行测试
-        start_urls = []
-        for page in range(1, max_page + 1):
-            url = f'https://ee.ofweek.com/CATList-2800-8100-ee-{page}.html'
-            start_urls.append(url)
-        self.logger.info(f"生成了 {len(start_urls)} 个起始URL")
-        # 生成请求
-        for url in start_urls:
-            self.logger.info(f"添加起始URL: {url}")
-            try:
-                yield scrapy.Request(
-                    url=url,
-                    callback=self.parse,
-                    headers=headers,
-                    cookies=cookies,
-                    dont_filter=True
-                )
-            except Exception as e:
-                self.logger.error(f"创建请求失败: {url}, 错误: {e}")
-        self.logger.info("start_requests方法执行完成")
-    def parse(self, response):
-        """解析列表页"""
-        self.logger.info(f'正在解析页面: {response.url}')
-        # 检查响应状态
-        if response.status != 200:
-            self.logger.warning(f"页面返回非200状态码: {response.status}, URL: {response.url}")
-            return
-        # ================== 数据提取 ==================
-        try:
-            rows = response.xpath('//div[@class="main_left"]/div[@class="list_model"]/div[@class="model_right model_right2"]')
-            self.logger.info(f"在页面 {response.url} 中找到 {len(rows)} 个条目")
-            for row in rows:
-                try:
-                    # 提取URL和标题
-                    url = row.xpath('./h3/a/@href').extract_first()
-                    title = row.xpath('./h3/a/text()').extract_first()
-                    # 容错处理
-                    if not url:
-                        self.logger.warning(f"条目缺少URL，跳过")
-                        continue
-                    if not title:
-                        self.logger.warning(f"条目缺少标题，跳过")
-                        continue
-                    # 确保 URL 是绝对路径
-                    absolute_url = response.urljoin(url)
-                    # 验证URL格式
-                    if not absolute_url.startswith(('http://', 'https://')):
-                        self.logger.warning(f"无效的URL格式，跳过: {absolute_url}")
-                        continue
-                    self.logger.info(f"提取到详情页链接: {absolute_url}, 标题: {title}")
-                    yield scrapy.Request(
-                        url=absolute_url,
-                        meta={
-                            "title": title.strip() if title else '',
-                            "parent_url": response.url
-                        },
-                        callback=self.parse_detail
-                    )
-                except Exception as e:
-                    self.logger.error(f"处理条目时出错: {e}")
-                    continue
-        except Exception as e:
-            self.logger.error(f"解析页面 {response.url} 时出错: {e}")
-    def parse_detail(self, response):
-        """解析详情页"""
-        self.logger.info(f'正在解析详情页: {response.url}')
-        # 检查响应状态
-        if response.status != 200:
-            self.logger.warning(f"详情页返回非200状态码: {response.status}, URL: {response.url}")
-            return
-        try:
-            title = response.meta.get('title', '')
-            # 提取内容，增加容错处理
-            content_elements = response.xpath('//div[@class="TRS_Editor"]|//*[@id="articleC"]')
-            if content_elements:
-                content = content_elements.xpath('.//text()').extract()
-                content = '\n'.join([text.strip() for text in content if text.strip()])
-            else:
-                content = ''
-                self.logger.warning(f"未找到内容区域: {response.url}")
-            # 提取发布时间
-            publish_time = response.xpath('//div[@class="time fl"]/text()').extract_first()
-            if publish_time:
-                publish_time = publish_time.strip()
-            source = response.xpath('//div[@class="source-name"]/text()').extract_first()
-            # 创建数据项
-            item = {
-                'title': title.strip() if title else '',
-                'publish_time': publish_time if publish_time else '',
-                'url': response.url,
-                'source': source if source else '',
-                'content': content
-            }
-            # 验证必要字段
-            if not item['title']:
-                self.logger.warning(f"详情页缺少标题: {response.url}")
-            self.logger.info(f"成功提取详情页数据: {item['title']}")
-            yield item
-        except Exception as e:
-            self.logger.error(f"解析详情页 {response.url} 时出错: {e}")

crawlo 1.4.5__py3-none-any.whl → 1.4.7__py3-none-any.whl

Potentially problematic release.

crawlo 1.4.5py3-none-any.whl → 1.4.7py3-none-any.whl