crawlo 1.4.3__py3-none-any.whl → 1.4.4__py3-none-any.whl

crawlo/templates/project/settings_simple.py.tmpl

@@ -6,75 +6,79 @@
 适合快速开始和简单项目。
 """
 
-# ============================== 项目基本信息 ==============================
+# =================================== 基础配置 ===================================
+
+# 项目基本信息
 PROJECT_NAME = '{{project_name}}'
 
-# ============================== 运行模式 ==============================
+# 运行模式
 RUN_MODE = 'standalone'
 
-# ============================== 并发配置 ==============================
+# 并发配置
 CONCURRENCY = 4
 MAX_RUNNING_SPIDERS = 1
 DOWNLOAD_DELAY = 1.0
 
-# ============================== 下载器配置 ==============================
-# 可选下载器:
-# DOWNLOADER = 'crawlo.downloader.aiohttp_downloader.AioHttpDownloader'
-# DOWNLOADER = 'crawlo.downloader.httpx_downloader.HttpXDownloader'
-# DOWNLOADER = 'crawlo.downloader.cffi_downloader.CurlCffiDownloader'
+# =================================== 核心组件配置 ===================================
+
+# 下载器配置
 DOWNLOADER = 'crawlo.downloader.aiohttp_downloader.AioHttpDownloader'
 
-# ============================== 队列配置 ==============================
+# 队列配置
 QUEUE_TYPE = 'memory'
 
-# ============================== 去重过滤器 ==============================
+# 去重过滤器
 FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
 
-# ============================== 默认去重管道 ==============================
+# 默认去重管道
 DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline'
 
-# ============================== 爬虫模块配置 ==============================
-SPIDER_MODULES = ['{{project_name}}.spiders']
+# =================================== 爬虫配置 ===================================
 
-# ============================== 中间件 ==============================
-# MIDDLEWARES = [
-#     'crawlo.middleware.simple_proxy.SimpleProxyMiddleware',
-# ]
+# 爬虫模块配置
+SPIDER_MODULES = ['{{project_name}}.spiders']
 
-# ============================== 默认请求头配置 ==============================
+# 默认请求头配置
 # 为DefaultHeaderMiddleware配置默认请求头
-DEFAULT_REQUEST_HEADERS = {
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
-    'Accept-Encoding': 'gzip, deflate, br',
-}
+# DEFAULT_REQUEST_HEADERS = {}
 
-# ============================== 允许的域名 ==============================
+# 允许的域名
 # 为OffsiteMiddleware配置允许的域名
-# ALLOWED_DOMAINS = ['example.com']
+# ALLOWED_DOMAINS = []
 
-# ============================== 数据管道 ==============================
+# 数据管道
+# 如需添加自定义管道，请取消注释并添加
 # PIPELINES = [
-#     'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',     # MySQL 存储（使用asyncmy异步库）
+#     'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',  # MySQL 存储（使用asyncmy异步库）
+#     # '{{project_name}}.pipelines.CustomPipeline',  # 用户自定义管道示例
 # ]
 
-# ============================== 扩展组件 ==============================
+# =================================== 系统配置 ===================================
+
+# 扩展组件
+# 如需添加自定义扩展，请取消注释并添加
 # EXTENSIONS = [
-#     'crawlo.extension.log_interval.LogIntervalExtension',
-#     'crawlo.extension.log_stats.LogStats',
-#     'crawlo.extension.logging_extension.CustomLoggerExtension',
+#     # '{{project_name}}.extensions.CustomExtension',  # 用户自定义扩展示例
+# ]
+
+# 中间件
+# 如需添加自定义中间件，请取消注释并添加
+# MIDDLEWARES = [
+#     # '{{project_name}}.middlewares.CustomMiddleware',  # 用户自定义中间件示例
 # ]
 
-# ============================== 日志配置 ==============================
+# 日志配置
 LOG_LEVEL = 'INFO'
 LOG_FILE = 'logs/{{project_name}}.log'
 LOG_ENCODING = 'utf-8'  # 明确指定日志文件编码
 STATS_DUMP = True
 
-# ============================== 输出配置 ==============================
+# 输出配置
 OUTPUT_DIR = 'output'
 
-# ============================== Redis配置 ==============================
+# =================================== 数据库配置 ===================================
+
+# Redis配置
 REDIS_HOST = '127.0.0.1'
 REDIS_PORT = 6379
 REDIS_PASSWORD = ''
@@ -86,7 +90,7 @@ if REDIS_PASSWORD:
 else:
     REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
 
-# ============================== MySQL配置 ==============================
+# MySQL配置
 MYSQL_HOST = '127.0.0.1'
 MYSQL_PORT = 3306
 MYSQL_USER = 'root'
@@ -96,7 +100,7 @@ MYSQL_TABLE = '{{project_name}}_data'
 MYSQL_BATCH_SIZE = 100
 MYSQL_USE_BATCH = False  # 是否启用批量插入
 
-# ============================== MongoDB配置 ==============================
+# MongoDB配置
 MONGO_URI = 'mongodb://localhost:27017'
 MONGO_DATABASE = '{{project_name}}_db'
 MONGO_COLLECTION = '{{project_name}}_items'
@@ -105,7 +109,9 @@ MONGO_MIN_POOL_SIZE = 20
 MONGO_BATCH_SIZE = 100  # 批量插入条数
 MONGO_USE_BATCH = False  # 是否启用批量插入
 
-# ============================== 代理配置 ==============================
+# =================================== 网络配置 ===================================
+
+# 代理配置
 # 代理功能默认不启用，如需使用请在项目配置文件中启用并配置相关参数
 PROXY_ENABLED = False  # 是否启用代理
 
@@ -124,7 +130,6 @@ PROXY_EXTRACTOR = "proxy"
 PROXY_REFRESH_INTERVAL = 60  # 代理刷新间隔（秒）
 PROXY_API_TIMEOUT = 10  # 请求代理 API 超时时间
 
-# ============================== Curl-Cffi 特有配置 ==============================
 # 浏览器指纹模拟（仅 CurlCffi 下载器有效）
 CURL_BROWSER_TYPE = "chrome"  # 可选: chrome, edge, safari, firefox 或版本如 chrome136
 
@@ -136,7 +141,7 @@ CURL_BROWSER_VERSION_MAP = {
     "firefox": "firefox135",
 }
 
-# ============================== 下载器优化配置 ==============================
+# 下载器优化配置
 # 下载器健康检查
 DOWNLOADER_HEALTH_CHECK = True  # 是否启用下载器健康检查
 HEALTH_CHECK_INTERVAL = 60  # 健康检查间隔（秒）
@@ -157,7 +162,7 @@ AIOHTTP_FORCE_CLOSE = False  # 是否强制关闭连接
 CONNECTION_TTL_DNS_CACHE = 300  # DNS缓存TTL（秒）
 CONNECTION_KEEPALIVE_TIMEOUT = 15  # Keep-Alive超时（秒）
 
-# ============================== 内存监控配置 ==============================
+# 内存监控配置
 # 内存监控扩展默认不启用，如需使用请在项目配置文件中启用
 MEMORY_MONITOR_ENABLED = False  # 是否启用内存监控
 MEMORY_MONITOR_INTERVAL = 60  # 内存监控检查间隔（秒）

crawlo/templates/run.py.tmpl

@@ -6,7 +6,7 @@
 基于 Crawlo 框架的简化爬虫启动器。
 
 框架会自动处理爬虫模块的导入和注册，用户无需手动导入。
-只需指定spider_modules参数，框架会自动扫描并导入所有爬虫。
+框架会自动从settings.py中读取SPIDER_MODULES配置。
 """
 import sys
 import asyncio
@@ -17,12 +17,8 @@ from crawlo.crawler import CrawlerProcess
 def main():
     """主函数：运行爬虫"""
     try:
-        # 指定爬虫模块路径，框架会自动导入并注册所有爬虫
-        spider_modules = ['{{project_name}}.spiders']
-        process = CrawlerProcess(spider_modules=spider_modules)
-
-        # TODO 运行指定的爬虫
-        asyncio.run(process.crawl('spider_name'))
+        # TODO: 请将 'spider_name' 替换为实际要运行的爬虫名称
+        asyncio.run(CrawlerProcess().crawl('spider_name'))
 
     except Exception as e:
         print(f"❌ 运行失败: {e}")

{crawlo-1.4.3.dist-info → crawlo-1.4.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.4.3
+Version: 1.4.4
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

{crawlo-1.4.3.dist-info → crawlo-1.4.4.dist-info}/RECORD

@@ -1,9 +1,9 @@
 crawlo/__init__.py,sha256=2Io5P9qJghOAjjD3YWdaiIq5laPLyLWVkEqgiVfUa3o,2381
-crawlo/__version__.py,sha256=adPldP4cMp2T8pbTFGYTXV50hu_smS3hxWkk5kLXpZE,23
+crawlo/__version__.py,sha256=2ik6wvURqg571WApVvR_ELhg_eclmC_WvbDLEPmoO4Q,23
 crawlo/cli.py,sha256=AQnAB5NMI-Ic1VPw_Jjng8L4AI4-wMozOwzE6CfXkZU,2402
 crawlo/config.py,sha256=EQIT7WpkXAlr2ocd5SYJYOKTSWUlQx2AkTHX7ErEWxw,9798
 crawlo/config_validator.py,sha256=oY4-2bwXUlwHAnGgkI-EznviDfML_dcxbWSGXNSxC2k,11516
-crawlo/crawler.py,sha256=i4rc9beEOilKGK633nRh5UxCNgciil9Lyfj38xgIauU,26998
+crawlo/crawler.py,sha256=E-fgYVtx6v2xEPixlQeWfNYVbW1oeWE0fQFZTQ6_K-I,27305
 crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
 crawlo/exceptions.py,sha256=YVIDnC1bKSMv3fXH_6tinWMuD9HmKHIaUfO4_fkX5sY,1247
 crawlo/framework.py,sha256=9gP6VN4MHqutGXaxnwpNMSULfVYbNp906UdZiJGywlQ,9458
@@ -14,7 +14,7 @@ crawlo/subscriber.py,sha256=h8fx69NJZeWem0ZkCmfHAi2kgfDGFObHpwN0aGNUM6Y,5115
 crawlo/task_manager.py,sha256=I9h3Rl0VRAfwqp24CHT3TuEAapNdTbVghkmuJhtM7jg,5966
 crawlo/commands/__init__.py,sha256=orvY6wLOBwGUEJKeF3h_T1fxj8AaQLjngBDd-3xKOE4,392
 crawlo/commands/check.py,sha256=TKDhI_sj7kErgiJpt2vCZ9QL-g6yWjrrPWKbgh8pgEU,23199
-crawlo/commands/genspider.py,sha256=7YGZdv12G341SWmkGbyDeMde2RgqGYxYXRExFy7KKNc,5088
+crawlo/commands/genspider.py,sha256=JB4ZuFpKsYwtjx3DSsxugH7e3kqxhDWPG5ZKfvM0isI,6041
 crawlo/commands/help.py,sha256=8xPC0iNCg1rRBoK2bb6noAEANc1JwrdM35eF-j6yeZM,5111
 crawlo/commands/list.py,sha256=trzcd3kG6DhkOqYZADcl3yR7M8iJBgRw5fE-g9e0gVM,5877
 crawlo/commands/run.py,sha256=EjpIilgCTkXGVSV4rEISbJubdhqrok9nNe5-xDfDK5E,13169
@@ -92,26 +92,26 @@ crawlo/pipelines/mysql_pipeline.py,sha256=Kjgu6cks1KD4FPXwlTnFaos2LG-N8LLaBDyKZ_
 crawlo/pipelines/pipeline_manager.py,sha256=R6MRb5d-caOit7PZoglJLHa3qQ68U5YAQlwt8KcjRxo,4393
 crawlo/pipelines/redis_dedup_pipeline.py,sha256=RB1kXLr8ZuWNrgZKYwt--tlmnWsQTbuwTsSt3pafol8,6077
 crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/queue/pqueue.py,sha256=Q4H2ePag7-pBjzJ0a4S1P4z_IT_G08T6l1uZBkuxO5A,1262
-crawlo/queue/queue_manager.py,sha256=Y2K4P-il5ACdP72X5jgTT-65Fu94_kQZ7H4c7AdRt-c,21547
+crawlo/queue/pqueue.py,sha256=bbgd3l1VfqYXfz-4VFaiWLmJit1LdB3qHalCtNqyrqI,1210
+crawlo/queue/queue_manager.py,sha256=8rKygMxr6DgSjnGsKFmvlTI5XAARvQIN_ENkAruHGXs,21532
 crawlo/queue/redis_priority_queue.py,sha256=vLvg2toKaRrXD1QyEdu1ZjTmANv7clFaBF7mCtstBmI,15995
 crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
-crawlo/settings/default_settings.py,sha256=DdoMvB7M_OyGKlBOr0lY-T5TN4V1bo6PlPvu4ZcQXnY,12836
+crawlo/settings/default_settings.py,sha256=IKh2eZ9WWXkAbHx5K5KX0whNtumATRpZ7ifFPZJFfBk,11827
 crawlo/settings/setting_manager.py,sha256=yI1tGaludevxKGGZO3Pn4aYofrg2cwYwvMZCFC5PPZw,8595
 crawlo/spider/__init__.py,sha256=QGhe_yNsnfnCF3G9nSoWEw23b8SkP5oSFU5W79C5DzI,21881
 crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
-crawlo/templates/run.py.tmpl,sha256=Mt4gcw7RJu9ri4eBRnsAzf0EDt2giFWzpX36OTqEUpQ,961
+crawlo/templates/run.py.tmpl,sha256=g8yst2hkqhKGNotR33fDxwmEsX6aEvhrXY_cfYos_vc,788
 crawlo/templates/spiders_init.py.tmpl,sha256=p6UK8KWr8FDydNxiAh6Iz29MY5WmgXIkf2z-buOGhOM,354
 crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
 crawlo/templates/project/items.py.tmpl,sha256=8_3DBA8HrS2XbfHzsMZNJiZbFY6fDJUUMFoFti_obJk,314
 crawlo/templates/project/middlewares.py.tmpl,sha256=fxHqi-Sjec5GiHJciprOU-6SAUTzM728NlZckIqf9hM,4278
 crawlo/templates/project/pipelines.py.tmpl,sha256=j9oqEhCezmmHlBhMWgYtlgup4jhWnMlv6AEiAOHODkg,2704
-crawlo/templates/project/settings.py.tmpl,sha256=gGYwPf-RCuw1pWyBZRUvL1BHzgoLYcIQ1sXUmWDGp1k,7050
-crawlo/templates/project/settings_distributed.py.tmpl,sha256=YSN7YcA05MT2yO5mg5MpryVyeEDhcaJEp9TU2NAUPS8,7147
-crawlo/templates/project/settings_gentle.py.tmpl,sha256=HS1qumqDciXFBTZYm-RS__ldphDVbuo6Poz8pTCHNEg,6797
-crawlo/templates/project/settings_high_performance.py.tmpl,sha256=u6n7S0KzSgJU23pkRifXnsS7AdQplJXefRyeG51v_QI,6887
-crawlo/templates/project/settings_minimal.py.tmpl,sha256=GJG44CJlop0_kDwtqhbKJlG_42VThHfICGVUNijkkRs,2550
-crawlo/templates/project/settings_simple.py.tmpl,sha256=fmy5PgrS_uVlXnIwXQFQ1Q28Ls121lbVg21Dbxrn6B4,6649
+crawlo/templates/project/settings.py.tmpl,sha256=mL9_JAyz8R35r-ywRHi4T-dtal7oczU5kodEWxldw40,5265
+crawlo/templates/project/settings_distributed.py.tmpl,sha256=RHzfWZITv-0ErCR9OYEswAZHpA5d9fYil0ZoGCtFt8g,5459
+crawlo/templates/project/settings_gentle.py.tmpl,sha256=pmjrBLjnpGcR90RkcJrM5O8PsTrRhUB92QR3R4TJyko,5733
+crawlo/templates/project/settings_high_performance.py.tmpl,sha256=9QhXSzfxIsMPyq0kZY9h2YBllyXGpGE37bMEbSrs_Ag,5823
+crawlo/templates/project/settings_minimal.py.tmpl,sha256=1qUPhSdHtvLSHTpytUJ8K63sMROhTwkz8e4tVg1fYoM,2222
+crawlo/templates/project/settings_simple.py.tmpl,sha256=sIyrCIVXsHSKl8Yjj8HkGs-ppMFH26a5yp6egVNlT2Q,5585
 crawlo/templates/project/spiders/__init__.py.tmpl,sha256=llhcIItXpm0TlEeumeLwp4fcYv2NHl8Iru7tLhDhxiE,216
 crawlo/templates/spider/spider.py.tmpl,sha256=KvU-9YpN6MifDE7XzejjyyQS7RUjLDLZ8zqJcLwSsu0,5198
 crawlo/tools/__init__.py,sha256=tOYfYPvZlrO8cmvnMWBjTma6UTLTFZN3qdC8pJwHrzI,4142
@@ -151,15 +151,6 @@ crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
 crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
 crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
 examples/__init__.py,sha256=NkRbV8_S1tb8S2AW6BE2U6P2-eGOPwMR1k0YQAwQpSE,130
-examples/test_project/__init__.py,sha256=BQ6FVVDjB00-Vyib8h2I3P-LV5tjsFzTSLC2rFDe7Gw,136
-examples/test_project/run.py,sha256=HHMFVY8D9ouWytp-xNvFDKB8NphrZGuFbVrah6-afG8,953
-examples/test_project/test_project/__init__.py,sha256=RU5IUwU1oeDABAj3ZsLdMo53r3XG4saYIYgO1SXo13g,57
-examples/test_project/test_project/items.py,sha256=L_bnTuJLW7xtuja4AaqYuSsQvVdaqMunZWdh1OTz4HY,310
-examples/test_project/test_project/middlewares.py,sha256=fE7AUI2Yb9_ZwA4vC4ngn-s231FtEYeYWjXOwd5mkzE,4270
-examples/test_project/test_project/pipelines.py,sha256=7EDGrvuNH6JY0XINRwzgPmB-X4Ax6J790jtxbkL2O_U,2696
-examples/test_project/test_project/settings.py,sha256=imfrkrwSJ1V3jINe2ZhynlT4_w5bUBkeI9DFK7DS2g0,7115
-examples/test_project/test_project/spiders/__init__.py,sha256=Lx_To88ShpAR6Pyd9PUMjct690MZH0gETxz6knRqPRY,212
-examples/test_project/test_project/spiders/of_week_dis.py,sha256=-gDLaKGoF5birxCoLL_CX82bYopXjo4QmOV6a7I-Ci0,5178
 tests/__init__.py,sha256=409aRX8hsPffiZCVjOogtxwhACzBp8G2UTJyUQSxhK0,136
 tests/advanced_tools_example.py,sha256=1_iitECKCuWUYMNNGo61l3lmwMRrWdA8F_Xw56UaGZY,9340
 tests/authenticated_proxy_example.py,sha256=fKmHXXxIxCJXjEplttCWRh7PZhbxkBSxJF91Bx-qOME,3019
@@ -260,6 +251,8 @@ tests/test_logging_system.py,sha256=LGfK14ZEWzRtl3_VkBGz-AaVa_dDtuC5zu40m8FvmMo,
 tests/test_middleware_debug.py,sha256=gtiaWCxBSTcaNkdqXirM7CsThr_HfiCueBdQCpp7rqg,4572
 tests/test_mode_change.py,sha256=GT53CBdxcG3-evcKz_OOfH4PBiq_oqQyuDjRXrvv1UU,2665
 tests/test_mode_consistency.py,sha256=t72WX0etC_AayaL2AT6e2lIgbfP-zxTgYAiTARSN2Jk,1276
+tests/test_multi_directory.py,sha256=sH9Y3B-fuESlc7J1aICa-AlBcCW8HFR-Q5j2anUr8l0,2196
+tests/test_multiple_spider_modules.py,sha256=M0wPyQW7HMasbMIgn_R78wjZEj4A_DgqaGHp0qF9Y0c,2567
 tests/test_offsite_middleware.py,sha256=njpXTdngOqBs60Wj6xgo5EEXlJnMHd7vtYGi9dVauW0,10602
 tests/test_offsite_middleware_simple.py,sha256=4MfDKSXGHcoFLYnnxCH2rmnzztWyN0xByYLoHtepyiA,7918
 tests/test_parsel.py,sha256=wuZqRFIm9xx1tt6o3Xi_OjvwhT_MPmHiUEj2ax06zlo,701
@@ -306,6 +299,7 @@ tests/test_retry_middleware_realistic.py,sha256=Sam5y4jCN8oeElU4xxeS5zjAyzS-P8si
 tests/test_scheduler.py,sha256=1fCu35QgK5gzgrhD0aUZj5lxL0QbokzPav-yEJxz9Ig,8182
 tests/test_scheduler_config_update.py,sha256=LuxjEbt20QrPyVkjSFxvTnFtUxwMaHB6TcqjFyo8bow,4261
 tests/test_simple_response.py,sha256=_ui2PuVZvJcAuLY7HZ8xcsy_tDBimgBqX0ukj3kE5J0,1549
+tests/test_spider_modules.py,sha256=wxPs28FtpGnQTemMY6r7WxVrwYo3bHnAd5dq94qj1K4,2797
 tests/test_telecom_spider_redis_key.py,sha256=c-gfixPul2VlYMQJGf0H5ZgYJ461fQgSKbCPrbAU45M,7625
 tests/test_template_content.py,sha256=2RgCdOA3pMUSOqC_JbTGeW7KonbTqJ0ySYJNWegU-v0,2903
 tests/test_template_redis_key.py,sha256=99-s0_-8MFJbIvGG_X__sH0qkXWTtJv8fdTdlftsq4I,4876
@@ -319,8 +313,8 @@ tests/verify_distributed.py,sha256=0IolM4ymuPOz_uTfHSWFO3Vxzp7Lo6i0zhSbzJhHFtI,4
 tests/verify_log_fix.py,sha256=7reyVl3MXTDASyChgU5BAYuzuxvFjSLG9HywAHso0qg,4336
 tests/scrapy_comparison/ofweek_scrapy.py,sha256=rhVds_WjYum1bLuWWe90HtXE51fZXEqhhPSc822ZasQ,5790
 tests/scrapy_comparison/scrapy_test.py,sha256=-IsGUHPBgEL0TmXjeLZl-TUA01B7Dsc2nRo4JZbFwZA,5599
-crawlo-1.4.3.dist-info/METADATA,sha256=PTbW7GP9xgBjFEX1adwjxeJHqM_x9VRU_ZLZsezaydU,4848
-crawlo-1.4.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.4.3.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.4.3.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.4.3.dist-info/RECORD,,
+crawlo-1.4.4.dist-info/METADATA,sha256=LAg9xmMfxLUwVUGPqw_p48hGJYZqsRC9Mc4KqDroAUQ,4848
+crawlo-1.4.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.4.4.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.4.4.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.4.4.dist-info/RECORD,,

tests/test_multi_directory.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+"""
+测试多个爬虫目录的支持
+"""
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.dirname(__file__))
+
+# 添加ofweek_standalone到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'examples', 'ofweek_standalone'))
+
+def test_multiple_spider_directories():
+    """测试多个爬虫目录的支持"""
+    print("测试多个爬虫目录的支持...")
+    
+    # 导入设置
+    import examples.ofweek_standalone.ofweek_standalone.settings as settings_module
+    
+    # 创建设置管理器
+    from crawlo.settings.setting_manager import SettingManager
+    settings = SettingManager()
+    settings.set_settings(settings_module)
+    
+    # 检查SPIDER_MODULES配置
+    spider_modules = settings.get('SPIDER_MODULES')
+    print(f"SPIDER_MODULES配置: {spider_modules}")
+    
+    # 创建CrawlerProcess实例
+    from crawlo.crawler import CrawlerProcess
+    process = CrawlerProcess(settings=settings)
+    
+    # 检查是否注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    # 验证期望的爬虫是否已注册
+    expected_spiders = ['of_week_standalone', 'test_spider']
+    registered_spiders = []
+    
+    for spider_name in expected_spiders:
+        if spider_name in spider_names:
+            print(f"✅ 成功: 爬虫 '{spider_name}' 已注册")
+            registered_spiders.append(spider_name)
+        else:
+            print(f"❌ 失败: 爬虫 '{spider_name}' 未找到")
+    
+    if len(registered_spiders) == len(expected_spiders):
+        print(f"🎉 所有爬虫都已成功注册!")
+        return True
+    else:
+        print(f"⚠️  部分爬虫未注册: {set(expected_spiders) - set(registered_spiders)}")
+        return False
+
+
+if __name__ == '__main__':
+    print("开始测试多个爬虫目录的支持...\n")
+    
+    success = test_multiple_spider_directories()
+    
+    if success:
+        print("\n🎉 测试通过!")
+        sys.exit(0)
+    else:
+        print("\n❌ 测试失败!")
+        sys.exit(1)

tests/test_multiple_spider_modules.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+"""
+测试多个SPIDER_MODULES目录的支持
+"""
+import sys
+import os
+import asyncio
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.dirname(__file__))
+
+# 添加ofweek_standalone到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'examples', 'ofweek_standalone'))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import get_spider_names
+
+
+def test_multiple_spider_modules():
+    """测试多个SPIDER_MODULES目录的支持"""
+    print("测试多个SPIDER_MODULES目录的支持...")
+    
+    # 模拟包含多个目录的SPIDER_MODULES配置
+    spider_modules = ['ofweek_standalone.spiders', 'ofweek_standalone.new_spiders']
+    
+    # 创建CrawlerProcess实例
+    process = CrawlerProcess(spider_modules=spider_modules)
+    
+    # 检查是否注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    # 验证期望的爬虫是否已注册
+    expected_spider = 'of_week_standalone'
+    if expected_spider in spider_names:
+        print(f"✅ 成功: 爬虫 '{expected_spider}' 已注册")
+        return True
+    else:
+        print(f"❌ 失败: 爬虫 '{expected_spider}' 未找到")
+        return False
+
+
+def test_settings_with_multiple_spider_modules():
+    """测试settings中配置多个SPIDER_MODULES目录"""
+    print("\n测试settings中配置多个SPIDER_MODULES目录...")
+    
+    # 创建模拟的settings对象
+    class MockSettings:
+        def get(self, key, default=None):
+            if key == 'SPIDER_MODULES':
+                return ['ofweek_standalone.spiders', 'ofweek_standalone.new_spiders']
+            return default
+    
+    settings = MockSettings()
+    
+    # 创建CrawlerProcess实例
+    process = CrawlerProcess(settings=settings)
+    
+    # 检查是否注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    return True
+
+
+if __name__ == '__main__':
+    print("开始测试多个SPIDER_MODULES目录的支持...\n")
+    
+    # 测试显式传递多个spider_modules参数
+    success1 = test_multiple_spider_modules()
+    
+    # 测试从settings中读取多个spider_modules配置
+    success2 = test_settings_with_multiple_spider_modules()
+    
+    if success1 and success2:
+        print("\n🎉 所有测试通过!")
+        sys.exit(0)
+    else:
+        print("\n❌ 部分测试失败!")
+        sys.exit(1)

tests/test_spider_modules.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+"""
+测试SPIDER_MODULES配置的自动读取功能
+"""
+import sys
+import os
+import asyncio
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.dirname(__file__))
+
+# 添加ofweek_standalone到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'examples', 'ofweek_standalone'))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import get_spider_names
+
+
+def test_spider_modules_auto_discovery():
+    """测试SPIDER_MODULES配置的自动读取功能"""
+    print("测试SPIDER_MODULES配置的自动读取功能...")
+    
+    # 导入设置
+    import examples.ofweek_standalone.ofweek_standalone.settings as settings_module
+    
+    # 创建设置管理器
+    from crawlo.settings.setting_manager import SettingManager
+    settings = SettingManager()
+    settings.set_settings(settings_module)
+    
+    # 创建CrawlerProcess实例，不显式传递spider_modules
+    process = CrawlerProcess(settings=settings)
+    
+    # 检查是否自动注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    # 验证期望的爬虫是否已注册
+    expected_spider = 'of_week_standalone'
+    if expected_spider in spider_names:
+        print(f"✅ 成功: 爬虫 '{expected_spider}' 已自动注册")
+        return True
+    else:
+        print(f"❌ 失败: 爬虫 '{expected_spider}' 未找到")
+        return False
+
+
+def test_crawler_process_with_explicit_spider_modules():
+    """测试显式传递spider_modules参数的功能"""
+    print("\n测试显式传递spider_modules参数的功能...")
+    
+    # 显式传递spider_modules参数
+    spider_modules = ['ofweek_standalone.spiders']
+    process = CrawlerProcess(spider_modules=spider_modules)
+    
+    # 检查是否注册了爬虫
+    spider_names = process.get_spider_names()
+    print(f"已注册的爬虫: {spider_names}")
+    
+    # 验证期望的爬虫是否已注册
+    expected_spider = 'of_week_standalone'
+    if expected_spider in spider_names:
+        print(f"✅ 成功: 爬虫 '{expected_spider}' 已注册")
+        return True
+    else:
+        print(f"❌ 失败: 爬虫 '{expected_spider}' 未找到")
+        return False
+
+
+if __name__ == '__main__':
+    print("开始测试SPIDER_MODULES配置功能...\n")
+    
+    # 测试自动发现功能
+    success1 = test_spider_modules_auto_discovery()
+    
+    # 测试显式传递参数功能
+    success2 = test_crawler_process_with_explicit_spider_modules()
+    
+    if success1 and success2:
+        print("\n🎉 所有测试通过!")
+        sys.exit(0)
+    else:
+        print("\n❌ 部分测试失败!")
+        sys.exit(1)

examples/test_project/__init__.py

@@ -1,7 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-09-27 23:36
-# @Author  :   crawl-coder
-# @Desc    :   None
-"""

examples/test_project/run.py

@@ -1,35 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: UTF-8 -*-
-"""
-test_project 项目运行脚本
-============================
-基于 Crawlo 框架的简化爬虫启动器。
-
-框架会自动处理爬虫模块的导入和注册，用户无需手动导入。
-只需指定spider_modules参数，框架会自动扫描并导入所有爬虫。
-"""
-import sys
-import asyncio
-
-from crawlo.crawler import CrawlerProcess
-
-
-def main():
-    """主函数：运行爬虫"""
-    try:
-        # 指定爬虫模块路径，框架会自动导入并注册所有爬虫
-        spider_modules = ['test_project.spiders']
-        process = CrawlerProcess(spider_modules=spider_modules)
-
-        # TODO 运行指定的爬虫
-        asyncio.run(process.crawl('of_week_dis'))
-
-    except Exception as e:
-        print(f"❌ 运行失败: {e}")
-        import traceback
-        traceback.print_exc()
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    main()

examples/test_project/test_project/__init__.py

@@ -1,4 +0,0 @@
-# -*- coding: UTF-8 -*-
-"""
-test_project 项目包
-"""

examples/test_project/test_project/items.py

@@ -1,18 +0,0 @@
-# -*- coding: UTF-8 -*-
-"""
-test_project.items
-======================
-定义你抓取的数据结构。
-"""
-
-from crawlo.items import Item, Field
-
-
-class ExampleItem(Item):
-    """
-    一个示例数据项。
-    """
-    id = Field()
-    # price = Field()
-    # description = Field()
-    pass