crawlo 1.3.1__py3-none-any.whl → 1.3.2__py3-none-any.whl

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = '1.3.1'
+__version__ = '1.3.2'

crawlo/middleware/offsite.py

@@ -28,8 +28,16 @@ class OffsiteMiddleware:
         创建中间件实例
         从爬虫设置中获取允许的域名列表
         """
-        # 从爬虫设置中获取允许的域名
-        allowed_domains = crawler.settings.get_list('ALLOWED_DOMAINS')
+        # 优先使用 Spider 实例的 allowed_domains，回退到全局设置中的 ALLOWED_DOMAINS
+        allowed_domains = []
+        
+        # 检查当前爬虫实例是否有 allowed_domains 属性
+        if hasattr(crawler, 'spider') and crawler.spider and hasattr(crawler.spider, 'allowed_domains'):
+            allowed_domains = getattr(crawler.spider, 'allowed_domains', [])
+        
+        # 如果 Spider 实例没有设置 allowed_domains，则从全局设置中获取
+        if not allowed_domains:
+            allowed_domains = crawler.settings.get_list('ALLOWED_DOMAINS')
         
         # 如果没有配置允许的域名，则禁用此中间件
         if not allowed_domains:

crawlo/mode_manager.py

@@ -40,6 +40,45 @@ class ModeManager:
             'DOWNLOAD_DELAY': 1.0,
         }
 
+    @staticmethod
+    def get_distributed_settings(
+            redis_host: str = '127.0.0.1',
+            redis_port: int = 6379,
+            redis_password: Optional[str] = None,
+            redis_db: int = 0,
+            project_name: str = 'crawlo'
+    ) -> Dict[str, Any]:
+        """获取分布式模式配置"""
+        # 构建 Redis URL
+        if redis_password:
+            redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}'
+        else:
+            redis_url = f'redis://{redis_host}:{redis_port}/{redis_db}'
+            
+        return {
+            'QUEUE_TYPE': 'redis',
+            'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
+            'DEFAULT_DEDUP_PIPELINE': 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline',
+            'REDIS_HOST': redis_host,
+            'REDIS_PORT': redis_port,
+            'REDIS_PASSWORD': redis_password,
+            'REDIS_DB': redis_db,
+            'REDIS_URL': redis_url,
+            'PROJECT_NAME': project_name,
+            'SCHEDULER_QUEUE_NAME': f'crawlo:{project_name}:queue:requests',
+            'CONCURRENCY': 16,
+            'MAX_RUNNING_SPIDERS': 10,
+            'DOWNLOAD_DELAY': 1.0,
+        }
+
+    @staticmethod
+    def get_auto_settings() -> Dict[str, Any]:
+        """获取自动检测模式配置"""
+        # 默认使用内存队列和过滤器
+        settings = ModeManager.get_standalone_settings()
+        settings['QUEUE_TYPE'] = 'auto'
+        return settings
+
     def resolve_mode_settings(
             self,
             mode: str = 'standalone',

crawlo/templates/project/settings.py.tmpl

@@ -156,6 +156,11 @@ MONGO_USE_BATCH = False  # 是否启用批量插入
 # FILTER_DEBUG = True
 # DECODE_RESPONSES = True
 
+# ============================== 域名过滤配置 ==============================
+# OffsiteMiddleware 配置，用于限制爬虫只爬取指定域名的页面
+# 如需启用域名过滤功能，请取消注释并配置允许的域名列表
+# ALLOWED_DOMAINS = ['example.com', 'www.example.com']
+
 # ============================== 用户自定义中间件配置 ==============================
 # 注意：框架默认中间件已自动加载，此处可添加或覆盖默认中间件
 # 如需启用代理功能，请取消注释 ProxyMiddleware 并配置代理相关参数

crawlo/templates/project/settings_distributed.py.tmpl

@@ -121,13 +121,18 @@ MONGO_USE_BATCH = True
 # 注意：框架已提供默认的去重配置，以下配置项通常无需修改
 # 如需自定义，请取消注释并修改相应值
 
-# 明确指定分布式模式下使用Redis去重管道
+# 明确指定分布式模式下使用Redis去重管道和过滤器
 # DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline'
 # FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'
 # REDIS_TTL = 0
 # CLEANUP_FP = 0
 # FILTER_DEBUG = True
 
+# ============================== 域名过滤配置 ==============================
+# OffsiteMiddleware 配置，用于限制爬虫只爬取指定域名的页面
+# 如需启用域名过滤功能，请取消注释并配置允许的域名列表
+# ALLOWED_DOMAINS = ['example.com', 'www.example.com']
+
 # ============================== 用户自定义中间件配置 ==============================
 # 注意：框架默认中间件已自动加载，此处可添加或覆盖默认中间件
 

crawlo/templates/project/settings_gentle.py.tmpl

@@ -1,43 +1,3 @@
-# -*- coding: UTF-8 -*-
-"""
-温和模式配置模板
-低负载配置，对目标网站友好
-"""
-
-# ============================== 项目基本信息 ==============================
-PROJECT_NAME = '{{project_name}}'
-
-# ============================== 温和运行模式 ==============================
-# 运行模式：'standalone'(单机), 'distributed'(分布式), 'auto'(自动检测)
-RUN_MODE = 'standalone'  # 单机模式 - 适用于开发和小规模数据采集
-
-# 并发配置
-CONCURRENCY = 2  # 极低并发数以减少目标网站压力
-DOWNLOAD_DELAY = 3.0  # 增加延迟以降低目标网站压力
-RANDOMNESS = True  # 启用随机延迟
-RANDOM_RANGE = (0.5, 2.0)  # 随机延迟范围
-
-# ============================== 队列配置 ==============================
-
-# 队列类型：'auto'（自动选择）, 'memory'（内存队列）, 'redis'（分布式队列）
-QUEUE_TYPE = 'auto'  # 自动检测，如果Redis可用则使用Redis队列
-SCHEDULER_MAX_QUEUE_SIZE = 500
-SCHEDULER_QUEUE_NAME = f'crawlo:{{project_name}}:queue:requests'
-QUEUE_MAX_RETRIES = 3
-QUEUE_TIMEOUT = 300
-
-# ============================== 去重过滤配置 ==============================
-
-# 温和模式下使用内存去重管道和过滤器
-DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline'
-FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
-
-# --- Redis 配置（用于分布式去重和队列） ---
-REDIS_HOST = '127.0.0.1'
-REDIS_PORT = 6379
-REDIS_PASSWORD = ''  # 如果有密码，请填写
-
-# 根据是否有密码生成 URL
 if REDIS_PASSWORD:
     REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/0'
 else:
@@ -55,6 +15,11 @@ CLEANUP_FP = 0
 FILTER_DEBUG = True
 DECODE_RESPONSES = True
 
+# ============================== 域名过滤配置 ==============================
+# OffsiteMiddleware 配置，用于限制爬虫只爬取指定域名的页面
+# 如需启用域名过滤功能，请取消注释并配置允许的域名列表
+# ALLOWED_DOMAINS = ['example.com', 'www.example.com']
+
 # ============================== 用户自定义中间件配置 ==============================
 # 注意：框架默认中间件已自动加载，此处可添加或覆盖默认中间件
 

crawlo/templates/project/settings_high_performance.py.tmpl

@@ -86,6 +86,11 @@ DECODE_RESPONSES = True
     # 'crawlo.extension.health_check.HealthCheckExtension',  # 健康检查
 # ]
 
+# ============================== 域名过滤配置 ==============================
+# OffsiteMiddleware 配置，用于限制爬虫只爬取指定域名的页面
+# 如需启用域名过滤功能，请取消注释并配置允许的域名列表
+# ALLOWED_DOMAINS = ['example.com', 'www.example.com']
+
 # ============================== 日志配置 ==============================
 
 LOG_LEVEL = 'INFO'

crawlo/templates/project/settings_minimal.py.tmpl

@@ -21,6 +21,11 @@ PIPELINES = [
     'crawlo.pipelines.json_pipeline.JsonPipeline',
 ]
 
+# ============================== 域名过滤配置 ==============================
+# OffsiteMiddleware 配置，用于限制爬虫只爬取指定域名的页面
+# 如需启用域名过滤功能，请取消注释并配置允许的域名列表
+# ALLOWED_DOMAINS = ['example.com', 'www.example.com']
+
 # ============================== 日志配置 ==============================
 LOG_LEVEL = 'INFO'
 LOG_FILE = f'logs/{{project_name}}.log'

crawlo/templates/project/settings_simple.py.tmpl

@@ -59,6 +59,11 @@ DOWNLOAD_DELAY = 1.0  # 增加延迟以降低目标网站压力
 # FILTER_DEBUG = True
 # DECODE_RESPONSES = True
 
+# ============================== 域名过滤配置 ==============================
+# OffsiteMiddleware 配置，用于限制爬虫只爬取指定域名的页面
+# 如需启用域名过滤功能，请取消注释并配置允许的域名列表
+# ALLOWED_DOMAINS = ['example.com', 'www.example.com']
+
 # ============================== 用户自定义中间件配置 ==============================
 # 注意：框架默认中间件已自动加载，此处可添加或覆盖默认中间件
 

crawlo/templates/run.py.tmpl

@@ -5,18 +5,9 @@
 ============================
 基于 Crawlo 框架的简化爬虫启动器。
 """
-
 import sys
-import os
 import asyncio
 
-# 添加项目根目录到 Python 路径
-project_root = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, project_root)
-
-# 切换到项目根目录
-os.chdir(project_root)
-
 from crawlo.crawler import CrawlerProcess
 
 

{crawlo-1.3.1.dist-info → crawlo-1.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.3.1
+Version: 1.3.2
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

{crawlo-1.3.1.dist-info → crawlo-1.3.2.dist-info}/RECORD

@@ -1,12 +1,12 @@
 crawlo/__init__.py,sha256=qZzTmb7hw5h_qcP2EYGUZcoSScxlKZFJ76CjSeS7UfA,1381
-crawlo/__version__.py,sha256=P2DFKJQEJRlJhF0IW0Lwt4G4uMYyJJ5ymhv-XrCcPGo,22
+crawlo/__version__.py,sha256=FVIvqGrcsQXkMjh8L0_Hc4T61ShpSr0KRWouUASp_pM,22
 crawlo/cli.py,sha256=OXprmcTUbFK02ptw_Gq8Gk4-ZCU-WEMJgzU1ztgP6Bk,2327
 crawlo/config.py,sha256=dNoNyTkXLe2msQ7bZx3YTQItk1m49nIg5-g89FQDNwE,9486
 crawlo/config_validator.py,sha256=gsiLqf5swWd9ISDvoLqCdG7iSXr-ZdBPD4iT6ug1ua4,11239
 crawlo/crawler.py,sha256=Fiu9O_eFHKCfzgzFe0O9gpzWGyneY-imI8-9O4hiWqU,42608
 crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
 crawlo/exceptions.py,sha256=sMay0wnWLfc_FXWslqxm60qz6b66LXs3EdN_w8ygE9k,1166
-crawlo/mode_manager.py,sha256=IN3CsWW1pzKnzHxvULxJTTx3Vw2kxJilfeLuDWfxm4Q,4890
+crawlo/mode_manager.py,sha256=QPVFZmsreysYAVxFRdtuknPqkwXM5mLtuLJxpPeI-sQ,6386
 crawlo/project.py,sha256=swSTcan4Ky7sYfCatpNLKsVxztmPkIVwjdo3u6dgcpI,11128
 crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
 crawlo/subscriber.py,sha256=D3hzE7Pc_zJjc-zR7lct5pt32bz6LsDYeC8uHlS4Hso,4986
@@ -52,7 +52,7 @@ crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0
 crawlo/middleware/default_header.py,sha256=wQ7BrUHd-hRosFoKsReV9hwNNr_jwK6V0ZfxL6MOGrk,5032
 crawlo/middleware/download_delay.py,sha256=zt9R5g2HWErWA_MAOnGcw_D8l6HD769Kyaw-Hv-vcTc,3438
 crawlo/middleware/middleware_manager.py,sha256=9Sj9rrWK6R9NZq9eT38sWRGuBKLKfjSgEAxu-5NCWgU,6278
-crawlo/middleware/offsite.py,sha256=cR0nVAygxAGbkmyI8yqiRtrZWTRLWddUiToMYGmhrfs,4084
+crawlo/middleware/offsite.py,sha256=R9e5haPpCs2Uw9Hm5MW93G4usRZ-DqTqa33eVMoaK-4,4557
 crawlo/middleware/proxy.py,sha256=NquB6tqHAgHs3-2_1_5220kJYfjNG5JyHRJyo_2j4wo,15636
 crawlo/middleware/request_ignore.py,sha256=xcyZ1c7r_HhbzR3r9pfjsLGW7L7FBVeYvlNt8cpP2wY,2577
 crawlo/middleware/response_code.py,sha256=-Aa9Mm9nJN-WdddN7iTanJRMA83_LYYgSEz3XLQGvMo,4934
@@ -82,17 +82,17 @@ crawlo/settings/default_settings.py,sha256=VKaCb8JnHx_B-Zi3hN6Mt0QIdv0YRhFlhVEZ2
 crawlo/settings/setting_manager.py,sha256=4uuMpGVYzxjmQjvlGqfZ8hDaoSh34OAoL0LCATsMCkI,7512
 crawlo/spider/__init__.py,sha256=ZnSAL9PXLZSIH-Jdv-P6RuWmQUdukr8KPLQK6SXZZaU,20435
 crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
-crawlo/templates/run.py.tmpl,sha256=v_g-LQMYJ6pC8TZgyWj0yB2yTTKrwy9lEJufAYCXyxY,1228
+crawlo/templates/run.py.tmpl,sha256=iICBXZAEkQnn2Z-72vBwnvYghBHK02u6I8uYncY-WPY,1033
 crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
 crawlo/templates/project/items.py.tmpl,sha256=mt1Mm--H2Ouos3r7JPkYh0r33rgYJf1YOMz0OZy8TYs,297
 crawlo/templates/project/middlewares.py.tmpl,sha256=T67p8j0laL4NJJ_3xzPM9yivgZRjTEMiEtEWLPwbkmw,4160
 crawlo/templates/project/pipelines.py.tmpl,sha256=GBHYU0Jx8sKDCdGJp44FMSH7u2slxoFg6a-R9Uwg_-I,2608
-crawlo/templates/project/settings.py.tmpl,sha256=K0WOyCJsiykbZjoZRhzmTVssoahETkYS2zb2q3Ai5Ts,9998
-crawlo/templates/project/settings_distributed.py.tmpl,sha256=BsSqtYl69NtFUTq-lDkXTzU7cwhqZhkna3x__pQx7oc,6692
-crawlo/templates/project/settings_gentle.py.tmpl,sha256=9S6l-v0yJOPDy3oxsCIpSDieXjxtHNPHhBqfGP28CG4,3975
-crawlo/templates/project/settings_high_performance.py.tmpl,sha256=zjlGESHvt3m_vsvAYDc5oE2Eui3eI9QEn3-uKFyHXpc,4706
-crawlo/templates/project/settings_minimal.py.tmpl,sha256=dFoz39BGkSzLDTwBN-mQh242SLzcP6g8MhI8Zk49jvw,909
-crawlo/templates/project/settings_simple.py.tmpl,sha256=iWQFaw1WxTJA2QF-kXH4nspDGuXvnGEAuqpGQHCfuew,4101
+crawlo/templates/project/settings.py.tmpl,sha256=LBILzRqj5oOOCZdaoWvHBoc_0ZFgcbvHaJ58N2S0d28,10296
+crawlo/templates/project/settings_distributed.py.tmpl,sha256=WrIA5k8Je-o4BX2sbv2YffzdskSwNeptKAywNOFMNTM,7002
+crawlo/templates/project/settings_gentle.py.tmpl,sha256=C2OB_dVLz2u1et5YNudOgl7PpF2lLgKiYoU-fzfvB3I,2710
+crawlo/templates/project/settings_high_performance.py.tmpl,sha256=cjS-0Q6yx7nPhGr2uaFVTKcQQdsoBcDEkwtkbtNFOAQ,5004
+crawlo/templates/project/settings_minimal.py.tmpl,sha256=sVs8eCeB2zGed1ttTWFKJOLzxWE9Gk3KV5Nx5e73e-0,1207
+crawlo/templates/project/settings_simple.py.tmpl,sha256=p6FWlTgLkiYiMbFolTWwS3BAAuR3rfCQrVB-55ve-Sk,4399
 crawlo/templates/project/spiders/__init__.py.tmpl,sha256=j_YKsw6HQMJyqlk3WUouP3bsr-XVxshRoSNakHBc00g,106
 crawlo/templates/spider/spider.py.tmpl,sha256=jMhzyxpIpV_KigB-pmN-5mGMiYtu4mfQIOvpZcCGGJI,5055
 crawlo/tools/__init__.py,sha256=8igeUXLD0vJ5ta2X91QyTvna6dOioKCn0z7EF4oHvHI,3942
@@ -212,8 +212,8 @@ tests/test_tools.py,sha256=9t9FXZ61MfdB70nck9NYzCq97yd3SLVlLiMybEAlClk,5345
 tests/test_user_agents.py,sha256=rUotyuE2iJDi2LQBrUh980U-dAMTs4ARPMJxICOoQFY,3231
 tests/tools_example.py,sha256=MtIypR-OFiWwi-skurwmq4fM0cGTt-GUX4hSekYs7BY,7739
 tests/verify_distributed.py,sha256=krnYYA5Qx9xXDMWc9YF5DxPSplGvawDg2n0l-3CAqoM,3928
-crawlo-1.3.1.dist-info/METADATA,sha256=6m0OV78Iso8DwQFiIB0_mWkYSDymuhjw69F9aVKwtCo,26813
-crawlo-1.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.3.1.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.3.1.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.3.1.dist-info/RECORD,,
+crawlo-1.3.2.dist-info/METADATA,sha256=VOvsyZ5e2RkIdzSCj9NPKOpqKG6SJ9UXleWwUSQqyxE,26813
+crawlo-1.3.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.3.2.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.3.2.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.3.2.dist-info/RECORD,,