crawlo 1.2.1__py3-none-any.whl → 1.2.3__py3-none-any.whl

crawlo/data/user_agents.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+User-Agent列表
+包含各种设备和浏览器的User-Agent字符串，用于爬虫伪装
+"""
+
+# 桌面浏览器User-Agent
+DESKTOP_USER_AGENTS = [
+    # Chrome
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    
+    # Firefox
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:125.0) Gecko/20100101 Firefox/125.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:125.0) Gecko/20100101 Firefox/125.0",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0",
+    
+    # Safari
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.15",
+    
+    # Edge
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.2478.80",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/123.0.2420.40",
+    
+    # Internet Explorer
+    "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
+]
+
+# 移动设备User-Agent
+MOBILE_USER_AGENTS = [
+    # iPhone
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1",
+    
+    # iPad
+    "Mozilla/5.0 (iPad; CPU OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (iPad; CPU OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1",
+    
+    # Android
+    "Mozilla/5.0 (Linux; Android 14; SM-S911B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Mobile Safari/537.36",
+    
+    # Android平板
+    "Mozilla/5.0 (Linux; Android 13; SM-X906C Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/124.0.6367.118 Safari/537.36",
+]
+
+# 爬虫/机器人User-Agent (用于测试)
+BOT_USER_AGENTS = [
+    "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+    "Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+    "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
+    "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
+]
+
+# 所有User-Agent的组合列表
+ALL_USER_AGENTS = DESKTOP_USER_AGENTS + MOBILE_USER_AGENTS
+
+# 按设备类型分类的User-Agent字典
+USER_AGENTS_BY_TYPE = {
+    "desktop": DESKTOP_USER_AGENTS,
+    "mobile": MOBILE_USER_AGENTS,
+    "bot": BOT_USER_AGENTS,
+    "all": ALL_USER_AGENTS,
+}
+
+def get_user_agents(device_type="all"):
+    """
+    获取指定类型的User-Agent列表
+    
+    Args:
+        device_type (str): 设备类型，可选值: "desktop", "mobile", "bot", "all"
+        
+    Returns:
+        list: User-Agent字符串列表
+    """
+    return USER_AGENTS_BY_TYPE.get(device_type, ALL_USER_AGENTS)
+
+def get_random_user_agent(device_type="all"):
+    """
+    获取随机User-Agent
+    
+    Args:
+        device_type (str): 设备类型，可选值: "desktop", "mobile", "bot", "all"
+        
+    Returns:
+        str: 随机User-Agent字符串
+    """
+    import random
+    user_agents = get_user_agents(device_type)
+    return random.choice(user_agents) if user_agents else ""
+
+# 导出常用的User-Agent列表
+__all__ = [
+    "DESKTOP_USER_AGENTS",
+    "MOBILE_USER_AGENTS", 
+    "BOT_USER_AGENTS",
+    "ALL_USER_AGENTS",
+    "USER_AGENTS_BY_TYPE",
+    "get_user_agents",
+    "get_random_user_agent"
+]

crawlo/downloader/hybrid_downloader.py

@@ -17,7 +17,6 @@
 4. 统一的接口和响应格式
 5. 自动资源管理和优化
 """
-import asyncio
 from typing import Optional, Dict, Type
 from urllib.parse import urlparse
 

crawlo/downloader/selenium_downloader.py

@@ -14,9 +14,9 @@ Selenium 下载器
 - 支持翻页操作（鼠标滑动、点击翻页）
 - 单浏览器多标签页模式
 """
-import asyncio
 import os
 import time
+import asyncio
 from typing import Optional, Dict, List
 
 from selenium import webdriver

crawlo/extension/health_check.py

@@ -1,11 +1,11 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 import asyncio
+from datetime import datetime
 from typing import Any, Optional, Dict
-from datetime import datetime, timedelta
 
-from crawlo.utils.log import get_logger
 from crawlo.event import spider_opened, spider_closed, response_received, request_scheduled
+from crawlo.utils.log import get_logger
 
 
 class HealthCheckExtension:

crawlo/items/fields.py

@@ -3,7 +3,6 @@
 """
 Field 类定义
 """
-
 from typing import Any, Optional, Type
 
 

crawlo/middleware/offsite.py

@@ -4,7 +4,6 @@
 OffsiteMiddleware 中间件
 用于过滤掉不在指定域名范围内的请求
 """
-
 import re
 from urllib.parse import urlparse
 

crawlo/middleware/proxy.py

@@ -1,10 +1,11 @@
 #!/usr/bin/python
 # -*- coding: UTF-8 -*-
+import time
 import asyncio
 import socket
-from typing import Optional, Dict, Any, Callable, Union, TYPE_CHECKING, List
 from urllib.parse import urlparse
-import time
+from typing import Optional, Dict, Any, Callable, Union, TYPE_CHECKING, List
+
 
 from crawlo import Request, Response
 from crawlo.exceptions import NotConfiguredError

crawlo/middleware/request_ignore.py

@@ -4,7 +4,6 @@
 RequestIgnoreMiddleware 中间件
 用于处理和记录被忽略的请求
 """
-
 from crawlo.utils.log import get_logger
 from crawlo.exceptions import IgnoreRequestError
 from crawlo.event import ignore_request

crawlo/middleware/response_code.py

@@ -4,7 +4,6 @@
 ResponseCodeMiddleware 中间件
 用于处理HTTP响应状态码，记录统计信息并支持特殊状态码处理
 """
-
 from crawlo.utils.log import get_logger
 
 

crawlo/middleware/response_filter.py

@@ -4,7 +4,6 @@
 ResponseFilterMiddleware 中间件
 用于过滤不符合要求的HTTP响应，支持自定义允许的状态码
 """
-
 from crawlo.utils.log import get_logger
 from crawlo.exceptions import IgnoreRequestError
 

crawlo/middleware/retry.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-from typing import List
 import asyncio
+from typing import List
 
 try:
     from anyio import EndOfStream

crawlo/mode_manager.py

@@ -10,10 +10,10 @@
 2. distributed - 分布式模式  
 3. auto - 自动检测模式
 """
-
-from typing import Dict, Any, Optional
-from enum import Enum
 import os
+from enum import Enum
+from typing import Dict, Any, Optional
+
 from crawlo.utils.log import get_logger
 
 

crawlo/pipelines/database_dedup_pipeline.py

@@ -11,15 +11,13 @@
 - 适用性广: 支持多种数据库后端
 - 可扩展: 支持自定义表结构和字段
 """
-
 import hashlib
-from typing import Dict, Any, Optional
 import aiomysql
 
 from crawlo import Item
+from crawlo.exceptions import DropItem
 from crawlo.spider import Spider
 from crawlo.utils.log import get_logger
-from crawlo.exceptions import DropItem
 
 
 class DatabaseDedupPipeline:

crawlo/pipelines/memory_dedup_pipeline.py

@@ -13,12 +13,12 @@
 """
 
 import hashlib
-from typing import Dict, Any, Set
+from typing import Set
 
 from crawlo import Item
+from crawlo.exceptions import DropItem
 from crawlo.spider import Spider
 from crawlo.utils.log import get_logger
-from crawlo.exceptions import DropItem
 
 
 class MemoryDedupPipeline:

crawlo/pipelines/mysql_pipeline.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 import asyncio
 import aiomysql
-from typing import Optional, List, Dict
 from asyncmy import create_pool
-from crawlo.utils.log import get_logger
+from typing import Optional, List, Dict
+
 from crawlo.exceptions import ItemDiscard
-from crawlo.utils.db_helper import make_insert_sql, make_batch_sql, logger
+from crawlo.utils.db_helper import make_insert_sql, make_batch_sql
+from crawlo.utils.log import get_logger
 
 
 class AsyncmyMySQLPipeline:

crawlo/pipelines/redis_dedup_pipeline.py

@@ -11,15 +11,13 @@
 - 可配置: 支持自定义 Redis 连接参数
 - 容错设计: 网络异常时不会丢失数据
 """
-
+import redis
 import hashlib
 from typing import Optional
 
-import redis
-
 from crawlo import Item
-from crawlo.exceptions import DropItem
 from crawlo.spider import Spider
+from crawlo.exceptions import DropItem
 from crawlo.utils.log import get_logger
 
 

crawlo/project.py

@@ -14,10 +14,10 @@ import sys
 import configparser
 from importlib import import_module
 from inspect import iscoroutinefunction
-from typing import Callable, Optional, Tuple
+from typing import Callable, Optional
 
-from crawlo.utils.log import get_logger
 from crawlo.settings.setting_manager import SettingManager
+from crawlo.utils.log import get_logger
 
 logger = get_logger(__name__)
 

crawlo/subscriber.py

@@ -1,10 +1,9 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 import asyncio
-import weakref
 from collections import defaultdict
 from inspect import iscoroutinefunction
-from typing import Dict, Set, Callable, Coroutine, Any, TypeAlias, List, Tuple
+from typing import Dict, Callable, Coroutine, Any, TypeAlias, List, Tuple
 
 
 class ReceiverTypeError(TypeError):

crawlo/templates/project/settings.py.tmpl

@@ -146,8 +146,6 @@ MONGO_USE_BATCH = False  # 是否启用批量插入
 REQUEST_DIR = '.'
 
 # 根据运行模式自动选择去重管道
-# 单机模式默认使用内存去重管道
-# 分布式模式默认使用Redis去重管道
 if RUN_MODE == 'distributed':
     # 分布式模式下默认使用Redis去重管道
     DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline'
@@ -157,7 +155,6 @@ else:
 
 # 去重过滤器（推荐分布式项目使用 Redis 过滤器）
 FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
-# FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'  # 分布式去重
 
 # --- Redis 配置（用于分布式去重和队列） ---
 REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
@@ -190,6 +187,7 @@ MIDDLEWARES = [
     'crawlo.middleware.download_delay.DownloadDelayMiddleware',
     'crawlo.middleware.default_header.DefaultHeaderMiddleware',
     'crawlo.middleware.proxy.ProxyMiddleware',
+    'crawlo.middleware.offsite.OffsiteMiddleware', 
     
     # === 响应处理阶段 ===
     'crawlo.middleware.retry.RetryMiddleware',

crawlo/templates/project/settings_distributed.py.tmpl

@@ -83,6 +83,8 @@ MIDDLEWARES = [
     'crawlo.middleware.download_delay.DownloadDelayMiddleware',
     'crawlo.middleware.default_header.DefaultHeaderMiddleware',
     'crawlo.middleware.proxy.ProxyMiddleware',
+    'crawlo.middleware.offsite.OffsiteMiddleware', 
+    
     'crawlo.middleware.retry.RetryMiddleware',
     'crawlo.middleware.response_code.ResponseCodeMiddleware',
     'crawlo.middleware.response_filter.ResponseFilterMiddleware',

crawlo/utils/batch_processor.py

@@ -5,12 +5,11 @@
 提供批量操作的统一接口和优化实现
 """
 import asyncio
-import time
-from typing import List, Callable, Any, Optional, Dict
 from functools import wraps
+from typing import List, Callable, Any, Optional, Dict
 
-from crawlo.utils.log import get_logger
 from crawlo.utils.error_handler import ErrorHandler
+from crawlo.utils.log import get_logger
 
 
 class BatchProcessor:

crawlo/utils/controlled_spider_mixin.py

@@ -6,8 +6,8 @@
 """
 import asyncio
 import time
-from typing import Generator, Optional, Set
 from collections import deque
+from typing import Generator, Optional
 
 from crawlo import Request
 from crawlo.utils.log import get_logger

crawlo/utils/enhanced_error_handler.py

@@ -1,16 +1,13 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 """
-增强版错误处理工具
+误处理工具
 提供更详细、更一致的错误处理和日志记录机制
 """
 import traceback
-import logging
-import sys
-import os
-from typing import Optional, Callable, Any, Dict, List
-from functools import wraps
 from datetime import datetime
+from functools import wraps
+from typing import Optional, Callable, Any, Dict, List
 
 from crawlo.utils.log import get_logger
 

crawlo/utils/env_config.py

@@ -5,7 +5,7 @@
 提供统一的环境变量读取和配置管理机制
 """
 import os
-from typing import Optional, Union, Any
+from typing import Any
 
 
 class EnvConfigManager:

crawlo/utils/error_handler.py

@@ -4,13 +4,11 @@
 统一错误处理工具
 提供一致的错误处理和日志记录机制
 """
-import traceback
-import logging
-from typing import Optional, Callable, Any
 from functools import wraps
+from typing import Callable, Any
 
-from crawlo.utils.log import get_logger
 from crawlo.utils.enhanced_error_handler import EnhancedErrorHandler, ErrorContext
+from crawlo.utils.log import get_logger
 
 
 class ErrorHandler:

crawlo/utils/large_scale_helper.py

@@ -6,7 +6,8 @@
 import asyncio
 import json
 import time
-from typing import Generator, Iterator, List, Set, Optional, Dict, Any
+from typing import Generator, List, Dict, Any
+
 from crawlo.utils.log import get_logger
 
 

crawlo/utils/performance_monitor.py

@@ -4,14 +4,15 @@
 性能监控工具
 提供系统性能监控和资源使用情况跟踪
 """
-import time
-import psutil
 import asyncio
-from typing import Dict, Any, Optional, Callable
+import time
 from functools import wraps
+from typing import Dict, Any
+
+import psutil
 
-from crawlo.utils.log import get_logger
 from crawlo.utils.error_handler import ErrorHandler
+from crawlo.utils.log import get_logger
 
 
 class PerformanceMonitor:

crawlo/utils/redis_connection_pool.py

@@ -4,13 +4,13 @@
 Redis连接池优化工具
 提供优化的Redis连接池管理和配置
 """
-import asyncio
-import redis.asyncio as aioredis
-from typing import Dict, Any, Optional, Union
 from contextlib import asynccontextmanager
+from typing import Dict, Any, Optional
+
+import redis.asyncio as aioredis
 
-from crawlo.utils.log import get_logger
 from crawlo.utils.error_handler import ErrorHandler
+from crawlo.utils.log import get_logger
 
 
 class OptimizedRedisConnectionPool:

crawlo/utils/redis_key_validator.py

@@ -5,9 +5,8 @@ Redis Key 验证工具
 =================
 提供 Redis Key 命名规范的验证功能
 """
-
-import re
 from typing import List, Tuple
+
 from crawlo.utils.log import get_logger
 
 

crawlo/utils/request_serializer.py

@@ -4,10 +4,9 @@
 Request 序列化工具类
 负责处理 Request 对象的序列化前清理工作，解决 logger 等不可序列化对象的问题
 """
+import gc
 import logging
 import pickle
-import gc
-from typing import Any, Dict
 
 from crawlo.utils.log import get_logger
 

crawlo/utils/spider_loader.py

@@ -1,5 +1,4 @@
 import importlib
-import inspect
 from pathlib import Path
 from typing import List, Type, Optional, Dict
 

{crawlo-1.2.1.dist-info → crawlo-1.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.2.1
+Version: 1.2.3
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder