crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

crawlo/tools/scenario_adapter.py

@@ -0,0 +1,263 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+场景适配器
+=========
+根据不同的动态加载场景，自动配置和优化下载器选择策略。
+
+支持的场景:
+1. 列表页、详情页都要动态加载
+2. 列表页使用协议请求、详情页使用动态加载
+3. 列表页使用动态加载，详情页使用协议请求
+
+使用示例:
+    # 场景1: 全动态加载
+    adapter = DynamicLoadingScenarioAdapter(scenario="all_dynamic")
+    
+    # 场景2: 列表页协议，详情页动态
+    adapter = DynamicLoadingScenarioAdapter(scenario="list_protocol_detail_dynamic")
+    
+    # 场景3: 列表页动态，详情页协议
+    adapter = DynamicLoadingScenarioAdapter(scenario="list_dynamic_detail_protocol")
+"""
+import re
+from typing import Dict, Any
+from urllib.parse import urlparse
+
+from crawlo.utils.log import get_logger
+
+
+class DynamicLoadingScenarioAdapter:
+    """
+    动态加载场景适配器
+    根据不同的业务场景自动配置下载器选择策略
+    """
+    
+    SCENARIOS = {
+        "all_dynamic": "列表页、详情页都要动态加载",
+        "list_protocol_detail_dynamic": "列表页使用协议请求、详情页使用动态加载",
+        "list_dynamic_detail_protocol": "列表页使用动态加载，详情页使用协议请求"
+    }
+    
+    def __init__(self, scenario: str = "list_protocol_detail_dynamic", **kwargs):
+        """
+        初始化场景适配器
+        
+        :param scenario: 场景类型
+        :param kwargs: 其他配置参数
+        """
+        self.scenario = scenario
+        self.logger = get_logger(self.__class__.__name__)
+        
+        # 验证场景类型
+        if scenario not in self.SCENARIOS:
+            raise ValueError(f"Unsupported scenario: {scenario}. Supported: {list(self.SCENARIOS.keys())}")
+            
+        self.logger.info(f"Initializing DynamicLoadingScenarioAdapter for scenario: {scenario}")
+        
+        # 配置参数
+        self.list_page_patterns = kwargs.get("list_page_patterns", [
+            r"/list", r"/search", r"/category", r"/page/\d+", r"\?page=", r"\?p=\d+"
+        ])
+        
+        self.detail_page_patterns = kwargs.get("detail_page_patterns", [
+            r"/detail", r"/item", r"/product", r"/article", r"/post", r"/view", r"/\d+"
+        ])
+        
+        self.list_domains = set(kwargs.get("list_domains", []))
+        self.detail_domains = set(kwargs.get("detail_domains", []))
+        
+        # 构建配置
+        self.config = self._build_config()
+        
+    def _build_config(self) -> Dict[str, Any]:
+        """根据场景构建配置"""
+        config = {
+            "HYBRID_DYNAMIC_URL_PATTERNS": [],
+            "HYBRID_PROTOCOL_URL_PATTERNS": [],
+            "HYBRID_DYNAMIC_DOMAINS": [],
+            "HYBRID_PROTOCOL_DOMAINS": []
+        }
+        
+        if self.scenario == "all_dynamic":
+            # 所有页面都使用动态加载
+            config["HYBRID_DYNAMIC_URL_PATTERNS"].extend(self.list_page_patterns)
+            config["HYBRID_DYNAMIC_URL_PATTERNS"].extend(self.detail_page_patterns)
+            
+        elif self.scenario == "list_protocol_detail_dynamic":
+            # 列表页使用协议请求，详情页使用动态加载
+            config["HYBRID_PROTOCOL_URL_PATTERNS"].extend(self.list_page_patterns)
+            config["HYBRID_DYNAMIC_URL_PATTERNS"].extend(self.detail_page_patterns)
+            
+            # 域名配置
+            config["HYBRID_PROTOCOL_DOMAINS"].extend(self.list_domains)
+            config["HYBRID_DYNAMIC_DOMAINS"].extend(self.detail_domains)
+            
+        elif self.scenario == "list_dynamic_detail_protocol":
+            # 列表页使用动态加载，详情页使用协议请求
+            config["HYBRID_DYNAMIC_URL_PATTERNS"].extend(self.list_page_patterns)
+            config["HYBRID_PROTOCOL_URL_PATTERNS"].extend(self.detail_page_patterns)
+            
+            # 域名配置
+            config["HYBRID_DYNAMIC_DOMAINS"].extend(self.list_domains)
+            config["HYBRID_PROTOCOL_DOMAINS"].extend(self.detail_domains)
+            
+        return config
+    
+    def get_settings(self) -> Dict[str, Any]:
+        """获取场景配置设置"""
+        return self.config
+    
+    def should_use_dynamic_loader(self, url: str) -> bool:
+        """
+        判断给定URL是否应该使用动态加载器
+        
+        :param url: 要判断的URL
+        :return: 是否使用动态加载器
+        """
+        parsed_url = urlparse(url)
+        domain = parsed_url.netloc.lower()
+        path = parsed_url.path.lower()
+        query = parsed_url.query.lower()
+        full_path = path + ("?" + query if query else "")
+        
+        # 检查域名
+        if domain in self.config["HYBRID_DYNAMIC_DOMAINS"]:
+            return True
+        if domain in self.config["HYBRID_PROTOCOL_DOMAINS"]:
+            return False
+            
+        # 检查URL模式
+        # 检查动态模式
+        for pattern in self.config["HYBRID_DYNAMIC_URL_PATTERNS"]:
+            if re.search(pattern, full_path):
+                return True
+                
+        # 检查协议模式
+        for pattern in self.config["HYBRID_PROTOCOL_URL_PATTERNS"]:
+            if re.search(pattern, full_path):
+                return False
+                
+        # 默认策略
+        if self.scenario == "all_dynamic":
+            return True
+        elif self.scenario == "list_protocol_detail_dynamic":
+            # 默认详情页使用动态加载
+            return True
+        elif self.scenario == "list_dynamic_detail_protocol":
+            # 默认详情页使用协议加载
+            return False
+            
+        # 默认返回False（使用协议加载器）
+        return False
+    
+    def get_loader_options(self, url: str) -> Dict[str, Any]:
+        """
+        获取特定URL的加载器选项
+        
+        :param url: URL
+        :return: 加载器选项
+        """
+        options = {}
+        
+        # 可以根据URL添加特定的加载器选项
+        # 例如，对于某些页面可能需要等待特定元素加载完成
+        if "product" in url:
+            options["wait_for_element"] = ".product-detail"
+        elif "article" in url:
+            options["wait_for_element"] = ".article-content"
+            
+        return options
+    
+    def adapt_request(self, request) -> None:
+        """
+        适配请求对象，根据场景自动设置加载器
+        
+        :param request: 请求对象
+        """
+        use_dynamic = self.should_use_dynamic_loader(request.url)
+        
+        if use_dynamic:
+            request.set_dynamic_loader(True, self.get_loader_options(request.url))
+            self.logger.debug(f"Adapted request {request.url} to use dynamic loader")
+        else:
+            request.set_protocol_loader()
+            self.logger.debug(f"Adapted request {request.url} to use protocol loader")
+
+
+# 便利函数
+def create_scenario_adapter(scenario: str = "list_protocol_detail_dynamic", **kwargs):
+    """
+    创建场景适配器的便利函数
+    
+    :param scenario: 场景类型
+    :param kwargs: 其他配置参数
+    :return: 场景适配器实例
+    """
+    return DynamicLoadingScenarioAdapter(scenario, **kwargs)
+
+
+def get_scenario_settings(scenario: str = "list_protocol_detail_dynamic", **kwargs) -> Dict[str, Any]:
+    """
+    获取场景配置设置的便利函数
+    
+    :param scenario: 场景类型
+    :param kwargs: 其他配置参数
+    :return: 配置设置字典
+    """
+    adapter = DynamicLoadingScenarioAdapter(scenario, **kwargs)
+    return adapter.get_settings()
+
+
+# 预定义场景配置
+SCENARIO_CONFIGS = {
+    "电商网站": {
+        "scenario": "list_protocol_detail_dynamic",
+        "list_page_patterns": [r"/list", r"/search", r"/category", r"/page/\d+"],
+        "detail_page_patterns": [r"/product", r"/item", r"/detail/\d+"],
+        "list_domains": [],
+        "detail_domains": []
+    },
+    
+    "新闻网站": {
+        "scenario": "list_protocol_detail_dynamic",
+        "list_page_patterns": [r"/list", r"/category", r"/page/\d+"],
+        "detail_page_patterns": [r"/article", r"/post", r"/news/\d+"],
+        "list_domains": [],
+        "detail_domains": []
+    },
+    
+    "社交平台": {
+        "scenario": "all_dynamic",
+        "list_page_patterns": [r"/feed", r"/timeline", r"/explore"],
+        "detail_page_patterns": [r"/post", r"/status", r"/photo"],
+        "list_domains": [],
+        "detail_domains": []
+    },
+    
+    "博客平台": {
+        "scenario": "list_dynamic_detail_protocol",
+        "list_page_patterns": [r"/blog", r"/posts", r"/archive"],
+        "detail_page_patterns": [r"/\d{4}/\d{2}/\d{2}/"],
+        "list_domains": [],
+        "detail_domains": []
+    }
+}
+
+
+def create_adapter_for_platform(platform: str, **custom_kwargs) -> DynamicLoadingScenarioAdapter:
+    """
+    为特定平台创建场景适配器
+    
+    :param platform: 平台名称
+    :param custom_kwargs: 自定义配置参数
+    :return: 场景适配器实例
+    """
+    if platform not in SCENARIO_CONFIGS:
+        raise ValueError(f"Unsupported platform: {platform}. Supported: {list(SCENARIO_CONFIGS.keys())}")
+    
+    config = SCENARIO_CONFIGS[platform].copy()
+    config.update(custom_kwargs)
+    scenario = config.pop("scenario")
+    
+    return DynamicLoadingScenarioAdapter(scenario, **config)

crawlo/utils/__init__.py

@@ -3,5 +3,33 @@
 """
 # @Time    :    2025-02-05 13:57
 # @Author  :   oscar
-# @Desc    :   None
+# @Desc    :   工具模块集合
 """
+
+from .date_tools import (
+    TimeUtils,
+    parse_time,
+    format_time,
+    time_diff,
+    to_timestamp,
+    to_datetime,
+    now,
+    to_timezone,
+    to_utc,
+    to_local,
+    from_timestamp_with_tz
+)
+
+__all__ = [
+    "TimeUtils",
+    "parse_time",
+    "format_time",
+    "time_diff",
+    "to_timestamp",
+    "to_datetime",
+    "now",
+    "to_timezone",
+    "to_utc",
+    "to_local",
+    "from_timestamp_with_tz"
+]

crawlo/utils/batch_processor.py

@@ -0,0 +1,261 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+批处理操作工具
+提供批量操作的统一接口和优化实现
+"""
+import asyncio
+import time
+from typing import List, Callable, Any, Optional, Dict
+from functools import wraps
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.error_handler import ErrorHandler
+
+
+class BatchProcessor:
+    """批处理处理器"""
+    
+    def __init__(self, batch_size: int = 100, max_concurrent_batches: int = 5):
+        self.batch_size = batch_size
+        self.max_concurrent_batches = max_concurrent_batches
+        self.logger = get_logger(self.__class__.__name__)
+        self.error_handler = ErrorHandler(self.__class__.__name__)
+    
+    async def process_batch(self, items: List[Any], processor_func: Callable, 
+                           *args, **kwargs) -> List[Any]:
+        """
+        处理一批数据项
+        
+        Args:
+            items: 要处理的数据项列表
+            processor_func: 处理函数
+            *args: 传递给处理函数的额外参数
+            **kwargs: 传递给处理函数的关键字参数
+            
+        Returns:
+            处理结果列表
+        """
+        results = []
+        semaphore = asyncio.Semaphore(self.max_concurrent_batches)
+        
+        async def process_item(item):
+            async with semaphore:
+                try:
+                    if asyncio.iscoroutinefunction(processor_func):
+                        return await processor_func(item, *args, **kwargs)
+                    else:
+                        return processor_func(item, *args, **kwargs)
+                except Exception as e:
+                    self.logger.error(f"处理单项失败: {e}")
+                    return None
+        
+        # 并发处理批次中的所有项
+        tasks = [process_item(item) for item in items]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        
+        # 过滤掉异常结果
+        return [result for result in results if not isinstance(result, Exception)]
+    
+    async def process_in_batches(self, items: List[Any], processor_func: Callable,
+                                *args, **kwargs) -> List[Any]:
+        """
+        分批处理大量数据项
+        
+        Args:
+            items: 要处理的数据项列表
+            processor_func: 处理函数
+            *args: 传递给处理函数的额外参数
+            **kwargs: 传递给处理函数的关键字参数
+            
+        Returns:
+            所有处理结果的列表
+        """
+        all_results = []
+        
+        # 将数据分批处理
+        for i in range(0, len(items), self.batch_size):
+            batch = items[i:i + self.batch_size]
+            self.logger.debug(f"处理批次 {i//self.batch_size + 1}/{(len(items)-1)//self.batch_size + 1}")
+            
+            try:
+                batch_results = await self.process_batch(batch, processor_func, *args, **kwargs)
+                all_results.extend(batch_results)
+            except Exception as e:
+                self.logger.error(f"处理批次失败: {e}")
+                # 继续处理下一个批次而不是中断
+        
+        return all_results
+    
+    def batch_process_decorator(self, batch_size: Optional[int] = None):
+        """
+        装饰器：将函数转换为批处理函数
+        
+        Args:
+            batch_size: 批次大小（如果为None则使用实例的batch_size）
+        """
+        def decorator(func):
+            @wraps(func)
+            async def async_wrapper(items: List[Any], *args, **kwargs):
+                actual_batch_size = batch_size or self.batch_size
+                processor = BatchProcessor(actual_batch_size, self.max_concurrent_batches)
+                return await processor.process_in_batches(items, func, *args, **kwargs)
+            
+            @wraps(func)
+            def sync_wrapper(items: List[Any], *args, **kwargs):
+                # 同步版本使用事件循环运行异步函数
+                return asyncio.run(async_wrapper(items, *args, **kwargs))
+            
+            # 根据原函数是否为异步函数返回相应的包装器
+            import inspect
+            if inspect.iscoroutinefunction(func):
+                return async_wrapper
+            else:
+                return sync_wrapper
+        
+        return decorator
+
+
+class RedisBatchProcessor:
+    """Redis批处理处理器"""
+    
+    def __init__(self, redis_client, batch_size: int = 100):
+        self.redis_client = redis_client
+        self.batch_size = batch_size
+        self.logger = get_logger(self.__class__.__name__)
+        self.error_handler = ErrorHandler(self.__class__.__name__)
+    
+    async def batch_set(self, items: List[Dict[str, Any]]) -> int:
+        """
+        批量设置Redis键值对
+        
+        Args:
+            items: 包含key和value的字典列表
+            
+        Returns:
+            成功设置的键值对数量
+        """
+        try:
+            pipe = self.redis_client.pipeline()
+            count = 0
+            
+            for item in items:
+                if 'key' in item and 'value' in item:
+                    pipe.set(item['key'], item['value'])
+                    count += 1
+                    
+                    # 每达到批次大小就执行一次
+                    if count % self.batch_size == 0:
+                        await pipe.execute()
+                        pipe = self.redis_client.pipeline()
+            
+            # 执行剩余的操作
+            if count % self.batch_size != 0:
+                await pipe.execute()
+            
+            self.logger.debug(f"批量设置 {count} 个键值对")
+            return count
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="Redis批量设置失败", 
+                raise_error=False
+            )
+            return 0
+    
+    async def batch_get(self, keys: List[str]) -> Dict[str, Any]:
+        """
+        批量获取Redis键值
+        
+        Args:
+            keys: 要获取的键列表
+            
+        Returns:
+            键值对字典
+        """
+        try:
+            # 使用管道批量获取
+            pipe = self.redis_client.pipeline()
+            for key in keys:
+                pipe.get(key)
+            
+            results = await pipe.execute()
+            
+            # 构建结果字典
+            result_dict = {}
+            for i, key in enumerate(keys):
+                if results[i] is not None:
+                    result_dict[key] = results[i]
+            
+            self.logger.debug(f"批量获取 {len(result_dict)} 个键值对")
+            return result_dict
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="Redis批量获取失败", 
+                raise_error=False
+            )
+            return {}
+    
+    async def batch_delete(self, keys: List[str]) -> int:
+        """
+        批量删除Redis键
+        
+        Args:
+            keys: 要删除的键列表
+            
+        Returns:
+            成功删除的键数量
+        """
+        try:
+            pipe = self.redis_client.pipeline()
+            count = 0
+            
+            for key in keys:
+                pipe.delete(key)
+                count += 1
+                
+                # 每达到批次大小就执行一次
+                if count % self.batch_size == 0:
+                    await pipe.execute()
+                    pipe = self.redis_client.pipeline()
+            
+            # 执行剩余的操作
+            if count % self.batch_size != 0:
+                await pipe.execute()
+            
+            self.logger.debug(f"批量删除 {count} 个键")
+            return count
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="Redis批量删除失败", 
+                raise_error=False
+            )
+            return 0
+
+
+# 全局批处理器实例
+default_batch_processor = BatchProcessor()
+
+
+def batch_process(items: List[Any], processor_func: Callable, 
+                 batch_size: int = 100, max_concurrent_batches: int = 5,
+                 *args, **kwargs) -> List[Any]:
+    """
+    便捷函数：批处理数据项
+    
+    Args:
+        items: 要处理的数据项列表
+        processor_func: 处理函数
+        batch_size: 批次大小
+        max_concurrent_batches: 最大并发批次数量
+        *args: 传递给处理函数的额外参数
+        **kwargs: 传递给处理函数的关键字参数
+        
+    Returns:
+        所有处理结果的列表
+    """
+    processor = BatchProcessor(batch_size, max_concurrent_batches)
+    import asyncio
+    return asyncio.run(processor.process_in_batches(items, processor_func, *args, **kwargs))

crawlo/utils/date_tools.py

@@ -7,13 +7,17 @@
 """
 import dateparser
 from typing import Optional, Union, Literal
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from dateutil.relativedelta import relativedelta
+import pytz
+from pytz import timezone as pytz_timezone
 
 # 支持的单位类型
 TimeUnit = Literal["seconds", "minutes", "hours", "days"]
 # 时间输入类型
 TimeType = Union[str, datetime]
+# 时区类型
+TimezoneType = Union[str, timezone, pytz_timezone]
 
 # 常见时间格式列表（作为 dateparser 的后备方案）
 COMMON_FORMATS = [
@@ -195,6 +199,43 @@ class TimeUtils:
             return None
         return dt.isoformat()
 
+    @classmethod
+    def to_timezone(cls, dt: TimeType, tz: TimezoneType) -> Optional[datetime]:
+        """将时间转换为指定时区"""
+        dt = cls.parse(dt)
+        if dt is None:
+            return None
+        
+        try:
+            if isinstance(tz, str):
+                tz = pytz_timezone(tz)
+            return dt.astimezone(tz)
+        except Exception:
+            return None
+
+    @classmethod
+    def to_utc(cls, dt: TimeType) -> Optional[datetime]:
+        """将时间转换为 UTC 时区"""
+        return cls.to_timezone(dt, pytz.UTC)
+
+    @classmethod
+    def to_local(cls, dt: TimeType) -> Optional[datetime]:
+        """将时间转换为本地时区"""
+        return cls.to_timezone(dt, pytz.timezone("Asia/Shanghai"))
+
+    @classmethod
+    def from_timestamp_with_tz(cls, ts: float, tz: TimezoneType = None) -> Optional[datetime]:
+        """从时间戳创建 datetime，并可选择指定时区"""
+        try:
+            dt = datetime.fromtimestamp(ts)
+            if tz:
+                if isinstance(tz, str):
+                    tz = pytz_timezone(tz)
+                dt = dt.replace(tzinfo=tz)
+            return dt
+        except Exception:
+            return None
+
 
 # =======================对外接口=======================
 
@@ -227,6 +268,22 @@ def now(fmt: Optional[str] = None) -> Union[datetime, str]:
     """获取当前时间"""
     return TimeUtils.now(fmt)
 
+def to_timezone(dt: TimeType, tz: TimezoneType) -> Optional[datetime]:
+    """将时间转换为指定时区"""
+    return TimeUtils.to_timezone(dt, tz)
+
+def to_utc(dt: TimeType) -> Optional[datetime]:
+    """将时间转换为 UTC 时区"""
+    return TimeUtils.to_utc(dt)
+
+def to_local(dt: TimeType) -> Optional[datetime]:
+    """将时间转换为本地时区"""
+    return TimeUtils.to_local(dt)
+
+def from_timestamp_with_tz(ts: float, tz: TimezoneType = None) -> Optional[datetime]:
+    """从时间戳创建 datetime，并可选择指定时区"""
+    return TimeUtils.from_timestamp_with_tz(ts, tz)
+
 
 if __name__ == '__main__':
     get_current_time = now(fmt="%Y-%m-%d %H:%M:%S")