crawlo 1.4.6__py3-none-any.whl → 1.4.7__py3-none-any.whl

crawlo/tools/network_diagnostic.py

@@ -1,365 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-网络诊断工具
-提供网络连接问题的诊断和解决方案
-"""
-
-import asyncio
-import socket
-import time
-from typing import Dict, List, Optional, Tuple
-from urllib.parse import urlparse
-
-import aiohttp
-from crawlo.utils.log import get_logger
-
-
-class NetworkDiagnostic:
-    """网络诊断工具类"""
-    
-    def __init__(self):
-        self.logger = get_logger(self.__class__.__name__)
-        self._dns_cache: Dict[str, str] = {}
-    
-    async def diagnose_url(self, url: str) -> Dict[str, any]:
-        """
-        诊断URL的网络连接问题
-        
-        Args:
-            url: 要诊断的URL
-            
-        Returns:
-            诊断结果字典
-        """
-        parsed = urlparse(url)
-        hostname = parsed.hostname
-        port = parsed.port or (443 if parsed.scheme == 'https' else 80)
-        
-        result = {
-            'url': url,
-            'hostname': hostname,
-            'port': port,
-            'dns_resolution': None,
-            'tcp_connection': None,
-            'http_response': None,
-            'recommendations': []
-        }
-        
-        # DNS解析测试
-        dns_result = await self._test_dns_resolution(hostname)
-        result['dns_resolution'] = dns_result
-        
-        if dns_result['success']:
-            # TCP连接测试
-            tcp_result = await self._test_tcp_connection(hostname, port)
-            result['tcp_connection'] = tcp_result
-            
-            if tcp_result['success']:
-                # HTTP响应测试
-                http_result = await self._test_http_response(url)
-                result['http_response'] = http_result
-        
-        # 生成建议
-        result['recommendations'] = self._generate_recommendations(result)
-        
-        return result
-    
-    async def _test_dns_resolution(self, hostname: str) -> Dict[str, any]:
-        """测试DNS解析"""
-        try:
-            start_time = time.time()
-            
-            # 使用asyncio的DNS解析
-            loop = asyncio.get_event_loop()
-            addr_info = await loop.getaddrinfo(hostname, None)
-            
-            resolution_time = time.time() - start_time
-            ip_addresses = list(set([addr[4][0] for addr in addr_info]))
-            
-            # 缓存DNS结果
-            if ip_addresses:
-                self._dns_cache[hostname] = ip_addresses[0]
-            
-            return {
-                'success': True,
-                'ip_addresses': ip_addresses,
-                'resolution_time': resolution_time,
-                'error': None
-            }
-            
-        except socket.gaierror as e:
-            return {
-                'success': False,
-                'ip_addresses': [],
-                'resolution_time': None,
-                'error': {
-                    'type': 'DNSError',
-                    'code': e.errno,
-                    'message': str(e)
-                }
-            }
-        except Exception as e:
-            return {
-                'success': False,
-                'ip_addresses': [],
-                'resolution_time': None,
-                'error': {
-                    'type': type(e).__name__,
-                    'message': str(e)
-                }
-            }
-    
-    async def _test_tcp_connection(self, hostname: str, port: int) -> Dict[str, any]:
-        """测试TCP连接"""
-        try:
-            start_time = time.time()
-            
-            # 尝试TCP连接
-            reader, writer = await asyncio.wait_for(
-                asyncio.open_connection(hostname, port),
-                timeout=10.0
-            )
-            
-            connection_time = time.time() - start_time
-            
-            # 关闭连接
-            writer.close()
-            await writer.wait_closed()
-            
-            return {
-                'success': True,
-                'connection_time': connection_time,
-                'error': None
-            }
-            
-        except asyncio.TimeoutError:
-            return {
-                'success': False,
-                'connection_time': None,
-                'error': {
-                    'type': 'TimeoutError',
-                    'message': 'Connection timeout'
-                }
-            }
-        except Exception as e:
-            return {
-                'success': False,
-                'connection_time': None,
-                'error': {
-                    'type': type(e).__name__,
-                    'message': str(e)
-                }
-            }
-    
-    async def _test_http_response(self, url: str) -> Dict[str, any]:
-        """测试HTTP响应"""
-        try:
-            start_time = time.time()
-            
-            timeout = aiohttp.ClientTimeout(total=30, connect=10)
-            async with aiohttp.ClientSession(timeout=timeout) as session:
-                async with session.get(url) as response:
-                    response_time = time.time() - start_time
-                    
-                    return {
-                        'success': True,
-                        'url': url,
-                        'status_code': response.status_code,  # 修复：使用status_code而不是status
-                        'response_time': response_time,
-                        'content_length': len(response.body) if response.body else 0,
-                        'headers': dict(response.headers)
-                    }
-                    
-        except aiohttp.ClientError as e:
-            return {
-                'success': False,
-                'status_code': None,
-                'response_time': None,
-                'headers': {},
-                'error': {
-                    'type': type(e).__name__,
-                    'message': str(e)
-                }
-            }
-        except Exception as e:
-            return {
-                'success': False,
-                'status_code': None,
-                'response_time': None,
-                'headers': {},
-                'error': {
-                    'type': type(e).__name__,
-                    'message': str(e)
-                }
-            }
-    
-    def _generate_recommendations(self, result: Dict[str, any]) -> List[str]:
-        """根据诊断结果生成建议"""
-        recommendations = []
-        
-        dns_result = result.get('dns_resolution', {})
-        tcp_result = result.get('tcp_connection', {})
-        http_result = result.get('http_response', {})
-        
-        # DNS问题建议
-        if not dns_result.get('success'):
-            error = dns_result.get('error', {})
-            if error.get('code') == 8:  # nodename nor servname provided, or not known
-                recommendations.extend([
-                    "DNS解析失败 - 检查域名是否正确",
-                    "检查网络连接是否正常",
-                    "尝试使用不同的DNS服务器（如8.8.8.8或1.1.1.1）",
-                    "检查本地hosts文件是否有相关配置",
-                    "确认域名是否可以从外部访问"
-                ])
-            elif error.get('code') == 2:  # Name or service not known
-                recommendations.extend([
-                    "域名不存在或无法解析",
-                    "检查域名拼写是否正确",
-                    "确认域名是否已注册且配置了DNS记录"
-                ])
-        
-        # TCP连接问题建议
-        elif not tcp_result.get('success'):
-            error = tcp_result.get('error', {})
-            if error.get('type') == 'TimeoutError':
-                recommendations.extend([
-                    "TCP连接超时 - 服务器可能无响应",
-                    "检查防火墙设置是否阻止了连接",
-                    "尝试增加连接超时时间",
-                    "检查代理设置"
-                ])
-        
-        # HTTP问题建议
-        elif not http_result.get('success'):
-            error = http_result.get('error', {})
-            recommendations.extend([
-                f"HTTP请求失败: {error.get('message', 'Unknown error')}",
-                "检查URL是否正确",
-                "确认服务器是否正常运行"
-            ])
-        
-        # 性能建议
-        if dns_result.get('success') and dns_result.get('resolution_time', 0) > 1.0:
-            recommendations.append("DNS解析时间较长，考虑使用DNS缓存或更快的DNS服务器")
-        
-        if tcp_result.get('success') and tcp_result.get('connection_time', 0) > 2.0:
-            recommendations.append("TCP连接时间较长，可能存在网络延迟问题")
-        
-        if http_result.get('success') and http_result.get('response_time', 0) > 5.0:
-            recommendations.append("HTTP响应时间较长，服务器可能负载较高")
-        
-        return recommendations
-    
-    async def batch_diagnose(self, urls: List[str]) -> Dict[str, Dict[str, any]]:
-        """批量诊断多个URL"""
-        tasks = []
-        for url in urls:
-            task = asyncio.create_task(self.diagnose_url(url))
-            tasks.append((url, task))
-        
-        results = {}
-        for url, task in tasks:
-            try:
-                result = await task
-                results[url] = result
-            except Exception as e:
-                results[url] = {
-                    'url': url,
-                    'error': f"诊断过程出错: {e}",
-                    'recommendations': ["诊断工具本身出现问题，请检查网络环境"]
-                }
-        
-        return results
-    
-    def format_diagnostic_report(self, result: Dict[str, any]) -> str:
-        """格式化诊断报告"""
-        lines = [
-            f"=== 网络诊断报告 ===",
-            f"URL: {result['url']}",
-            f"主机: {result['hostname']}:{result['port']}",
-            ""
-        ]
-        
-        # DNS解析结果
-        dns = result.get('dns_resolution', {})
-        if dns.get('success'):
-            lines.extend([
-                "✅ DNS解析: 成功",
-                f"   IP地址: {', '.join(dns['ip_addresses'])}",
-                f"   解析时间: {dns['resolution_time']:.3f}秒"
-            ])
-        else:
-            error = dns.get('error', {})
-            lines.extend([
-                "❌ DNS解析: 失败",
-                f"   错误类型: {error.get('type', 'Unknown')}",
-                f"   错误信息: {error.get('message', 'Unknown error')}"
-            ])
-        
-        lines.append("")
-        
-        # TCP连接结果
-        tcp = result.get('tcp_connection', {})
-        if tcp and tcp.get('success'):
-            lines.extend([
-                "✅ TCP连接: 成功",
-                f"   连接时间: {tcp['connection_time']:.3f}秒"
-            ])
-        elif tcp:
-            error = tcp.get('error', {})
-            lines.extend([
-                "❌ TCP连接: 失败",
-                f"   错误类型: {error.get('type', 'Unknown')}",
-                f"   错误信息: {error.get('message', 'Unknown error')}"
-            ])
-        
-        lines.append("")
-        
-        # HTTP响应结果
-        http = result.get('http_response', {})
-        if http and http.get('success'):
-            lines.extend([
-                "✅ HTTP响应: 成功",
-                f"   状态码: {http['status_code']}",
-                f"   响应时间: {http['response_time']:.3f}秒"
-            ])
-        elif http:
-            error = http.get('error', {})
-            lines.extend([
-                "❌ HTTP响应: 失败",
-                f"   错误类型: {error.get('type', 'Unknown')}",
-                f"   错误信息: {error.get('message', 'Unknown error')}"
-            ])
-        
-        # 建议
-        recommendations = result.get('recommendations', [])
-        if recommendations:
-            lines.extend([
-                "",
-                "🔧 建议:",
-            ])
-            for i, rec in enumerate(recommendations, 1):
-                lines.append(f"   {i}. {rec}")
-        
-        return "\n".join(lines)
-
-
-# 便捷函数
-async def diagnose_url(url: str) -> Dict[str, any]:
-    """诊断单个URL的网络问题"""
-    diagnostic = NetworkDiagnostic()
-    return await diagnostic.diagnose_url(url)
-
-
-async def diagnose_urls(urls: List[str]) -> Dict[str, Dict[str, any]]:
-    """批量诊断URL的网络问题"""
-    diagnostic = NetworkDiagnostic()
-    return await diagnostic.batch_diagnose(urls)
-
-
-def format_report(result: Dict[str, any]) -> str:
-    """格式化诊断报告"""
-    diagnostic = NetworkDiagnostic()
-    return diagnostic.format_diagnostic_report(result)

crawlo/tools/request_tools.py

@@ -1,83 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-20 10:00
-# @Author  : crawl-coder
-# @Desc    : 请求处理工具
-"""
-from typing import Dict, Any, Optional
-from urllib.parse import urlparse, parse_qs, urlencode, urlunparse
-from copy import deepcopy
-
-
-def build_url(base_url: str, path: str = "", query_params: Optional[Dict[str, Any]] = None) -> str:
-    """
-    构建完整的URL
-    
-    :param base_url: 基础URL
-    :param path: 路径
-    :param query_params: 查询参数字典
-    :return: 完整的URL
-    """
-    # 解析基础URL
-    parsed = urlparse(base_url)
-    
-    # 合并路径
-    if path:
-        # 确保路径以/开头
-        if not path.startswith('/'):
-            path = '/' + path
-        # 合并路径
-        new_path = parsed.path.rstrip('/') + path
-    else:
-        new_path = parsed.path
-    
-    # 处理查询参数
-    if query_params:
-        # 解析现有查询参数
-        existing_params = parse_qs(parsed.query)
-        # 合并新参数
-        for key, value in query_params.items():
-            existing_params[key] = [str(value)]
-        # 编码查询参数
-        new_query = urlencode(existing_params, doseq=True)
-    else:
-        new_query = parsed.query
-    
-    # 构建新的URL
-    new_parsed = parsed._replace(path=new_path, query=new_query)
-    return urlunparse(new_parsed)
-
-
-def add_query_params(url: str, params: Dict[str, Any]) -> str:
-    """
-    向URL添加查询参数
-    
-    :param url: 原始URL
-    :param params: 要添加的参数字典
-    :return: 添加参数后的URL
-    """
-    parsed = urlparse(url)
-    # 解析现有查询参数
-    existing_params = parse_qs(parsed.query)
-    # 添加新参数
-    for key, value in params.items():
-        existing_params[key] = [str(value)]
-    # 编码查询参数
-    new_query = urlencode(existing_params, doseq=True)
-    # 构建新的URL
-    new_parsed = parsed._replace(query=new_query)
-    return urlunparse(new_parsed)
-
-
-def merge_headers(base_headers: Dict[str, str], additional_headers: Dict[str, str]) -> Dict[str, str]:
-    """
-    合并请求头
-    
-    :param base_headers: 基础请求头
-    :param additional_headers: 要添加的请求头
-    :return: 合并后的请求头
-    """
-    merged = deepcopy(base_headers)
-    merged.update(additional_headers)
-    return merged

crawlo/tools/retry_mechanism.py

@@ -1,224 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 重试机制工具
-"""
-
-import time
-import random
-import asyncio
-from typing import Callable, Any, Optional, Tuple, Set
-from functools import wraps
-
-
-class RetryMechanism:
-    """重试机制工具类"""
-
-    # 默认应该重试的HTTP状态码
-    DEFAULT_RETRY_STATUS_CODES = {429, 500, 502, 503, 504}
-
-    # 默认应该重试的异常类型
-    DEFAULT_RETRY_EXCEPTIONS = (
-        ConnectionError,
-        TimeoutError,
-        asyncio.TimeoutError,
-    )
-
-    def __init__(self, max_retries: int = 3,
-                 retry_status_codes: Optional[Set[int]] = None,
-                 retry_exceptions: Optional[Tuple[type, ...]] = None):
-        """
-        初始化重试机制
-        
-        Args:
-            max_retries (int): 最大重试次数
-            retry_status_codes (Optional[Set[int]]): 应该重试的HTTP状态码
-            retry_exceptions (Optional[Tuple[type, ...]]): 应该重试的异常类型
-        """
-        self.max_retries = max_retries
-        self.retry_status_codes = retry_status_codes or self.DEFAULT_RETRY_STATUS_CODES
-        self.retry_exceptions = retry_exceptions or self.DEFAULT_RETRY_EXCEPTIONS
-
-    def should_retry(self, status_code: Optional[int] = None,
-                     exception: Optional[Exception] = None) -> bool:
-        """
-        判断是否应该重试
-        
-        Args:
-            status_code (Optional[int]): HTTP状态码
-            exception (Optional[Exception]): 异常对象
-            
-        Returns:
-            bool: 是否应该重试
-        """
-        # 如果有状态码，检查是否在重试列表中
-        if status_code is not None and status_code in self.retry_status_codes:
-            return True
-
-        # 如果有异常，检查是否在重试列表中
-        if exception is not None and isinstance(exception, self.retry_exceptions):
-            return True
-
-        return False
-
-    def exponential_backoff(self, attempt: int, base_delay: float = 1.0,
-                            max_delay: float = 60.0) -> float:
-        """
-        计算指数退避延迟时间
-        
-        Args:
-            attempt (int): 当前重试次数
-            base_delay (float): 基础延迟时间（秒）
-            max_delay (float): 最大延迟时间（秒）
-            
-        Returns:
-            float: 延迟时间（秒）
-        """
-        # 计算基本延迟：base_delay * (2 ^ attempt)
-        delay = base_delay * (2 ** attempt)
-
-        # 添加随机抖动，避免惊群效应
-        jitter = random.uniform(0, 0.1) * delay
-
-        # 返回最终延迟时间，不超过最大延迟
-        return min(delay + jitter, max_delay)
-
-    async def async_retry(self, func: Callable, *args, **kwargs) -> Any:
-        """
-        异步重试执行函数
-        
-        Args:
-            func (Callable): 要执行的函数
-            *args: 函数参数
-            **kwargs: 函数关键字参数
-            
-        Returns:
-            Any: 函数执行结果
-            
-        Raises:
-            Exception: 如果超过最大重试次数仍未成功，则抛出最后一个异常
-        """
-        last_exception = None
-
-        for attempt in range(self.max_retries + 1):
-            try:
-                result = await func(*args, **kwargs)
-
-                # 如果函数返回状态码，检查是否需要重试
-                if hasattr(result, 'status') and self.should_retry(status_code=result.status):
-                    if attempt < self.max_retries:
-                        delay = self.exponential_backoff(attempt)
-                        await asyncio.sleep(delay)
-                        continue
-                    else:
-                        raise Exception(f"HTTP {result.status} after {self.max_retries} retries")
-
-                return result
-
-            except Exception as e:
-                last_exception = e
-
-                # 检查是否应该重试
-                if self.should_retry(exception=e) and attempt < self.max_retries:
-                    delay = self.exponential_backoff(attempt)
-                    await asyncio.sleep(delay)
-                    continue
-                else:
-                    raise e
-
-        # 如果到达这里，说明所有重试都失败了
-        raise last_exception
-
-    def sync_retry(self, func: Callable, *args, **kwargs) -> Any:
-        """
-        同步重试执行函数
-        
-        Args:
-            func (Callable): 要执行的函数
-            *args: 函数参数
-            **kwargs: 函数关键字参数
-            
-        Returns:
-            Any: 函数执行结果
-            
-        Raises:
-            Exception: 如果超过最大重试次数仍未成功，则抛出最后一个异常
-        """
-        last_exception = None
-
-        for attempt in range(self.max_retries + 1):
-            try:
-                result = func(*args, **kwargs)
-
-                # 如果函数返回状态码，检查是否需要重试
-                if hasattr(result, 'status') and self.should_retry(status_code=result.status):
-                    if attempt < self.max_retries:
-                        delay = self.exponential_backoff(attempt)
-                        time.sleep(delay)
-                        continue
-                    else:
-                        raise Exception(f"HTTP {result.status} after {self.max_retries} retries")
-
-                return result
-
-            except Exception as e:
-                last_exception = e
-
-                # 检查是否应该重试
-                if self.should_retry(exception=e) and attempt < self.max_retries:
-                    delay = self.exponential_backoff(attempt)
-                    time.sleep(delay)
-                    continue
-                else:
-                    raise e
-
-        # 如果到达这里，说明所有重试都失败了
-        raise last_exception
-
-
-def retry(max_retries: int = 3,
-          retry_status_codes: Optional[Set[int]] = None,
-          retry_exceptions: Optional[Tuple[type, ...]] = None):
-    """
-    重试装饰器
-    
-    Args:
-        max_retries (int): 最大重试次数
-        retry_status_codes (Optional[Set[int]]): 应该重试的HTTP状态码
-        retry_exceptions (Optional[Tuple[type, ...]]): 应该重试的异常类型
-    """
-
-    def decorator(func: Callable) -> Callable:
-        retry_mechanism = RetryMechanism(max_retries, retry_status_codes, retry_exceptions)
-
-        if asyncio.iscoroutinefunction(func):
-            @wraps(func)
-            async def async_wrapper(*args, **kwargs):
-                return await retry_mechanism.async_retry(func, *args, **kwargs)
-
-            return async_wrapper
-        else:
-            @wraps(func)
-            def sync_wrapper(*args, **kwargs):
-                return retry_mechanism.sync_retry(func, *args, **kwargs)
-
-            return sync_wrapper
-
-    return decorator
-
-
-# 便捷函数
-def should_retry(status_code: Optional[int] = None,
-                 exception: Optional[Exception] = None) -> bool:
-    """判断是否应该重试"""
-    retry_mechanism = RetryMechanism()
-    return retry_mechanism.should_retry(status_code, exception)
-
-
-def exponential_backoff(attempt: int, base_delay: float = 1.0,
-                        max_delay: float = 60.0) -> float:
-    """计算指数退避延迟时间"""
-    retry_mechanism = RetryMechanism()
-    return retry_mechanism.exponential_backoff(attempt, base_delay, max_delay)