crawlo 1.4.6__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/tools/authenticated_proxy.py

@@ -1,241 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-带认证代理工具
-=============
-支持带用户名密码认证的代理和非认证代理的统一处理工具
-
-功能特性:
-- 支持HTTP/HTTPS/SOCKS代理
-- 支持带认证和不带认证的代理
-- 统一的代理格式处理
-- 代理有效性检测
-"""
-
-from typing import Dict, Any, Optional, Union
-from urllib.parse import urlparse, urlunparse
-
-
-class AuthenticatedProxy:
-    """带认证代理处理类"""
-
-    def __init__(self, proxy_url: str):
-        """
-        初始化代理对象
-        
-        Args:
-            proxy_url (str): 代理URL，支持带认证信息的格式
-                - 带认证: http://username:password@proxy.example.com:8080
-                - 无认证: http://proxy.example.com:8080
-        """
-        self.proxy_url = proxy_url
-        self.parsed = urlparse(proxy_url)
-        
-        # 提取认证信息
-        self.username = self.parsed.username
-        self.password = self.parsed.password
-        
-        # 构造不带认证信息的URL
-        self.clean_url = urlunparse((
-            self.parsed.scheme,
-            f"{self.parsed.hostname}:{self.parsed.port}" if self.parsed.port else self.parsed.hostname,
-            self.parsed.path,
-            self.parsed.params,
-            self.parsed.query,
-            self.parsed.fragment
-        ))
-        
-        # 构造下载器兼容的代理字典
-        self.proxy_dict = {
-            "http": self.clean_url,
-            "https": self.clean_url
-        }
-        
-        # 如果有认证信息，构造认证字符串
-        if self.username and self.password:
-            self.auth_string = f"{self.username}:{self.password}"
-        else:
-            self.auth_string = None
-
-    def get_proxy_for_downloader(self) -> Union[str, Dict[str, str]]:
-        """
-        获取适用于下载器的代理配置
-        
-        Returns:
-            Union[str, Dict[str, str]]: 代理配置
-                - 对于AioHttp/CurlCffi: 返回字典格式 {"http": "...", "https": "..."}
-                - 对于HttpX: 可以直接使用字符串或字典格式
-        """
-        return self.proxy_dict
-
-    def get_auth_credentials(self) -> Optional[Dict[str, str]]:
-        """
-        获取认证凭据
-        
-        Returns:
-            Optional[Dict[str, str]]: 认证凭据 {"username": "...", "password": "..."}
-        """
-        if self.username and self.password:
-            return {
-                "username": self.username,
-                "password": self.password
-            }
-        return None
-
-    def get_auth_header(self) -> Optional[str]:
-        """
-        获取Basic Auth认证头
-        
-        Returns:
-            Optional[str]: Basic Auth头信息
-        """
-        if self.username and self.password:
-            import base64
-            credentials = f"{self.username}:{self.password}"
-            encoded = base64.b64encode(credentials.encode()).decode()
-            return f"Basic {encoded}"
-        return None
-
-    def is_valid(self) -> bool:
-        """
-        检查代理URL是否有效
-        
-        Returns:
-            bool: 代理URL是否有效
-        """
-        # 检查协议
-        if self.parsed.scheme not in ["http", "https", "socks4", "socks5"]:
-            return False
-            
-        # 检查主机名
-        if not self.parsed.hostname:
-            return False
-            
-        # 检查端口（如果指定）
-        if self.parsed.port and (self.parsed.port < 1 or self.parsed.port > 65535):
-            return False
-            
-        return True
-
-    def __str__(self) -> str:
-        return self.proxy_url
-
-    def __repr__(self) -> str:
-        return f"AuthenticatedProxy(url='{self.proxy_url}', username={self.username is not None})"
-
-
-def create_proxy_config(proxy_url: str) -> Dict[str, Any]:
-    """
-    创建代理配置，兼容各种下载器
-    
-    Args:
-        proxy_url (str): 代理URL
-        
-    Returns:
-        Dict[str, Any]: 代理配置字典
-    """
-    proxy = AuthenticatedProxy(proxy_url)
-    
-    if not proxy.is_valid():
-        raise ValueError(f"Invalid proxy URL: {proxy_url}")
-    
-    config = {
-        "url": proxy.clean_url,
-        "proxy_dict": proxy.proxy_dict,
-        "has_auth": proxy.auth_string is not None
-    }
-    
-    if proxy.auth_string:
-        config["auth"] = proxy.get_auth_credentials()
-        config["auth_header"] = proxy.get_auth_header()
-        
-    return config
-
-
-def format_proxy_for_request(proxy_config: Dict[str, Any], downloader_type: str = "aiohttp") -> Dict[str, Any]:
-    """
-    格式化代理配置以适配特定下载器
-    
-    Args:
-        proxy_config (Dict[str, Any]): 代理配置
-        downloader_type (str): 下载器类型 (aiohttp, httpx, curl_cffi)
-        
-    Returns:
-        Dict[str, Any]: 适配下载器的代理配置
-    """
-    formatted = {}
-    
-    if downloader_type.lower() == "aiohttp":
-        # AioHttp使用proxy和proxy_auth参数
-        formatted["proxy"] = proxy_config["url"]
-        if proxy_config.get("has_auth") and proxy_config.get("auth"):
-            from aiohttp import BasicAuth
-            auth = proxy_config["auth"]
-            formatted["proxy_auth"] = BasicAuth(auth["username"], auth["password"])
-            
-    elif downloader_type.lower() == "httpx":
-        # HttpX可以直接使用代理URL字符串或字典
-        formatted["proxy"] = proxy_config["url"]
-        
-    elif downloader_type.lower() == "curl_cffi":
-        # CurlCffi使用proxies字典
-        formatted["proxies"] = proxy_config["proxy_dict"]
-        # 认证信息包含在URL中或通过headers传递
-        if proxy_config.get("auth_header"):
-            formatted["headers"] = {"Proxy-Authorization": proxy_config["auth_header"]}
-            
-    return formatted
-
-
-# 便捷函数
-def parse_proxy_url(proxy_url: str) -> Dict[str, Any]:
-    """
-    解析代理URL并返回详细信息
-    
-    Args:
-        proxy_url (str): 代理URL
-        
-    Returns:
-        Dict[str, Any]: 代理详细信息
-    """
-    return create_proxy_config(proxy_url)
-
-
-def validate_proxy_url(proxy_url: str) -> bool:
-    """
-    验证代理URL是否有效
-    
-    Args:
-        proxy_url (str): 代理URL
-        
-    Returns:
-        bool: 是否有效
-    """
-    try:
-        proxy = AuthenticatedProxy(proxy_url)
-        return proxy.is_valid()
-    except:
-        return False
-
-
-def get_proxy_info(proxy_url: str) -> Dict[str, Any]:
-    """
-    获取代理详细信息
-    
-    Args:
-        proxy_url (str): 代理URL
-        
-    Returns:
-        Dict[str, Any]: 代理详细信息
-    """
-    proxy = AuthenticatedProxy(proxy_url)
-    return {
-        "original_url": proxy.proxy_url,
-        "clean_url": proxy.clean_url,
-        "scheme": proxy.parsed.scheme,
-        "hostname": proxy.parsed.hostname,
-        "port": proxy.parsed.port,
-        "has_auth": proxy.auth_string is not None,
-        "username": proxy.username,
-        "is_valid": proxy.is_valid()
-    }

crawlo/tools/data_formatter.py

@@ -1,226 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 数据格式化工具
-"""
-import re
-from typing import Any, Optional, Union
-from decimal import Decimal, InvalidOperation
-
-
-class DataFormatter:
-    """
-    数据格式化工具类，提供各种数据格式化功能。
-    特别适用于爬虫中处理各种数据类型的格式化需求。
-    """
-
-    @staticmethod
-    def format_number(value: Any, 
-                      precision: int = 2, 
-                      thousand_separator: bool = False) -> Optional[str]:
-        """
-        格式化数字
-        
-        :param value: 数字值
-        :param precision: 小数点精度
-        :param thousand_separator: 是否使用千位分隔符
-        :return: 格式化后的数字字符串
-        """
-        if value is None:
-            return None
-            
-        try:
-            # 转换为Decimal以避免浮点数精度问题
-            decimal_value = Decimal(str(value))
-            
-            if thousand_separator:
-                # 使用千位分隔符
-                formatted = f"{decimal_value:,.{precision}f}"
-            else:
-                # 不使用千位分隔符
-                formatted = f"{decimal_value:.{precision}f}"
-                
-            return formatted
-        except (ValueError, InvalidOperation):
-            return None
-
-    @staticmethod
-    def format_currency(value: Any, 
-                        currency_symbol: str = "¥", 
-                        precision: int = 2) -> Optional[str]:
-        """
-        格式化货币
-        
-        :param value: 货币值
-        :param currency_symbol: 货币符号
-        :param precision: 小数点精度
-        :return: 格式化后的货币字符串
-        """
-        formatted_number = DataFormatter.format_number(value, precision, thousand_separator=True)
-        if formatted_number is None:
-            return None
-            
-        return f"{currency_symbol}{formatted_number}"
-
-    @staticmethod
-    def format_percentage(value: Any, 
-                          precision: int = 2, 
-                          multiply_100: bool = True) -> Optional[str]:
-        """
-        格式化百分比
-        
-        :param value: 百分比值
-        :param precision: 小数点精度
-        :param multiply_100: 是否乘以100（如果原始值是小数）
-        :return: 格式化后的百分比字符串
-        """
-        if value is None:
-            return None
-            
-        try:
-            decimal_value = Decimal(str(value))
-            
-            if multiply_100:
-                decimal_value *= 100
-                
-            formatted = f"{decimal_value:.{precision}f}%"
-            return formatted
-        except (ValueError, InvalidOperation):
-            return None
-
-    @staticmethod
-    def format_phone_number(phone: str, 
-                            country_code: str = "+86", 
-                            format_type: str = "international") -> Optional[str]:
-        """
-        格式化电话号码
-        
-        :param phone: 电话号码
-        :param country_code: 国家代码
-        :param format_type: 格式类型 ('international', 'domestic', 'plain')
-        :return: 格式化后的电话号码
-        """
-        if not isinstance(phone, str):
-            phone = str(phone)
-            
-        # 移除所有非数字字符
-        digits = re.sub(r'\D', '', phone)
-        
-        if not digits:
-            return None
-            
-        # 如果是11位中国手机号
-        if len(digits) == 11 and digits.startswith('1'):
-            if format_type == "international":
-                return f"{country_code} {digits[:3]} {digits[3:7]} {digits[7:]}"
-            elif format_type == "domestic":
-                return f"{digits[:3]}-{digits[3:7]}-{digits[7:]}"
-            else:  # plain
-                return digits
-        else:
-            # 其他情况简单处理
-            if format_type == "international" and country_code:
-                return f"{country_code} {digits}"
-            else:
-                return digits
-
-    @staticmethod
-    def format_chinese_id_card(id_card: str) -> Optional[str]:
-        """
-        格式化中国身份证号码（隐藏中间部分）
-        
-        :param id_card: 身份证号码
-        :return: 格式化后的身份证号码
-        """
-        if not isinstance(id_card, str):
-            id_card = str(id_card)
-            
-        # 移除空格
-        id_card = id_card.replace(" ", "")
-        
-        if len(id_card) == 18:
-            # 18位身份证号
-            return f"{id_card[:6]}********{id_card[-4:]}"
-        elif len(id_card) == 15:
-            # 15位身份证号
-            return f"{id_card[:6]}******{id_card[-3:]}"
-        else:
-            return None
-
-    @staticmethod
-    def capitalize_words(text: str, 
-                         delimiter: str = " ", 
-                         preserve_articles: bool = True) -> str:
-        """
-        单词首字母大写
-        
-        :param text: 文本
-        :param delimiter: 单词分隔符
-        :param preserve_articles: 是否保留冠词小写
-        :return: 首字母大写后的文本
-        """
-        if not isinstance(text, str):
-            return str(text)
-            
-        # 常见的冠词和介词
-        articles = {'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'}
-        
-        words = text.split(delimiter)
-        capitalized_words = []
-        
-        for i, word in enumerate(words):
-            if not word:
-                capitalized_words.append(word)
-                continue
-                
-            # 第一个单词和最后一个单词总是大写
-            if i == 0 or i == len(words) - 1 or not preserve_articles or word.lower() not in articles:
-                capitalized_words.append(word.capitalize())
-            else:
-                capitalized_words.append(word.lower())
-                
-        return delimiter.join(capitalized_words)
-
-
-# =======================对外接口=======================
-
-def format_number(value: Any, 
-                  precision: int = 2, 
-                  thousand_separator: bool = False) -> Optional[str]:
-    """格式化数字"""
-    return DataFormatter.format_number(value, precision, thousand_separator)
-
-
-def format_currency(value: Any, 
-                    currency_symbol: str = "¥", 
-                    precision: int = 2) -> Optional[str]:
-    """格式化货币"""
-    return DataFormatter.format_currency(value, currency_symbol, precision)
-
-
-def format_percentage(value: Any, 
-                      precision: int = 2, 
-                      multiply_100: bool = True) -> Optional[str]:
-    """格式化百分比"""
-    return DataFormatter.format_percentage(value, precision, multiply_100)
-
-
-def format_phone_number(phone: str, 
-                        country_code: str = "+86", 
-                        format_type: str = "international") -> Optional[str]:
-    """格式化电话号码"""
-    return DataFormatter.format_phone_number(phone, country_code, format_type)
-
-
-def format_chinese_id_card(id_card: str) -> Optional[str]:
-    """格式化中国身份证号码"""
-    return DataFormatter.format_chinese_id_card(id_card)
-
-
-def capitalize_words(text: str, 
-                     delimiter: str = " ", 
-                     preserve_articles: bool = True) -> str:
-    """单词首字母大写"""
-    return DataFormatter.capitalize_words(text, delimiter, preserve_articles)

crawlo/tools/data_validator.py

@@ -1,181 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 数据验证工具
-"""
-
-import re
-from typing import Any, Union, Dict, List
-from datetime import datetime
-from urllib.parse import urlparse
-
-
-class DataValidator:
-    """数据验证工具类"""
-
-    @staticmethod
-    def validate_email(email: str) -> bool:
-        """
-        验证邮箱地址格式
-        
-        Args:
-            email (str): 邮箱地址
-            
-        Returns:
-            bool: 验证结果
-        """
-        pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
-        return bool(re.match(pattern, email))
-
-    @staticmethod
-    def validate_phone(phone: str, country_code: str = "CN") -> bool:
-        """
-        验证电话号码格式
-        
-        Args:
-            phone (str): 电话号码
-            country_code (str): 国家代码，默认为"CN"
-            
-        Returns:
-            bool: 验证结果
-        """
-        if country_code == "CN":
-            # 中国手机号码格式
-            pattern = r'^1[3-9]\d{9}$'
-            return bool(re.match(pattern, phone))
-        else:
-            # 通用格式，只检查是否全为数字且长度在7-15之间
-            pattern = r'^\d{7,15}$'
-            return bool(re.match(pattern, phone))
-
-    @staticmethod
-    def validate_url(url: str) -> bool:
-        """
-        验证URL格式
-        
-        Args:
-            url (str): URL地址
-            
-        Returns:
-            bool: 验证结果
-        """
-        try:
-            result = urlparse(url)
-            return all([result.scheme, result.netloc])
-        except Exception:
-            return False
-
-    @staticmethod
-    def validate_chinese_id_card(id_card: str) -> bool:
-        """
-        验证中国身份证号码格式
-        
-        Args:
-            id_card (str): 身份证号码
-            
-        Returns:
-            bool: 验证结果
-        """
-        # 18位身份证号码格式
-        pattern = r'^[1-9]\d{5}(18|19|20)\d{2}((0[1-9])|(1[0-2]))(([0-2][1-9])|10|20|30|31)\d{3}[0-9Xx]$'
-        return bool(re.match(pattern, id_card))
-
-    @staticmethod
-    def validate_date(date_str: str, date_format: str = "%Y-%m-%d") -> bool:
-        """
-        验证日期格式
-        
-        Args:
-            date_str (str): 日期字符串
-            date_format (str): 日期格式，默认为"%Y-%m-%d"
-            
-        Returns:
-            bool: 验证结果
-        """
-        try:
-            datetime.strptime(date_str, date_format)
-            return True
-        except ValueError:
-            return False
-
-    @staticmethod
-    def validate_number_range(value: Union[int, float], min_val: Union[int, float], 
-                             max_val: Union[int, float]) -> bool:
-        """
-        验证数值是否在指定范围内
-        
-        Args:
-            value (Union[int, float]): 要验证的数值
-            min_val (Union[int, float]): 最小值
-            max_val (Union[int, float]): 最大值
-            
-        Returns:
-            bool: 验证结果
-        """
-        return min_val <= value <= max_val
-
-    @staticmethod
-    def check_data_integrity(data: Dict[str, Any], required_fields: List[str]) -> Dict[str, Any]:
-        """
-        检查数据完整性，确保关键字段不为空
-        
-        Args:
-            data (Dict[str, Any]): 要检查的数据
-            required_fields (List[str]): 必需字段列表
-            
-        Returns:
-            Dict[str, Any]: 检查结果，包含缺失字段和空值字段
-        """
-        missing_fields = []
-        empty_fields = []
-        
-        for field in required_fields:
-            if field not in data:
-                missing_fields.append(field)
-            elif data[field] is None or data[field] == "":
-                empty_fields.append(field)
-                
-        return {
-            "is_valid": len(missing_fields) == 0 and len(empty_fields) == 0,
-            "missing_fields": missing_fields,
-            "empty_fields": empty_fields
-        }
-
-
-# 便捷函数
-def validate_email(email: str) -> bool:
-    """验证邮箱地址格式"""
-    return DataValidator.validate_email(email)
-
-
-def validate_phone(phone: str, country_code: str = "CN") -> bool:
-    """验证电话号码格式"""
-    return DataValidator.validate_phone(phone, country_code)
-
-
-def validate_url(url: str) -> bool:
-    """验证URL格式"""
-    return DataValidator.validate_url(url)
-
-
-def validate_chinese_id_card(id_card: str) -> bool:
-    """验证中国身份证号码格式"""
-    return DataValidator.validate_chinese_id_card(id_card)
-
-
-def validate_date(date_str: str, date_format: str = "%Y-%m-%d") -> bool:
-    """验证日期格式"""
-    return DataValidator.validate_date(date_str, date_format)
-
-
-def validate_number_range(value: Union[int, float], min_val: Union[int, float], 
-                         max_val: Union[int, float]) -> bool:
-    """验证数值是否在指定范围内"""
-    return DataValidator.validate_number_range(value, min_val, max_val)
-
-
-def check_data_integrity(data: Dict[str, Any], required_fields: List[str]) -> Dict[str, Any]:
-    """检查数据完整性"""
-    return DataValidator.check_data_integrity(data, required_fields)