crawlo 1.4.6__py3-none-any.whl → 1.4.7__py3-none-any.whl

crawlo/tools/data_formatter.py

@@ -1,226 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 数据格式化工具
-"""
-import re
-from typing import Any, Optional, Union
-from decimal import Decimal, InvalidOperation
-
-
-class DataFormatter:
-    """
-    数据格式化工具类，提供各种数据格式化功能。
-    特别适用于爬虫中处理各种数据类型的格式化需求。
-    """
-
-    @staticmethod
-    def format_number(value: Any, 
-                      precision: int = 2, 
-                      thousand_separator: bool = False) -> Optional[str]:
-        """
-        格式化数字
-        
-        :param value: 数字值
-        :param precision: 小数点精度
-        :param thousand_separator: 是否使用千位分隔符
-        :return: 格式化后的数字字符串
-        """
-        if value is None:
-            return None
-            
-        try:
-            # 转换为Decimal以避免浮点数精度问题
-            decimal_value = Decimal(str(value))
-            
-            if thousand_separator:
-                # 使用千位分隔符
-                formatted = f"{decimal_value:,.{precision}f}"
-            else:
-                # 不使用千位分隔符
-                formatted = f"{decimal_value:.{precision}f}"
-                
-            return formatted
-        except (ValueError, InvalidOperation):
-            return None
-
-    @staticmethod
-    def format_currency(value: Any, 
-                        currency_symbol: str = "¥", 
-                        precision: int = 2) -> Optional[str]:
-        """
-        格式化货币
-        
-        :param value: 货币值
-        :param currency_symbol: 货币符号
-        :param precision: 小数点精度
-        :return: 格式化后的货币字符串
-        """
-        formatted_number = DataFormatter.format_number(value, precision, thousand_separator=True)
-        if formatted_number is None:
-            return None
-            
-        return f"{currency_symbol}{formatted_number}"
-
-    @staticmethod
-    def format_percentage(value: Any, 
-                          precision: int = 2, 
-                          multiply_100: bool = True) -> Optional[str]:
-        """
-        格式化百分比
-        
-        :param value: 百分比值
-        :param precision: 小数点精度
-        :param multiply_100: 是否乘以100（如果原始值是小数）
-        :return: 格式化后的百分比字符串
-        """
-        if value is None:
-            return None
-            
-        try:
-            decimal_value = Decimal(str(value))
-            
-            if multiply_100:
-                decimal_value *= 100
-                
-            formatted = f"{decimal_value:.{precision}f}%"
-            return formatted
-        except (ValueError, InvalidOperation):
-            return None
-
-    @staticmethod
-    def format_phone_number(phone: str, 
-                            country_code: str = "+86", 
-                            format_type: str = "international") -> Optional[str]:
-        """
-        格式化电话号码
-        
-        :param phone: 电话号码
-        :param country_code: 国家代码
-        :param format_type: 格式类型 ('international', 'domestic', 'plain')
-        :return: 格式化后的电话号码
-        """
-        if not isinstance(phone, str):
-            phone = str(phone)
-            
-        # 移除所有非数字字符
-        digits = re.sub(r'\D', '', phone)
-        
-        if not digits:
-            return None
-            
-        # 如果是11位中国手机号
-        if len(digits) == 11 and digits.startswith('1'):
-            if format_type == "international":
-                return f"{country_code} {digits[:3]} {digits[3:7]} {digits[7:]}"
-            elif format_type == "domestic":
-                return f"{digits[:3]}-{digits[3:7]}-{digits[7:]}"
-            else:  # plain
-                return digits
-        else:
-            # 其他情况简单处理
-            if format_type == "international" and country_code:
-                return f"{country_code} {digits}"
-            else:
-                return digits
-
-    @staticmethod
-    def format_chinese_id_card(id_card: str) -> Optional[str]:
-        """
-        格式化中国身份证号码（隐藏中间部分）
-        
-        :param id_card: 身份证号码
-        :return: 格式化后的身份证号码
-        """
-        if not isinstance(id_card, str):
-            id_card = str(id_card)
-            
-        # 移除空格
-        id_card = id_card.replace(" ", "")
-        
-        if len(id_card) == 18:
-            # 18位身份证号
-            return f"{id_card[:6]}********{id_card[-4:]}"
-        elif len(id_card) == 15:
-            # 15位身份证号
-            return f"{id_card[:6]}******{id_card[-3:]}"
-        else:
-            return None
-
-    @staticmethod
-    def capitalize_words(text: str, 
-                         delimiter: str = " ", 
-                         preserve_articles: bool = True) -> str:
-        """
-        单词首字母大写
-        
-        :param text: 文本
-        :param delimiter: 单词分隔符
-        :param preserve_articles: 是否保留冠词小写
-        :return: 首字母大写后的文本
-        """
-        if not isinstance(text, str):
-            return str(text)
-            
-        # 常见的冠词和介词
-        articles = {'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'}
-        
-        words = text.split(delimiter)
-        capitalized_words = []
-        
-        for i, word in enumerate(words):
-            if not word:
-                capitalized_words.append(word)
-                continue
-                
-            # 第一个单词和最后一个单词总是大写
-            if i == 0 or i == len(words) - 1 or not preserve_articles or word.lower() not in articles:
-                capitalized_words.append(word.capitalize())
-            else:
-                capitalized_words.append(word.lower())
-                
-        return delimiter.join(capitalized_words)
-
-
-# =======================对外接口=======================
-
-def format_number(value: Any, 
-                  precision: int = 2, 
-                  thousand_separator: bool = False) -> Optional[str]:
-    """格式化数字"""
-    return DataFormatter.format_number(value, precision, thousand_separator)
-
-
-def format_currency(value: Any, 
-                    currency_symbol: str = "¥", 
-                    precision: int = 2) -> Optional[str]:
-    """格式化货币"""
-    return DataFormatter.format_currency(value, currency_symbol, precision)
-
-
-def format_percentage(value: Any, 
-                      precision: int = 2, 
-                      multiply_100: bool = True) -> Optional[str]:
-    """格式化百分比"""
-    return DataFormatter.format_percentage(value, precision, multiply_100)
-
-
-def format_phone_number(phone: str, 
-                        country_code: str = "+86", 
-                        format_type: str = "international") -> Optional[str]:
-    """格式化电话号码"""
-    return DataFormatter.format_phone_number(phone, country_code, format_type)
-
-
-def format_chinese_id_card(id_card: str) -> Optional[str]:
-    """格式化中国身份证号码"""
-    return DataFormatter.format_chinese_id_card(id_card)
-
-
-def capitalize_words(text: str, 
-                     delimiter: str = " ", 
-                     preserve_articles: bool = True) -> str:
-    """单词首字母大写"""
-    return DataFormatter.capitalize_words(text, delimiter, preserve_articles)

crawlo/tools/data_validator.py

@@ -1,181 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 数据验证工具
-"""
-
-import re
-from typing import Any, Union, Dict, List
-from datetime import datetime
-from urllib.parse import urlparse
-
-
-class DataValidator:
-    """数据验证工具类"""
-
-    @staticmethod
-    def validate_email(email: str) -> bool:
-        """
-        验证邮箱地址格式
-        
-        Args:
-            email (str): 邮箱地址
-            
-        Returns:
-            bool: 验证结果
-        """
-        pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
-        return bool(re.match(pattern, email))
-
-    @staticmethod
-    def validate_phone(phone: str, country_code: str = "CN") -> bool:
-        """
-        验证电话号码格式
-        
-        Args:
-            phone (str): 电话号码
-            country_code (str): 国家代码，默认为"CN"
-            
-        Returns:
-            bool: 验证结果
-        """
-        if country_code == "CN":
-            # 中国手机号码格式
-            pattern = r'^1[3-9]\d{9}$'
-            return bool(re.match(pattern, phone))
-        else:
-            # 通用格式，只检查是否全为数字且长度在7-15之间
-            pattern = r'^\d{7,15}$'
-            return bool(re.match(pattern, phone))
-
-    @staticmethod
-    def validate_url(url: str) -> bool:
-        """
-        验证URL格式
-        
-        Args:
-            url (str): URL地址
-            
-        Returns:
-            bool: 验证结果
-        """
-        try:
-            result = urlparse(url)
-            return all([result.scheme, result.netloc])
-        except Exception:
-            return False
-
-    @staticmethod
-    def validate_chinese_id_card(id_card: str) -> bool:
-        """
-        验证中国身份证号码格式
-        
-        Args:
-            id_card (str): 身份证号码
-            
-        Returns:
-            bool: 验证结果
-        """
-        # 18位身份证号码格式
-        pattern = r'^[1-9]\d{5}(18|19|20)\d{2}((0[1-9])|(1[0-2]))(([0-2][1-9])|10|20|30|31)\d{3}[0-9Xx]$'
-        return bool(re.match(pattern, id_card))
-
-    @staticmethod
-    def validate_date(date_str: str, date_format: str = "%Y-%m-%d") -> bool:
-        """
-        验证日期格式
-        
-        Args:
-            date_str (str): 日期字符串
-            date_format (str): 日期格式，默认为"%Y-%m-%d"
-            
-        Returns:
-            bool: 验证结果
-        """
-        try:
-            datetime.strptime(date_str, date_format)
-            return True
-        except ValueError:
-            return False
-
-    @staticmethod
-    def validate_number_range(value: Union[int, float], min_val: Union[int, float], 
-                             max_val: Union[int, float]) -> bool:
-        """
-        验证数值是否在指定范围内
-        
-        Args:
-            value (Union[int, float]): 要验证的数值
-            min_val (Union[int, float]): 最小值
-            max_val (Union[int, float]): 最大值
-            
-        Returns:
-            bool: 验证结果
-        """
-        return min_val <= value <= max_val
-
-    @staticmethod
-    def check_data_integrity(data: Dict[str, Any], required_fields: List[str]) -> Dict[str, Any]:
-        """
-        检查数据完整性，确保关键字段不为空
-        
-        Args:
-            data (Dict[str, Any]): 要检查的数据
-            required_fields (List[str]): 必需字段列表
-            
-        Returns:
-            Dict[str, Any]: 检查结果，包含缺失字段和空值字段
-        """
-        missing_fields = []
-        empty_fields = []
-        
-        for field in required_fields:
-            if field not in data:
-                missing_fields.append(field)
-            elif data[field] is None or data[field] == "":
-                empty_fields.append(field)
-                
-        return {
-            "is_valid": len(missing_fields) == 0 and len(empty_fields) == 0,
-            "missing_fields": missing_fields,
-            "empty_fields": empty_fields
-        }
-
-
-# 便捷函数
-def validate_email(email: str) -> bool:
-    """验证邮箱地址格式"""
-    return DataValidator.validate_email(email)
-
-
-def validate_phone(phone: str, country_code: str = "CN") -> bool:
-    """验证电话号码格式"""
-    return DataValidator.validate_phone(phone, country_code)
-
-
-def validate_url(url: str) -> bool:
-    """验证URL格式"""
-    return DataValidator.validate_url(url)
-
-
-def validate_chinese_id_card(id_card: str) -> bool:
-    """验证中国身份证号码格式"""
-    return DataValidator.validate_chinese_id_card(id_card)
-
-
-def validate_date(date_str: str, date_format: str = "%Y-%m-%d") -> bool:
-    """验证日期格式"""
-    return DataValidator.validate_date(date_str, date_format)
-
-
-def validate_number_range(value: Union[int, float], min_val: Union[int, float], 
-                         max_val: Union[int, float]) -> bool:
-    """验证数值是否在指定范围内"""
-    return DataValidator.validate_number_range(value, min_val, max_val)
-
-
-def check_data_integrity(data: Dict[str, Any], required_fields: List[str]) -> Dict[str, Any]:
-    """检查数据完整性"""
-    return DataValidator.check_data_integrity(data, required_fields)

crawlo/tools/encoding_converter.py

@@ -1,127 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 编码转换工具
-"""
-try:
-    import chardet
-
-    HAS_CHARDET = True
-except ImportError:
-    HAS_CHARDET = False
-from typing import Optional, Union
-
-
-class EncodingConverter:
-    """
-    编码转换工具类，提供各种编码转换功能。
-    特别适用于爬虫中处理不同编码的网页内容。
-    """
-
-    @staticmethod
-    def detect_encoding(data: Union[str, bytes]) -> Optional[str]:
-        """
-        检测数据编码
-        
-        :param data: 数据（字符串或字节）
-        :return: 检测到的编码
-        """
-        if isinstance(data, str):
-            # 如果是字符串，直接返回
-            return 'utf-8'
-
-        if not isinstance(data, bytes):
-            return None
-
-        if HAS_CHARDET:
-            try:
-                # 使用chardet检测编码
-                result = chardet.detect(data)
-                return result['encoding']
-            except Exception:
-                return None
-        else:
-            # 如果没有chardet，返回None
-            return None
-
-    @staticmethod
-    def to_utf8(data: Union[str, bytes], source_encoding: Optional[str] = None) -> Optional[str]:
-        """
-        转换为UTF-8编码的字符串
-        
-        :param data: 数据（字符串或字节）
-        :param source_encoding: 源编码（如果为None则自动检测）
-        :return: UTF-8编码的字符串
-        """
-        if isinstance(data, str):
-            # 如果已经是字符串，假设它已经是UTF-8
-            return data
-
-        if not isinstance(data, bytes):
-            return None
-
-        try:
-            if source_encoding is None:
-                # 自动检测编码
-                source_encoding = EncodingConverter.detect_encoding(data)
-                if source_encoding is None:
-                    # 如果检测失败，尝试常见编码
-                    for encoding in ['utf-8', 'gbk', 'gb2312', 'latin1']:
-                        try:
-                            decoded = data.decode(encoding)
-                            return decoded
-                        except UnicodeDecodeError:
-                            continue
-                    return None
-            else:
-                # 使用指定编码
-                return data.decode(source_encoding)
-
-            # 使用检测到的编码解码
-            return data.decode(source_encoding)
-        except Exception:
-            return None
-
-    @staticmethod
-    def convert_encoding(data: Union[str, bytes],
-                         source_encoding: Optional[str] = None,
-                         target_encoding: str = 'utf-8') -> Optional[bytes]:
-        """
-        编码转换
-        
-        :param data: 数据（字符串或字节）
-        :param source_encoding: 源编码（如果为None则自动检测）
-        :param target_encoding: 目标编码
-        :return: 转换后的字节数据
-        """
-        # 先转换为UTF-8字符串
-        utf8_str = EncodingConverter.to_utf8(data, source_encoding)
-        if utf8_str is None:
-            return None
-
-        try:
-            # 再转换为目标编码
-            return utf8_str.encode(target_encoding)
-        except Exception:
-            return None
-
-
-# =======================对外接口=======================
-
-def detect_encoding(data: Union[str, bytes]) -> Optional[str]:
-    """检测数据编码"""
-    return EncodingConverter.detect_encoding(data)
-
-
-def to_utf8(data: Union[str, bytes], source_encoding: Optional[str] = None) -> Optional[str]:
-    """转换为UTF-8编码的字符串"""
-    return EncodingConverter.to_utf8(data, source_encoding)
-
-
-def convert_encoding(data: Union[str, bytes],
-                     source_encoding: Optional[str] = None,
-                     target_encoding: str = 'utf-8') -> Optional[bytes]:
-    """编码转换"""
-    return EncodingConverter.convert_encoding(data, source_encoding, target_encoding)