crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

crawlo/__init__.py

@@ -9,7 +9,21 @@ from crawlo.network.request import Request
 from crawlo.network.response import Response
 from crawlo.downloader import DownloaderBase
 from crawlo.middleware import BaseMiddleware
-
+from crawlo.utils import (
+    TimeUtils,
+    parse_time,
+    format_time,
+    time_diff,
+    to_timestamp,
+    to_datetime,
+    now,
+    to_timezone,
+    to_utc,
+    to_local,
+    from_timestamp_with_tz
+)
+from crawlo import cleaners
+from crawlo import tools
 
 # 版本号：优先从元数据读取
 try:
@@ -31,5 +45,18 @@ __all__ = [
     'Response',
     'DownloaderBase',
     'BaseMiddleware',
+    'TimeUtils',
+    'parse_time',
+    'format_time',
+    'time_diff',
+    'to_timestamp',
+    'to_datetime',
+    'now',
+    'to_timezone',
+    'to_utc',
+    'to_local',
+    'from_timestamp_with_tz',
+    'cleaners',
+    'tools',
     '__version__',
 ]

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = "1.1.3"
+__version__ = "1.1.5"

crawlo/cleaners/__init__.py

@@ -0,0 +1,61 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-10 22:00
+# @Author  : crawl-coder
+# @Desc    : 数据清洗工具包
+"""
+
+from .text_cleaner import (
+    TextCleaner,
+    remove_html_tags,
+    decode_html_entities,
+    remove_extra_whitespace,
+    remove_special_chars,
+    normalize_unicode,
+    clean_text,
+    extract_numbers,
+    extract_emails,
+    extract_urls
+)
+
+from .data_formatter import (
+    DataFormatter,
+    format_number,
+    format_currency,
+    format_percentage,
+    format_phone_number,
+    format_chinese_id_card,
+    capitalize_words
+)
+
+from .encoding_converter import (
+    EncodingConverter,
+    detect_encoding,
+    to_utf8,
+    convert_encoding
+)
+
+__all__ = [
+    "TextCleaner",
+    "DataFormatter",
+    "EncodingConverter",
+    "remove_html_tags",
+    "decode_html_entities",
+    "remove_extra_whitespace",
+    "remove_special_chars",
+    "normalize_unicode",
+    "clean_text",
+    "extract_numbers",
+    "extract_emails",
+    "extract_urls",
+    "format_number",
+    "format_currency",
+    "format_percentage",
+    "format_phone_number",
+    "format_chinese_id_card",
+    "capitalize_words",
+    "detect_encoding",
+    "to_utf8",
+    "convert_encoding"
+]

crawlo/cleaners/data_formatter.py

@@ -0,0 +1,226 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-10 22:00
+# @Author  : crawl-coder
+# @Desc    : 数据格式化工具
+"""
+import re
+from typing import Any, Optional, Union
+from decimal import Decimal, InvalidOperation
+
+
+class DataFormatter:
+    """
+    数据格式化工具类，提供各种数据格式化功能。
+    特别适用于爬虫中处理各种数据类型的格式化需求。
+    """
+
+    @staticmethod
+    def format_number(value: Any, 
+                      precision: int = 2, 
+                      thousand_separator: bool = False) -> Optional[str]:
+        """
+        格式化数字
+        
+        :param value: 数字值
+        :param precision: 小数点精度
+        :param thousand_separator: 是否使用千位分隔符
+        :return: 格式化后的数字字符串
+        """
+        if value is None:
+            return None
+            
+        try:
+            # 转换为Decimal以避免浮点数精度问题
+            decimal_value = Decimal(str(value))
+            
+            if thousand_separator:
+                # 使用千位分隔符
+                formatted = f"{decimal_value:,.{precision}f}"
+            else:
+                # 不使用千位分隔符
+                formatted = f"{decimal_value:.{precision}f}"
+                
+            return formatted
+        except (ValueError, InvalidOperation):
+            return None
+
+    @staticmethod
+    def format_currency(value: Any, 
+                        currency_symbol: str = "¥", 
+                        precision: int = 2) -> Optional[str]:
+        """
+        格式化货币
+        
+        :param value: 货币值
+        :param currency_symbol: 货币符号
+        :param precision: 小数点精度
+        :return: 格式化后的货币字符串
+        """
+        formatted_number = DataFormatter.format_number(value, precision, thousand_separator=True)
+        if formatted_number is None:
+            return None
+            
+        return f"{currency_symbol}{formatted_number}"
+
+    @staticmethod
+    def format_percentage(value: Any, 
+                          precision: int = 2, 
+                          multiply_100: bool = True) -> Optional[str]:
+        """
+        格式化百分比
+        
+        :param value: 百分比值
+        :param precision: 小数点精度
+        :param multiply_100: 是否乘以100（如果原始值是小数）
+        :return: 格式化后的百分比字符串
+        """
+        if value is None:
+            return None
+            
+        try:
+            decimal_value = Decimal(str(value))
+            
+            if multiply_100:
+                decimal_value *= 100
+                
+            formatted = f"{decimal_value:.{precision}f}%"
+            return formatted
+        except (ValueError, InvalidOperation):
+            return None
+
+    @staticmethod
+    def format_phone_number(phone: str, 
+                            country_code: str = "+86", 
+                            format_type: str = "international") -> Optional[str]:
+        """
+        格式化电话号码
+        
+        :param phone: 电话号码
+        :param country_code: 国家代码
+        :param format_type: 格式类型 ('international', 'domestic', 'plain')
+        :return: 格式化后的电话号码
+        """
+        if not isinstance(phone, str):
+            phone = str(phone)
+            
+        # 移除所有非数字字符
+        digits = re.sub(r'\D', '', phone)
+        
+        if not digits:
+            return None
+            
+        # 如果是11位中国手机号
+        if len(digits) == 11 and digits.startswith('1'):
+            if format_type == "international":
+                return f"{country_code} {digits[:3]} {digits[3:7]} {digits[7:]}"
+            elif format_type == "domestic":
+                return f"{digits[:3]}-{digits[3:7]}-{digits[7:]}"
+            else:  # plain
+                return digits
+        else:
+            # 其他情况简单处理
+            if format_type == "international" and country_code:
+                return f"{country_code} {digits}"
+            else:
+                return digits
+
+    @staticmethod
+    def format_chinese_id_card(id_card: str) -> Optional[str]:
+        """
+        格式化中国身份证号码（隐藏中间部分）
+        
+        :param id_card: 身份证号码
+        :return: 格式化后的身份证号码
+        """
+        if not isinstance(id_card, str):
+            id_card = str(id_card)
+            
+        # 移除空格
+        id_card = id_card.replace(" ", "")
+        
+        if len(id_card) == 18:
+            # 18位身份证号
+            return f"{id_card[:6]}********{id_card[-4:]}"
+        elif len(id_card) == 15:
+            # 15位身份证号
+            return f"{id_card[:6]}******{id_card[-3:]}"
+        else:
+            return None
+
+    @staticmethod
+    def capitalize_words(text: str, 
+                         delimiter: str = " ", 
+                         preserve_articles: bool = True) -> str:
+        """
+        单词首字母大写
+        
+        :param text: 文本
+        :param delimiter: 单词分隔符
+        :param preserve_articles: 是否保留冠词小写
+        :return: 首字母大写后的文本
+        """
+        if not isinstance(text, str):
+            return str(text)
+            
+        # 常见的冠词和介词
+        articles = {'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'}
+        
+        words = text.split(delimiter)
+        capitalized_words = []
+        
+        for i, word in enumerate(words):
+            if not word:
+                capitalized_words.append(word)
+                continue
+                
+            # 第一个单词和最后一个单词总是大写
+            if i == 0 or i == len(words) - 1 or not preserve_articles or word.lower() not in articles:
+                capitalized_words.append(word.capitalize())
+            else:
+                capitalized_words.append(word.lower())
+                
+        return delimiter.join(capitalized_words)
+
+
+# =======================对外接口=======================
+
+def format_number(value: Any, 
+                  precision: int = 2, 
+                  thousand_separator: bool = False) -> Optional[str]:
+    """格式化数字"""
+    return DataFormatter.format_number(value, precision, thousand_separator)
+
+
+def format_currency(value: Any, 
+                    currency_symbol: str = "¥", 
+                    precision: int = 2) -> Optional[str]:
+    """格式化货币"""
+    return DataFormatter.format_currency(value, currency_symbol, precision)
+
+
+def format_percentage(value: Any, 
+                      precision: int = 2, 
+                      multiply_100: bool = True) -> Optional[str]:
+    """格式化百分比"""
+    return DataFormatter.format_percentage(value, precision, multiply_100)
+
+
+def format_phone_number(phone: str, 
+                        country_code: str = "+86", 
+                        format_type: str = "international") -> Optional[str]:
+    """格式化电话号码"""
+    return DataFormatter.format_phone_number(phone, country_code, format_type)
+
+
+def format_chinese_id_card(id_card: str) -> Optional[str]:
+    """格式化中国身份证号码"""
+    return DataFormatter.format_chinese_id_card(id_card)
+
+
+def capitalize_words(text: str, 
+                     delimiter: str = " ", 
+                     preserve_articles: bool = True) -> str:
+    """单词首字母大写"""
+    return DataFormatter.capitalize_words(text, delimiter, preserve_articles)

crawlo/cleaners/encoding_converter.py

@@ -0,0 +1,126 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-10 22:00
+# @Author  : crawl-coder
+# @Desc    : 编码转换工具
+"""
+try:
+    import chardet
+    HAS_CHARDET = True
+except ImportError:
+    HAS_CHARDET = False
+from typing import Optional, Union
+
+
+class EncodingConverter:
+    """
+    编码转换工具类，提供各种编码转换功能。
+    特别适用于爬虫中处理不同编码的网页内容。
+    """
+
+    @staticmethod
+    def detect_encoding(data: Union[str, bytes]) -> Optional[str]:
+        """
+        检测数据编码
+        
+        :param data: 数据（字符串或字节）
+        :return: 检测到的编码
+        """
+        if isinstance(data, str):
+            # 如果是字符串，直接返回
+            return 'utf-8'
+        
+        if not isinstance(data, bytes):
+            return None
+            
+        if HAS_CHARDET:
+            try:
+                # 使用chardet检测编码
+                result = chardet.detect(data)
+                return result['encoding']
+            except Exception:
+                return None
+        else:
+            # 如果没有chardet，返回None
+            return None
+
+    @staticmethod
+    def to_utf8(data: Union[str, bytes], source_encoding: Optional[str] = None) -> Optional[str]:
+        """
+        转换为UTF-8编码的字符串
+        
+        :param data: 数据（字符串或字节）
+        :param source_encoding: 源编码（如果为None则自动检测）
+        :return: UTF-8编码的字符串
+        """
+        if isinstance(data, str):
+            # 如果已经是字符串，假设它已经是UTF-8
+            return data
+            
+        if not isinstance(data, bytes):
+            return None
+            
+        try:
+            if source_encoding is None:
+                # 自动检测编码
+                source_encoding = EncodingConverter.detect_encoding(data)
+                if source_encoding is None:
+                    # 如果检测失败，尝试常见编码
+                    for encoding in ['utf-8', 'gbk', 'gb2312', 'latin1']:
+                        try:
+                            decoded = data.decode(encoding)
+                            return decoded
+                        except UnicodeDecodeError:
+                            continue
+                    return None
+            else:
+                # 使用指定编码
+                return data.decode(source_encoding)
+                
+            # 使用检测到的编码解码
+            return data.decode(source_encoding)
+        except Exception:
+            return None
+
+    @staticmethod
+    def convert_encoding(data: Union[str, bytes], 
+                        source_encoding: Optional[str] = None, 
+                        target_encoding: str = 'utf-8') -> Optional[bytes]:
+        """
+        编码转换
+        
+        :param data: 数据（字符串或字节）
+        :param source_encoding: 源编码（如果为None则自动检测）
+        :param target_encoding: 目标编码
+        :return: 转换后的字节数据
+        """
+        # 先转换为UTF-8字符串
+        utf8_str = EncodingConverter.to_utf8(data, source_encoding)
+        if utf8_str is None:
+            return None
+            
+        try:
+            # 再转换为目标编码
+            return utf8_str.encode(target_encoding)
+        except Exception:
+            return None
+
+
+# =======================对外接口=======================
+
+def detect_encoding(data: Union[str, bytes]) -> Optional[str]:
+    """检测数据编码"""
+    return EncodingConverter.detect_encoding(data)
+
+
+def to_utf8(data: Union[str, bytes], source_encoding: Optional[str] = None) -> Optional[str]:
+    """转换为UTF-8编码的字符串"""
+    return EncodingConverter.to_utf8(data, source_encoding)
+
+
+def convert_encoding(data: Union[str, bytes], 
+                    source_encoding: Optional[str] = None, 
+                    target_encoding: str = 'utf-8') -> Optional[bytes]:
+    """编码转换"""
+    return EncodingConverter.convert_encoding(data, source_encoding, target_encoding)

crawlo/cleaners/text_cleaner.py

@@ -0,0 +1,233 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-10 22:00
+# @Author  : crawl-coder
+# @Desc    : 文本清洗工具
+"""
+import re
+import html
+from typing import Optional, Union, List
+import unicodedata
+
+
+class TextCleaner:
+    """
+    文本清洗工具类，提供各种文本清洗功能。
+    特别适用于爬虫中处理网页内容的清洗需求。
+    """
+
+    @staticmethod
+    def remove_html_tags(text: str) -> str:
+        """
+        移除HTML标签
+        
+        :param text: 包含HTML标签的文本
+        :return: 移除HTML标签后的文本
+        """
+        if not isinstance(text, str):
+            return str(text)
+        
+        # 使用正则表达式移除HTML标签
+        clean_text = re.sub(r'<[^>]+>', '', text)
+        return clean_text.strip()
+
+    @staticmethod
+    def decode_html_entities(text: str) -> str:
+        """
+        解码HTML实体字符
+        
+        :param text: 包含HTML实体字符的文本
+        :return: 解码后的文本
+        """
+        if not isinstance(text, str):
+            return str(text)
+        
+        return html.unescape(text)
+
+    @staticmethod
+    def remove_extra_whitespace(text: str) -> str:
+        """
+        移除多余的空白字符（包括空格、制表符、换行符等）
+        
+        :param text: 文本
+        :return: 清理后的文本
+        """
+        if not isinstance(text, str):
+            return str(text)
+        
+        # 将多个连续的空白字符替换为单个空格
+        clean_text = re.sub(r'\s+', ' ', text)
+        return clean_text.strip()
+
+    @staticmethod
+    def remove_special_chars(text: str, chars: str = '') -> str:
+        """
+        移除特殊字符
+        
+        :param text: 文本
+        :param chars: 要移除的特殊字符
+        :return: 清理后的文本
+        """
+        if not isinstance(text, str):
+            return str(text)
+        
+        # 移除常见的特殊字符
+        special_chars = r'[^\w\s\u4e00-\u9fff' + chars + r']'
+        clean_text = re.sub(special_chars, '', text)
+        return clean_text
+
+    @staticmethod
+    def normalize_unicode(text: str) -> str:
+        """
+        标准化Unicode字符
+        
+        :param text: 文本
+        :return: 标准化后的文本
+        """
+        if not isinstance(text, str):
+            return str(text)
+        
+        return unicodedata.normalize('NFKC', text)
+
+    @staticmethod
+    def clean_text(text: str, 
+                   remove_html: bool = True,
+                   decode_entities: bool = True,
+                   remove_whitespace: bool = True,
+                   remove_special: bool = False,
+                   normalize: bool = True) -> str:
+        """
+        综合文本清洗方法
+        
+        :param text: 原始文本
+        :param remove_html: 是否移除HTML标签
+        :param decode_entities: 是否解码HTML实体
+        :param remove_whitespace: 是否移除多余空白字符
+        :param remove_special: 是否移除特殊字符
+        :param normalize: 是否标准化Unicode字符
+        :return: 清洗后的文本
+        """
+        if not isinstance(text, str):
+            text = str(text)
+        
+        if not text:
+            return text
+            
+        # 按顺序进行清洗
+        if remove_html:
+            text = TextCleaner.remove_html_tags(text)
+        
+        if decode_entities:
+            text = TextCleaner.decode_html_entities(text)
+        
+        if normalize:
+            text = TextCleaner.normalize_unicode(text)
+        
+        if remove_whitespace:
+            text = TextCleaner.remove_extra_whitespace(text)
+        
+        if remove_special:
+            text = TextCleaner.remove_special_chars(text)
+        
+        return text
+
+    @staticmethod
+    def extract_numbers(text: str) -> List[str]:
+        """
+        从文本中提取数字
+        
+        :param text: 文本
+        :return: 数字列表
+        """
+        if not isinstance(text, str):
+            return []
+        
+        # 匹配整数和小数
+        numbers = re.findall(r'-?\d+\.?\d*', text)
+        return numbers
+
+    @staticmethod
+    def extract_emails(text: str) -> List[str]:
+        """
+        从文本中提取邮箱地址
+        
+        :param text: 文本
+        :return: 邮箱地址列表
+        """
+        if not isinstance(text, str):
+            return []
+        
+        # 匹配邮箱地址
+        emails = re.findall(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', text)
+        return emails
+
+    @staticmethod
+    def extract_urls(text: str) -> List[str]:
+        """
+        从文本中提取URL
+        
+        :param text: 文本
+        :return: URL列表
+        """
+        if not isinstance(text, str):
+            return []
+        
+        # 匹配URL
+        urls = re.findall(
+            r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
+            text
+        )
+        return urls
+
+
+# =======================对外接口=======================
+
+def remove_html_tags(text: str) -> str:
+    """移除HTML标签"""
+    return TextCleaner.remove_html_tags(text)
+
+
+def decode_html_entities(text: str) -> str:
+    """解码HTML实体字符"""
+    return TextCleaner.decode_html_entities(text)
+
+
+def remove_extra_whitespace(text: str) -> str:
+    """移除多余的空白字符"""
+    return TextCleaner.remove_extra_whitespace(text)
+
+
+def remove_special_chars(text: str, chars: str = '') -> str:
+    """移除特殊字符"""
+    return TextCleaner.remove_special_chars(text, chars)
+
+
+def normalize_unicode(text: str) -> str:
+    """标准化Unicode字符"""
+    return TextCleaner.normalize_unicode(text)
+
+
+def clean_text(text: str, 
+               remove_html: bool = True,
+               decode_entities: bool = True,
+               remove_whitespace: bool = True,
+               remove_special: bool = False,
+               normalize: bool = True) -> str:
+    """综合文本清洗"""
+    return TextCleaner.clean_text(text, remove_html, decode_entities, remove_whitespace, remove_special, normalize)
+
+
+def extract_numbers(text: str) -> List[str]:
+    """提取数字"""
+    return TextCleaner.extract_numbers(text)
+
+
+def extract_emails(text: str) -> List[str]:
+    """提取邮箱地址"""
+    return TextCleaner.extract_emails(text)
+
+
+def extract_urls(text: str) -> List[str]:
+    """提取URL"""
+    return TextCleaner.extract_urls(text)