crawlo 1.2.8__py3-none-any.whl → 1.3.0__py3-none-any.whl

crawlo/{cleaners → tools}/encoding_converter.py

@@ -1,126 +1,127 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 编码转换工具
-"""
-try:
-    import chardet
-    HAS_CHARDET = True
-except ImportError:
-    HAS_CHARDET = False
-from typing import Optional, Union
-
-
-class EncodingConverter:
-    """
-    编码转换工具类，提供各种编码转换功能。
-    特别适用于爬虫中处理不同编码的网页内容。
-    """
-
-    @staticmethod
-    def detect_encoding(data: Union[str, bytes]) -> Optional[str]:
-        """
-        检测数据编码
-        
-        :param data: 数据（字符串或字节）
-        :return: 检测到的编码
-        """
-        if isinstance(data, str):
-            # 如果是字符串，直接返回
-            return 'utf-8'
-        
-        if not isinstance(data, bytes):
-            return None
-            
-        if HAS_CHARDET:
-            try:
-                # 使用chardet检测编码
-                result = chardet.detect(data)
-                return result['encoding']
-            except Exception:
-                return None
-        else:
-            # 如果没有chardet，返回None
-            return None
-
-    @staticmethod
-    def to_utf8(data: Union[str, bytes], source_encoding: Optional[str] = None) -> Optional[str]:
-        """
-        转换为UTF-8编码的字符串
-        
-        :param data: 数据（字符串或字节）
-        :param source_encoding: 源编码（如果为None则自动检测）
-        :return: UTF-8编码的字符串
-        """
-        if isinstance(data, str):
-            # 如果已经是字符串，假设它已经是UTF-8
-            return data
-            
-        if not isinstance(data, bytes):
-            return None
-            
-        try:
-            if source_encoding is None:
-                # 自动检测编码
-                source_encoding = EncodingConverter.detect_encoding(data)
-                if source_encoding is None:
-                    # 如果检测失败，尝试常见编码
-                    for encoding in ['utf-8', 'gbk', 'gb2312', 'latin1']:
-                        try:
-                            decoded = data.decode(encoding)
-                            return decoded
-                        except UnicodeDecodeError:
-                            continue
-                    return None
-            else:
-                # 使用指定编码
-                return data.decode(source_encoding)
-                
-            # 使用检测到的编码解码
-            return data.decode(source_encoding)
-        except Exception:
-            return None
-
-    @staticmethod
-    def convert_encoding(data: Union[str, bytes], 
-                        source_encoding: Optional[str] = None, 
-                        target_encoding: str = 'utf-8') -> Optional[bytes]:
-        """
-        编码转换
-        
-        :param data: 数据（字符串或字节）
-        :param source_encoding: 源编码（如果为None则自动检测）
-        :param target_encoding: 目标编码
-        :return: 转换后的字节数据
-        """
-        # 先转换为UTF-8字符串
-        utf8_str = EncodingConverter.to_utf8(data, source_encoding)
-        if utf8_str is None:
-            return None
-            
-        try:
-            # 再转换为目标编码
-            return utf8_str.encode(target_encoding)
-        except Exception:
-            return None
-
-
-# =======================对外接口=======================
-
-def detect_encoding(data: Union[str, bytes]) -> Optional[str]:
-    """检测数据编码"""
-    return EncodingConverter.detect_encoding(data)
-
-
-def to_utf8(data: Union[str, bytes], source_encoding: Optional[str] = None) -> Optional[str]:
-    """转换为UTF-8编码的字符串"""
-    return EncodingConverter.to_utf8(data, source_encoding)
-
-
-def convert_encoding(data: Union[str, bytes], 
-                    source_encoding: Optional[str] = None, 
-                    target_encoding: str = 'utf-8') -> Optional[bytes]:
-    """编码转换"""
-    return EncodingConverter.convert_encoding(data, source_encoding, target_encoding)
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-10 22:00
+# @Author  : crawl-coder
+# @Desc    : 编码转换工具
+"""
+try:
+    import chardet
+
+    HAS_CHARDET = True
+except ImportError:
+    HAS_CHARDET = False
+from typing import Optional, Union
+
+
+class EncodingConverter:
+    """
+    编码转换工具类，提供各种编码转换功能。
+    特别适用于爬虫中处理不同编码的网页内容。
+    """
+
+    @staticmethod
+    def detect_encoding(data: Union[str, bytes]) -> Optional[str]:
+        """
+        检测数据编码
+        
+        :param data: 数据（字符串或字节）
+        :return: 检测到的编码
+        """
+        if isinstance(data, str):
+            # 如果是字符串，直接返回
+            return 'utf-8'
+
+        if not isinstance(data, bytes):
+            return None
+
+        if HAS_CHARDET:
+            try:
+                # 使用chardet检测编码
+                result = chardet.detect(data)
+                return result['encoding']
+            except Exception:
+                return None
+        else:
+            # 如果没有chardet，返回None
+            return None
+
+    @staticmethod
+    def to_utf8(data: Union[str, bytes], source_encoding: Optional[str] = None) -> Optional[str]:
+        """
+        转换为UTF-8编码的字符串
+        
+        :param data: 数据（字符串或字节）
+        :param source_encoding: 源编码（如果为None则自动检测）
+        :return: UTF-8编码的字符串
+        """
+        if isinstance(data, str):
+            # 如果已经是字符串，假设它已经是UTF-8
+            return data
+
+        if not isinstance(data, bytes):
+            return None
+
+        try:
+            if source_encoding is None:
+                # 自动检测编码
+                source_encoding = EncodingConverter.detect_encoding(data)
+                if source_encoding is None:
+                    # 如果检测失败，尝试常见编码
+                    for encoding in ['utf-8', 'gbk', 'gb2312', 'latin1']:
+                        try:
+                            decoded = data.decode(encoding)
+                            return decoded
+                        except UnicodeDecodeError:
+                            continue
+                    return None
+            else:
+                # 使用指定编码
+                return data.decode(source_encoding)
+
+            # 使用检测到的编码解码
+            return data.decode(source_encoding)
+        except Exception:
+            return None
+
+    @staticmethod
+    def convert_encoding(data: Union[str, bytes],
+                         source_encoding: Optional[str] = None,
+                         target_encoding: str = 'utf-8') -> Optional[bytes]:
+        """
+        编码转换
+        
+        :param data: 数据（字符串或字节）
+        :param source_encoding: 源编码（如果为None则自动检测）
+        :param target_encoding: 目标编码
+        :return: 转换后的字节数据
+        """
+        # 先转换为UTF-8字符串
+        utf8_str = EncodingConverter.to_utf8(data, source_encoding)
+        if utf8_str is None:
+            return None
+
+        try:
+            # 再转换为目标编码
+            return utf8_str.encode(target_encoding)
+        except Exception:
+            return None
+
+
+# =======================对外接口=======================
+
+def detect_encoding(data: Union[str, bytes]) -> Optional[str]:
+    """检测数据编码"""
+    return EncodingConverter.detect_encoding(data)
+
+
+def to_utf8(data: Union[str, bytes], source_encoding: Optional[str] = None) -> Optional[str]:
+    """转换为UTF-8编码的字符串"""
+    return EncodingConverter.to_utf8(data, source_encoding)
+
+
+def convert_encoding(data: Union[str, bytes],
+                     source_encoding: Optional[str] = None,
+                     target_encoding: str = 'utf-8') -> Optional[bytes]:
+    """编码转换"""
+    return EncodingConverter.convert_encoding(data, source_encoding, target_encoding)

crawlo/tools/request_tools.py

@@ -0,0 +1,83 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-20 10:00
+# @Author  : crawl-coder
+# @Desc    : 请求处理工具
+"""
+from typing import Dict, Any, Optional
+from urllib.parse import urlparse, parse_qs, urlencode, urlunparse
+from copy import deepcopy
+
+
+def build_url(base_url: str, path: str = "", query_params: Optional[Dict[str, Any]] = None) -> str:
+    """
+    构建完整的URL
+    
+    :param base_url: 基础URL
+    :param path: 路径
+    :param query_params: 查询参数字典
+    :return: 完整的URL
+    """
+    # 解析基础URL
+    parsed = urlparse(base_url)
+    
+    # 合并路径
+    if path:
+        # 确保路径以/开头
+        if not path.startswith('/'):
+            path = '/' + path
+        # 合并路径
+        new_path = parsed.path.rstrip('/') + path
+    else:
+        new_path = parsed.path
+    
+    # 处理查询参数
+    if query_params:
+        # 解析现有查询参数
+        existing_params = parse_qs(parsed.query)
+        # 合并新参数
+        for key, value in query_params.items():
+            existing_params[key] = [str(value)]
+        # 编码查询参数
+        new_query = urlencode(existing_params, doseq=True)
+    else:
+        new_query = parsed.query
+    
+    # 构建新的URL
+    new_parsed = parsed._replace(path=new_path, query=new_query)
+    return urlunparse(new_parsed)
+
+
+def add_query_params(url: str, params: Dict[str, Any]) -> str:
+    """
+    向URL添加查询参数
+    
+    :param url: 原始URL
+    :param params: 要添加的参数字典
+    :return: 添加参数后的URL
+    """
+    parsed = urlparse(url)
+    # 解析现有查询参数
+    existing_params = parse_qs(parsed.query)
+    # 添加新参数
+    for key, value in params.items():
+        existing_params[key] = [str(value)]
+    # 编码查询参数
+    new_query = urlencode(existing_params, doseq=True)
+    # 构建新的URL
+    new_parsed = parsed._replace(query=new_query)
+    return urlunparse(new_parsed)
+
+
+def merge_headers(base_headers: Dict[str, str], additional_headers: Dict[str, str]) -> Dict[str, str]:
+    """
+    合并请求头
+    
+    :param base_headers: 基础请求头
+    :param additional_headers: 要添加的请求头
+    :return: 合并后的请求头
+    """
+    merged = deepcopy(base_headers)
+    merged.update(additional_headers)
+    return merged