crawlo 1.1.0__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/utils/request_serializer.py

@@ -0,0 +1,220 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Request 序列化工具类
+负责处理 Request 对象的序列化前清理工作，解决 logger 等不可序列化对象的问题
+"""
+import logging
+import pickle
+import gc
+from typing import Any, Dict
+
+from crawlo.utils.log import get_logger
+
+
+class RequestSerializer:
+    """Request 序列化工具类"""
+    
+    def __init__(self):
+        self.logger = get_logger(self.__class__.__name__)
+    
+    def prepare_for_serialization(self, request):
+        """
+        为序列化准备 Request 对象
+        移除不可序列化的属性，保存必要信息用于恢复
+        """
+        try:
+            # 处理 callback
+            self._handle_callback(request)
+            
+            # 清理 meta 中的 logger
+            if hasattr(request, 'meta') and request.meta:
+                self._clean_dict_recursive(request.meta)
+            
+            # 清理 cb_kwargs 中的 logger
+            if hasattr(request, 'cb_kwargs') and request.cb_kwargs:
+                self._clean_dict_recursive(request.cb_kwargs)
+            
+            # 清理其他可能的 logger 引用
+            for attr_name in ['headers', 'cookies']:
+                if hasattr(request, attr_name):
+                    attr_value = getattr(request, attr_name)
+                    if isinstance(attr_value, dict):
+                        self._clean_dict_recursive(attr_value)
+            
+            # 最终验证
+            if not self._test_serialization(request):
+                self.logger.warning("⚠️ 常规清理无效，使用深度清理")
+                request = self._deep_clean_request(request)
+                
+            return request
+            
+        except Exception as e:
+            self.logger.error(f"❌ Request 序列化准备失败: {e}")
+            # 最后的保险：重建 Request
+            return self._rebuild_clean_request(request)
+    
+    def restore_after_deserialization(self, request, spider=None):
+        """
+        反序列化后恢复 Request 对象
+        恢复 callback 等必要信息
+        """
+        if not request:
+            return request
+            
+        # 恢复 callback
+        if hasattr(request, 'meta') and '_callback_info' in request.meta:
+            callback_info = request.meta.pop('_callback_info')
+            
+            if spider:
+                spider_class_name = callback_info.get('spider_class')
+                method_name = callback_info.get('method_name')
+                
+                if (spider.__class__.__name__ == spider_class_name and 
+                    hasattr(spider, method_name)):
+                    request.callback = getattr(spider, method_name)
+                    
+                    # 确保 spider 有有效的 logger
+                    if not hasattr(spider, 'logger') or spider.logger is None:
+                        spider.logger = get_logger(spider.name or spider.__class__.__name__)
+        
+        return request
+    
+    def _handle_callback(self, request):
+        """处理 callback 相关的清理"""
+        if hasattr(request, 'callback') and request.callback is not None:
+            callback = request.callback
+            
+            # 如果是绑定方法，保存信息并移除引用
+            if hasattr(callback, '__self__') and hasattr(callback, '__name__'):
+                spider_instance = callback.__self__
+                
+                # 保存 callback 信息
+                if not hasattr(request, 'meta') or request.meta is None:
+                    request.meta = {}
+                request.meta['_callback_info'] = {
+                    'spider_class': spider_instance.__class__.__name__,
+                    'method_name': callback.__name__
+                }
+                
+                # 移除 callback 引用
+                request.callback = None
+    
+    def _clean_dict_recursive(self, data, depth=0):
+        """递归清理字典中的 logger"""
+        if depth > 5 or not isinstance(data, dict):
+            return
+        
+        keys_to_remove = []
+        for key, value in list(data.items()):
+            if isinstance(value, logging.Logger):
+                keys_to_remove.append(key)
+            elif isinstance(key, str) and 'logger' in key.lower():
+                keys_to_remove.append(key)
+            elif isinstance(value, dict):
+                self._clean_dict_recursive(value, depth + 1)
+            elif isinstance(value, (list, tuple)):
+                for item in value:
+                    if isinstance(item, dict):
+                        self._clean_dict_recursive(item, depth + 1)
+        
+        for key in keys_to_remove:
+            data.pop(key, None)
+    
+    def _test_serialization(self, request):
+        """测试是否可以序列化"""
+        try:
+            pickle.dumps(request)
+            return True
+        except Exception:
+            return False
+    
+    def _deep_clean_request(self, request):
+        """深度清理 Request 对象"""
+        import logging
+        
+        def recursive_clean(target, visited=None, depth=0):
+            if depth > 5 or not target:
+                return
+            if visited is None:
+                visited = set()
+                
+            obj_id = id(target)
+            if obj_id in visited:
+                return
+            visited.add(obj_id)
+            
+            # 处理对象属性
+            if hasattr(target, '__dict__'):
+                attrs_to_clean = []
+                for attr_name, attr_value in list(target.__dict__.items()):
+                    if isinstance(attr_value, logging.Logger):
+                        attrs_to_clean.append(attr_name)
+                    elif isinstance(attr_name, str) and 'logger' in attr_name.lower():
+                        attrs_to_clean.append(attr_name)
+                    elif hasattr(attr_value, '__dict__'):
+                        recursive_clean(attr_value, visited, depth + 1)
+                
+                for attr_name in attrs_to_clean:
+                    try:
+                        setattr(target, attr_name, None)
+                    except (AttributeError, TypeError):
+                        pass
+            
+            # 处理字典
+            elif isinstance(target, dict):
+                self._clean_dict_recursive(target, depth)
+        
+        recursive_clean(request)
+        gc.collect()
+        return request
+    
+    def _rebuild_clean_request(self, original_request):
+        """重建一个干净的 Request 对象"""
+        from crawlo.network.request import Request
+        
+        try:
+            # 提取安全的属性
+            safe_meta = {}
+            if hasattr(original_request, 'meta') and original_request.meta:
+                for key, value in original_request.meta.items():
+                    if not isinstance(value, logging.Logger):
+                        try:
+                            pickle.dumps(value)
+                            safe_meta[key] = value
+                        except Exception:
+                            try:
+                                safe_meta[key] = str(value)
+                            except Exception:
+                                continue
+            
+            # 安全地获取其他属性
+            safe_headers = {}
+            if hasattr(original_request, 'headers') and original_request.headers:
+                for k, v in original_request.headers.items():
+                    try:
+                        safe_headers[str(k)] = str(v)
+                    except Exception:
+                        continue
+            
+            # 创建干净的 Request
+            clean_request = Request(
+                url=str(original_request.url),
+                method=getattr(original_request, 'method', 'GET'),
+                headers=safe_headers,
+                meta=safe_meta,
+                priority=-getattr(original_request, 'priority', 0),
+                dont_filter=getattr(original_request, 'dont_filter', False),
+                timeout=getattr(original_request, 'timeout', None),
+                encoding=getattr(original_request, 'encoding', 'utf-8')
+            )
+            
+            # 验证新 Request 可以序列化
+            pickle.dumps(clean_request)
+            return clean_request
+            
+        except Exception as e:
+            self.logger.error(f"❌ 重建 Request 失败: {e}")
+            # 最简单的 fallback
+            from crawlo.network.request import Request
+            return Request(url=str(original_request.url))

crawlo/utils/spider_loader.py

@@ -1,63 +1,63 @@
-import importlib
-import inspect
-from pathlib import Path
-from typing import List, Type, Optional, Dict
-
-from crawlo.spider import Spider
-from crawlo.utils.log import get_logger
-
-logger = get_logger(__name__)
-
-
-class SpiderLoader:
-    """爬虫加载器，负责发现和加载爬虫"""
-
-    def __init__(self, project_package: str):
-        self.project_package = project_package
-        self._spiders: Dict[str, Type[Spider]] = {}
-        self._load_spiders()
-
-    def _load_spiders(self):
-        """加载所有爬虫"""
-        spiders_dir = Path.cwd() / self.project_package / 'spiders'
-        if not spiders_dir.exists():
-            logger.warning(f"Spiders directory not found: {spiders_dir}")
-            return
-
-        for py_file in spiders_dir.glob("*.py"):
-            if py_file.name.startswith('_'):
-                continue
-
-            module_name = py_file.stem
-            spider_module_path = f"{self.project_package}.spiders.{module_name}"
-
-            try:
-                module = importlib.import_module(spider_module_path)
-            except ImportError as e:
-                logger.debug(f"Skip module {module_name}: {e}")
-                continue
-
-            # 查找所有 Spider 子类
-            for attr_name in dir(module):
-                attr_value = getattr(module, attr_name)
-                if (isinstance(attr_value, type) and
-                        issubclass(attr_value, Spider) and
-                        attr_value != Spider and
-                        hasattr(attr_value, 'name')):
-
-                    spider_name = getattr(attr_value, 'name')
-                    if spider_name in self._spiders:
-                        logger.warning(f"Duplicate spider name '{spider_name}' found")
-                    self._spiders[spider_name] = attr_value
-
-    def load(self, spider_name: str) -> Optional[Type[Spider]]:
-        """通过 name 加载爬虫"""
-        return self._spiders.get(spider_name)
-
-    def list(self) -> List[str]:
-        """列出所有可用的爬虫名称"""
-        return list(self._spiders.keys())
-
-    def get_all(self) -> Dict[str, Type[Spider]]:
-        """获取所有爬虫"""
+import importlib
+import inspect
+from pathlib import Path
+from typing import List, Type, Optional, Dict
+
+from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+
+logger = get_logger(__name__)
+
+
+class SpiderLoader:
+    """爬虫加载器，负责发现和加载爬虫"""
+
+    def __init__(self, project_package: str):
+        self.project_package = project_package
+        self._spiders: Dict[str, Type[Spider]] = {}
+        self._load_spiders()
+
+    def _load_spiders(self):
+        """加载所有爬虫"""
+        spiders_dir = Path.cwd() / self.project_package / 'spiders'
+        if not spiders_dir.exists():
+            logger.warning(f"Spiders directory not found: {spiders_dir}")
+            return
+
+        for py_file in spiders_dir.glob("*.py"):
+            if py_file.name.startswith('_'):
+                continue
+
+            module_name = py_file.stem
+            spider_module_path = f"{self.project_package}.spiders.{module_name}"
+
+            try:
+                module = importlib.import_module(spider_module_path)
+            except ImportError as e:
+                logger.debug(f"Skip module {module_name}: {e}")
+                continue
+
+            # 查找所有 Spider 子类
+            for attr_name in dir(module):
+                attr_value = getattr(module, attr_name)
+                if (isinstance(attr_value, type) and
+                        issubclass(attr_value, Spider) and
+                        attr_value != Spider and
+                        hasattr(attr_value, 'name')):
+
+                    spider_name = getattr(attr_value, 'name')
+                    if spider_name in self._spiders:
+                        logger.warning(f"Duplicate spider name '{spider_name}' found")
+                    self._spiders[spider_name] = attr_value
+
+    def load(self, spider_name: str) -> Optional[Type[Spider]]:
+        """通过 name 加载爬虫"""
+        return self._spiders.get(spider_name)
+
+    def list(self) -> List[str]:
+        """列出所有可用的爬虫名称"""
+        return list(self._spiders.keys())
+
+    def get_all(self) -> Dict[str, Type[Spider]]:
+        """获取所有爬虫"""
         return self._spiders.copy()

crawlo/utils/system.py

@@ -1,11 +1,11 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import platform
-
-system_name = platform.system().lower()
-if system_name == 'windows':
-    import asyncio
-    asyncio.set_event_loop_policy(
-        asyncio.WindowsSelectorEventLoopPolicy()
-    )
-
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import platform
+
+system_name = platform.system().lower()
+if system_name == 'windows':
+    import asyncio
+    asyncio.set_event_loop_policy(
+        asyncio.WindowsSelectorEventLoopPolicy()
+    )
+

crawlo/utils/tools.py

@@ -1,5 +1,5 @@
-def custom_extractor_proxy(data: dict, key: str='proxy') -> dict | str | None:
-    """只负责从 API 返回数据中提取代理部分"""
-    if data.get("status") == 0:
-        return data.get(key)  # 返回 {"http": "...", "https": "..."} 整个字典
+def custom_extractor_proxy(data: dict, key: str='proxy') -> dict | str | None:
+    """只负责从 API 返回数据中提取代理部分"""
+    if data.get("status") == 0:
+        return data.get(key)  # 返回 {"http": "...", "https": "..."} 整个字典
     return None

crawlo/utils/url.py

@@ -1,40 +1,40 @@
-from urllib.parse import urldefrag
-from w3lib.url import add_or_replace_parameter
-
-
-def escape_ajax(url: str) -> str:
-    """
-    根据Google AJAX爬取规范转换URL（处理哈希片段#!）：
-    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
-
-    规则说明：
-    1. 仅当URL包含 `#!` 时才转换（表示这是AJAX可爬取页面）
-    2. 将 `#!key=value` 转换为 `?_escaped_fragment_=key%3Dvalue`
-    3. 保留原始查询参数（如果有）
-
-    示例：
-    >>> escape_ajax("www.example.com/ajax.html#!key=value")
-    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html?k1=v1#!key=value")
-    'www.example.com/ajax.html?k1=v1&_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html#!")
-    'www.example.com/ajax.html?_escaped_fragment_='
-
-    非AJAX可爬取的URL（无#!）原样返回：
-    >>> escape_ajax("www.example.com/ajax.html#normal")
-    'www.example.com/ajax.html#normal'
-    """
-    # 分离URL的基础部分和哈希片段
-    de_frag, frag = urldefrag(url)
-
-    # 仅处理以"!"开头的哈希片段（Google规范）
-    if not frag.startswith("!"):
-        return url  # 不符合规则则原样返回
-
-    # 调用辅助函数添加 `_escaped_fragment_` 参数
-    return add_or_replace_parameter(de_frag, "_escaped_fragment_", frag[1:])
-
-
-if __name__ == '__main__':
-    f = escape_ajax('http://example.com/page#!')
+from urllib.parse import urldefrag
+from w3lib.url import add_or_replace_parameter
+
+
+def escape_ajax(url: str) -> str:
+    """
+    根据Google AJAX爬取规范转换URL（处理哈希片段#!）：
+    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+
+    规则说明：
+    1. 仅当URL包含 `#!` 时才转换（表示这是AJAX可爬取页面）
+    2. 将 `#!key=value` 转换为 `?_escaped_fragment_=key%3Dvalue`
+    3. 保留原始查询参数（如果有）
+
+    示例：
+    >>> escape_ajax("www.example.com/ajax.html#!key=value")
+    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html?k1=v1#!key=value")
+    'www.example.com/ajax.html?k1=v1&_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html#!")
+    'www.example.com/ajax.html?_escaped_fragment_='
+
+    非AJAX可爬取的URL（无#!）原样返回：
+    >>> escape_ajax("www.example.com/ajax.html#normal")
+    'www.example.com/ajax.html#normal'
+    """
+    # 分离URL的基础部分和哈希片段
+    de_frag, frag = urldefrag(url)
+
+    # 仅处理以"!"开头的哈希片段（Google规范）
+    if not frag.startswith("!"):
+        return url  # 不符合规则则原样返回
+
+    # 调用辅助函数添加 `_escaped_fragment_` 参数
+    return add_or_replace_parameter(de_frag, "_escaped_fragment_", frag[1:])
+
+
+if __name__ == '__main__':
+    f = escape_ajax('http://example.com/page#!')
     print(f)