crawlo 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

crawlo/network/request.py

@@ -1,52 +1,234 @@
 #!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import hashlib
+# -*- coding: UTF-8 -*-
+import json
 from copy import deepcopy
-from typing import Dict, Optional, Callable
+from urllib.parse import urlencode
+from w3lib.url import safe_url_string
+from typing import Dict, Optional, Callable, Union, Any, TypeVar, List
 
+from crawlo.utils.url import escape_ajax
 
-class Request(object):
+_Request = TypeVar("_Request", bound="Request")
 
-    def __init__(
-            self,
-            url: str,
-            *,
-            callback: Optional[Callable] = None,
-            headers: Optional[Dict[str, str]] = None,
-            body: Optional[bytes] = None,
-            method: Optional[str] = 'GET',
-            cookies: Optional[Dict[str, str]] = None,
-            priority: int = 0,
-            encoding: Optional[str] = 'UTF-8',
-            meta: Optional[Dict[str, str]] = None
 
+class RequestPriority:
+    HIGH = -100
+    NORMAL = 0
+    LOW = 100
+
+
+class Request:
+    """
+    封装一个 HTTP 请求对象，用于爬虫框架中表示一个待抓取的请求任务。
+    支持设置回调函数、请求头、请求体、优先级、元数据等。
+    """
+
+    __slots__ = (
+        '_url',
+        '_meta',
+        'callback',
+        'cb_kwargs',
+        'err_back',
+        'headers',
+        'body',
+        'method',
+        'cookies',
+        'priority',
+        'encoding',
+        'dont_filter',
+        'timeout',
+        'proxy',
+        'allow_redirects',
+        'auth',
+        'verify',
+        'flags'
+    )
+
+    def __init__(
+        self,
+        url: str,
+        callback: Optional[Callable] = None,
+        method: Optional[str] = 'GET',
+        headers: Optional[Dict[str, str]] = None,
+        body: Optional[Union[Dict, bytes, str]] = None,
+        form_data: Optional[Dict] = None,
+        json_body: Optional[Dict] = None,   # ✅ 参数名从 json 改为 json_body
+        cb_kwargs: Optional[Dict[str, Any]] = None,
+        err_back: Optional[Callable] = None,
+        cookies: Optional[Dict[str, str]] = None,
+        meta: Optional[Dict[str, Any]] = None,
+        priority: int = RequestPriority.NORMAL,
+        dont_filter: bool = False,
+        timeout: Optional[float] = None,
+        proxy: Optional[str] = None,
+        allow_redirects: bool = True,
+        auth: Optional[tuple] = None,
+        verify: bool = True,
+        flags: Optional[List[str]] = None,
+        encoding: str = 'utf-8'
     ):
-        self.url = url
+        """
+        初始化请求对象。
+
+        参数说明:
+        :param url: 请求的 URL 地址（必须）
+        :param callback: 响应处理回调函数（可选）
+        :param method: HTTP 请求方法，默认为 GET
+        :param headers: 请求头（可选）
+        :param body: 请求体（可为 dict、bytes 或 str）
+        :param form_data 表单数据，自动设置为 POST 并构造 x-www-form-urlencoded 请求体
+        :param json_body: 用于构造 JSON 请求体，自动设置 Content-Type 为 application/json
+        :param cb_kwargs: 传递给回调函数的额外参数（可选）
+        :param err_back: 请求失败时的错误回调函数（可选）
+        :param cookies: 请求 cookies（可选）
+        :param meta: 元数据字典，用于在请求间传递数据
+        :param priority: 请求优先级，数值越小优先级越高（默认为 0）
+        :param dont_filter: 是否跳过去重过滤（默认为 False）
+        :param timeout: 请求超时时间（秒）
+        :param proxy: 代理地址（如：http://127.0.0.1:8080）
+        :param allow_redirects: 是否允许重定向（默认为 True）
+        :param auth: 认证信息，格式为 (username, password)
+        :param verify: 是否验证 SSL 证书（默认为 True）
+        :param flags: 请求标记（调试、重试等用途）
+        """
         self.callback = callback
-        self.headers = headers if headers else {}
+        self.method = str(method).upper()
+        self.headers = headers or {}
         self.body = body
-        self.method = method
-        self.cookies = cookies
-        self.priority = priority
+        self.cb_kwargs = cb_kwargs or {}
+        self.err_back = err_back
+        self.cookies = cookies or {}
+        self.priority = -priority  # 高优先级值更小，便于排序
+        self._meta = deepcopy(meta) if meta is not None else {}
+        self.timeout = self._meta.get('download_timeout', timeout)
+        self.proxy = proxy
+        self.allow_redirects = allow_redirects
+        self.auth = auth
+        self.verify = verify
+        self.flags = flags or []
+
+        # 默认编码
         self.encoding = encoding
-        self._meta = meta if meta is not None else {}
 
-    def copy(self):
-        return deepcopy(self)
+        # 优先使用 json_body 参数
+        if json_body is not None:
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/json'
+            self.body = json.dumps(json_body, ensure_ascii=False).encode(self.encoding)
+            if self.method == 'GET':
+                self.method = 'POST'
+
+        # 其次使用 form_data
+        elif form_data is not None:
+            if self.method == 'GET':
+                self.method = 'POST'
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+            self.body = urlencode(form_data)
+
+        # 最后处理 body 为 dict 的情况
+        elif isinstance(self.body, dict):
+            if 'Content-Type' not in self.headers:
+                self.headers['Content-Type'] = 'application/json'
+            self.body = json.dumps(self.body, ensure_ascii=False).encode(self.encoding)
+
+        self.dont_filter = dont_filter
+        self._set_url(url)
+
+    def copy(self: _Request) -> _Request:
+        """
+        创建当前 Request 的副本，用于避免引用共享数据。
+
+        :return: 一个新的 Request 实例
+        """
+        return type(self)(
+            url=self.url,
+            callback=self.callback,
+            method=self.method,
+            headers=self.headers.copy(),
+            body=self.body,
+            form_data=None,  # form_data 不参与复制
+            json_body=None,   # json_body 参数也不参与复制
+            cb_kwargs=deepcopy(self.cb_kwargs),
+            err_back=self.err_back,
+            cookies=self.cookies.copy(),
+            meta=deepcopy(self._meta),
+            priority=-self.priority,
+            dont_filter=self.dont_filter,
+            timeout=self.timeout,
+            proxy=self.proxy,
+            allow_redirects=self.allow_redirects,
+            auth=self.auth,
+            verify=self.verify,
+            flags=self.flags.copy(),
+        )
 
-    def fingerprint(self) -> str:
-        data = f"{self.url}{self.method}{self.body or b''}".encode()
-        return hashlib.sha256(data).hexdigest()
+    def set_meta(self, key: str, value: Any) -> None:
+        """
+        设置 meta 中的某个键值对。
 
-    def set_meta(self, key: str, value: str):
+        :param key: 要设置的键
+        :param value: 对应的值
+        """
         self._meta[key] = value
 
+    def _set_url(self, url: str) -> None:
+        """
+        设置并验证 URL，确保其格式正确且包含 scheme。
+
+        :param url: 原始 URL 字符串
+        :raises TypeError: 如果传入的不是字符串
+        :raises ValueError: 如果 URL 没有 scheme
+        """
+        if not isinstance(url, str):
+            raise TypeError(f"Request url 必须为字符串类型，当前类型为 {type(url).__name__}")
+
+        s = safe_url_string(url, self.encoding)
+        escaped_url = escape_ajax(s)
+        self._url = escaped_url
+
+        if not self._url.startswith(('http://', 'https://', 'about:', '')):
+            raise ValueError(f"请求 URL 缺少 scheme（如 http://）: {self._url}")
+
+    @property
+    def url(self) -> str:
+        """
+        获取请求的 URL。
+
+        :return: 当前请求的 URL 字符串
+        """
+        return self._url
+
     @property
-    def meta(self):
+    def meta(self) -> Dict[str, Any]:
+        """
+        获取请求的元数据。
+
+        :return: 元数据字典
+        """
         return self._meta
 
-    def __str__(self):
-        return f'<Request url={self.url}> method={self.method}>'
+    def __str__(self) -> str:
+        """
+        返回对象的字符串表示，用于调试和日志输出。
+
+        :return: 字符串 <Request url=... method=...>
+        """
+        return f'<Request url={self.url} method={self.method}>'
+
+    def __repr__(self) -> str:
+        """
+        返回对象的官方字符串表示。
+
+        :return: 字符串，与 __str__ 相同
+        """
+        return str(self)
+
+    def __lt__(self, other: _Request) -> bool:
+        """
+        比较两个请求的优先级，用于排序。
 
-    def __lt__(self, other):
-        return self.priority < other.priority
+        :param other: 另一个 Request 对象
+        :return: 如果当前请求优先级更高（数值更小）返回 True
+        """
+        return self.priority < other.priority

crawlo/network/response.py

@@ -2,8 +2,8 @@
 # -*- coding:UTF-8 -*-
 import re
 import ujson
-from typing import Dict
-from parsel import Selector
+from typing import Dict, Any, List, Optional
+from parsel import Selector, SelectorList
 from http.cookies import SimpleCookie
 from urllib.parse import urljoin as _urljoin
 
@@ -11,17 +11,20 @@ from crawlo import Request
 from crawlo.exceptions import DecodeError
 
 
-class Response(object):
+class Response:
+    """
+    HTTP响应的封装，提供数据解析的便捷方法。
+    """
 
     def __init__(
-            self,
-            url: str,
-            *,
-            headers: Dict,
-            body: bytes = b"",
-            method: str = 'GET',
-            request: Request = None,
-            status_code: int = 200,
+        self,
+        url: str,
+        *,
+        headers: Dict[str, Any],
+        body: bytes = b"",
+        method: str = 'GET',
+        request: Request = None,
+        status_code: int = 200,
     ):
         self.url = url
         self.headers = headers
@@ -29,65 +32,131 @@ class Response(object):
         self.method = method
         self.request = request
         self.status_code = status_code
-        self.encoding = request.encoding
-        self._selector = None
+        self.encoding = self.request.encoding if self.request else None
         self._text_cache = None
+        self._selector_instance = None  # 修改变量名，避免与 @property 冲突
 
     @property
-    def text(self):
-        # 请求缓存
-        if self._text_cache:
+    def text(self) -> str:
+        """将响应体(body)以正确的编码解码为字符串，并缓存结果。"""
+        if self._text_cache is not None:
             return self._text_cache
+
+        encoding = self.encoding
         try:
-            self._text_cache = self.body.decode(self.encoding)
-        except UnicodeDecodeError:
-            try:
-                _encoding_re = re.compile(r"charset=([\w-]+)", flags=re.I)
-                _encoding_string = self.headers.get('Content-Type', '') or self.headers.get('content-type', '')
-                _encoding = _encoding_re.search(_encoding_string)
-                if _encoding:
-                    _encoding = _encoding.group(1)
-                    self._text_cache = self.body.decode(_encoding)
-                else:
-                    raise DecodeError(f"{self.request} {self.request.encoding} error.")
-            except UnicodeDecodeError as exp:
-                raise UnicodeDecodeError(
-                    exp.encoding, exp.object, exp.start, exp.end, f"{self.request} error."
-                )
-        return self._text_cache
-
-    def json(self):
+            # 优先使用 request 提供的编码
+            if encoding:
+                self._text_cache = self.body.decode(encoding)
+                return self._text_cache
+
+            # 从 Content-Type 中提取编码
+            content_type = self.headers.get("Content-Type", "")
+            charset_match = re.search(r"charset=([\w-]+)", content_type, re.I)
+            if charset_match:
+                encoding = charset_match.group(1)
+                self._text_cache = self.body.decode(encoding)
+                return self._text_cache
+
+            # 默认尝试 UTF-8
+            self._text_cache = self.body.decode("utf-8")
+            return self._text_cache
+
+        except UnicodeDecodeError as e:
+            raise DecodeError(f"Failed to decode response from {self.url}: {e}")
+
+    def json(self) -> Any:
+        """将响应文本解析为 JSON 对象。"""
         return ujson.loads(self.text)
 
-    def urljoin(self, url):
+    def urljoin(self, url: str) -> str:
+        """拼接 URL，自动处理相对路径。"""
         return _urljoin(self.url, url)
 
-    def xpath(self, xpath_str):
-        if self._selector is None:
-            self._selector = Selector(self.text)
-        return self._selector.xpath(xpath_str)
+    @property
+    def _selector(self) -> Selector:
+        """懒加载 Selector 实例"""
+        if self._selector_instance is None:
+            self._selector_instance = Selector(self.text)
+        return self._selector_instance
+
+    def xpath(self, query: str) -> SelectorList:
+        """使用 XPath 选择器查询文档。"""
+        return self._selector.xpath(query)
+
+    def css(self, query: str) -> SelectorList:
+        """使用 CSS 选择器查询文档。"""
+        return self._selector.css(query)
+
+    def xpath_text(self, query: str) -> str:
+        """使用 XPath 提取并返回纯文本。"""
+        fragments = self.xpath(f"{query}//text()").getall()
+        return " ".join(text.strip() for text in fragments if text.strip())
+
+    def css_text(self, query: str) -> str:
+        """使用 CSS 选择器提取并返回纯文本。"""
+        fragments = self.css(f"{query} ::text").getall()
+        return " ".join(text.strip() for text in fragments if text.strip())
+
+    def get_text(self, xpath_or_css: str, join_str: str = " ") -> str:
+        """
+        获取指定节点的纯文本(自动拼接子节点文本)
+
+        参数:
+            xpath_or_css: XPath或CSS选择器
+            join_str: 文本拼接分隔符(默认为空格)
+
+        返回:
+            拼接后的纯文本字符串
+        """
+        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
+        texts = elements.xpath('.//text()').getall()
+        return join_str.join(text.strip() for text in texts if text.strip())
+
+    def get_all_text(self, xpath_or_css: str, join_str: str = " ") -> List[str]:
+        """
+        获取多个节点的纯文本列表
+
+        参数:
+            xpath_or_css: XPath或CSS选择器
+            join_str: 单个节点内文本拼接分隔符
 
-    def css(self, css_str):
-        if self._selector is None:
-            self._selector = Selector(self.text)
-        return self._selector.css(css_str)
+        返回:
+            纯文本列表(每个元素对应一个节点的文本)
+        """
+        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
+        result = []
+        for element in elements:
+            texts = element.xpath('.//text()').getall()
+            clean_text = join_str.join(text.strip() for text in texts if text.strip())
+            if clean_text:
+                result.append(clean_text)
+        return result
 
-    def re_search(self, pattern, flags=re.DOTALL):
+    def re_search(self, pattern: str, flags: int = re.DOTALL) -> Optional[re.Match]:
+        """在响应文本上执行正则表达式搜索。"""
+        if not isinstance(pattern, str):
+            raise TypeError("Pattern must be a string")
         return re.search(pattern, self.text, flags=flags)
 
-    def re_findall(self, pattern, flags=re.DOTALL):
+    def re_findall(self, pattern: str, flags: int = re.DOTALL) -> List[Any]:
+        """在响应文本上执行正则表达式查找。"""
+        if not isinstance(pattern, str):
+            raise TypeError("Pattern must be a string")
         return re.findall(pattern, self.text, flags=flags)
 
-    def get_cookies(self):
-        cookie_headers = self.headers.getlist('Set-Cookie') or []
+    def get_cookies(self) -> Dict[str, str]:
+        """从响应头中解析并返回Cookies。"""
+        cookie_header = self.headers.get("Set-Cookie", "")
+        if isinstance(cookie_header, list):
+            cookie_header = ", ".join(cookie_header)
         cookies = SimpleCookie()
-        for header in cookie_headers:
-            cookies.load(header)
-        return {k: v.value for k, v in cookies.items()}
+        cookies.load(cookie_header)
+        return {key: morsel.value for key, morsel in cookies.items()}
 
     @property
-    def meta(self):
-        return self.request.meta
+    def meta(self) -> Dict:
+        """获取关联的 Request 对象的 meta 字典。"""
+        return self.request.meta if self.request else {}
 
     def __str__(self):
-        return f"{self.url} {self.status_code} {self.request.encoding} "
+        return f"<{self.status_code} {self.url}>"

crawlo/pipelines/console_pipeline.py

@@ -1,20 +1,40 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+from typing import Dict, Any
+
 from crawlo import Item
 from crawlo.spider import Spider
 from crawlo.utils.log import get_logger
 
 
-class DebugPipeline:
+class ConsolePipeline:
+    """将Item内容输出到控制台的管道"""
 
-    def __init__(self, logger):
-        self.logger = logger
+    def __init__(self, log_level: str = "DEBUG"):
+        self.logger = get_logger(self.__class__.__name__, log_level)
 
     @classmethod
-    def create_instance(cls, crawler):
-        logger = get_logger(cls.__name__, crawler.settings.get('LOG_LEVEL'))
-        return cls(logger)
+    def from_crawler(cls, crawler):
+        """从crawler实例创建管道"""
+        return cls(
+            log_level=crawler.settings.get('LOG_LEVEL', 'DEBUG')
+        )
 
     async def process_item(self, item: Item, spider: Spider) -> Item:
-        self.logger.debug(item.to_dict())
-        return item
+        """处理Item并输出到日志"""
+        try:
+            item_dict = self._convert_to_serializable(item)
+            self.logger.info(f"Item processed: {item_dict}")
+            return item
+        except Exception as e:
+            self.logger.error(f"Error processing item: {e}", exc_info=True)
+            raise
+
+    @staticmethod
+    def _convert_to_serializable(item: Item) -> Dict[str, Any]:
+        """将Item转换为可序列化的字典"""
+        try:
+            return item.to_dict()
+        except AttributeError:
+            # 兼容没有to_dict方法的Item实现
+            return dict(item)

crawlo/pipelines/mongo_pipeline.py

@@ -1,5 +1,117 @@
 # -*- coding: utf-8 -*-
+from typing import Optional
+from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo.errors import PyMongoError
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import ItemDiscard
 
 
-class MongoPipeline(object):
-    pass
+class MongoPipeline:
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+
+        # 初始化连接参数
+        self.client = None
+        self.db = None
+        self.collection = None
+
+        # 配置默认值
+        self.mongo_uri = self.settings.get('MONGO_URI', 'mongodb://localhost:27017')
+        self.db_name = self.settings.get('MONGO_DATABASE', 'scrapy_db')
+        self.collection_name = self.settings.get('MONGO_COLLECTION', crawler.spider.name)
+
+        # 注册关闭事件
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    async def _ensure_connection(self):
+        """确保连接已建立"""
+        if self.client is None:
+            self.client = AsyncIOMotorClient(self.mongo_uri)
+            self.db = self.client[self.db_name]
+            self.collection = self.db[self.collection_name]
+            self.logger.info(f"MongoDB连接建立 (集合: {self.collection_name})")
+
+    async def process_item(self, item, spider) -> Optional[dict]:
+        """处理item的核心方法"""
+        try:
+            await self._ensure_connection()
+
+            item_dict = dict(item)
+            result = await self.collection.insert_one(item_dict)
+
+            # 统计计数
+            self.crawler.stats.inc_value('mongodb/inserted')
+            self.logger.debug(f"插入文档ID: {result.inserted_id}")
+
+            return item
+
+        except Exception as e:
+            self.crawler.stats.inc_value('mongodb/failed')
+            self.logger.error(f"MongoDB插入失败: {e}")
+            raise ItemDiscard(f"MongoDB操作失败: {e}")
+
+    async def spider_closed(self):
+        """关闭爬虫时清理资源"""
+        if self.client:
+            self.client.close()
+            self.logger.info("MongoDB连接已关闭")
+
+
+class MongoPoolPipeline:
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+
+        # 连接池配置
+        self.client = AsyncIOMotorClient(
+            self.settings.get('MONGO_URI', 'mongodb://localhost:27017'),
+            maxPoolSize=self.settings.getint('MONGO_MAX_POOL_SIZE', 100),
+            minPoolSize=self.settings.getint('MONGO_MIN_POOL_SIZE', 10),
+            connectTimeoutMS=5000,
+            socketTimeoutMS=30000
+        )
+
+        self.db = self.client[self.settings.get('MONGO_DATABASE', 'scrapy_db')]
+        self.collection = self.db[self.settings.get('MONGO_COLLECTION', crawler.spider.name)]
+
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+        self.logger.info(f"MongoDB连接池已初始化 (集合: {self.collection.name})")
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls(crawler)
+
+    async def process_item(self, item, spider) -> Optional[dict]:
+        """处理item方法（带重试机制）"""
+        try:
+            item_dict = dict(item)
+
+            # 带重试的插入操作
+            for attempt in range(3):
+                try:
+                    result = await self.collection.insert_one(item_dict)
+                    self.crawler.stats.inc_value('mongodb/insert_success')
+                    self.logger.debug(f"插入成功 [attempt {attempt + 1}]: {result.inserted_id}")
+                    return item
+                except PyMongoError as e:
+                    if attempt == 2:  # 最后一次尝试仍失败
+                        raise
+                    self.logger.warning(f"插入重试中 [attempt {attempt + 1}]: {e}")
+
+        except Exception as e:
+            self.crawler.stats.inc_value('mongodb/insert_failed')
+            self.logger.error(f"MongoDB操作最终失败: {e}")
+            raise ItemDiscard(f"MongoDB操作失败: {e}")
+
+    async def spider_closed(self):
+        """资源清理"""
+        if hasattr(self, 'client'):
+            self.client.close()
+            self.logger.info("MongoDB连接池已释放")